From da9f8798418288ca8b1bffec802e00b23f2cc9cd Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 08:01:21 -0500 Subject: [PATCH 001/154] updated merge --- .jenkins/lsu/env-clang-12.sh | 2 + .jenkins/lsu/env-clang-13.sh | 2 + .jenkins/lsu/env-clang-14.sh | 2 + .jenkins/lsu/env-clang-15.sh | 7 + .jenkins/lsu/env-gcc-10.sh | 2 + .jenkins/lsu/env-gcc-12.sh | 2 + .jenkins/lsu/env-gcc-13.sh | 2 + CMakeLists.txt | 20 + cmake/FindOpenShmemPmi.cmake | 65 ++ cmake/HPX_SetupOpenSHMEM.cmake | 1002 +++++++++++++++++ cmake/templates/HPXConfig.cmake.in | 5 + cmake/templates/hpxrun.py.in | 37 +- libs/core/CMakeLists.txt | 1 + libs/core/openshmem_base/CMakeLists.txt | 31 + .../openshmem_base/examples/CMakeLists.txt | 16 + .../include/hpx/openshmem_base/openshmem.hpp | 36 + .../openshmem_base/openshmem_environment.hpp | 126 +++ .../src/openshmem_environment.cpp | 378 +++++++ libs/core/openshmem_base/tests/CMakeLists.txt | 44 + .../tests/performance/CMakeLists.txt | 5 + .../tests/regressions/CMakeLists.txt | 5 + .../openshmem_base/tests/unit/CMakeLists.txt | 5 + libs/core/version/CMakeLists.txt | 7 + libs/core/version/src/version.cpp | 37 + libs/full/CMakeLists.txt | 1 + .../full/command_line_handling/CMakeLists.txt | 5 + .../src/command_line_handling.cpp | 15 + libs/full/parcelport_openshmem/CMakeLists.txt | 45 + .../examples/CMakeLists.txt | 18 + .../hpx/parcelport_openshmem/header.hpp | 150 +++ .../hpx/parcelport_openshmem/locality.hpp | 71 ++ .../hpx/parcelport_openshmem/receiver.hpp | 170 +++ .../receiver_connection.hpp | 269 +++++ .../hpx/parcelport_openshmem/sender.hpp | 164 +++ .../sender_connection.hpp | 325 ++++++ .../hpx/parcelport_openshmem/tag_provider.hpp | 61 + .../parcelport_openshmem/src/locality.cpp | 36 + .../src/parcelport_openshmem.cpp | 313 +++++ .../parcelport_openshmem/tests/CMakeLists.txt | 42 + .../tests/performance/CMakeLists.txt | 5 + .../tests/regressions/CMakeLists.txt | 5 + .../tests/unit/CMakeLists.txt | 5 + 42 files changed, 3516 insertions(+), 23 deletions(-) create mode 100644 cmake/FindOpenShmemPmi.cmake create mode 100644 cmake/HPX_SetupOpenSHMEM.cmake create mode 100644 libs/core/openshmem_base/CMakeLists.txt create mode 100644 libs/core/openshmem_base/examples/CMakeLists.txt create mode 100644 libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp create mode 100644 libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp create mode 100644 libs/core/openshmem_base/src/openshmem_environment.cpp create mode 100644 libs/core/openshmem_base/tests/CMakeLists.txt create mode 100644 libs/core/openshmem_base/tests/performance/CMakeLists.txt create mode 100644 libs/core/openshmem_base/tests/regressions/CMakeLists.txt create mode 100644 libs/core/openshmem_base/tests/unit/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/examples/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp create mode 100644 libs/full/parcelport_openshmem/src/locality.cpp create mode 100644 libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp create mode 100644 libs/full/parcelport_openshmem/tests/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index c9b5e4a5ee10..9cbbcc72cec6 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 76df09cfd93b..a679e9b4e135 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 144c17e7820d..98aefa4dfa97 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 1df2a9fd23b3..f9b4db596066 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -25,6 +25,13 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +<<<<<<< HEAD +======= +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +>>>>>>> a9eef08e69 (initial import) configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 9c4064825a0a..b33db50c9d5b 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 48a6887c1ba6..3e086927c117 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -24,6 +24,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 4a10458aa535..526916453018 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -24,6 +24,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" diff --git a/CMakeLists.txt b/CMakeLists.txt index 19bcb7181786..fd2ee0042711 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1228,6 +1228,26 @@ if(HPX_WITH_NETWORKING) ) endif() + hpx_option( + HPX_WITH_PARCELPORT_OPENSHMEM BOOL "Enable the OPENSHMEM based parcelport." + OFF CATEGORY "Parcelport" + ) + hpx_option( + HPX_WITH_FETCH_OPENSHMEM + BOOL + "Download an OpenSHMEM if one is not available" OFF + CATEGORY "Build Targets" + ADVANCED + ) + if(HPX_WITH_PARCELPORT_OPENSHMEM) + hpx_add_config_define(HPX_HAVE_PARCELPORT_OPENSHMEM) + hpx_option( + HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT STRING + "Define which conduit to use for the OpenSHMEM parcelport" "sos" + STRINGS "sos;ucx" + ) + endif() + hpx_option( HPX_WITH_PARCELPORT_TCP BOOL "Enable the TCP based parcelport." ON CATEGORY "Parcelport" diff --git a/cmake/FindOpenShmemPmi.cmake b/cmake/FindOpenShmemPmi.cmake new file mode 100644 index 000000000000..5f6814a509db --- /dev/null +++ b/cmake/FindOpenShmemPmi.cmake @@ -0,0 +1,65 @@ +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +find_package(PkgConfig QUIET) +# look for cray pmi... +pkg_check_modules(PC_PMI_CRAY QUIET cray-pmi) +# look for the rest if we couldn't find the cray package +if(NOT PC_PMI_CRAY_FOUND) + pkg_check_modules(PC_PMI QUIET pmi) +endif() + +find_path( + PMI_INCLUDE_DIR pmi2.h + HINTS ${PMI_ROOT} + ENV + PMI_ROOT + ${PMI_DIR} + ENV + PMI_DIR + ${PC_PMI_CRAY_INCLUDEDIR} + ${PC_PMI_CRAY_INCLUDE_DIRS} + ${PC_PMI_INCLUDEDIR} + ${PC_PMI_INCLUDE_DIRS} + PATH_SUFFIXES include +) + +find_library( + PMI_LIBRARY + NAMES pmi + HINTS ${PMI_ROOT} + ENV + PMI_ROOT + ${PC_PMI_CRAY_LIBDIR} + ${PC_PMI_CRAY_LIBRARY_DIRS} + ${PC_PMI_LIBDIR} + ${PC_PMI_LIBRARY_DIRS} + PATH_SUFFIXES lib lib64 +) + +# Set PMI_ROOT in case the other hints are used +if(PMI_ROOT) + # The call to file is for compatibility with windows paths + file(TO_CMAKE_PATH ${PMI_ROOT} PMI_ROOT) +elseif("$ENV{PMI_ROOT}") + file(TO_CMAKE_PATH $ENV{PMI_ROOT} PMI_ROOT) +else() + file(TO_CMAKE_PATH "${PMI_INCLUDE_DIR}" PMI_INCLUDE_DIR) + string(REPLACE "/include" "" PMI_ROOT "${PMI_INCLUDE_DIR}") +endif() + +if(NOT PMI_LIBRARY OR NOT PMI_INCLUDE_DIR) + set(PMI_FOUND=OFF) + return() +endif() + +# hpx_error( "PMI_LIBRARY OR PMI_INCLUDE_DIR not found, please install PMI or +# set \ the right PMI_ROOT path" ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(PMI DEFAULT_MSG PMI_LIBRARY PMI_INCLUDE_DIR) + +mark_as_advanced(PMI_ROOT PMI_LIBRARY PMI_INCLUDE_DIR) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake new file mode 100644 index 000000000000..9c603343ef33 --- /dev/null +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -0,0 +1,1002 @@ +# Copyright (c) 2019-2023 Ste||ar Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) + +macro(hpx_setup_openshmem) + + if(NOT TARGET PkgConfig::OPENSHMEM) + + set(OPENSHMEM_PC "") + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + set(OPENSHMEM_PC "ucx") + + pkg_search_module(UCX IMPORTED_TARGET GLOBAL ucx) + + if(NOT UCX_FOUND) + message( + FATAL + "HPX_WITH_PARCELPORT_OPENSHMEM=ucx selected but UCX is unavailable" + ) + endif() + + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") + set(OPENSHMEM_PC + "sandia-openshmem"") + + pkg_search_module( + OPENSHMEM IMPORTED_TARGET GLOBAL + sandia-openshmem + ) + endif() + endif() + + if(NOT OPENSHMEM_FOUND) + + find_program(MAKE_EXECUTABLE NAMES gmake make mingw32-make REQUIRED) + + include(FindOpenShmemPmi) + + set(PMI_AUTOCONF_OPTS) + if(NOT PMI_LIBRARY OR NOT PMI_FOUND) + set(PMI_AUTOCONF_OPTS --enable-pmi-simple) + else() + set(PMI_AUTOCONF_OPTS "--with-pmi=${PMI_INCLUDE_DIR} + --with-pmi-libdir=${PMI_LIBRARY}") + endif() + + include(FetchContent) + + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + + message(STATUS "Fetching + OSSS-UCX-OpenSHMEM") + + fetchcontent_declare( + openshmem + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz + ) + + message(STATUS "Building + OSSS-UCX + (OpenSHMEM on UCX) + and + installing + into + ${CMAKE_INSTALL_PREFIX}") + + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") + + message(STATUS "Fetching + Sandia-OpenSHMEM") + + fetchcontent_declare( + openshmem + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://github.com/Sandia-OpenSHMEM/SOS/archive/refs/tags/v1.5.2.tar.gz + ) + + message(STATUS "Building + and + installing + Sandia + OpenSHMEM + into + ${CMAKE_INSTALL_PREFIX}") + + else() + message(FATAL_ERROR "HPX_WITH_PARCELPORT_OPENSHMEM + is + not + set + to + `ucx` + or + `sos`") + endif() + + fetchcontent_getproperties(openshmem) + if(NOT openshmem) + fetchcontent_populate(openshmem) + endif() + + set(CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + + set(OPENSHMEM_DIR "${openshmem_SOURCE_DIR}") + set(OPENSHMEM_BUILD_OUTPUT "${OPENSHMEM_DIR}/build.log") + set(OPENSHMEM_ERROR_FILE "${OPENSHMEM_DIR}/error.log") + + execute_process( + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} + ./autogen.sh + && + CC=${CMAKE_C_COMPILER} + ./configure + --prefix=${OPENSHMEM_DIR}/install + --enable-shared + ${PMI_AUTOCONF_OPTS} + && + make + && + make + install" + WORKING_DIRECTORY ${OPENSHMEM_DIR} + RESULT_VARIABLE OPENSHMEM_BUILD_STATUS + OUTPUT_FILE ${OPENSHMEM_BUILD_OUTPUT} + ERROR_FILE ${OPENSHMEM_ERROR_FILE} + ) + + if(OPENSHMEM_BUILD_STATUS) + message(FATAL_ERROR "OpenSHMEM + build + result + = + ${OPENSHMEM_SRC_BUILD_STATUS} + - + see + ${OPENSHMEM_SRC_BUILD_OUTPUT} + for + more + details") + else() + + find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND + ${OPENSHMEM_PC} + ${OPENSHMEM_DIR}/install/lib/pkgconfig + ) + + if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) + message( + FATAL_ERROR + "PKG-CONFIG + ERROR + (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) + -> + CANNOT + FIND + COMPILED + OpenSHMEM: + ${OPENSHMEMT_DIR}/install/lib/pkgconfig" + ) + endif() + + install(CODE "set + (OPENSHMEM_PATH \"${OPENSHMEM_DIR}\") + ") + + install( + CODE [[ + file( + READ + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PC} + OPENSHMEM_PKGCONFIG_FILE_CONTENT + ) + + if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) + message(FATAL_ERROR "ERROR + INSTALLING + OPENSHMEM") + endif() + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_PKGCONFIG_FILE_CONTENT + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} + ) + + file( + WRITE + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PC} + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} + ) + + file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) + + if(NOT OPENSHMEM_FILES) + message(STATUS "ERROR + INSTALLING + OPENSHMEM") + endif() + + foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) + set(OPENSHMEM_FILE_CACHED "${OPENSHMEM_FILE}") + + string(REGEX MATCH " + (^ \/.*\/) + " OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE}) + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE ${OPENSHMEM_FILE} + ) + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE_PATH} + ) + + file(MAKE_DIRECTORY ${OPENSHMEM_FILE_PATH}) + + string(LENGTH ${OPENSHMEM_FILE_PATH} OPENSHMEM_FILE_PATH_SIZE) + math(EXPR OPENSHMEM_FILE_PATH_SIZE "${OPENSHMEM_FILE_PATH_SIZE}-1") + + string(SUBSTRING ${OPENSHMEM_FILE_PATH} 0 ${OPENSHMEM_FILE_PATH_SIZE} + OPENSHMEM_FILE_PATH + ) + + file(COPY ${OPENSHMEM_FILE_CACHED} DESTINATION ${OPENSHMEM_FILE_PATH}) + endforeach() + ]] + ) + + # install(FILES ${OPENSHMEM_FILES} DESTINATION ${CMAKE_INSTALL_PREFIX}) + endif() + + set(CMAKE_PREFIX_PATH "${OPENSHMEM_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") + + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + pkg_search_module( + OPENSHMEM IMPORTED_TARGET GLOBAL + osss-ucx + ) + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") + pkg_search_module( + OPENSHMEM IMPORTED_TARGET GLOBAL + sandia-openshmem + ) + endif() + + if(NOT OPENSHMEM_FOUND) + message(FATAL_ERROR "OpenSHMEM + downloaded, + compiled, + but + cannot + be + found + in + ${CMAKE_INSTALL_PREFIX}") + endif() + + endif() + + if(OPENSHMEM_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS "${X}") + endforeach() + endif() + + if(OPENSHMEM_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(OPENSHMEM_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(SKIP 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + endif() + + if(OPENSHMEM_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS "${X}") + endforeach() + endif() + + if(OPENSHMEM_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(OPENSHMEM_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} + ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_DIR) + list(TRANSFORM OPENSHMEM_CFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LDFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LIBRARY_DIRS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + else() + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS} + " + ) + endif() + + endif() +endmacro() diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 5c2c7260f96e..a0dba9ef2e95 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -52,6 +52,11 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) endif() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() +endif() + if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) include(HPX_SetupGasnet) hpx_setup_gasnet() diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 3bc50eacd078..d48403112373 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -195,19 +195,13 @@ def run_mpi(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) -def run_gasnet(cmd, localities, verbose): - os.environ['GASNET_PSHM_NODES'] = str(localities) - os.environ['GASNET_QUIET'] = 'yes' - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - run_mpi(cmd, localities, verbose) - -def run_gasnet_smp(cmd, localities, verbose): - os.environ['GASNET_PSHM_NODES'] = str(localities) - os.environ['GASNET_BARRIER'] = 'PSHM' - os.environ['GASNET_QUIET'] = 'yes' - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - # No launcher needed - run_none(cmd, localities, nodes, verbose) +# Run with oshrun +# This is executing amudprun with the "-np" option set to the number of localities +def run_openshmem(cmd, localities, verbose): + exec_cmd = ['oshrun', '-np', str(localities)] + cmd + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) # Run with srun # This is executing srun with the '-n' option set to the number of localities @@ -232,12 +226,9 @@ def run(cmd, runwrapper, localities, nodes, verbose): if runwrapper == 'mpi': assert nodes is None, "nodes option only valid with tcp parcelport." run_mpi(cmd, localities, verbose) - if runwrapper == 'gasnet': + if runwrapper == 'openshmem': assert nodes is None, "nodes option only valid with tcp parcelport." - run_gasnet(cmd, localities, verbose) - if runwrapper == 'gasnet-smp': - assert nodes is None, "nodes option only valid with smp parcelport." - run_gasnet_smp(cmd, localities, verbose) + run_openshmem(cmd, localities, verbose) if runwrapper == 'srun': assert nodes is None, "nodes option only valid with tcp parcelport." run_srun(cmd, localities, verbose) @@ -259,7 +250,7 @@ def build_cmd(options, args): select_parcelport = (lambda pp: ['--hpx:ini=hpx.parcel.mpi.priority=1000', '--hpx:ini=hpx.parcel.mpi.enable=1', '--hpx:ini=hpx.parcel.bootstrap=mpi'] if pp == 'mpi' else ['--hpx:ini=hpx.parcel.lci.priority=1000', '--hpx:ini=hpx.parcel.lci.enable=1', '--hpx:ini=hpx.parcel.bootstrap=lci'] if pp == 'lci' - else ['--hpx:ini=hpx.parcel.gasnet.priority=1000', '--hpx:ini=hpx.parcel.gasnet.enable=1', '--hpx:ini=hpx.parcel.bootstrap=gasnet'] if pp == 'gasnet' + else ['--hpx:ini=hpx.parcel.gasnet.priority=1000', '--hpx:ini=hpx.parcel.openshmem.enable=1', '--hpx:ini=hpx.parcel.bootstrap=openshmem'] if pp == 'openshmem' else ['--hpx:ini=hpx.parcel.tcp.priority=1000', '--hpx:ini=hpx.parcel.tcp.enable=1'] if pp == 'tcp' else []) cmd += select_parcelport(options.parcelport) @@ -296,14 +287,14 @@ def check_options(parser, options, args): print('Can not start less than one thread per locality', sys.stderr) sys.exit(1) - check_valid_parcelport = (lambda x: x == 'mpi' or x == 'lci' or x == 'gasnet' or x == 'tcp' or x == 'none'); + check_valid_parcelport = (lambda x: x == 'mpi' or x == 'lci' or x == 'openshmem' or x == 'tcp' or x == 'none'); if not check_valid_parcelport(options.parcelport): print('Error: Parcelport option not valid\n', sys.stderr) parser.print_help() sys.exit(1) check_valid_runwrapper = (lambda x: - x == 'none' or x == 'mpi' or x == 'srun' or x =='jsrun' or x == 'gasnet' or x == 'gasnet-smp'); + x == 'none' or x == 'mpi' or x == 'srun' or x =='jsrun' or x == 'openshmem'); if not check_valid_runwrapper(options.runwrapper): print('Error: Runwrapper option not valid\n', sys.stderr) parser.print_help() @@ -382,13 +373,13 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') - , help='Which parcelport to use (Options are: mpi, lci, gasnet, tcp) ' + , help='Which parcelport to use (Options are: mpi, lci, openshmem, tcp) ' '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' , action='store', type='string' , dest='runwrapper', default=default_env('HPXRUN_RUNWRAPPER', 'none') - , help='Which runwrapper to use (Options are: none, mpi, gasnet, gasnet-smp, srun, jsrun) ' + , help='Which runwrapper to use (Options are: none, mpi, openshmem, srun, jsrun) ' '(environment variable HPXRUN_RUNWRAPPER)') parser.add_option('-e', '--expected' diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index 3e6550e93f25..a55d1eaac8e9 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -56,6 +56,7 @@ set(_hpx_core_modules logging memory mpi_base + openshmem_base pack_traversal plugin prefix diff --git a/libs/core/openshmem_base/CMakeLists.txt b/libs/core/openshmem_base/CMakeLists.txt new file mode 100644 index 000000000000..c6164fcfa68e --- /dev/null +++ b/libs/core/openshmem_base/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM)) + return() +endif() + +include(HPX_SetupOpenSHMEM) +hpx_setup_openshmem() + +# Default location is $HPX_ROOT/libs/openshmem_base/include +set(openshmem_base_headers hpx/openshmem_base/openshmem.hpp + hpx/openshmem_base/openshmem_environment.hpp +) + +set(openshmem_base_sources openshmem_environment.cpp) + +include(HPX_AddModule) +add_hpx_module( + core openshmem_base + GLOBAL_HEADER_GEN ON + SOURCES ${openshmem_base_sources} + HEADERS ${openshmem_base_headers} + DEPENDENCIES PkgConfig::OPENSHMEM + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_string_util + hpx_threading_base hpx_util + CMAKE_SUBDIRS examples tests +) diff --git a/libs/core/openshmem_base/examples/CMakeLists.txt b/libs/core/openshmem_base/examples/CMakeLists.txt new file mode 100644 index 000000000000..ff6189b08a6f --- /dev/null +++ b/libs/core/openshmem_base/examples/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(HPX_WITH_EXAMPLES) + add_hpx_pseudo_target(examples.modules.openshmem_base) + add_hpx_pseudo_dependencies(examples.modules examples.modules.openshmem_base) + if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) + add_hpx_pseudo_target(tests.examples.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.examples.modules tests.examples.modules.openshmem_base + ) + endif() +endif() diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp new file mode 100644 index 000000000000..e3f6bcb3a94f --- /dev/null +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -0,0 +1,36 @@ +// Copyright (c) 2023 Christoper Taylor +// Copyright (c) 2017 Mikael Simberg +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifdef __cplusplus +} // extern "C" +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +#endif diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp new file mode 100644 index 000000000000..af57d81b0fe4 --- /dev/null +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -0,0 +1,126 @@ +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#if !defined(OPENSHMEM_PAGESIZE) +// GASNet default pagesize is 16384 +#define OPENSHMEM_PER_RANK_PAGESIZE 16384 +#endif + +namespace hpx { namespace util { + + struct openshmem_seginfo_t + { + std::uint8_t* addr; + std::uint8_t* size; + }; + + struct HPX_CORE_EXPORT openshmem_environment + { + static bool check_openshmem_environment( + runtime_configuration const& cfg); + + static int init(int* argc, char*** argv, int& provided); + static void init(int* argc, char*** argv, runtime_configuration& cfg); + static void finalize(); + + static bool enabled(); + static bool multi_threaded(); + static bool has_called_init(); + + static int rank(); + static int size(); + + static std::string get_processor_name(); + + static void put_signal(const std::uint8_t* addr, const int rank, + std::uint8_t* raddr, const std::size_t size, unsigned int* sigaddr); + + static void wait_until(const unsigned int value, unsigned int* sigaddr); + + static void get(std::uint8_t* addr, const int rank, + const std::uint8_t* raddr, const std::size_t size); + + static void global_barrier(); + + struct HPX_CORE_EXPORT scoped_lock + { + scoped_lock(); + scoped_lock(scoped_lock const&) = delete; + scoped_lock& operator=(scoped_lock const&) = delete; + ~scoped_lock(); + void unlock(); + }; + + struct HPX_CORE_EXPORT scoped_try_lock + { + scoped_try_lock(); + scoped_try_lock(scoped_try_lock const&) = delete; + scoped_try_lock& operator=(scoped_try_lock const&) = delete; + ~scoped_try_lock(); + void unlock(); + bool locked; + }; + + typedef hpx::spinlock mutex_type; + + public: + static hpx::spinlock pollingLock; + static hpx::mutex mtx_; + + static bool enabled_; + static bool has_called_init_; + static int provided_threading_flag_; + + static int is_initialized_; + + static hpx::mutex dshm_mut; + static int init_val_; + static hpx::mutex* segment_mutex; + static openshmem_seginfo_t* segments; + static std::uint8_t* shmem_buffer; + static unsigned int rcv; + static unsigned int xmt; + }; +}} // namespace hpx::util + +#include + +#else + +#include + +#include + +namespace hpx { namespace util { + struct HPX_CORE_EXPORT openshmem_environment + { + static bool check_openshmem_environment( + runtime_configuration const& cfg); + }; +}} // namespace hpx::util + +#include + +#endif diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp new file mode 100644 index 000000000000..1274c201c487 --- /dev/null +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -0,0 +1,378 @@ +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2020 Google +// Copyright (c) 2022 Patrick Diehl +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +namespace hpx::util { + + namespace detail { + + bool detect_openshmem_environment( + util::runtime_configuration const& cfg, char const* default_env) + { + std::string openshmem_environment_strings = + cfg.get_entry("hpx.parcel.openshmem.env", default_env); + + hpx::string_util::char_separator sep(";,: "); + hpx::string_util::tokenizer tokens( + openshmem_environment_strings, sep); + for (auto const& tok : tokens) + { + char* env = std::getenv(tok.c_str()); + if (env) + { + LBT_(debug) + << "Found OPENSHMEM environment variable: " << tok + << "=" << std::string(env) + << ", enabling OPENSHMEM support\n"; + return true; + } + } + + LBT_(info) + << "No known OPENSHMEM environment variable found, disabling " + "OPENSHMEM support\n"; + return false; + } + } // namespace detail + + bool openshmem_environment::check_openshmem_environment( + util::runtime_configuration const& cfg) + { +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + // We disable the OPENSHMEM parcelport if any of these hold: + // + // - The parcelport is explicitly disabled + // - The application is not run in an OPENSHMEM environment + // - The TCP parcelport is enabled and has higher priority + // + if (get_entry_as(cfg, "hpx.parcel.openshmem.enable", 1) == 0 || + (get_entry_as(cfg, "hpx.parcel.tcp.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.tcp.priority", 1) > + get_entry_as(cfg, "hpx.parcel.openshmem.priority", 0))) || + (get_entry_as(cfg, "hpx.parcel.openshmem.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.openshmem.priority", 1) > + get_entry_as(cfg, "hpx.parcel.mpi.priority", 0)))) + { + LBT_(info) + << "OpenSHMEM support disabled via configuration settings\n"; + return false; + } + + return true; +#else + return false; +#endif + } +} // namespace hpx::util + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_OPENSHMEM_BASE)) + +namespace hpx::util { + + hpx::spinlock openshmem_environment::pollingLock{}; + hpx::mutex openshmem_environment::dshm_mut{}; + hpx::mutex openshmem_environment::mtx_{}; + bool openshmem_environment::enabled_ = false; + bool openshmem_environment::has_called_init_ = false; + int openshmem_environment::provided_threading_flag_ = 0; + int openshmem_environment::is_initialized_ = -1; + int openshmem_environment::init_val_ = 0; + hpx::mutex* openshmem_environment::segment_mutex = nullptr; + openshmem_seginfo_t* openshmem_environment::segments = nullptr; + std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; + unsigned int openshmem_environment::rcv = 0; + unsigned int openshmem_environment::xmt = 0; + + /////////////////////////////////////////////////////////////////////////// + int openshmem_environment::init(int* argc, char*** argv, int& provided) + { + if (!has_called_init_) + { + shmem_init(); + openshmem_environment::init_val_ = 1; + has_called_init_ = true; + } + + if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM initialization error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM resource error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM bad argument error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM not ready error"); + } + + // create a number of segments equal to the number of hardware + // threads per machine (locality) + // + //segments.resize(hpx::threads::hardware_concurrency() * size()); + // + openshmem_environment::segments = new openshmem_seginfo_t[size()]; + openshmem_environment::segment_mutex = new hpx::mutex[size()]; + + hpx::util::openshmem_environment::shmem_buffer = + static_cast(shmem_calloc( + OPENSHMEM_PER_RANK_PAGESIZE, sizeof(std::uint8_t))); + + for (int i = 0; i < size(); ++i) + { + segments[i].addr = hpx::util::openshmem_environment::shmem_buffer + + (i * OPENSHMEM_PER_RANK_PAGESIZE); + segments[i].size = static_cast(segments[i].addr) + + OPENSHMEM_PER_RANK_PAGESIZE; + } + + shmem_barrier_all(); + + return openshmem_environment::init_val_; + } + + /////////////////////////////////////////////////////////////////////////// + void openshmem_environment::init( + int* argc, char*** argv, util::runtime_configuration& rtcfg) + { + if (enabled_) + return; // don't call twice + + int this_rank = -1; + has_called_init_ = false; + + // We assume to use the OpenSHMEM parcelport if it is not explicitly disabled + enabled_ = check_openshmem_environment(rtcfg); + if (!enabled_) + { + rtcfg.add_entry("hpx.parcel.openshmem.enable", "0"); + return; + } + + rtcfg.add_entry("hpx.parcel.bootstrap", "openshmem"); + + int retval = init(argc, argv, provided_threading_flag_); + if (1 != retval) + { + // explicitly disable openshmem if not run by openshmemrun + rtcfg.add_entry("hpx.parcel.openshmem.enable", "0"); + + enabled_ = false; + + std::string msg( + "openshmem_environment::init: openshmem_init failed"); + throw std::runtime_error(msg.c_str()); + } + + if (provided_threading_flag_ != 1) + { + // explicitly disable openshmem if not run by openshmemrun + rtcfg.add_entry("hpx.parcel.openshmem.multithreaded", "0"); + } + + this_rank = rank(); + +#if defined(HPX_HAVE_NETWORKING) + if (this_rank == 0) + { + rtcfg.mode_ = hpx::runtime_mode::console; + } + else + { + rtcfg.mode_ = hpx::runtime_mode::worker; + } +#elif defined(HPX_HAVE_DISTRIBUTED_RUNTIME) + rtcfg.mode_ = hpx::runtime_mode::console; +#else + rtcfg.mode_ = hpx::runtime_mode::local; +#endif + + rtcfg.add_entry("hpx.parcel.openshmem.rank", std::to_string(this_rank)); + rtcfg.add_entry( + "hpx.parcel.openshmem.processorname", get_processor_name()); + } + + std::string openshmem_environment::get_processor_name() + { + char name[1024 + 1] = {'\0'}; + const std::string rnkstr = std::to_string(rank()); + const int len = rnkstr.size(); + if (1025 < len) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::get_processor_name", + "openshmem processor name is larger than 1025"); + } + std::copy(std::begin(rnkstr), std::end(rnkstr), name); + return name; + } + + void openshmem_environment::put_signal(const std::uint8_t* addr, + const int node, std::uint8_t* raddr, const std::size_t size, + unsigned int* sigaddr) + { + if (rank() == node) + { + const std::lock_guard lk(segment_mutex[node]); + std::memmove(raddr, addr, size); + } + else + { + const std::lock_guard lk(segment_mutex[node]); + shmem_uint8_put_signal(raddr, addr, size, + reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, + node); + } + } + + void openshmem_environment::wait_until( + const unsigned int value, unsigned int* sigaddr) + { + shmem_uint_wait_until(sigaddr, SHMEM_CMP_EQ, value); + } + + void openshmem_environment::get(std::uint8_t* addr, const int node, + const std::uint8_t* raddr, const std::size_t size) + { + if (rank() == node) + { + std::memmove(addr, raddr, size); + } + else + { + shmem_uint8_get( + addr, raddr, size, node); // dest, node, src, size + } + } + + void openshmem_environment::global_barrier() + { + shmem_barrier_all(); + } + + void openshmem_environment::finalize() + { + if (enabled() && has_called_init()) + { + shmem_finalize(); + + delete segments; + delete segment_mutex; + shmem_free(hpx::util::openshmem_environment::shmem_buffer); + segments = nullptr; + segment_mutex = nullptr; + shmem_buffer = nullptr; + } + } + + bool openshmem_environment::enabled() + { + return enabled_; + } + + bool openshmem_environment::multi_threaded() + { + return provided_threading_flag_ != 0; + } + + bool openshmem_environment::has_called_init() + { + return has_called_init_; + } + + int openshmem_environment::size() + { + int res(-1); + if (enabled()) + res = static_cast(shmem_n_pes()); + return res; + } + + int openshmem_environment::rank() + { + int res(-1); + if (enabled()) + res = static_cast(shmem_my_pe()); + return res; + } + + openshmem_environment::scoped_lock::scoped_lock() + { + if (!multi_threaded()) + mtx_.lock(); + } + + openshmem_environment::scoped_lock::~scoped_lock() + { + if (!multi_threaded()) + mtx_.unlock(); + } + + void openshmem_environment::scoped_lock::unlock() + { + if (!multi_threaded()) + mtx_.unlock(); + } + + openshmem_environment::scoped_try_lock::scoped_try_lock() + : locked(true) + { + if (!multi_threaded()) + { + locked = mtx_.try_lock(); + } + } + + openshmem_environment::scoped_try_lock::~scoped_try_lock() + { + if (!multi_threaded() && locked) + mtx_.unlock(); + } + + void openshmem_environment::scoped_try_lock::unlock() + { + if (!multi_threaded() && locked) + { + locked = false; + mtx_.unlock(); + } + } +} // namespace hpx::util + +#endif diff --git a/libs/core/openshmem_base/tests/CMakeLists.txt b/libs/core/openshmem_base/tests/CMakeLists.txt new file mode 100644 index 000000000000..76ce5efbb376 --- /dev/null +++ b/libs/core/openshmem_base/tests/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) +include(HPX_Option) + +if(HPX_WITH_TESTS) + if(HPX_WITH_TESTS_UNIT) + add_hpx_pseudo_target(tests.unit.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.openshmem_base + ) + add_subdirectory(unit) + endif() + + if(HPX_WITH_TESTS_REGRESSIONS) + add_hpx_pseudo_target(tests.regressions.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.openshmem_base + ) + add_subdirectory(regressions) + endif() + + if(HPX_WITH_TESTS_BENCHMARKS) + add_hpx_pseudo_target(tests.performance.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.openshmem_base + ) + add_subdirectory(performance) + endif() + + if(HPX_WITH_TESTS_HEADERS) + add_hpx_header_tests( + modules.openshmem_base + HEADERS ${openshmem_base_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + NOLIBS + DEPENDENCIES hpx_openshmem_base + ) + endif() +endif() diff --git a/libs/core/openshmem_base/tests/performance/CMakeLists.txt b/libs/core/openshmem_base/tests/performance/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/performance/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/openshmem_base/tests/regressions/CMakeLists.txt b/libs/core/openshmem_base/tests/regressions/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/regressions/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/openshmem_base/tests/unit/CMakeLists.txt b/libs/core/openshmem_base/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/unit/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index 5a82861e5ec2..f0886e68d6e4 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -20,6 +20,13 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() +if(HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM +) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index c54c21e54919..634b57e1d0f0 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -46,6 +46,14 @@ #include #endif +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) +#include + +#define OPENSHMEM_CONDUIT_NAME_STR_HELPER(x) #x +#define OPENSHMEM_CONDUIT_NAME_STR \ + OPENSHMEM_CONDUIT_NAME_STR_HELPER(HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT) +#endif + #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) #include @@ -136,6 +144,25 @@ namespace hpx { } #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + std::string openshmem_version() + { + int major = 0; + int minor = 0; + shmem_info_get_version(&major, &minor); + char vendor_cstr[SHMEM_MAX_NAME_LEN]; + shmem_info_get_name(vendor_cstr); + + std::ostringstream strm; + strm << "OPENSHMEM_VENDOR:" << vendor_cstr << ':' << major << ':' + << minor << '-' + << "OPENSHMEM_CONDUIT:" << OPENSHMEM_CONDUIT_NAME_STR; + return strm.str(); + } +#endif + #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) std::string gasnet_version() @@ -312,6 +339,11 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) " LCI: {}\n" #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + " OPENSHMEM: {}\n" +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) " GASNET: {}\n" @@ -332,6 +364,11 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) lci_version(), #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + openshmem_version(), +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) gasnet_version(), diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index e4a7eb24b923..d4f1b25c0e69 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -35,6 +35,7 @@ set(_hpx_full_modules parcelport_lci parcelport_libfabric parcelport_mpi + parcelport_openshmem parcelport_tcp parcelports parcelset diff --git a/libs/full/command_line_handling/CMakeLists.txt b/libs/full/command_line_handling/CMakeLists.txt index 966ba221406e..cf4cd5d16212 100644 --- a/libs/full/command_line_handling/CMakeLists.txt +++ b/libs/full/command_line_handling/CMakeLists.txt @@ -34,6 +34,11 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() +if(HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_PARCELPORT_GASNET) include(HPX_SetupGasnet) hpx_setup_gasnet() diff --git a/libs/full/command_line_handling/src/command_line_handling.cpp b/libs/full/command_line_handling/src/command_line_handling.cpp index b8b7abbf5743..23d5b914b5f4 100644 --- a/libs/full/command_line_handling/src/command_line_handling.cpp +++ b/libs/full/command_line_handling/src/command_line_handling.cpp @@ -21,6 +21,9 @@ #if defined(HPX_HAVE_MODULE_LCI_BASE) #include #endif +#if defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) +#include +#endif #if defined(HPX_HAVE_MODULE_GASNET_BASE) #include #endif @@ -1006,6 +1009,18 @@ namespace hpx::util { node_ = static_cast(util::lci_environment::rank()); } #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + if (util::openshmem_environment::check_openshmem_environment(rtcfg_)) + { + util::openshmem_environment::init(&argc, &argv, rtcfg_); + num_localities_ = + static_cast(util::openshmem_environment::size()); + node_ = + static_cast(util::openshmem_environment::rank()); + } +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) // better to put GASNET init after MPI init, since GASNET will also diff --git a/libs/full/parcelport_openshmem/CMakeLists.txt b/libs/full/parcelport_openshmem/CMakeLists.txt new file mode 100644 index 000000000000..78575d783503 --- /dev/null +++ b/libs/full/parcelport_openshmem/CMakeLists.txt @@ -0,0 +1,45 @@ +# Copyright (c) 2023 Chistopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT + (HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM + AND HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT) +) + return() +endif() + +include(HPX_SetupOpenSHMEM) +hpx_setup_openshmem() + +set(parcelport_openshmem_headers + hpx/parcelport_openshmem/header.hpp + hpx/parcelport_openshmem/locality.hpp + hpx/parcelport_openshmem/receiver.hpp + hpx/parcelport_openshmem/receiver_connection.hpp + hpx/parcelport_openshmem/sender.hpp + hpx/parcelport_openshmem/sender_connection.hpp + hpx/parcelport_openshmem/tag_provider.hpp +) + +set(parcelport_openshmem_sources locality.cpp parcelport_openshmem.cpp) + +include(HPX_AddModule) +add_hpx_module( + full parcelport_openshmem + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_openshmem_sources} + HEADERS ${parcelport_openshmem_headers} + DEPENDENCIES hpx_core hpx_openshmem_base PkgConfig::OPENSHMEM + ${openshmem_additional_dependencies} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests +) + +set(HPX_STATIC_PARCELPORT_PLUGINS + ${HPX_STATIC_PARCELPORT_PLUGINS} parcelport_openshmem + CACHE INTERNAL "" FORCE +) diff --git a/libs/full/parcelport_openshmem/examples/CMakeLists.txt b/libs/full/parcelport_openshmem/examples/CMakeLists.txt new file mode 100644 index 000000000000..d14c5d1103c2 --- /dev/null +++ b/libs/full/parcelport_openshmem/examples/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(HPX_WITH_EXAMPLES) + add_hpx_pseudo_target(examples.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + examples.modules examples.modules.parcelport_openshmem + ) + if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) + add_hpx_pseudo_target(tests.examples.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.examples.modules tests.examples.modules.parcelport_openshmem + ) + endif() +endif() diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp new file mode 100644 index 000000000000..3712581d08e2 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp @@ -0,0 +1,150 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2013-2021 Hartmut Kaiser +// Copyright (c) 2013-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include + +#include + +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct header + { + using value_type = int; + + enum data_pos + { + pos_tag = 0 * sizeof(value_type), + pos_size = 1 * sizeof(value_type), + pos_numbytes = 2 * sizeof(value_type), + pos_numchunks_first = 3 * sizeof(value_type), + pos_numchunks_second = 4 * sizeof(value_type), + pos_piggy_back_flag = 5 * sizeof(value_type), + pos_piggy_back_data = 5 * sizeof(value_type) + 1 + }; + + static constexpr int data_size_ = 512; + + template + header(Buffer const& buffer, int tag) noexcept + { + std::int64_t size = static_cast(buffer.size_); + std::int64_t numbytes = + static_cast(buffer.data_size_); + + HPX_ASSERT(size <= (std::numeric_limits::max)()); + HPX_ASSERT(numbytes <= (std::numeric_limits::max)()); + + set(tag); + set(static_cast(size)); + set(static_cast(numbytes)); + set( + static_cast(buffer.num_chunks_.first)); + set( + static_cast(buffer.num_chunks_.second)); + + if (buffer.data_.size() <= (data_size_ - pos_piggy_back_data)) + { + data_[pos_piggy_back_flag] = 1; + std::memcpy(&data_[pos_piggy_back_data], &buffer.data_[0], + buffer.data_.size()); + } + else + { + data_[pos_piggy_back_flag] = 0; + } + } + + header() noexcept + { + reset(); + } + + void reset() noexcept + { + std::memset(&data_[0], -1, data_size_); + data_[pos_piggy_back_flag] = 1; + } + + bool valid() const noexcept + { + return data_[0] != -1; + } + + void assert_valid() const noexcept + { + HPX_ASSERT(tag() != -1); + HPX_ASSERT(size() != -1); + HPX_ASSERT(numbytes() != -1); + HPX_ASSERT(num_chunks().first != -1); + HPX_ASSERT(num_chunks().second != -1); + } + + constexpr char* data() noexcept + { + return &data_[0]; + } + + value_type tag() const noexcept + { + return get(); + } + + value_type size() const noexcept + { + return get(); + } + + value_type numbytes() const noexcept + { + return get(); + } + + std::pair num_chunks() const noexcept + { + return std::make_pair( + get(), get()); + } + + constexpr char* piggy_back() noexcept + { + if (data_[pos_piggy_back_flag]) + return &data_[pos_piggy_back_data]; + return nullptr; + } + + private: + std::array data_; + + template + void set(T const& t) noexcept + { + std::memcpy(&data_[Pos], &t, sizeof(t)); + } + + template + value_type get() const noexcept + { + value_type res; + std::memcpy(&res, &data_[Pos], sizeof(res)); + return res; + } + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp new file mode 100644 index 000000000000..d0fe8b8eecf4 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp @@ -0,0 +1,71 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2013-2014 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include + +namespace hpx::parcelset::policies::openshmem { + + class locality + { + public: + constexpr locality() noexcept + : rank_(-1) + { + } + + explicit constexpr locality(std::int32_t rank) noexcept + : rank_(rank) + { + } + + constexpr std::int32_t rank() const noexcept + { + return static_cast(rank_); + } + + static constexpr const char* type() noexcept + { + return "openshmem"; + } + + explicit constexpr operator bool() const noexcept + { + return rank_ != -1; + } + + HPX_EXPORT void save(serialization::output_archive& ar) const; + HPX_EXPORT void load(serialization::input_archive& ar); + + private: + friend bool operator==( + locality const& lhs, locality const& rhs) noexcept + { + return lhs.rank_ == rhs.rank_; + } + + friend bool operator<(locality const& lhs, locality const& rhs) noexcept + { + return lhs.rank_ < rhs.rank_; + } + + friend HPX_EXPORT std::ostream& operator<<( + std::ostream& os, locality const& loc) noexcept; + + int rank_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp new file mode 100644 index 000000000000..2c828f54e643 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -0,0 +1,170 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + template + struct receiver + { + using header_list = std::list>; + using handles_header_type = std::set>; + using connection_type = receiver_connection; + using connection_ptr = std::shared_ptr; + using connection_list = std::deque; + + struct exp_backoff + { + int numTries; + const static int maxRetries = 10; + + void operator()(unsigned int* addr) + { + if (numTries <= maxRetries) + { + if (shmem_uint_test(addr, SHMEM_CMP_EQ, 1)) + { + return; + } + hpx::this_thread::suspend( + std::chrono::microseconds(1 << numTries)); + } + else + { + numTries = 0; + } + } + }; + + explicit constexpr receiver(Parcelport& pp) noexcept + : pp_(pp) + , bo() + { + } + + void run() noexcept + { + util::openshmem_environment::scoped_lock l; + new_header(); + } + + bool background_work() noexcept + { + // We first try to accept a new connection + connection_ptr connection = accept(); + + // If we don't have a new connection, try to handle one of the + // already accepted ones. + if (!connection) + { + std::unique_lock l(connections_mtx_, std::try_to_lock); + if (l.owns_lock() && !connections_.empty()) + { + connection = HPX_MOVE(connections_.front()); + connections_.pop_front(); + } + } + + if (connection) + { + receive_messages(HPX_MOVE(connection)); + return true; + } + + return false; + } + + void receive_messages(connection_ptr connection) noexcept + { + if (!connection->receive()) + { + std::unique_lock l(connections_mtx_); + connections_.push_back(HPX_MOVE(connection)); + } + } + + connection_ptr accept() noexcept + { + std::unique_lock l(headers_mtx_, std::try_to_lock); + if (l.owns_lock()) + { + return accept_locked(l); + } + return connection_ptr(); + } + + template + connection_ptr accept_locked(Lock& header_lock) noexcept + { + connection_ptr res; + util::openshmem_environment::scoped_try_lock l; + + if (l.locked) + { + header h = new_header(); + l.unlock(); + header_lock.unlock(); + + // remote localities 'put' into the openshmem shared + // memory segment on this machine + // + res.reset(new connection_type( + hpx::util::openshmem_environment::rank(), h, pp_)); + return res; + } + return res; + } + + header new_header() noexcept + { + header h = rcv_header_; + rcv_header_.reset(); + + while (rcv_header_.data() == 0) + { + bo(&hpx::util::openshmem_environment::rcv); + } + + hpx::util::openshmem_environment::rcv = 0; + return h; + } + + Parcelport& pp_; + + hpx::spinlock headers_mtx_; + header rcv_header_; + + hpx::spinlock handles_header_mtx_; + handles_header_type handles_header_; + + hpx::spinlock connections_mtx_; + connection_list connections_; + exp_backoff bo; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp new file mode 100644 index 000000000000..b2166090b437 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -0,0 +1,269 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2007-2021 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + template + struct receiver_connection + { + private: + enum connection_state + { + initialized, + rcvd_transmission_chunks, + rcvd_data, + rcvd_chunks, + sent_release_tag + }; + + using data_type = std::vector; + using buffer_type = parcel_buffer; + + public: + receiver_connection(int src, header h, Parcelport& pp) noexcept + : state_(initialized) + , src_(src) + , tag_(h.tag()) + , header_(h) + , request_ptr_(false) + , chunks_idx_(0) + , pp_(pp) + { + header_.assert_valid(); +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds(); + data.bytes_ = static_cast(header_.numbytes()); +#endif + buffer_.data_.resize(static_cast(header_.size())); + buffer_.num_chunks_ = header_.num_chunks(); + } + + bool receive(std::size_t num_thread = -1) + { + switch (state_) + { + case initialized: + return receive_transmission_chunks(num_thread); + + case rcvd_transmission_chunks: + return receive_data(num_thread); + + case rcvd_data: + return receive_chunks(num_thread); + + case rcvd_chunks: + return send_release_tag(num_thread); + + case sent_release_tag: + return done(); + + default: + HPX_ASSERT(false); + } + return false; + } + + bool receive_transmission_chunks(std::size_t num_thread = -1) + { + auto self_ = hpx::util::openshmem_environment::rank(); + + // determine the size of the chunk buffer + std::size_t num_zero_copy_chunks = static_cast( + static_cast(buffer_.num_chunks_.first)); + std::size_t num_non_zero_copy_chunks = static_cast( + static_cast(buffer_.num_chunks_.second)); + buffer_.transmission_chunks_.resize( + num_zero_copy_chunks + num_non_zero_copy_chunks); + if (num_zero_copy_chunks != 0) + { + buffer_.chunks_.resize(num_zero_copy_chunks); + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast( + buffer_.transmission_chunks_.data()), + self_, + hpx::util::openshmem_environment::segments[self_].addr, + static_cast(buffer_.transmission_chunks_.size() * + sizeof(buffer_type::transmission_chunk_type))); + + request_ptr_ = true; + } + } + + state_ = rcvd_transmission_chunks; + + return receive_data(num_thread); + } + + bool receive_data(std::size_t num_thread = -1) + { + if (!request_done()) + { + return false; + } + + char* piggy_back = header_.piggy_back(); + if (piggy_back) + { + std::memcpy( + &buffer_.data_[0], piggy_back, buffer_.data_.size()); + } + else + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(buffer_.data_.data()), + self_, + hpx::util::openshmem_environment::segments[self_].addr, + buffer_.data_.size()); + + request_ptr_ = true; + } + + state_ = rcvd_data; + + return receive_chunks(num_thread); + } + + bool receive_chunks(std::size_t num_thread = -1) + { + while (chunks_idx_ < buffer_.chunks_.size()) + { + if (!request_done()) + { + return false; + } + + std::size_t idx = chunks_idx_++; + std::size_t chunk_size = + buffer_.transmission_chunks_[idx].second; + + data_type& c = buffer_.chunks_[idx]; + c.resize(chunk_size); + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(c.data()), self_, + hpx::util::openshmem_environment::segments[self_].addr, + c.size()); + + hpx::util::openshmem_environment::put_signal(nullptr, src_, + nullptr, 0, &hpx::util::openshmem_environment::xmt); + + request_ptr_ = true; + } + } + + state_ = rcvd_chunks; + + return send_release_tag(num_thread); + } + + bool send_release_tag(std::size_t num_thread = -1) + { + if (!request_done()) + { + return false; + } +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds() - data.time_; +#endif + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(&tag_), self_, + hpx::util::openshmem_environment::segments[self_].addr, + sizeof(int)); + + request_ptr_ = true; + } + + decode_parcels(pp_, HPX_MOVE(buffer_), num_thread); + + state_ = sent_release_tag; + + return done(); + } + + bool done() noexcept + { + return request_done(); + } + + bool request_done() noexcept + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (!l.locked) + { + return false; + } + + return request_ptr_; + } + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + hpx::chrono::high_resolution_timer timer_; +#endif + connection_state state_; + + int src_; + int tag_; + header header_; + buffer_type buffer_; + + bool request_ptr_; + std::size_t chunks_idx_; + + Parcelport& pp_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp new file mode 100644 index 000000000000..b4a9bba0d479 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -0,0 +1,164 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct sender + { + using connection_type = sender_connection; + using connection_ptr = std::shared_ptr; + using connection_list = std::deque; + + // different versions of clang-format disagree + // clang-format off + sender() noexcept + : next_free_tag_request_(-1) + , next_free_tag_(-1) + { + } + // clang-format on + + void run() noexcept + { + hpx::util::openshmem_environment::scoped_lock l; + get_next_free_tag(); + } + + connection_ptr create_connection(int dest, parcelset::parcelport* pp) + { + return std::make_shared(this, dest, pp); + } + + void add(connection_ptr const& ptr) + { + std::unique_lock l(connections_mtx_); + connections_.push_back(ptr); + } + + int acquire_tag() noexcept + { + return tag_provider_.acquire(); + } + + void send_messages(connection_ptr connection) + { + // Check if sending has been completed.... + if (connection->send()) + { + error_code ec(throwmode::lightweight); + hpx::move_only_function + postprocess_handler; + std::swap( + postprocess_handler, connection->postprocess_handler_); + postprocess_handler(ec, connection->destination(), connection); + } + else + { + std::unique_lock l(connections_mtx_); + connections_.push_back(HPX_MOVE(connection)); + } + } + + bool background_work() noexcept + { + connection_ptr connection; + { + std::unique_lock l(connections_mtx_, std::try_to_lock); + if (l && !connections_.empty()) + { + connection = HPX_MOVE(connections_.front()); + connections_.pop_front(); + } + } + + bool has_work = false; + if (connection) + { + send_messages(HPX_MOVE(connection)); + has_work = true; + } + next_free_tag(); + return has_work; + } + + private: + tag_provider tag_provider_; + + void next_free_tag() noexcept + { + int next_free = -1; + { + std::unique_lock l(next_free_tag_mtx_, std::try_to_lock); + if (l.owns_lock()) + { + next_free = next_free_tag_locked(); + } + } + + if (next_free != -1) + { + HPX_ASSERT(next_free > 1); + tag_provider_.release(next_free); + } + } + + int next_free_tag_locked() noexcept + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (l.locked) + { + return get_next_free_tag(); + } + return -1; + } + + int get_next_free_tag() noexcept + { + int next_free = next_free_tag_; + + hpx::util::openshmem_environment::scoped_lock l; + std::memcpy(&next_free, + hpx::util::openshmem_environment::segments + [hpx::util::openshmem_environment::rank()] + .addr, + sizeof(int)); + + return next_free; + } + + hpx::spinlock connections_mtx_; + connection_list connections_; + + hpx::spinlock next_free_tag_mtx_; + int next_free_tag_request_; + int next_free_tag_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp new file mode 100644 index 000000000000..eb1047405a61 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -0,0 +1,325 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct sender; + struct sender_connection; + + int acquire_tag(sender*) noexcept; + void add_connection(sender*, std::shared_ptr const&); + + struct sender_connection + : parcelset::parcelport_connection> + { + private: + using sender_type = sender; + + using write_handler_type = + hpx::function; + + using data_type = std::vector; + + enum connection_state + { + initialized, + sent_header, + sent_transmission_chunks, + sent_data, + sent_chunks + }; + + using base_type = + parcelset::parcelport_connection; + + public: + sender_connection(sender_type* s, int dst, parcelset::parcelport* pp) + : state_(initialized) + , sender_(s) + , tag_(-1) + , dst_(dst) + , chunks_idx_(0) + , ack_(0) + , pp_(pp) + , there_(parcelset::locality(locality(dst_))) + { + } + + parcelset::locality const& destination() const noexcept + { + return there_; + } + + constexpr void verify_( + parcelset::locality const& /* parcel_locality_id */) const noexcept + { + } + + template + void async_write( + Handler&& handler, ParcelPostprocess&& parcel_postprocess) + { + HPX_ASSERT(!handler_); + HPX_ASSERT(!postprocess_handler_); + HPX_ASSERT(!buffer_.data_.empty()); + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + buffer_.data_point_.time_ = + hpx::chrono::high_resolution_clock::now(); +#endif + chunks_idx_ = 0; + tag_ = acquire_tag(sender_); + header_ = header(buffer_, tag_); + header_.assert_valid(); + + state_ = initialized; + + handler_ = HPX_FORWARD(Handler, handler); + + if (!send()) + { + postprocess_handler_ = + HPX_FORWARD(ParcelPostprocess, parcel_postprocess); + add_connection(sender_, shared_from_this()); + } + else + { + HPX_ASSERT(!handler_); + error_code ec; + parcel_postprocess(ec, there_, shared_from_this()); + } + } + + bool send() + { + switch (state_) + { + case initialized: + return send_header(); + + case sent_header: + return send_transmission_chunks(); + + case sent_transmission_chunks: + return send_data(); + + case sent_data: + return send_chunks(); + + case sent_chunks: + return done(); + + default: + HPX_ASSERT(false); + } + return false; + } + + bool send_header() + { + { + hpx::util::openshmem_environment::scoped_lock l; + HPX_ASSERT(state_ == initialized); + + // compute + send the number of OPENSHMEM_PAGEs to send and the + // remainder number of bytes to a OPENSHMEM_PAGE + // + std::size_t chunks[] = {static_cast(header_.data_size_ / + OPENSHMEM_PER_RANK_PAGESIZE), + static_cast( + header_.data_size_ % OPENSHMEM_PER_RANK_PAGESIZE)}; + const std::size_t sizeof_chunks = sizeof(chunks); + + // put from this localities openshmem shared memory segment + // into the remote locality (dst_)'s shared memory segment + // + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(chunks), dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_].addr), + sizeof_chunks, &hpx::util::openshmem_environment::rcv); + } + + state_ = sent_header; + return send_transmission_chunks(); + } + + bool send_transmission_chunks() + { + HPX_ASSERT(state_ == sent_header); + if (!request_done()) + { + return false; + } + + std::vector& + chunks = buffer_.transmission_chunks_; + if (!chunks.empty()) + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(chunks.data()), dst_, + static_cast( + hpx::util::openshmem_environment::segments + [hpx::util::openshmem_environment::rank()] + .addr), + static_cast(chunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type)), + &hpx::util::openshmem_environment::rcv); + } + + state_ = sent_transmission_chunks; + return send_data(); + } + + bool send_data() + { + HPX_ASSERT(state_ == sent_transmission_chunks); + if (!request_done()) + { + return false; + } + + if (!header_.piggy_back()) + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(buffer_.data_.data()), dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_].addr), + buffer_.data_.size(), + &hpx::util::openshmem_environment::rcv); + } + state_ = sent_data; + + return send_chunks(); + } + + bool send_chunks() + { + HPX_ASSERT(state_ == sent_data); + + while (chunks_idx_ < buffer_.chunks_.size()) + { + serialization::serialization_chunk& c = + buffer_.chunks_[chunks_idx_]; + if (c.type_ == serialization::chunk_type::chunk_type_pointer) + { + if (!request_done()) + { + return false; + } + + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(c.data_.cpos_), + dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_] + .addr), + static_cast(c.size_), + &hpx::util::openshmem_environment::rcv); + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::xmt); + hpx::util::openshmem_environment::xmt = 0; + } + + ++chunks_idx_; + } + + state_ = sent_chunks; + + return done(); + } + + bool done() + { + if (!request_done()) + { + return false; + } + + error_code ec(throwmode::lightweight); + handler_(ec); + handler_.reset(); +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + buffer_.data_point_.time_ = + hpx::chrono::high_resolution_clock::now() - + buffer_.data_point_.time_; + pp_->add_sent_data(buffer_.data_point_); +#endif + buffer_.clear(); + + state_ = initialized; + + return true; + } + + bool request_done() + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (!l.locked) + { + return false; + } + + return true; + } + + connection_state state_; + sender_type* sender_; + int tag_; + int dst_; + + using handler_type = hpx::move_only_function; + handler_type handler_; + + using post_handler_type = hpx::move_only_function)>; + post_handler_type postprocess_handler_; + + header header_; + + std::size_t chunks_idx_; + char ack_; + + parcelset::parcelport* pp_; + + parcelset::locality there_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp new file mode 100644 index 000000000000..6e75ac5e5482 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp @@ -0,0 +1,61 @@ +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct tag_provider + { + tag_provider() noexcept + : next_tag_(2) + { + } + + int acquire() noexcept + { + int tag = -1; + std::lock_guard l(mtx_); + if (free_tags_.empty()) + { + HPX_ASSERT(next_tag_ < (std::numeric_limits::max)()); + tag = next_tag_++; + } + else + { + tag = free_tags_.front(); + free_tags_.pop_front(); + } + HPX_ASSERT(tag > 1); + return tag; + } + + void release(int tag) + { + HPX_ASSERT(tag > 1); + std::lock_guard l(mtx_); + HPX_ASSERT(tag < next_tag_); + + free_tags_.push_back(tag); + } + + hpx::spinlock mtx_; + int next_tag_; + std::deque free_tags_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/src/locality.cpp b/libs/full/parcelport_openshmem/src/locality.cpp new file mode 100644 index 000000000000..e9e22c16acbe --- /dev/null +++ b/libs/full/parcelport_openshmem/src/locality.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2013-2014 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + void locality::save(serialization::output_archive& ar) const + { + ar << rank_; + } + + void locality::load(serialization::input_archive& ar) + { + ar >> rank_; + } + + std::ostream& operator<<(std::ostream& os, locality const& loc) noexcept + { + hpx::util::ios_flags_saver ifs(os); + os << loc.rank_; + return os; + } +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp new file mode 100644 index 000000000000..27b2f2de4a3e --- /dev/null +++ b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp @@ -0,0 +1,313 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2020 Google +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace hpx::parcelset { + + namespace policies::openshmem { + class HPX_EXPORT parcelport; + } // namespace policies::openshmem + + template <> + struct connection_handler_traits + { + using connection_type = policies::openshmem::sender_connection; + using send_early_parcel = std::true_type; + using do_background_work = std::true_type; + using send_immediate_parcels = std::false_type; + using is_connectionless = std::false_type; + + static constexpr const char* type() noexcept + { + return "openshmem"; + } + + static constexpr const char* pool_name() noexcept + { + return "parcel-pool-openshmem"; + } + + static constexpr const char* pool_name_postfix() noexcept + { + return "-openshmem"; + } + }; + + namespace policies::openshmem { + + int acquire_tag(sender* s) noexcept + { + return s->acquire_tag(); + } + + void add_connection( + sender* s, std::shared_ptr const& ptr) + { + s->add(ptr); + } + + class HPX_EXPORT parcelport : public parcelport_impl + { + using base_type = parcelport_impl; + + static parcelset::locality here() + { + return parcelset::locality( + locality(util::openshmem_environment::enabled() ? + util::openshmem_environment::rank() : + -1)); + } + + static std::size_t max_connections( + util::runtime_configuration const& ini) + { + return hpx::util::get_entry_as(ini, + "hpx.parcel.openshmem.max_connections", + HPX_PARCEL_MAX_CONNECTIONS); + } + + static std::size_t background_threads( + util::runtime_configuration const& ini) + { + /* + return hpx::util::get_entry_as(ini, + "hpx.parcel.openshmem.background_threads", + HPX_HAVE_PARCELPORT_OPENSHMEM_BACKGROUND_THREADS); + */ + return 1UL; + } + + public: + parcelport(util::runtime_configuration const& ini, + threads::policies::callback_notifier const& notifier) + : base_type(ini, here(), notifier) + , stopped_(false) + , receiver_(*this) + , background_threads_(background_threads(ini)) + { + } + + parcelport(parcelport const&) = delete; + parcelport(parcelport&&) = delete; + parcelport& operator=(parcelport const&) = delete; + parcelport& operator=(parcelport&&) = delete; + + ~parcelport() + { + util::openshmem_environment::finalize(); + } + + // Start the handling of connections. + bool do_run() + { + receiver_.run(); + sender_.run(); + + for (std::size_t i = 0; i != io_service_pool_.size(); ++i) + { + io_service_pool_.get_io_service(int(i)).post( + hpx::bind(&parcelport::io_service_work, this)); + } + return true; + } + + // Stop the handling of connections. + void do_stop() + { + while (do_background_work(0, parcelport_background_mode_all)) + { + if (threads::get_self_ptr()) + hpx::this_thread::suspend( + hpx::threads::thread_schedule_state::pending, + "openshmem::parcelport::do_stop"); + } + + bool expected = false; + if (stopped_.compare_exchange_strong(expected, true)) + { + stopped_ = true; + hpx::util::openshmem_environment::global_barrier(); + } + } + + /// Return the name of this locality + std::string get_locality_name() const override + { + return util::openshmem_environment::get_processor_name(); + } + + std::shared_ptr create_connection( + parcelset::locality const& l, error_code&) + { + int dest_rank = l.get().rank(); + return sender_.create_connection(dest_rank, this); + } + + parcelset::locality agas_locality( + util::runtime_configuration const&) const override + { + return parcelset::locality( + locality(util::openshmem_environment::enabled() ? 0 : -1)); + } + + parcelset::locality create_locality() const override + { + return parcelset::locality(locality()); + } + + bool background_work( + std::size_t num_thread, parcelport_background_mode mode) + { + if (stopped_.load(std::memory_order_acquire) || + num_thread >= background_threads_) + { + return false; + } + + bool has_work = false; + if (mode & parcelport_background_mode_send) + { + has_work = sender_.background_work(); + } + if (mode & parcelport_background_mode_receive) + { + has_work = receiver_.background_work() || has_work; + } + return has_work; + } + + private: + std::atomic stopped_; + + sender sender_; + receiver receiver_; + + void io_service_work() + { + std::size_t k = 0; + + // We only execute work on the IO service while HPX is starting + while (hpx::is_starting()) + { + bool has_work = sender_.background_work(); + has_work = receiver_.background_work() || has_work; + if (has_work) + { + k = 0; + } + else + { + ++k; + util::detail::yield_k(k, + "hpx::parcelset::policies::openshmem::parcelport::" + "io_service_work"); + } + } + } + + std::size_t background_threads_; + + void early_write_handler(std::error_code const& ec, parcel const& p) + { + if (ec) + { + // all errors during early parcel handling are fatal + std::exception_ptr exception = hpx::detail::get_exception( + hpx::exception(ec), "openshmem::early_write_handler", + __FILE__, __LINE__, + "error while handling early parcel: " + ec.message() + + "(" + std::to_string(ec.value()) + ")" + + parcelset::dump_parcel(p)); + + hpx::report_error(exception); + } + } + }; + } // namespace policies::openshmem +} // namespace hpx::parcelset + +namespace hpx::traits { + + // Inject additional configuration data into the factory registry for this + // type. This information ends up in the system wide configuration database + // under the plugin specific section: + // + // [hpx.parcel.openshmem] + // ... + // priority = 1 + // + template <> + struct plugin_config_data + { + static constexpr char const* priority() noexcept + { + return "1"; + } + + static void init( + int* argc, char*** argv, util::command_line_handling& cfg) + { + util::openshmem_environment::init(argc, argv, cfg.rtcfg_); + cfg.num_localities_ = + static_cast(util::openshmem_environment::size()); + cfg.node_ = + static_cast(util::openshmem_environment::rank()); + } + + // by default no additional initialization using the resource + // partitioner is required + static constexpr void init(hpx::resource::partitioner&) noexcept {} + + static void destroy() noexcept + { + util::openshmem_environment::finalize(); + } + + static constexpr char const* call() noexcept + { + return ""; + } + }; +} // namespace hpx::traits + +HPX_REGISTER_PARCELPORT( + hpx::parcelset::policies::openshmem::parcelport, openshmem) + +#endif diff --git a/libs/full/parcelport_openshmem/tests/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/CMakeLists.txt new file mode 100644 index 000000000000..5206a56dbe16 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) + +if(HPX_WITH_TESTS) + if(HPX_WITH_TESTS_UNIT) + add_hpx_pseudo_target(tests.unit.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.parcelport_openshmem + ) + add_subdirectory(unit) + endif() + + if(HPX_WITH_TESTS_REGRESSIONS) + add_hpx_pseudo_target(tests.regressions.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.parcelport_openshmem + ) + add_subdirectory(regressions) + endif() + + if(HPX_WITH_TESTS_BENCHMARKS) + add_hpx_pseudo_target(tests.performance.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.parcelport_openshmem + ) + add_subdirectory(performance) + endif() + + if(HPX_WITH_TESTS_HEADERS) + add_hpx_header_tests( + modules.parcelport_openshmem + HEADERS ${parcelport_openshmem_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + DEPENDENCIES hpx_parcelport_openshmem + ) + endif() +endif() diff --git a/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) From 684aa7a92e142b06cf2d53e7442dbb78905dc62f Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 23 Oct 2023 22:27:05 -0400 Subject: [PATCH 002/154] fixes to cmake script --- cmake/HPX_SetupOpenSHMEM.cmake | 126 +++++++++++++-------------------- 1 file changed, 51 insertions(+), 75 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 9c603343ef33..ec8448d8f4b8 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -26,7 +26,7 @@ macro(hpx_setup_openshmem) pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") set(OPENSHMEM_PC - "sandia-openshmem"") + "sandia-openshmem") pkg_search_module( OPENSHMEM IMPORTED_TARGET GLOBAL @@ -35,7 +35,19 @@ macro(hpx_setup_openshmem) endif() endif() - if(NOT OPENSHMEM_FOUND) + if((NOT OPENSHMEM_FOUND) AND HPX_WITH_FETCH_OPENSHMEM) + if(NOT CMAKE_C_COMPILER) + message( + FATAL_ERROR + "HPX_WITH_FETCH_OPENSHMEM requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset." + ) + endif() + if(NOT CMAKE_CXX_COMPILER) + message( + FATAL_ERROR + "HPX_WITH_FETCH_OPENSHMEM requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset." + ) + endif() find_program(MAKE_EXECUTABLE NAMES gmake make mingw32-make REQUIRED) @@ -53,8 +65,10 @@ macro(hpx_setup_openshmem) if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") - message(STATUS "Fetching - OSSS-UCX-OpenSHMEM") + message( + STATUS + "Fetching OSSS-UCX-OpenSHMEM" + ) fetchcontent_declare( openshmem @@ -62,18 +76,17 @@ macro(hpx_setup_openshmem) URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz ) - message(STATUS "Building - OSSS-UCX - (OpenSHMEM on UCX) - and - installing - into - ${CMAKE_INSTALL_PREFIX}") + message( + STATUS + "Building OSSS-UCX (OpenSHMEM on UCX) and installing into ${CMAKE_INSTALL_PREFIX}" + ) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - message(STATUS "Fetching - Sandia-OpenSHMEM") + message( + STATUS + "Fetching Sandia-OpenSHMEM" + ) fetchcontent_declare( openshmem @@ -81,23 +94,16 @@ macro(hpx_setup_openshmem) URL https://github.com/Sandia-OpenSHMEM/SOS/archive/refs/tags/v1.5.2.tar.gz ) - message(STATUS "Building - and - installing - Sandia - OpenSHMEM - into - ${CMAKE_INSTALL_PREFIX}") + message( + STATUS + "Building and installing Sandia OpenSHMEM into ${CMAKE_INSTALL_PREFIX}" + ) else() - message(FATAL_ERROR "HPX_WITH_PARCELPORT_OPENSHMEM - is - not - set - to - `ucx` - or - `sos`") + message( + FATAL_ERROR + "HPX_WITH_PARCELPORT_OPENSHMEM is not set to `ucx` or `sos`" + ) endif() fetchcontent_getproperties(openshmem) @@ -115,19 +121,7 @@ macro(hpx_setup_openshmem) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} - ./autogen.sh - && - CC=${CMAKE_C_COMPILER} - ./configure - --prefix=${OPENSHMEM_DIR}/install - --enable-shared - ${PMI_AUTOCONF_OPTS} - && - make - && - make - install" + "./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared ${PMI_AUTOCONF_OPTS} && make && make install" WORKING_DIRECTORY ${OPENSHMEM_DIR} RESULT_VARIABLE OPENSHMEM_BUILD_STATUS OUTPUT_FILE ${OPENSHMEM_BUILD_OUTPUT} @@ -135,17 +129,10 @@ macro(hpx_setup_openshmem) ) if(OPENSHMEM_BUILD_STATUS) - message(FATAL_ERROR "OpenSHMEM - build - result - = - ${OPENSHMEM_SRC_BUILD_STATUS} - - - see - ${OPENSHMEM_SRC_BUILD_OUTPUT} - for - more - details") + message( + FATAL_ERROR + "OpenSHMEM build result = ${OPENSHMEM_SRC_BUILD_STATUS} - see ${OPENSHMEM_SRC_BUILD_OUTPUT} for more details" + ) else() find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND @@ -155,22 +142,12 @@ macro(hpx_setup_openshmem) if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) message( - FATAL_ERROR - "PKG-CONFIG - ERROR - (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) - -> - CANNOT - FIND - COMPILED - OpenSHMEM: - ${OPENSHMEMT_DIR}/install/lib/pkgconfig" + FATAL_ERROR + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEMT_DIR}/install/lib/pkgconfig" ) endif() - install(CODE "set - (OPENSHMEM_PATH \"${OPENSHMEM_DIR}\") - ") + install(CODE "set(OPENSHMEM_PATH \"${OPENSHMEM_DIR}\")") install( CODE [[ @@ -181,9 +158,10 @@ macro(hpx_setup_openshmem) ) if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) - message(FATAL_ERROR "ERROR - INSTALLING - OPENSHMEM") + message( + FATAL_ERROR + "ERROR INSTALLING OPENSHMEM" + ) endif() string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" @@ -200,9 +178,10 @@ macro(hpx_setup_openshmem) file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) if(NOT OPENSHMEM_FILES) - message(STATUS "ERROR - INSTALLING - OPENSHMEM") + message( + STATUS + "ERROR INSTALLING OPENSHMEM" + ) endif() foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) @@ -992,11 +971,8 @@ macro(hpx_setup_openshmem) PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS "${OPENSHMEM_LDFLAGS}" ) set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES - "${OPENSHMEM_LIBRARY_DIRS} - " + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES "${OPENSHMEM_LIBRARY_DIRS}" ) endif() - endif() endmacro() From b9707c117a01f1cbf44628e0646e1eb46adc8729 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 23 Oct 2023 22:33:00 -0400 Subject: [PATCH 003/154] cmake-format applied --- cmake/HPX_SetupOpenSHMEM.cmake | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index ec8448d8f4b8..6c10f6eace72 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -232,15 +232,9 @@ macro(hpx_setup_openshmem) endif() if(NOT OPENSHMEM_FOUND) - message(FATAL_ERROR "OpenSHMEM - downloaded, - compiled, - but - cannot - be - found - in - ${CMAKE_INSTALL_PREFIX}") + message( + FATAL_ERROR "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" + ) endif() endif() From 66d37694d08bed1e49c6dbd76e0c986432118184 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 25 Oct 2023 10:37:00 -0400 Subject: [PATCH 004/154] fixed misquote --- cmake/HPX_SetupOpenSHMEM.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 6c10f6eace72..54133f93154a 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -53,9 +53,9 @@ macro(hpx_setup_openshmem) include(FindOpenShmemPmi) - set(PMI_AUTOCONF_OPTS) + set(PMI_AUTOCONF_OPTS "") if(NOT PMI_LIBRARY OR NOT PMI_FOUND) - set(PMI_AUTOCONF_OPTS --enable-pmi-simple) + set(PMI_AUTOCONF_OPTS "--enable-pmi-simple") else() set(PMI_AUTOCONF_OPTS "--with-pmi=${PMI_INCLUDE_DIR} --with-pmi-libdir=${PMI_LIBRARY}") @@ -143,7 +143,7 @@ macro(hpx_setup_openshmem) if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) message( FATAL_ERROR - "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEMT_DIR}/install/lib/pkgconfig" + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" ) endif() From f28d8b4d8080364c22191bbcf10090696ca46378 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 25 Oct 2023 15:03:10 -0400 Subject: [PATCH 005/154] cmake formatting reapplied --- CMakeLists.txt | 3 +- cmake/HPX_SetupOpenSHMEM.cmake | 1186 ++++++++++++++++---------------- 2 files changed, 596 insertions(+), 593 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fd2ee0042711..c13725149de9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1233,8 +1233,7 @@ if(HPX_WITH_NETWORKING) OFF CATEGORY "Parcelport" ) hpx_option( - HPX_WITH_FETCH_OPENSHMEM - BOOL + HPX_WITH_FETCH_OPENSHMEM BOOL "Download an OpenSHMEM if one is not available" OFF CATEGORY "Build Targets" ADVANCED diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 54133f93154a..5046968a45df 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -25,13 +25,9 @@ macro(hpx_setup_openshmem) pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - set(OPENSHMEM_PC - "sandia-openshmem") + set(OPENSHMEM_PC "sandia-openshmem") - pkg_search_module( - OPENSHMEM IMPORTED_TARGET GLOBAL - sandia-openshmem - ) + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL sandia-openshmem) endif() endif() @@ -58,35 +54,30 @@ macro(hpx_setup_openshmem) set(PMI_AUTOCONF_OPTS "--enable-pmi-simple") else() set(PMI_AUTOCONF_OPTS "--with-pmi=${PMI_INCLUDE_DIR} - --with-pmi-libdir=${PMI_LIBRARY}") + --with-pmi-libdir=${PMI_LIBRARY}" + ) endif() include(FetchContent) if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") - message( - STATUS - "Fetching OSSS-UCX-OpenSHMEM" - ) + message(STATUS "Fetching OSSS-UCX-OpenSHMEM") fetchcontent_declare( openshmem DOWNLOAD_EXTRACT_TIMESTAMP TRUE - URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz + URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz ) message( - STATUS - "Building OSSS-UCX (OpenSHMEM on UCX) and installing into ${CMAKE_INSTALL_PREFIX}" + STATUS + "Building OSSS-UCX (OpenSHMEM on UCX) and installing into ${CMAKE_INSTALL_PREFIX}" ) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - message( - STATUS - "Fetching Sandia-OpenSHMEM" - ) + message(STATUS "Fetching Sandia-OpenSHMEM") fetchcontent_declare( openshmem @@ -95,15 +86,14 @@ macro(hpx_setup_openshmem) ) message( - STATUS - "Building and installing Sandia OpenSHMEM into ${CMAKE_INSTALL_PREFIX}" + STATUS + "Building and installing Sandia OpenSHMEM into ${CMAKE_INSTALL_PREFIX}" ) else() - message( - FATAL_ERROR - "HPX_WITH_PARCELPORT_OPENSHMEM is not set to `ucx` or `sos`" - ) + message( + FATAL_ERROR "HPX_WITH_PARCELPORT_OPENSHMEM is not set to `ucx` or `sos`" + ) endif() fetchcontent_getproperties(openshmem) @@ -130,20 +120,19 @@ macro(hpx_setup_openshmem) if(OPENSHMEM_BUILD_STATUS) message( - FATAL_ERROR - "OpenSHMEM build result = ${OPENSHMEM_SRC_BUILD_STATUS} - see ${OPENSHMEM_SRC_BUILD_OUTPUT} for more details" + FATAL_ERROR + "OpenSHMEM build result = ${OPENSHMEM_SRC_BUILD_STATUS} - see ${OPENSHMEM_SRC_BUILD_OUTPUT} for more details" ) else() - find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND - ${OPENSHMEM_PC} - ${OPENSHMEM_DIR}/install/lib/pkgconfig + find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND ${OPENSHMEM_PC} + ${OPENSHMEM_DIR}/install/lib/pkgconfig ) if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) message( - FATAL_ERROR - "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" + FATAL_ERROR + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" ) endif() @@ -220,118 +209,115 @@ macro(hpx_setup_openshmem) set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") - pkg_search_module( - OPENSHMEM IMPORTED_TARGET GLOBAL - osss-ucx - ) + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - pkg_search_module( - OPENSHMEM IMPORTED_TARGET GLOBAL - sandia-openshmem - ) + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL sandia-openshmem) endif() if(NOT OPENSHMEM_FOUND) message( - FATAL_ERROR "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" + FATAL_ERROR + "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" ) endif() endif() - if(OPENSHMEM_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" + if(OPENSHMEM_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH OPENSHMEM_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_CFLAGS NEWPARAM) - endforeach() + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_CFLAGS "${X}") - endforeach() - endif() + list(LENGTH OPENSHMEM_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS NEWPARAM) + endforeach() - if(OPENSHMEM_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS "${X}") + endforeach() + endif() - foreach(X IN ITEMS ${OPENSHMEM_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" + if(OPENSHMEM_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() - list(LENGTH OPENSHMEM_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_CFLAGS_OTHER NEWPARAM) - endforeach() + list(LENGTH OPENSHMEM_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS_OTHER NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_CFLAGS_OTHER "${X}") - endforeach() - endif() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS_OTHER "${X}") + endforeach() + endif() - if(OPENSHMEM_LDFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(SKIP 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-l" LIDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") + if(OPENSHMEM_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(SKIP 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") @@ -345,7 +331,8 @@ macro(hpx_setup_openshmem) set(IS_PARAM "0") elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") + ${X}" + ) set(NEWPARAM "") set(IS_PARAM "0") elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") @@ -361,112 +348,114 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() - endif() - endforeach() + endif() + endforeach() - set(IDX 0) - list(LENGTH OPENSHMEM_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) - endforeach() + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_LDFLAGS "${X}") - endforeach() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS "${X}") + endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive" - "${NEWLINK}" IDX + ) + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") endif() endif() endif() + endif() - if(OPENSHMEM_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(SKIP 0) - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") + if(OPENSHMEM_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") @@ -480,7 +469,8 @@ macro(hpx_setup_openshmem) set(IS_PARAM "0") elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") + ${X}" + ) set(NEWPARAM "") set(IS_PARAM "0") elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") @@ -496,187 +486,191 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() - endif() - endforeach() + endif() + endforeach() - set(IDX 0) - list(LENGTH OPENSHMEM_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_LDFLAGS_OTHER NEWPARAM) - endforeach() + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS_OTHER NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_LDFLAGS_OTHER "${X}") - endforeach() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${X}") + endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive" - "${NEWLINK}" IDX + ) + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") endif() endif() - endif() - if(OPENSHMEM_STATIC_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") + endif() - foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" + if(OPENSHMEM_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH OPENSHMEM_STATIC_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_STATIC_CFLAGS NEWPARAM) - endforeach() + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_STATIC_CFLAGS "${X}") - endforeach() - endif() + list(LENGTH OPENSHMEM_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS NEWPARAM) + endforeach() - if(OPENSHMEM_STATIC_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS "${X}") + endforeach() + endif() - foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" + if(OPENSHMEM_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() - list(LENGTH OPENSHMEM_STATIC_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_STATIC_CFLAGS_OTHER NEWPARAM) - endforeach() + list(LENGTH OPENSHMEM_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_STATIC_CFLAGS_OTHER "${X}") - endforeach() - endif() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() - if(OPENSHMEM_STATIC_LDFLAGS) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(SKIP 0) - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") + if(OPENSHMEM_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") @@ -690,7 +684,8 @@ macro(hpx_setup_openshmem) set(IS_PARAM "0") elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") + ${X}" + ) set(NEWPARAM "") set(IS_PARAM "0") elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") @@ -706,113 +701,115 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() - endif() - endforeach() + endif() + endforeach() - set(IDX 0) - list(LENGTH OPENSHMEM_STATIC_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS NEWPARAM) - endforeach() + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_STATIC_LDFLAGS "${X}") - endforeach() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${X}") + endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive" - "${NEWLINK}" IDX + ) + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_STATIC_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") endif() endif() endif() + endif() - if(OPENSHMEM_STATIC_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(SKIP 0) - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") + if(OPENSHMEM_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") @@ -826,7 +823,8 @@ macro(hpx_setup_openshmem) set(IS_PARAM "0") elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") + ${X}" + ) set(NEWPARAM "") set(IS_PARAM "0") elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") @@ -842,131 +840,137 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() - endif() - endforeach() + endif() + endforeach() - set(IDX 0) - list(LENGTH OPENSHMEM_STATIC_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS_OTHER NEWPARAM) - endforeach() + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${X}") - endforeach() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - message(STATUS "${FOUND_LIB} - ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} + ${X}" + ) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive" - "${NEWLINK}" IDX + ) + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") endif() endif() endif() + endif() - if(OPENSHMEM_DIR) - list(TRANSFORM OPENSHMEM_CFLAGS - REPLACE "${OPENSHMEM_DIR}/install" - "$" - ) - list(TRANSFORM OPENSHMEM_LDFLAGS - REPLACE "${OPENSHMEM_DIR}/install" - "$" - ) - list(TRANSFORM OPENSHMEM_LIBRARY_DIRS - REPLACE "${OPENSHMEM_DIR}/install" - "$" - ) + if(OPENSHMEM_DIR) + list(TRANSFORM OPENSHMEM_CFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LDFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LIBRARY_DIRS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS - "${OPENSHMEM_CFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS - "${OPENSHMEM_LDFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES - "${OPENSHMEM_LIBRARY_DIRS}" - ) - else() - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS "${OPENSHMEM_CFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS "${OPENSHMEM_LDFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES "${OPENSHMEM_LIBRARY_DIRS}" - ) - endif() + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + else() + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + endif() endmacro() From 740420c8fe2f0bdbca361eadaa7f70970413b845 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 10:30:52 -0400 Subject: [PATCH 006/154] fixed bugs in cmake script --- cmake/HPX_SetupOpenSHMEM.cmake | 108 ++++++++++++++------------------- 1 file changed, 47 insertions(+), 61 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 5046968a45df..fba9048b521a 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -101,17 +101,16 @@ macro(hpx_setup_openshmem) fetchcontent_populate(openshmem) endif() - set(CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") - set(ENV{PKG_CONFIG_PATH} "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") - set(OPENSHMEM_DIR "${openshmem_SOURCE_DIR}") set(OPENSHMEM_BUILD_OUTPUT "${OPENSHMEM_DIR}/build.log") set(OPENSHMEM_ERROR_FILE "${OPENSHMEM_DIR}/error.log") + set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") + execute_process( COMMAND bash -c - "./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared ${PMI_AUTOCONF_OPTS} && make && make install" + "./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared --disable-fortran ${PMI_AUTOCONF_OPTS} && make && make install" WORKING_DIRECTORY ${OPENSHMEM_DIR} RESULT_VARIABLE OPENSHMEM_BUILD_STATUS OUTPUT_FILE ${OPENSHMEM_BUILD_OUTPUT} @@ -125,94 +124,80 @@ macro(hpx_setup_openshmem) ) else() - find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND ${OPENSHMEM_PC} + find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND + ${OPENSHMEM_PC}.pc ${OPENSHMEM_DIR}/install/lib/pkgconfig ) if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) message( FATAL_ERROR - "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}; ${OPENSHMEM_PC}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" ) endif() + install(CODE "set(OPENSHMEM_PKGCONF \"${OPENSHMEM_PC}\")") install(CODE "set(OPENSHMEM_PATH \"${OPENSHMEM_DIR}\")") install( CODE [[ - file( - READ - ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PC} - OPENSHMEM_PKGCONFIG_FILE_CONTENT - ) + file(READ + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PKGCONF}.pc + OPENSHMEM_PKGCONFIG_FILE_CONTENT) - if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) - message( - FATAL_ERROR - "ERROR INSTALLING OPENSHMEM" - ) - endif() + if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) + message(FATAL_ERROR "ERROR INSTALLING OPENSHMEM") + endif() - string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" - OPENSHMEM_PKGCONFIG_FILE_CONTENT - ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} - ) + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_PKGCONFIG_FILE_CONTENT + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT}) - file( - WRITE - ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PC} - ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} - ) + file(WRITE + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PKGCONF}.pc + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} + ) - file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) + file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) - if(NOT OPENSHMEM_FILES) - message( - STATUS - "ERROR INSTALLING OPENSHMEM" - ) - endif() + if(NOT OPENSHMEM_FILES) + message(STATUS "ERROR INSTALLING OPENSHMEM") + endif() - foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) - set(OPENSHMEM_FILE_CACHED "${OPENSHMEM_FILE}") + foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) + set(OPENSHMEM_FILE_CACHED "${OPENSHMEM_FILE}") - string(REGEX MATCH " - (^ \/.*\/) - " OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE}) + string(REGEX MATCH "(^\/.*\/)" OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE}) - string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" - OPENSHMEM_FILE ${OPENSHMEM_FILE} - ) + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE ${OPENSHMEM_FILE} + ) - string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" - OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE_PATH} - ) + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE_PATH} + ) - file(MAKE_DIRECTORY ${OPENSHMEM_FILE_PATH}) + file(MAKE_DIRECTORY ${OPENSHMEM_FILE_PATH}) - string(LENGTH ${OPENSHMEM_FILE_PATH} OPENSHMEM_FILE_PATH_SIZE) - math(EXPR OPENSHMEM_FILE_PATH_SIZE "${OPENSHMEM_FILE_PATH_SIZE}-1") + string(LENGTH ${OPENSHMEM_FILE_PATH} OPENSHMEM_FILE_PATH_SIZE) + math(EXPR OPENSHMEM_FILE_PATH_SIZE "${OPENSHMEM_FILE_PATH_SIZE}-1") - string(SUBSTRING ${OPENSHMEM_FILE_PATH} 0 ${OPENSHMEM_FILE_PATH_SIZE} - OPENSHMEM_FILE_PATH - ) + string(SUBSTRING ${OPENSHMEM_FILE_PATH} 0 ${OPENSHMEM_FILE_PATH_SIZE} + OPENSHMEM_FILE_PATH) - file(COPY ${OPENSHMEM_FILE_CACHED} DESTINATION ${OPENSHMEM_FILE_PATH}) - endforeach() - ]] + file(COPY ${OPENSHMEM_FILE_CACHED} DESTINATION ${OPENSHMEM_FILE_PATH}) + endforeach() + ]] ) - - # install(FILES ${OPENSHMEM_FILES} DESTINATION ${CMAKE_INSTALL_PREFIX}) endif() set(CMAKE_PREFIX_PATH "${OPENSHMEM_DIR}/install/lib/pkgconfig") set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") - if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) - elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL sandia-openshmem) - endif() + pkg_search_module( + OPENSHMEM REQUIRED IMPORTED_TARGET GLOBAL + ${OPENSHMEM_PC} + ) if(NOT OPENSHMEM_FOUND) message( @@ -220,7 +205,8 @@ macro(hpx_setup_openshmem) "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" ) endif() - + elseif((NOT OPENSHMEM_FOUND) AND (NOT HPX_WITH_FETCH_OPENSHMEM)) + message(FATAL_ERROR "OpenSHMEM not found and HPX_WITH_FETCH_OPENSHMEM not set!") endif() if(OPENSHMEM_CFLAGS) From a7a97b47b66f5e03a37cc2016f5ef13f40053f4d Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 10:48:33 -0400 Subject: [PATCH 007/154] cmake formatting updates --- cmake/HPX_SetupOpenSHMEM.cmake | 12 +++++------- libs/core/version/CMakeLists.txt | 4 +--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index fba9048b521a..06d6bfaa9ed8 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -124,8 +124,7 @@ macro(hpx_setup_openshmem) ) else() - find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND - ${OPENSHMEM_PC}.pc + find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND ${OPENSHMEM_PC}.pc ${OPENSHMEM_DIR}/install/lib/pkgconfig ) @@ -194,10 +193,7 @@ macro(hpx_setup_openshmem) set(CMAKE_PREFIX_PATH "${OPENSHMEM_DIR}/install/lib/pkgconfig") set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") - pkg_search_module( - OPENSHMEM REQUIRED IMPORTED_TARGET GLOBAL - ${OPENSHMEM_PC} - ) + pkg_search_module(OPENSHMEM REQUIRED IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) if(NOT OPENSHMEM_FOUND) message( @@ -206,7 +202,9 @@ macro(hpx_setup_openshmem) ) endif() elseif((NOT OPENSHMEM_FOUND) AND (NOT HPX_WITH_FETCH_OPENSHMEM)) - message(FATAL_ERROR "OpenSHMEM not found and HPX_WITH_FETCH_OPENSHMEM not set!") + message( + FATAL_ERROR "OpenSHMEM not found and HPX_WITH_FETCH_OPENSHMEM not set!" + ) endif() if(OPENSHMEM_CFLAGS) diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index f0886e68d6e4..08fe9e1b6a06 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -20,9 +20,7 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() -if(HPX_WITH_NETWORKING - AND HPX_WITH_PARCELPORT_OPENSHMEM -) +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM) include(HPX_SetupOpenSHMEM) hpx_setup_openshmem() set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) From 90386c15aa78d02e49eb6c2aadad114fce180a38 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 12:04:51 -0400 Subject: [PATCH 008/154] fixed arguments --- .jenkins/lsu/env-clang-12.sh | 1 + .jenkins/lsu/env-clang-13.sh | 1 + .jenkins/lsu/env-clang-14.sh | 1 + .jenkins/lsu/env-clang-15.sh | 1 + .jenkins/lsu/env-gcc-10.sh | 1 + .jenkins/lsu/env-gcc-12.sh | 1 + .jenkins/lsu/env-gcc-13.sh | 1 + CMakeLists.txt | 4 ++++ 8 files changed, 11 insertions(+) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index 9cbbcc72cec6..3ec7f7fab212 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -22,6 +22,7 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" +configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index a679e9b4e135..52a81cd0c77d 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -22,6 +22,7 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" +configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 98aefa4dfa97..cdbffa0dfc05 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -22,6 +22,7 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" +configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index f9b4db596066..ca5470b6478d 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -22,6 +22,7 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" +configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index b33db50c9d5b..3562894f5f72 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -22,6 +22,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_ASIO_TAG=asio-1-12-0" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" +configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 3e086927c117..4da5ae194c88 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -21,6 +21,7 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" +configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 526916453018..56a658f99d4f 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -21,6 +21,7 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" +configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/CMakeLists.txt b/CMakeLists.txt index c13725149de9..d45c90fab925 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2257,6 +2257,10 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) include(HPX_SetupGasnet) hpx_setup_gasnet() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() +endif() # Setup packages and subprojects include(HPX_SetupSYCL) From af3c83a8682a99e36b55bbbbdda95a32afc5fcf7 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 13:14:56 -0400 Subject: [PATCH 009/154] added debugging for build failures --- cmake/HPX_SetupOpenSHMEM.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 06d6bfaa9ed8..774f0bcf573d 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -118,9 +118,11 @@ macro(hpx_setup_openshmem) ) if(OPENSHMEM_BUILD_STATUS) + file(READ ${OPENSHMEM_BUILD_OUTPUT} OPENSHMEM_BUILD_OUTPUT_CONTENT) + message( FATAL_ERROR - "OpenSHMEM build result = ${OPENSHMEM_SRC_BUILD_STATUS} - see ${OPENSHMEM_SRC_BUILD_OUTPUT} for more details" + "OpenSHMEM build result = ${OPENSHMEM_BUILD_STATUS} - see ${OPENSHMEM_BUILD_OUTPUT} for more details\nOpenSHMEM build command\t:./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared --disable-fortran ${PMI_AUTOCONF_OPTS} && make && make install\n${OPENSHMEM_BUILD_OUTPUT_CONTENT}" ) else() From 3a0091b9bb40405de58dfb45d98f29219e83aff2 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 14:05:41 -0400 Subject: [PATCH 010/154] correctly post-processing PMI paths --- cmake/HPX_SetupOpenSHMEM.cmake | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 774f0bcf573d..30bc43357bf4 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -53,8 +53,12 @@ macro(hpx_setup_openshmem) if(NOT PMI_LIBRARY OR NOT PMI_FOUND) set(PMI_AUTOCONF_OPTS "--enable-pmi-simple") else() - set(PMI_AUTOCONF_OPTS "--with-pmi=${PMI_INCLUDE_DIR} - --with-pmi-libdir=${PMI_LIBRARY}" + string(REGEX MATCH "(^\/[^\/]+)" PMI_INCLUDE_DIR_ROOT_PATH + ${PMI_INCLUDE_DIR} + ) + string(REGEX MATCH "(^\/[^\/]+)" PMI_LIBRARY_ROOT_PATH ${PMI_LIBRARY}) + set(PMI_AUTOCONF_OPTS + "--with-pmi=${PMI_INCLUDE_DIR_ROOT_PATH} --with-pmi-libdir=${PMI_LIBRARY_ROOT_PATH}" ) endif() From a151f0e9023a12c6d5840c93fc8acccc12e904fe Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 14:57:37 -0400 Subject: [PATCH 011/154] rm'd include of shmem --- libs/core/openshmem_base/src/openshmem_environment.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 1274c201c487..82970b495d8a 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -16,8 +16,6 @@ #include #include -#include - #include #include #include From 9048070d759091493fbfc62299741b600e9734d1 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 15:37:05 -0400 Subject: [PATCH 012/154] shmem is already extern "C" --- .../include/hpx/openshmem_base/openshmem.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp index e3f6bcb3a94f..06410c30e317 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -17,16 +17,8 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif -#ifdef __cplusplus -extern "C" { -#endif - #include -#ifdef __cplusplus -} // extern "C" -#endif - #if defined(__clang__) #pragma clang diagnostic pop #elif defined(__GNUC__) From b46ad1c2818b3fe09b01afc74c0a9dcf0f046916 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 16:36:53 -0400 Subject: [PATCH 013/154] fixing header issues --- .../openshmem_base/include/hpx/openshmem_base/openshmem.hpp | 2 +- libs/core/openshmem_base/src/openshmem_environment.cpp | 1 + .../include/hpx/parcelport_openshmem/sender.hpp | 4 +--- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp index 06410c30e317..b8432b0ebc9d 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -7,7 +7,7 @@ #pragma once -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) +#if defined(HPX_HAVE_PARCELPORT_OPENSHMEM) #if defined(__clang__) #pragma clang diagnostic push diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 82970b495d8a..da5d01efdc42 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -253,6 +253,7 @@ namespace hpx::util { else { const std::lock_guard lk(segment_mutex[node]); + shmem_uint8_put_signal(raddr, addr, size, reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, node); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp index b4a9bba0d479..fe75167a4208 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -37,8 +37,7 @@ namespace hpx::parcelset::policies::openshmem { // different versions of clang-format disagree // clang-format off sender() noexcept - : next_free_tag_request_(-1) - , next_free_tag_(-1) + : next_free_tag_(-1) { } // clang-format on @@ -156,7 +155,6 @@ namespace hpx::parcelset::policies::openshmem { connection_list connections_; hpx::spinlock next_free_tag_mtx_; - int next_free_tag_request_; int next_free_tag_; }; } // namespace hpx::parcelset::policies::openshmem From 2c4b944457920312d8981e02387d30129eff3503 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 20:55:12 -0400 Subject: [PATCH 014/154] builder include issues --- .../openshmem_base/include/hpx/openshmem_base/openshmem.hpp | 4 +++- libs/core/openshmem_base/src/openshmem_environment.cpp | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp index b8432b0ebc9d..fa6c6e1332fa 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -7,7 +7,7 @@ #pragma once -#if defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) #if defined(__clang__) #pragma clang diagnostic push @@ -17,7 +17,9 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif +#ifndef SHMEM_H #include +#endif #if defined(__clang__) #pragma clang diagnostic pop diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index da5d01efdc42..6ec06a60d99e 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -16,6 +16,10 @@ #include #include +#ifndef SHMEM_H +#include +#endif + #include #include #include From 43d9d1975cfe87f3093f5c7a605945b0da03b764 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 21:39:51 -0400 Subject: [PATCH 015/154] adding maybe_unused to init arguments --- libs/core/openshmem_base/src/openshmem_environment.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 6ec06a60d99e..8b85f47c0031 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -62,7 +62,7 @@ namespace hpx::util { } // namespace detail bool openshmem_environment::check_openshmem_environment( - util::runtime_configuration const& cfg) + [[maybe_unused]] util::runtime_configuration const& cfg) { #if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) // We disable the OPENSHMEM parcelport if any of these hold: @@ -110,7 +110,7 @@ namespace hpx::util { unsigned int openshmem_environment::xmt = 0; /////////////////////////////////////////////////////////////////////////// - int openshmem_environment::init(int* argc, char*** argv, int& provided) + int openshmem_environment::init([[maybe_unused]] int* argc, [[maybe_unused]] char*** argv, [[maybe_unused]] int& provided) { if (!has_called_init_) { @@ -171,7 +171,7 @@ namespace hpx::util { /////////////////////////////////////////////////////////////////////////// void openshmem_environment::init( - int* argc, char*** argv, util::runtime_configuration& rtcfg) + [[maybe_unused]] int* argc, [[maybe_unused]] char*** argv, [[maybe_unused]] util::runtime_configuration& rtcfg) { if (enabled_) return; // don't call twice From 420bed60fd1703e8ea64e467ea6583c77e333785 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 21:45:26 -0400 Subject: [PATCH 016/154] cmake formatting --- libs/core/openshmem_base/src/openshmem_environment.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 8b85f47c0031..001d8ddea740 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -110,7 +110,8 @@ namespace hpx::util { unsigned int openshmem_environment::xmt = 0; /////////////////////////////////////////////////////////////////////////// - int openshmem_environment::init([[maybe_unused]] int* argc, [[maybe_unused]] char*** argv, [[maybe_unused]] int& provided) + int openshmem_environment::init([[maybe_unused]] int* argc, + [[maybe_unused]] char*** argv, [[maybe_unused]] int& provided) { if (!has_called_init_) { @@ -170,8 +171,9 @@ namespace hpx::util { } /////////////////////////////////////////////////////////////////////////// - void openshmem_environment::init( - [[maybe_unused]] int* argc, [[maybe_unused]] char*** argv, [[maybe_unused]] util::runtime_configuration& rtcfg) + void openshmem_environment::init([[maybe_unused]] int* argc, + [[maybe_unused]] char*** argv, + [[maybe_unused]] util::runtime_configuration& rtcfg) { if (enabled_) return; // don't call twice From c512b26ee4565ab9328adf02c9a92a3a63d5e52a Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 28 Oct 2023 08:20:52 -0400 Subject: [PATCH 017/154] added maybe unused; corrected missing definition --- libs/core/openshmem_base/src/openshmem_environment.cpp | 5 +++++ libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 001d8ddea740..25c9d60e0dcc 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -18,6 +18,11 @@ #ifndef SHMEM_H #include + +#ifndef SHMEM_SIGNAL_SET +#define SHMEM_SIGNAL_SET 0 +#endif + #endif #include diff --git a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp index 27b2f2de4a3e..7915e9d9bb3a 100644 --- a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp +++ b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp @@ -104,7 +104,7 @@ namespace hpx::parcelset { } static std::size_t background_threads( - util::runtime_configuration const& ini) + [[maybe_unused]] util::runtime_configuration const& ini) { /* return hpx::util::get_entry_as(ini, From 90882d8ba4d5f8db16faeda028f3449dfb094f8e Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 28 Oct 2023 11:17:17 -0400 Subject: [PATCH 018/154] working to sort out why is being declared undefined --- .../openshmem_base/include/hpx/openshmem_base/openshmem.hpp | 2 +- libs/core/openshmem_base/src/openshmem_environment.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp index fa6c6e1332fa..60413db0dbdc 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -18,7 +18,7 @@ #endif #ifndef SHMEM_H -#include +#include #endif #if defined(__clang__) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 25c9d60e0dcc..b2ddc8013a98 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -17,10 +17,11 @@ #include #ifndef SHMEM_H -#include +#define SHMEM_H +#include #ifndef SHMEM_SIGNAL_SET -#define SHMEM_SIGNAL_SET 0 +#define SHMEM_SIGNAL_SET 0 #endif #endif From ab55f2d4baaaec1ed3e540d9dce2773e8ca92925 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 28 Oct 2023 14:01:28 -0400 Subject: [PATCH 019/154] removed preprocessor guards --- libs/core/openshmem_base/src/openshmem_environment.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index b2ddc8013a98..25f3821bab02 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -10,22 +10,25 @@ #include #include #include +#include #include #include #include #include #include +/* #ifndef SHMEM_H #define SHMEM_H +*/ #include - +/* #ifndef SHMEM_SIGNAL_SET #define SHMEM_SIGNAL_SET 0 #endif #endif - +*/ #include #include #include From 71b7656e9affceb3059b2d58399d5719075f9a7e Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 28 Oct 2023 19:43:43 -0400 Subject: [PATCH 020/154] added def header file --- .../core/openshmem_base/src/openshmem_environment.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 25f3821bab02..655eee97186e 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -17,18 +17,9 @@ #include #include -/* -#ifndef SHMEM_H -#define SHMEM_H -*/ +#include #include -/* -#ifndef SHMEM_SIGNAL_SET -#define SHMEM_SIGNAL_SET 0 -#endif -#endif -*/ #include #include #include From 761f753fefaacaa723a1fee9ae317fb80a90af91 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 28 Oct 2023 19:49:56 -0400 Subject: [PATCH 021/154] added def header file --- libs/core/openshmem_base/src/openshmem_environment.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 655eee97186e..a7c039ae897c 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include From a8315fd2417f75ece5f8839c293484bb2fd1c236 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 28 Oct 2023 22:58:05 -0400 Subject: [PATCH 022/154] adding support for openmpi's openshmem --- CMakeLists.txt | 2 +- cmake/HPX_SetupOpenSHMEM.cmake | 39 ++++++++++++++++++- .../src/openshmem_environment.cpp | 3 -- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d45c90fab925..6a6ff3b6c5d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1243,7 +1243,7 @@ if(HPX_WITH_NETWORKING) hpx_option( HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT STRING "Define which conduit to use for the OpenSHMEM parcelport" "sos" - STRINGS "sos;ucx" + STRINGS "sos;ucx;mpi" ) endif() diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 30bc43357bf4..a9e93fa30526 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -11,7 +11,42 @@ macro(hpx_setup_openshmem) if(NOT TARGET PkgConfig::OPENSHMEM) set(OPENSHMEM_PC "") - if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + if(HPX_WITH_PARCELPORT_MPI AND HPX_WITH_PARCELPORT_OPENSHMEM AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi"))) + message(FATAL "HPX_WITH_PARCELPORT_MPI & HPX_WITH_PARCELPORT_OPENSHMEM both set to ON and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT != 'mpi'") + + elseif(HPX_WITH_PARCELPORT_MPI AND HPX_WITH_PARCELPORT_OPENSHMEM AND HPX_WITH_FETCH_OPENSHMEM AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + message( + FATAL_ERROR + "HPX_WITH_FETCH_OPENSHMEM for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi' not supported" + ) + + elseif(HPX_WITH_PARCELPORT_MPI AND (MPI_FOUND OR Mpi::mpi) AND HPX_WITH_PARCELPORT_OPENSHMEM AND (NOT HPX_WITH_FETCH_OPENSHMEM) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + set(OPENSHMEM_PC "oshmem-cxx") + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem-cxx) + + if(NOT OPENSHMEM_FOUND) + message( + FATAL_ERROR + "oshmem-cxx not found for HPX_WITH_PARCELPORT_MPI, HPX_WITH_PARCELPORT_OPENSHMEM, and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'" + ) + endif() + + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + + return() + + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") set(OPENSHMEM_PC "ucx") pkg_search_module(UCX IMPORTED_TARGET GLOBAL ucx) @@ -31,7 +66,7 @@ macro(hpx_setup_openshmem) endif() endif() - if((NOT OPENSHMEM_FOUND) AND HPX_WITH_FETCH_OPENSHMEM) + if((NOT OPENSHMEM_FOUND) AND HPX_WITH_FETCH_OPENSHMEM AND NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) if(NOT CMAKE_C_COMPILER) message( FATAL_ERROR diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index a7c039ae897c..42ccd23233e6 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -16,9 +16,6 @@ #include #include -#include -#include - #include #include #include From e0f15b88f6ed35825140e6948eaeffe3657a5ed5 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 08:02:40 -0500 Subject: [PATCH 023/154] updated merge --- .jenkins/lsu/env-clang-12.sh | 2 +- .jenkins/lsu/env-clang-13.sh | 2 +- .jenkins/lsu/env-clang-14.sh | 2 +- .jenkins/lsu/env-clang-15.sh | 6 +----- .jenkins/lsu/env-gcc-10.sh | 2 +- .jenkins/lsu/env-gcc-12.sh | 2 +- .jenkins/lsu/env-gcc-13.sh | 2 +- 7 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index 3ec7f7fab212..fb84eabbefb3 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -27,7 +27,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 52a81cd0c77d..db258a801cef 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -27,7 +27,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index cdbffa0dfc05..7ffb8b05f6cb 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -27,7 +27,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index ca5470b6478d..a3302263d017 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -26,13 +26,9 @@ configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -<<<<<<< HEAD -======= -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" ->>>>>>> a9eef08e69 (initial import) +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 3562894f5f72..b83df8358073 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -27,7 +27,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 4da5ae194c88..db70f6ecc344 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -26,7 +26,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 56a658f99d4f..4b68406d756f 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -26,7 +26,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" From bdbebada6fc235f0c4ca4f4bdfd67725b4db47e1 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 28 Oct 2023 23:16:53 -0400 Subject: [PATCH 024/154] cmake format applied --- cmake/HPX_SetupOpenSHMEM.cmake | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index a9e93fa30526..72cfbda63484 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -11,16 +11,33 @@ macro(hpx_setup_openshmem) if(NOT TARGET PkgConfig::OPENSHMEM) set(OPENSHMEM_PC "") - if(HPX_WITH_PARCELPORT_MPI AND HPX_WITH_PARCELPORT_OPENSHMEM AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi"))) - message(FATAL "HPX_WITH_PARCELPORT_MPI & HPX_WITH_PARCELPORT_OPENSHMEM both set to ON and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT != 'mpi'") + if(HPX_WITH_PARCELPORT_MPI + AND HPX_WITH_PARCELPORT_OPENSHMEM + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) + message( + FATAL + "HPX_WITH_PARCELPORT_MPI & HPX_WITH_PARCELPORT_OPENSHMEM both set to ON and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT != 'mpi'" + ) - elseif(HPX_WITH_PARCELPORT_MPI AND HPX_WITH_PARCELPORT_OPENSHMEM AND HPX_WITH_FETCH_OPENSHMEM AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + elseif( + HPX_WITH_PARCELPORT_MPI + AND HPX_WITH_PARCELPORT_OPENSHMEM + AND HPX_WITH_FETCH_OPENSHMEM + AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + ) message( FATAL_ERROR - "HPX_WITH_FETCH_OPENSHMEM for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi' not supported" + "HPX_WITH_FETCH_OPENSHMEM for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi' not supported" ) - elseif(HPX_WITH_PARCELPORT_MPI AND (MPI_FOUND OR Mpi::mpi) AND HPX_WITH_PARCELPORT_OPENSHMEM AND (NOT HPX_WITH_FETCH_OPENSHMEM) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + elseif( + HPX_WITH_PARCELPORT_MPI + AND (MPI_FOUND OR Mpi::mpi) + AND HPX_WITH_PARCELPORT_OPENSHMEM + AND (NOT HPX_WITH_FETCH_OPENSHMEM) + AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + ) set(OPENSHMEM_PC "oshmem-cxx") pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem-cxx) @@ -66,7 +83,10 @@ macro(hpx_setup_openshmem) endif() endif() - if((NOT OPENSHMEM_FOUND) AND HPX_WITH_FETCH_OPENSHMEM AND NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + if((NOT OPENSHMEM_FOUND) + AND HPX_WITH_FETCH_OPENSHMEM + AND NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + ) if(NOT CMAKE_C_COMPILER) message( FATAL_ERROR From 9c6de43e7f480d3f425775314d5b4d484ef468de Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 28 Oct 2023 23:24:45 -0400 Subject: [PATCH 025/154] cmake format applied --- cmake/HPX_SetupOpenSHMEM.cmake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 72cfbda63484..aaa7ea278721 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -32,9 +32,7 @@ macro(hpx_setup_openshmem) ) elseif( - HPX_WITH_PARCELPORT_MPI - AND (MPI_FOUND OR Mpi::mpi) - AND HPX_WITH_PARCELPORT_OPENSHMEM + HPX_WITH_PARCELPORT_OPENSHMEM AND (NOT HPX_WITH_FETCH_OPENSHMEM) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") ) @@ -44,7 +42,7 @@ macro(hpx_setup_openshmem) if(NOT OPENSHMEM_FOUND) message( FATAL_ERROR - "oshmem-cxx not found for HPX_WITH_PARCELPORT_MPI, HPX_WITH_PARCELPORT_OPENSHMEM, and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'" + "oshmem-cxx not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'" ) endif() From a1753e6b175493ad61e49cd3befb02db7fcb5283 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sun, 29 Oct 2023 21:06:42 -0400 Subject: [PATCH 026/154] openmpi/openshmem pkgconfig file --- cmake/HPX_SetupOpenSHMEM.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index aaa7ea278721..842ca0aa80ca 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -36,8 +36,8 @@ macro(hpx_setup_openshmem) AND (NOT HPX_WITH_FETCH_OPENSHMEM) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") ) - set(OPENSHMEM_PC "oshmem-cxx") - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem-cxx) + set(OPENSHMEM_PC "oshmem") + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) if(NOT OPENSHMEM_FOUND) message( From 70a6a49bc34ed03336d01efe3f6da8af02a21d83 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sun, 29 Oct 2023 21:41:39 -0400 Subject: [PATCH 027/154] openmpi/openshmem pkgconfig file --- cmake/HPX_SetupOpenSHMEM.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 842ca0aa80ca..67a47138e389 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -37,12 +37,13 @@ macro(hpx_setup_openshmem) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") ) set(OPENSHMEM_PC "oshmem") + set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/../lib/pkgconfig") pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) if(NOT OPENSHMEM_FOUND) message( FATAL_ERROR - "oshmem-cxx not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'" + "oshmem not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'" ) endif() From b0ab5cae3b144a275dfd7aa2c3fdbba842f84e14 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sun, 29 Oct 2023 21:44:43 -0400 Subject: [PATCH 028/154] openmpi/openshmem pkgconfig file --- cmake/HPX_SetupOpenSHMEM.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 67a47138e389..d2975df21c27 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -37,7 +37,9 @@ macro(hpx_setup_openshmem) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") ) set(OPENSHMEM_PC "oshmem") - set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/../lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} + "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/../lib/pkgconfig" + ) pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) if(NOT OPENSHMEM_FOUND) From 491e9185b9199e2b665ed9efe95f0ea43ddb395f Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 10:29:08 -0400 Subject: [PATCH 029/154] pkgconfig checks for openmpi's oshmem --- cmake/HPX_SetupOpenSHMEM.cmake | 64 +++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index d2975df21c27..2999f6447001 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -36,17 +36,65 @@ macro(hpx_setup_openshmem) AND (NOT HPX_WITH_FETCH_OPENSHMEM) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") ) - set(OPENSHMEM_PC "oshmem") + set(ENV{PKG_CONFIG_PATH} - "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/../lib/pkgconfig" + "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/pkgconfig" ) - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) + + set(OPENSHMEM_PC "oshmem") + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) if(NOT OPENSHMEM_FOUND) - message( - FATAL_ERROR - "oshmem not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'" + find_program(OSHMEM_INFO NAMES oshmem_info ompi_info REQUIRED) + + if(NOT OSHMEM_INFO) + message( + FATAL_ERROR + "oshmem_info and/or ompi_info not found! pkg-config cannot find OpenMPI's `${OPENSHMEM_PC}.pc`" + ) + endif() + + set(OSHMEM_INFO_OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_stdout.log") + set(OSHMEM_INFO_ERROR "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_error.log") + + execute_process( + COMMAND + bash -c + "${OSHMEM_INFO} --path libdir" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE OSHMEM_INFO_STATUS + OUTPUT_FILE ${OSHMEM_INFO_OUTPUT} + ERROR_FILE ${OSHMEM_INFO_ERROR} ) + + if(NOT OSHMEM_INFO_STATUS) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Program status code: ${OSHMEM_INFO_STATUS}" + ) + endif() + + file(READ ${OSHMEM_INFO_OUTPUT} OSHMEM_INFO_OUTPUT_CONTENT) + + if(NOT OSHMEM_INFO_OUTPUT_CONTENT) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}" + ) + endif() + + string(REGEX MATCH "(\/.*)" OSHMEM_LIBDIR_PATH ${OSHMEM_INFO_OUTPUT_CONTENT}) + + set(ENV{PKG_CONFIG_PATH} "${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig") + + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) + + if(NOT OPENSHMEM_FOUND) + message( + FATAL_ERROR + "oshmem not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'" + ) + endif() endif() set_target_properties( @@ -67,7 +115,7 @@ macro(hpx_setup_openshmem) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") set(OPENSHMEM_PC "ucx") - pkg_search_module(UCX IMPORTED_TARGET GLOBAL ucx) + pkg_search_module(UCX IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) if(NOT UCX_FOUND) message( @@ -80,7 +128,7 @@ macro(hpx_setup_openshmem) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") set(OPENSHMEM_PC "sandia-openshmem") - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL sandia-openshmem) + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) endif() endif() From ee0fce6e93dc15c0a82dbd4844662ac46f54bdde Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 10:35:55 -0400 Subject: [PATCH 030/154] cmake formatting --- cmake/HPX_SetupOpenSHMEM.cmake | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 2999f6447001..6b13d3d2967e 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -37,9 +37,7 @@ macro(hpx_setup_openshmem) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") ) - set(ENV{PKG_CONFIG_PATH} - "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/pkgconfig" - ) + set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/pkgconfig") set(OPENSHMEM_PC "oshmem") pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) @@ -54,16 +52,18 @@ macro(hpx_setup_openshmem) ) endif() - set(OSHMEM_INFO_OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_stdout.log") - set(OSHMEM_INFO_ERROR "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_error.log") + set(OSHMEM_INFO_OUTPUT + "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_stdout.log" + ) + set(OSHMEM_INFO_ERROR + "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_error.log" + ) execute_process( - COMMAND - bash -c - "${OSHMEM_INFO} --path libdir" + COMMAND bash -c "${OSHMEM_INFO} --path libdir" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE OSHMEM_INFO_STATUS - OUTPUT_FILE ${OSHMEM_INFO_OUTPUT} + OUTPUT_FILE ${OSHMEM_INFO_OUTPUT} ERROR_FILE ${OSHMEM_INFO_ERROR} ) @@ -78,12 +78,13 @@ macro(hpx_setup_openshmem) if(NOT OSHMEM_INFO_OUTPUT_CONTENT) message( - FATAL_ERROR - "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}" + FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}" ) endif() - string(REGEX MATCH "(\/.*)" OSHMEM_LIBDIR_PATH ${OSHMEM_INFO_OUTPUT_CONTENT}) + string(REGEX MATCH "(\/.*)" OSHMEM_LIBDIR_PATH + ${OSHMEM_INFO_OUTPUT_CONTENT} + ) set(ENV{PKG_CONFIG_PATH} "${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig") From 8dd543e5135b93f272b17c7f031298cfff0a8496 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 10:38:55 -0400 Subject: [PATCH 031/154] added error handling and proper PKG_CONFIG_PATH appending --- cmake/HPX_SetupOpenSHMEM.cmake | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 6b13d3d2967e..9a0e2d41021d 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -86,14 +86,15 @@ macro(hpx_setup_openshmem) ${OSHMEM_INFO_OUTPUT_CONTENT} ) - set(ENV{PKG_CONFIG_PATH} "${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig") pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) if(NOT OPENSHMEM_FOUND) + file(READ ${OSHMEM_INFO_ERROR} OSHMEM_INFO_ERROR_CONTENT) message( FATAL_ERROR - "oshmem not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'" + "oshmem not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'\n${OSHMEM_INFO_ERROR_CONTENT}\n${${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig}" ) endif() endif() @@ -214,7 +215,7 @@ macro(hpx_setup_openshmem) set(OPENSHMEM_BUILD_OUTPUT "${OPENSHMEM_DIR}/build.log") set(OPENSHMEM_ERROR_FILE "${OPENSHMEM_DIR}/error.log") - set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${OPENSHMEM_DIR}/install/lib/pkgconfig") execute_process( COMMAND @@ -302,7 +303,7 @@ macro(hpx_setup_openshmem) endif() set(CMAKE_PREFIX_PATH "${OPENSHMEM_DIR}/install/lib/pkgconfig") - set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${OPENSHMEM_DIR}/install/lib/pkgconfig") pkg_search_module(OPENSHMEM REQUIRED IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) From 11b1e4b1f8ce0957cfeb0dd988de40d4bbcd6062 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 10:44:27 -0400 Subject: [PATCH 032/154] cmake formatting --- cmake/HPX_SetupOpenSHMEM.cmake | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 9a0e2d41021d..0b3bea597f7e 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -86,7 +86,9 @@ macro(hpx_setup_openshmem) ${OSHMEM_INFO_OUTPUT_CONTENT} ) - set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig") + set(ENV{PKG_CONFIG_PATH} + "$ENV{PKG_CONFIG_PATH}:${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig" + ) pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) @@ -215,7 +217,9 @@ macro(hpx_setup_openshmem) set(OPENSHMEM_BUILD_OUTPUT "${OPENSHMEM_DIR}/build.log") set(OPENSHMEM_ERROR_FILE "${OPENSHMEM_DIR}/error.log") - set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${OPENSHMEM_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} + "$ENV{PKG_CONFIG_PATH}:${OPENSHMEM_DIR}/install/lib/pkgconfig" + ) execute_process( COMMAND @@ -303,7 +307,9 @@ macro(hpx_setup_openshmem) endif() set(CMAKE_PREFIX_PATH "${OPENSHMEM_DIR}/install/lib/pkgconfig") - set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${OPENSHMEM_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} + "$ENV{PKG_CONFIG_PATH}:${OPENSHMEM_DIR}/install/lib/pkgconfig" + ) pkg_search_module(OPENSHMEM REQUIRED IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) From 14532ba5f54d9572a4b40d1dad5c894b53b00a37 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 10:55:50 -0400 Subject: [PATCH 033/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 0b3bea597f7e..827b0d10a656 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -67,7 +67,7 @@ macro(hpx_setup_openshmem) ERROR_FILE ${OSHMEM_INFO_ERROR} ) - if(NOT OSHMEM_INFO_STATUS) + if(OSHMEM_INFO_STATUS) message( FATAL_ERROR "${OSHMEM_INFO} Failed! Program status code: ${OSHMEM_INFO_STATUS}" From 259037e4c681106660fca09a5bd7d5f2bfd10286 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 12:03:22 -0400 Subject: [PATCH 034/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 827b0d10a656..e4b60ec4fc32 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -87,7 +87,12 @@ macro(hpx_setup_openshmem) ) set(ENV{PKG_CONFIG_PATH} - "$ENV{PKG_CONFIG_PATH}:${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig" + "$ENV{PKG_CONFIG_PATH}:${OSHMEM_LIBDIR_PATH}/pkgconfig" + ) + + messsage( + STATUS + "1)${OSHMEM_INFO} 2)${OSHMEM_INFO_OUTPUT} 3)${OSHMEM_INFO_OUTPUT_CONTENT} 4)${OSHMEM_LIBDIR_PATH}" ) pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) @@ -96,7 +101,7 @@ macro(hpx_setup_openshmem) file(READ ${OSHMEM_INFO_ERROR} OSHMEM_INFO_ERROR_CONTENT) message( FATAL_ERROR - "oshmem not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'\n${OSHMEM_INFO_ERROR_CONTENT}\n${${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig}" + "oshmem not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'\n${OSHMEM_INFO_ERROR_CONTENT}\n${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig" ) endif() endif() From b6f769c207c093e38de194c006742f9120bb6426 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 12:08:44 -0400 Subject: [PATCH 035/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index e4b60ec4fc32..88fc56ba6779 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -92,10 +92,10 @@ macro(hpx_setup_openshmem) messsage( STATUS - "1)${OSHMEM_INFO} 2)${OSHMEM_INFO_OUTPUT} 3)${OSHMEM_INFO_OUTPUT_CONTENT} 4)${OSHMEM_LIBDIR_PATH}" + "1) ${OSHMEM_INFO} 2) ${OSHMEM_INFO_OUTPUT} 3) ${OSHMEM_INFO_OUTPUT_CONTENT} 4) ${OSHMEM_LIBDIR_PATH}" ) - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL oshmem) + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) if(NOT OPENSHMEM_FOUND) file(READ ${OSHMEM_INFO_ERROR} OSHMEM_INFO_ERROR_CONTENT) From d82bdb9a4fbbabac09183d0dc7aa9f50d72b55af Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 12:14:37 -0400 Subject: [PATCH 036/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 88fc56ba6779..7e6de0ab2aa9 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -90,14 +90,15 @@ macro(hpx_setup_openshmem) "$ENV{PKG_CONFIG_PATH}:${OSHMEM_LIBDIR_PATH}/pkgconfig" ) - messsage( - STATUS - "1) ${OSHMEM_INFO} 2) ${OSHMEM_INFO_OUTPUT} 3) ${OSHMEM_INFO_OUTPUT_CONTENT} 4) ${OSHMEM_LIBDIR_PATH}" - ) - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) if(NOT OPENSHMEM_FOUND) + + messsage( + STATUS + "1) ${OSHMEM_INFO} 2) ${OSHMEM_INFO_OUTPUT} 3) ${OSHMEM_INFO_OUTPUT_CONTENT} 4) ${OSHMEM_LIBDIR_PATH}" + ) + file(READ ${OSHMEM_INFO_ERROR} OSHMEM_INFO_ERROR_CONTENT) message( FATAL_ERROR From f653dfd3d4c13f7afc5da8eab7f1391ace32df7e Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 12:46:37 -0400 Subject: [PATCH 037/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 7e6de0ab2aa9..1e3a9bfd8974 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -94,7 +94,7 @@ macro(hpx_setup_openshmem) if(NOT OPENSHMEM_FOUND) - messsage( + message( STATUS "1) ${OSHMEM_INFO} 2) ${OSHMEM_INFO_OUTPUT} 3) ${OSHMEM_INFO_OUTPUT_CONTENT} 4) ${OSHMEM_LIBDIR_PATH}" ) From a094610de83cce70a4f09227643467dbd0dbc2a1 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 13:37:13 -0400 Subject: [PATCH 038/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 1e3a9bfd8974..231ebb904a99 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -86,6 +86,8 @@ macro(hpx_setup_openshmem) ${OSHMEM_INFO_OUTPUT_CONTENT} ) + string(STRIP ${OSHMEM_LIBDIR_PATH} OSHMEM_LIBDIR_PATH) + set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${OSHMEM_LIBDIR_PATH}/pkgconfig" ) @@ -96,7 +98,7 @@ macro(hpx_setup_openshmem) message( STATUS - "1) ${OSHMEM_INFO} 2) ${OSHMEM_INFO_OUTPUT} 3) ${OSHMEM_INFO_OUTPUT_CONTENT} 4) ${OSHMEM_LIBDIR_PATH}" + "1) ${OSHMEM_INFO} 2) ${OSHMEM_INFO_OUTPUT} 3) ${OSHMEM_INFO_OUTPUT_CONTENT} 4) ${OSHMEM_LIBDIR_PATH}" ) file(READ ${OSHMEM_INFO_ERROR} OSHMEM_INFO_ERROR_CONTENT) From 600f6d3e21e2e1402f937ac9383cf52ae044a4d5 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 14:43:44 -0400 Subject: [PATCH 039/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 42 ++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 231ebb904a99..ad5ef441ad8f 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -96,16 +96,44 @@ macro(hpx_setup_openshmem) if(NOT OPENSHMEM_FOUND) - message( - STATUS - "1) ${OSHMEM_INFO} 2) ${OSHMEM_INFO_OUTPUT} 3) ${OSHMEM_INFO_OUTPUT_CONTENT} 4) ${OSHMEM_LIBDIR_PATH}" + execute_process( + COMMAND bash -c "${OSHMEM_INFO} --path incdir" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE OSHMEM_INFO_INCSTATUS + OUTPUT_FILE ${OSHMEM_INFO_INCOUTPUT} + ERROR_FILE ${OSHMEM_INFO_INCERROR} ) - file(READ ${OSHMEM_INFO_ERROR} OSHMEM_INFO_ERROR_CONTENT) - message( - FATAL_ERROR - "oshmem not found for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi'\n${OSHMEM_INFO_ERROR_CONTENT}\n${OSHMEM_INFO_OUTPUT_CONTENT}/pkgconfig" + if(OSHMEM_INFO_INCSTATUS) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Program status code: ${OSHMEM_INFO_INCSTATUS}" + ) + endif() + + file(READ ${OSHMEM_INFO_INCOUTPUT} OSHMEM_INFO_OUTPUT_INCCONTENT) + + if(NOT OSHMEM_INFO_OUTPUT_INCCONTENT) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_INCERROR}" + ) + endif() + + string(REGEX MATCH "(\/.*)" OSHMEM_INCDIR_PATH + ${OSHMEM_INFO_OUTPUT_INCCONTENT} ) + + string(STRIP ${OSHMEM_INCDIR_PATH} OSHMEM_INCDIR_PATH) + + set(OPENSHMEM_CFLAGS + "-I${OSHMEM_INCDIR_PATH} -pthread -I${OSHMEM_LIBDIR_PATH}" + ) + set(OPENSHMEM_LDFLAGS + "-Wl,-rpath -Wl,${OSHMEM_LIBDIR_PATH} -Wl,--enable-new-dtags -loshmem -lmpi" + ) + set(OPENSHMEM_LIBRARY_DIRS "-L${OSHMEM_LIBDIR_PATH}") + endif() endif() From c34abaf24482ac1d44605a0364bb4589c10710ee Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 15:00:44 -0400 Subject: [PATCH 040/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index ad5ef441ad8f..8b69b753bbd0 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -96,6 +96,13 @@ macro(hpx_setup_openshmem) if(NOT OPENSHMEM_FOUND) + set(OSHMEM_INFO_INCOUTPUT + "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_stdout_inc.log" + ) + set(OSHMEM_INFO_INCERROR + "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_error_inc.log" + ) + execute_process( COMMAND bash -c "${OSHMEM_INFO} --path incdir" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From 15e1c4d2a3b21603190deba9d5f457aa042c8fec Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 15:25:59 -0400 Subject: [PATCH 041/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 8b69b753bbd0..965217193842 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -141,6 +141,8 @@ macro(hpx_setup_openshmem) ) set(OPENSHMEM_LIBRARY_DIRS "-L${OSHMEM_LIBDIR_PATH}") + add_library(PkgConfig::OPENSHMEM INTERFACE IMPORTED GLOBAL) + endif() endif() From 858b293cb975de9f00468330301ed93b01c3f377 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 30 Oct 2023 15:51:46 -0400 Subject: [PATCH 042/154] cmake logic error --- cmake/HPX_SetupOpenSHMEM.cmake | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 965217193842..b0d658a35817 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -143,24 +143,22 @@ macro(hpx_setup_openshmem) add_library(PkgConfig::OPENSHMEM INTERFACE IMPORTED GLOBAL) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + return() endif() endif() - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS - "${OPENSHMEM_CFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS - "${OPENSHMEM_LDFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES - "${OPENSHMEM_LIBRARY_DIRS}" - ) - - return() - elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") set(OPENSHMEM_PC "ucx") From 3323702783743c75233876071da5edde3e73f753 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 08:03:54 -0500 Subject: [PATCH 043/154] updated merge --- cmake/templates/hpxrun.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index d48403112373..9aed278a204b 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -250,7 +250,7 @@ def build_cmd(options, args): select_parcelport = (lambda pp: ['--hpx:ini=hpx.parcel.mpi.priority=1000', '--hpx:ini=hpx.parcel.mpi.enable=1', '--hpx:ini=hpx.parcel.bootstrap=mpi'] if pp == 'mpi' else ['--hpx:ini=hpx.parcel.lci.priority=1000', '--hpx:ini=hpx.parcel.lci.enable=1', '--hpx:ini=hpx.parcel.bootstrap=lci'] if pp == 'lci' - else ['--hpx:ini=hpx.parcel.gasnet.priority=1000', '--hpx:ini=hpx.parcel.openshmem.enable=1', '--hpx:ini=hpx.parcel.bootstrap=openshmem'] if pp == 'openshmem' + else ['--hpx:ini=hpx.parcel.openshmem.priority=1000', '--hpx:ini=hpx.parcel.openshmem.enable=1', '--hpx:ini=hpx.parcel.bootstrap=openshmem'] if pp == 'openshmem' else ['--hpx:ini=hpx.parcel.tcp.priority=1000', '--hpx:ini=hpx.parcel.tcp.enable=1'] if pp == 'tcp' else []) cmd += select_parcelport(options.parcelport) From 6a0bd44ff7567650591bb57b2debd417759438fb Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 2 Nov 2023 10:11:13 -0400 Subject: [PATCH 044/154] formatting --- cmake/HPX_SetupOpenSHMEM.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index b0d658a35817..6e73cff9e100 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -36,7 +36,6 @@ macro(hpx_setup_openshmem) AND (NOT HPX_WITH_FETCH_OPENSHMEM) AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") ) - set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/pkgconfig") set(OPENSHMEM_PC "oshmem") @@ -158,7 +157,6 @@ macro(hpx_setup_openshmem) return() endif() endif() - elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") set(OPENSHMEM_PC "ucx") From 68bcbf6db7ece283773bf68e2cc2fcfb3c98c458 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 2 Nov 2023 10:30:59 -0400 Subject: [PATCH 045/154] added documentation --- docs/sphinx/manual/building_hpx.rst | 6 ++ .../manual/using_the_openshmem_parcelport.rst | 85 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 docs/sphinx/manual/using_the_openshmem_parcelport.rst diff --git a/docs/sphinx/manual/building_hpx.rst b/docs/sphinx/manual/building_hpx.rst index 3e974517ddfa..7032b15d9167 100644 --- a/docs/sphinx/manual/building_hpx.rst +++ b/docs/sphinx/manual/building_hpx.rst @@ -94,6 +94,12 @@ used CMake options. this experimental parcelport may provide better performance than the MPI parcelport. Please refer to :ref:`using_the_lci_parcelport` for more information about the LCI parcelport. +.. option:: HPX_WITH_PARCELPORT_OPENSHMEM + + Enable the OPENSHMEM parcelport. This enables the use of OpenSHMEM for the networking operations in the HPX runtime. + The default value is ``OFF`` because it's not available on all systems and/or requires another dependency. Please + refer to :ref:`using_the_openshmem_parcelport` for more information about the OpenSHMEM parcelport. + .. option:: HPX_WITH_APEX Enable APEX integration. `APEX `_ can be used to profile |hpx| diff --git a/docs/sphinx/manual/using_the_openshmem_parcelport.rst b/docs/sphinx/manual/using_the_openshmem_parcelport.rst new file mode 100644 index 000000000000..2582fd54eb17 --- /dev/null +++ b/docs/sphinx/manual/using_the_openshmem_parcelport.rst @@ -0,0 +1,85 @@ +.. + Copyright (c) 2023 Christopher Taylor + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _using_the_openshmem_parcelport: + +======================== +Using the OpenSHMEM parcelport +======================== + +.. _info_openshmem: + +Basic information +================= + +OpenSHMEM is an effort to create an API specification standardizing parallel +programming in the Partitioned Global Address Space model. The specification +effort also creates a reference implementation of the API. This implementation +aims to be portable, allowing it to be deployed in multiple environments, and +to be a starting point for implementations targeted to particular hardware +platforms. The reference implementation will serves as a springboard for future +development of the API. + +OpenSHMEM provides one-sided communications, atomics, collectives, and again +implements a partitioned global address space (PGAS) distributed memory +computing model. The parcelport uses put_signal (put and atomic_put), wait_until, +and get from the OpenSHMEM API. + +.. _`OpenSHMEM`: http://www.openshmem.org/site/ + +.. _build_openshmem_pp: + +Build |hpx| with the OpenSHMEM parcelport +=================================== + +While building |hpx|, you can specify a set of |cmake| variables to enable +and configure the OpenSHMEM parcelport. Below, there is a set of the most important +and frequently used CMake variables. + +.. option:: HPX_WITH_PARCELPORT_OPENSHMEM + + Enable the OpenSHMEM parcelport. This enables the use of OpenSHMEM for networking operations in the |hpx| runtime. + The default value is ``OFF`` because it's not available on all systems and/or requires another dependency. + You must set this variable to ``ON`` in order to use the OpenSHMEM parcelport. All the following variables only + make sense when this variable is set to ``ON``. + +.. option:: HPX_WITH_PARCELPORT_OPENSHMEM + + Defines which OpenSHMEM to utilize. The options are `sos;ucx;mpi`. This feature tells cmake how to compile the + parcelport against a specific implementation of OpenSHMEM. + +.. option:: HPX_WITH_FETCH_OPENSHMEM + + Use FetchContent to fetch OpenSHMEM. The default value is ``OFF``. + If this option is set to ``OFF``. This feature tells |hpx| to fetch and build OpenSHMEM for you. + |hpx| will download either `ucx` or `sos` OpenSHMEM based on the value provided in + `HPX_WITH_PARCELPORT_OPENSHMEM`. This feature requires the user to set `CMAKE_C_COMPILER`. The + OpenSHMEM downloaded will be installed into `CMAKE_INSTALL_PREFIX`. PMI support will be compiled + into the parcelport if it's available. If PMI is not available a barebones implementation for SMP + systems will be used. + +.. _run_openshmem_pp: + +Run |hpx| with the OpenSHMEM parcelport +================================= + +We use the same mechanisms as MPI to launch OpenSHMEM, so you can use the same way you run MPI parcelport to run OpenSHMEM +parcelport. Typically, it would be ``hpxrun``, ``mpirun``, ``srun``, or ``oshrun``. + +If you are using ``hpxrun.py``, just pass ``--parcelport openshmem`` to the scripts. + +If you are using ``mpirun``, ``oshrun``, or ``srun``, you can just pass +``--hpx:ini=hpx.parcel.openshmem.priority=1000``, ``--hpx:ini=hpx.parcel.openshmem.enable=1``, and +``--hpx:ini=hpx.parcel.bootstrap=openshmem`` to the |hpx| applications. + +If you are running on a Cray machine, you need to pass `--mpi=pmix` or `--mpi=pmi2` to srun +to enable the PMIx or PMI2 support of SLURM since OpenSHMEM does not support the default Cray PMI. +For example, + +.. code-block:: shell-session + + $ srun --mpi=pmix [hpx application] From aedb8afb0958d917e28f52e4f8f1a9280f90aded Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 2 Nov 2023 13:32:37 -0400 Subject: [PATCH 046/154] fixed documentation --- docs/sphinx/manual/using_the_openshmem_parcelport.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/manual/using_the_openshmem_parcelport.rst b/docs/sphinx/manual/using_the_openshmem_parcelport.rst index 2582fd54eb17..4557bb92ea86 100644 --- a/docs/sphinx/manual/using_the_openshmem_parcelport.rst +++ b/docs/sphinx/manual/using_the_openshmem_parcelport.rst @@ -47,7 +47,7 @@ and frequently used CMake variables. You must set this variable to ``ON`` in order to use the OpenSHMEM parcelport. All the following variables only make sense when this variable is set to ``ON``. -.. option:: HPX_WITH_PARCELPORT_OPENSHMEM +.. option:: HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT Defines which OpenSHMEM to utilize. The options are `sos;ucx;mpi`. This feature tells cmake how to compile the parcelport against a specific implementation of OpenSHMEM. @@ -57,7 +57,7 @@ and frequently used CMake variables. Use FetchContent to fetch OpenSHMEM. The default value is ``OFF``. If this option is set to ``OFF``. This feature tells |hpx| to fetch and build OpenSHMEM for you. |hpx| will download either `ucx` or `sos` OpenSHMEM based on the value provided in - `HPX_WITH_PARCELPORT_OPENSHMEM`. This feature requires the user to set `CMAKE_C_COMPILER`. The + `HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT`. This feature requires the user to set `CMAKE_C_COMPILER`. The OpenSHMEM downloaded will be installed into `CMAKE_INSTALL_PREFIX`. PMI support will be compiled into the parcelport if it's available. If PMI is not available a barebones implementation for SMP systems will be used. From e58a00f94457d31a2f36de8def8823eab8f6714d Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 2 Nov 2023 13:34:23 -0400 Subject: [PATCH 047/154] rm'd extraneous cmake variable --- libs/full/parcelport_openshmem/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/full/parcelport_openshmem/CMakeLists.txt b/libs/full/parcelport_openshmem/CMakeLists.txt index 78575d783503..064fd9d70301 100644 --- a/libs/full/parcelport_openshmem/CMakeLists.txt +++ b/libs/full/parcelport_openshmem/CMakeLists.txt @@ -34,7 +34,6 @@ add_hpx_module( SOURCES ${parcelport_openshmem_sources} HEADERS ${parcelport_openshmem_headers} DEPENDENCIES hpx_core hpx_openshmem_base PkgConfig::OPENSHMEM - ${openshmem_additional_dependencies} MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset CMAKE_SUBDIRS examples tests ) From b8a9634c6d024238c9c2677971c55bedf475db51 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 2 Nov 2023 14:32:16 -0400 Subject: [PATCH 048/154] updated with missing openshmem testing information --- cmake/HPX_AddTest.cmake | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index cd90dc0d0972..cd3238b98ef6 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -211,6 +211,30 @@ function(add_hpx_test category name) endif() endif() endif() + if(HPX_WITH_PARCELPORT_OPENSHMEM AND NOT ${name}_NO_PARCELPORT_OPENSHMEM) + set(_add_test FALSE) + if(DEFINED ${name}_PARCELPORTS) + set(PP_FOUND -1) + list(FIND ${name}_PARCELPORTS "openshmem" PP_FOUND) + if(NOT PP_FOUND EQUAL -1) + set(_add_test TRUE) + endif() + else() + set(_add_test TRUE) + endif() + if(_add_test) + set(_full_name "${category}.distributed.openshmem.${name}") + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "openshmem" "-r" + "openshmem" ${args} + ) + set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) + if(${name}_TIMEOUT) + set_tests_properties( + "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} + ) + endif() + endif() + endif() if(HPX_WITH_PARCELPORT_TCP AND NOT ${${name}_NO_PARCELPORT_TCP}) set(_add_test FALSE) if(DEFINED ${name}_PARCELPORTS) From 20cc1e30ff2c1546bd676e4141f67154cc991e28 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 2 Nov 2023 14:59:05 -0400 Subject: [PATCH 049/154] updated with missing openshmem testing information --- cmake/HPX_AddTest.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index cd3238b98ef6..66882b48f455 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -6,7 +6,7 @@ function(add_hpx_test category name) set(options FAILURE_EXPECTED RUN_SERIAL NO_PARCELPORT_TCP NO_PARCELPORT_MPI - NO_PARCELPORT_LCI NO_PARCELPORT_GASNET + NO_PARCELPORT_LCI NO_PARCELPORT_GASNET NO_PARCELPORT_OPENSHMEM ) set(one_value_args EXECUTABLE LOCALITIES THREADS_PER_LOCALITY TIMEOUT RUNWRAPPER From 82048a81d268ce44406f3b913c618e100b726621 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 08:06:36 -0500 Subject: [PATCH 050/154] updated merge --- .jenkins/lsu/env-clang-16.sh | 5 +++++ .jenkins/lsu/env-clang-17.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.jenkins/lsu/env-clang-16.sh b/.jenkins/lsu/env-clang-16.sh index 393e42c03e54..fa32b317d27a 100644 --- a/.jenkins/lsu/env-clang-16.sh +++ b/.jenkins/lsu/env-clang-16.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" +configure_extra_options+=" -DCMAKE_C_COMPILER=clang" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-17.sh b/.jenkins/lsu/env-clang-17.sh index 1b6ac8d26294..0e3dd4f9d663 100644 --- a/.jenkins/lsu/env-clang-17.sh +++ b/.jenkins/lsu/env-clang-17.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" +configure_extra_options+=" -DCMAKE_C_COMPILER=clang" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" From bf65478dbbc915e55a474fcb4b34590980161c18 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 2 Nov 2023 15:55:00 -0400 Subject: [PATCH 051/154] updated with missing openshmem testing information --- cmake/HPX_AddTest.cmake | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index 66882b48f455..ba314546f4a6 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -5,8 +5,14 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) function(add_hpx_test category name) - set(options FAILURE_EXPECTED RUN_SERIAL NO_PARCELPORT_TCP NO_PARCELPORT_MPI - NO_PARCELPORT_LCI NO_PARCELPORT_GASNET NO_PARCELPORT_OPENSHMEM + set(options + FAILURE_EXPECTED + RUN_SERIAL + NO_PARCELPORT_TCP + NO_PARCELPORT_MPI + NO_PARCELPORT_LCI + NO_PARCELPORT_GASNET + NO_PARCELPORT_OPENSHMEM ) set(one_value_args EXECUTABLE LOCALITIES THREADS_PER_LOCALITY TIMEOUT RUNWRAPPER From 601a0e6b66232ba946f41ed27d5dd538637082db Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 3 Nov 2023 12:07:31 -0400 Subject: [PATCH 052/154] rm'd unnecessary compilation flag --- .jenkins/lsu/env-clang-12.sh | 1 - .jenkins/lsu/env-clang-13.sh | 1 - .jenkins/lsu/env-clang-14.sh | 1 - .jenkins/lsu/env-clang-15.sh | 1 - .jenkins/lsu/env-gcc-10.sh | 1 - .jenkins/lsu/env-gcc-12.sh | 1 - .jenkins/lsu/env-gcc-13.sh | 1 - 7 files changed, 7 deletions(-) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index fb84eabbefb3..be90ef9b8ce1 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -22,7 +22,6 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index db258a801cef..75c858bafbbd 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -22,7 +22,6 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 7ffb8b05f6cb..f456d73e982c 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -22,7 +22,6 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index a3302263d017..584045aead6f 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -22,7 +22,6 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index b83df8358073..43d342336190 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -22,7 +22,6 @@ configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_ASIO_TAG=asio-1-12-0" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index db70f6ecc344..bec417214edc 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -21,7 +21,6 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 4b68406d756f..0538c016118e 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -21,7 +21,6 @@ configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_NETWORKING=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" From c5d4ef160b1a1bc27bf717350fea348caf3e04d6 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 08:07:22 -0500 Subject: [PATCH 053/154] updated merge --- .jenkins/lsu/env-clang-13.sh | 1 + .jenkins/lsu/env-clang-14.sh | 1 + .jenkins/lsu/env-clang-15.sh | 1 + .jenkins/lsu/env-clang-16.sh | 1 + .jenkins/lsu/env-clang-17.sh | 1 + .jenkins/lsu/env-gcc-11.sh | 1 + .jenkins/lsu/env-gcc-12.sh | 1 + .jenkins/lsu/env-gcc-13.sh | 1 + 8 files changed, 8 insertions(+) diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 75c858bafbbd..e3e00b4ac9a5 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -30,6 +30,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" +configure_extra_options+=" -DHPX_WITH_TESTS=ON" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index f456d73e982c..327ecf42178f 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -32,6 +32,7 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" +configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 584045aead6f..a6f622e06320 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -33,6 +33,7 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" +configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-clang-16.sh b/.jenkins/lsu/env-clang-16.sh index fa32b317d27a..455981a09bb4 100644 --- a/.jenkins/lsu/env-clang-16.sh +++ b/.jenkins/lsu/env-clang-16.sh @@ -35,6 +35,7 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" +configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-clang-17.sh b/.jenkins/lsu/env-clang-17.sh index 0e3dd4f9d663..9203dd8ef36d 100644 --- a/.jenkins/lsu/env-clang-17.sh +++ b/.jenkins/lsu/env-clang-17.sh @@ -35,6 +35,7 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" +configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-11.sh b/.jenkins/lsu/env-gcc-11.sh index 3276a6f9c97f..02d391b8ff9b 100644 --- a/.jenkins/lsu/env-gcc-11.sh +++ b/.jenkins/lsu/env-gcc-11.sh @@ -23,6 +23,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_NETWORKING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=STD_EXPERIMENTAL_SIMD" +configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index bec417214edc..3601a8c0d837 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -31,6 +31,7 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" +configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 0538c016118e..9e1f61e238b6 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -31,6 +31,7 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" +configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" From f89240ac8416f7a437eea29104195c0c01554659 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 13:39:13 -0500 Subject: [PATCH 054/154] fixed early return issue --- cmake/HPX_SetupOpenSHMEM.cmake | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 6e73cff9e100..7c8597d862e7 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -154,7 +154,6 @@ macro(hpx_setup_openshmem) PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES "${OPENSHMEM_LIBRARY_DIRS}" ) - return() endif() endif() elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") @@ -455,7 +454,11 @@ macro(hpx_setup_openshmem) foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + string(FIND "${X}" "-loshmem" IDX) + else() + string(FIND "${X}" "-lsma" IDX) + endif() string(FIND "${X}" "-l" LIDX) string(FIND "${X}" "-L" DIRIDX) string(FIND "${X}" "-Wl" SKIP) @@ -594,7 +597,11 @@ macro(hpx_setup_openshmem) foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + string(FIND "${X}" "-loshmem" IDX) + else() + string(FIND "${X}" "-lsma" IDX) + endif() string(FIND "${X}" "-L" DIRIDX) string(FIND "${X}" "-Wl" SKIP) @@ -809,7 +816,11 @@ macro(hpx_setup_openshmem) foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS}) string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + string(FIND "${X}" "-loshmem" IDX) + else() + string(FIND "${X}" "-lsma" IDX) + endif() string(FIND "${X}" "-L" DIRIDX) string(FIND "${X}" "-Wl" SKIP) @@ -948,7 +959,11 @@ macro(hpx_setup_openshmem) foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS_OTHER}) string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + string(FIND "${X}" "-loshmem" IDX) + else() + string(FIND "${X}" "-lsma" IDX) + endif() string(FIND "${X}" "-L" DIRIDX) string(FIND "${X}" "-Wl" SKIP) From a4b40647132c1753ce3f3b666dd20d682a210462 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 20:51:56 -0500 Subject: [PATCH 055/154] updated merge --- .jenkins/lsu/env-clang-13.sh | 1 - .jenkins/lsu/env-clang-14.sh | 1 - .jenkins/lsu/env-clang-15.sh | 1 - .jenkins/lsu/env-clang-16.sh | 1 - .jenkins/lsu/env-clang-17.sh | 1 - .jenkins/lsu/env-gcc-11.sh | 1 - .jenkins/lsu/env-gcc-12.sh | 1 - .jenkins/lsu/env-gcc-13.sh | 1 - 8 files changed, 8 deletions(-) diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index e3e00b4ac9a5..75c858bafbbd 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -30,7 +30,6 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" -configure_extra_options+=" -DHPX_WITH_TESTS=ON" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 327ecf42178f..f456d73e982c 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -32,7 +32,6 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" -configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index a6f622e06320..584045aead6f 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -33,7 +33,6 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" -configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-clang-16.sh b/.jenkins/lsu/env-clang-16.sh index 455981a09bb4..fa32b317d27a 100644 --- a/.jenkins/lsu/env-clang-16.sh +++ b/.jenkins/lsu/env-clang-16.sh @@ -35,7 +35,6 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" -configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-clang-17.sh b/.jenkins/lsu/env-clang-17.sh index 9203dd8ef36d..0e3dd4f9d663 100644 --- a/.jenkins/lsu/env-clang-17.sh +++ b/.jenkins/lsu/env-clang-17.sh @@ -35,7 +35,6 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" -configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-11.sh b/.jenkins/lsu/env-gcc-11.sh index 02d391b8ff9b..3276a6f9c97f 100644 --- a/.jenkins/lsu/env-gcc-11.sh +++ b/.jenkins/lsu/env-gcc-11.sh @@ -23,7 +23,6 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_NETWORKING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=STD_EXPERIMENTAL_SIMD" -configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 3601a8c0d837..bec417214edc 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -31,7 +31,6 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" -configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 9e1f61e238b6..0538c016118e 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -31,7 +31,6 @@ configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" -configure_extra_options+=" -DHPX_WITH_TESTS=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" From f353b99bc8ae08b7166c4c4f4388e5df499b8230 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 14:02:36 -0500 Subject: [PATCH 056/154] fixed outstanding issue --- cmake/HPX_SetupOpenSHMEM.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 7c8597d862e7..aa67dddfd177 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -154,6 +154,8 @@ macro(hpx_setup_openshmem) PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES "${OPENSHMEM_LIBRARY_DIRS}" ) + + set(OPENSHMEM_FOUND ON) endif() endif() elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") From 5a7589e944e7c4a8161e085e7a49401a7e3049da Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 15:43:26 -0500 Subject: [PATCH 057/154] fixed outstanding issue --- cmake/HPX_SetupOpenSHMEM.cmake | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index aa67dddfd177..6d81d17b19a4 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -142,19 +142,6 @@ macro(hpx_setup_openshmem) add_library(PkgConfig::OPENSHMEM INTERFACE IMPORTED GLOBAL) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS - "${OPENSHMEM_CFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS - "${OPENSHMEM_LDFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES - "${OPENSHMEM_LIBRARY_DIRS}" - ) - set(OPENSHMEM_FOUND ON) endif() endif() From 005d34ff5b0588eac94a50fe16cd200354cf767c Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 16:39:35 -0500 Subject: [PATCH 058/154] fixed outstanding issue --- cmake/HPX_SetupOpenSHMEM.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 6d81d17b19a4..b599394c79f7 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -138,7 +138,7 @@ macro(hpx_setup_openshmem) set(OPENSHMEM_LDFLAGS "-Wl,-rpath -Wl,${OSHMEM_LIBDIR_PATH} -Wl,--enable-new-dtags -loshmem -lmpi" ) - set(OPENSHMEM_LIBRARY_DIRS "-L${OSHMEM_LIBDIR_PATH}") + set(OPENSHMEM_LIBRARY_DIRS "${OSHMEM_LIBDIR_PATH}") add_library(PkgConfig::OPENSHMEM INTERFACE IMPORTED GLOBAL) From 78a92001b4b1ea84746fb339d39a7be65a1cd86e Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 18:21:25 -0500 Subject: [PATCH 059/154] added debugging message --- cmake/HPX_SetupOpenSHMEM.cmake | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index b599394c79f7..d9ae6fad93ed 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -1093,6 +1093,10 @@ macro(hpx_setup_openshmem) "$" ) + message(STATUS "OPENSHMEM_CFLAGS:\t${OPENSHMEM_CFLAGS}") + message(STATUS "OPENSHMEM_LDFLAGS:\t${OPENSHMEM_LDFLAGS}") + message(STATUS "OPENSHMEM_LIBRARY_DIRS:\t${OPENSHMEM_LIBRARY_DIRS}") + set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS "${OPENSHMEM_CFLAGS}" @@ -1106,6 +1110,10 @@ macro(hpx_setup_openshmem) "${OPENSHMEM_LIBRARY_DIRS}" ) else() + message(STATUS "OPENSHMEM_CFLAGS:\t${OPENSHMEM_CFLAGS}") + message(STATUS "OPENSHMEM_LDFLAGS:\t${OPENSHMEM_LDFLAGS}") + message(STATUS "OPENSHMEM_LIBRARY_DIRS:\t${OPENSHMEM_LIBRARY_DIRS}") + set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS "${OPENSHMEM_CFLAGS}" From bc63e5b1c88287503359d6cce0249af19d16860b Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 18:30:47 -0500 Subject: [PATCH 060/154] fixed undefined macro issue --- libs/core/openshmem_base/src/openshmem_environment.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 42ccd23233e6..bb21c7b42fe6 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -23,6 +23,10 @@ #include #include +#if !defined(SHMEM_SIGNAL_SET) +#define SHMEM_SIGNAL_SET (1<<0) +#endif + /////////////////////////////////////////////////////////////////////////////// namespace hpx::util { From c2da321bc91d24d3e61daf941d1baf81e203c762 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 18:40:39 -0500 Subject: [PATCH 061/154] rollback to a semantically equivalent implementation if put_signal is not present --- .../openshmem_base/src/openshmem_environment.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index bb21c7b42fe6..4734907d7108 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -23,10 +23,6 @@ #include #include -#if !defined(SHMEM_SIGNAL_SET) -#define SHMEM_SIGNAL_SET (1<<0) -#endif - /////////////////////////////////////////////////////////////////////////////// namespace hpx::util { @@ -260,9 +256,20 @@ namespace hpx::util { { const std::lock_guard lk(segment_mutex[node]); +#if !defined(SHMEM_SIGNAL_SET) + shmem_uint8_put(raddr, addr, size, node); + shmem_fence(); + shmem_uint8_put( + reinterpret_cast(sigaddr), + reinterpret_cast(sigaddr), + sizeof(sigaddr), + node + ); +#else shmem_uint8_put_signal(raddr, addr, size, reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, node); +#endif } } From 4fb5279b6e5d6b30988e9b0d656bbbd8d719f775 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 18:42:48 -0500 Subject: [PATCH 062/154] updated with clang-formatting --- libs/core/openshmem_base/src/openshmem_environment.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 4734907d7108..6345a722c5ad 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -259,12 +259,9 @@ namespace hpx::util { #if !defined(SHMEM_SIGNAL_SET) shmem_uint8_put(raddr, addr, size, node); shmem_fence(); - shmem_uint8_put( - reinterpret_cast(sigaddr), - reinterpret_cast(sigaddr), - sizeof(sigaddr), - node - ); + shmem_uint8_put(reinterpret_cast(sigaddr), + reinterpret_cast(sigaddr), sizeof(sigaddr), + node); #else shmem_uint8_put_signal(raddr, addr, size, reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, From f2ecbc692086b152206e31ca5d14ed49cfd53658 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 20:00:14 -0500 Subject: [PATCH 063/154] additional fixes for openmpi-openshmem --- cmake/HPX_SetupOpenSHMEM.cmake | 48 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index d9ae6fad93ed..9c234fbf88b0 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -354,7 +354,9 @@ macro(hpx_setup_openshmem) ) endif() - if(OPENSHMEM_CFLAGS) + if(OPENSHMEM_CFLAGS AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" + STREQUAL "mpi")) + ) set(IS_PARAM "0") set(PARAM_FOUND "0") set(NEWPARAM "") @@ -392,7 +394,9 @@ macro(hpx_setup_openshmem) endforeach() endif() - if(OPENSHMEM_CFLAGS_OTHER) + if(OPENSHMEM_CFLAGS_OTHER AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" + STREQUAL "mpi")) + ) set(IS_PARAM "0") set(PARAM_FOUND "0") set(NEWPARAM "") @@ -430,7 +434,9 @@ macro(hpx_setup_openshmem) endforeach() endif() - if(OPENSHMEM_LDFLAGS) + if(OPENSHMEM_LDFLAGS AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" + STREQUAL "mpi")) + ) set(IS_PARAM "0") set(PARAM_FOUND "0") set(NEWPARAM "") @@ -443,11 +449,7 @@ macro(hpx_setup_openshmem) foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) string(FIND "${X}" "--param" PARAM_FOUND) - if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") - string(FIND "${X}" "-loshmem" IDX) - else() - string(FIND "${X}" "-lsma" IDX) - endif() + string(FIND "${X}" "-lsma" IDX) string(FIND "${X}" "-l" LIDX) string(FIND "${X}" "-L" DIRIDX) string(FIND "${X}" "-Wl" SKIP) @@ -572,7 +574,9 @@ macro(hpx_setup_openshmem) endif() endif() - if(OPENSHMEM_LDFLAGS_OTHER) + if(OPENSHMEM_LDFLAGS_OTHER + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) unset(FOUND_LIB) set(IS_PARAM "0") set(PARAM_FOUND "0") @@ -586,11 +590,7 @@ macro(hpx_setup_openshmem) foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) string(FIND "${X}" "--param" PARAM_FOUND) - if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") - string(FIND "${X}" "-loshmem" IDX) - else() - string(FIND "${X}" "-lsma" IDX) - endif() + string(FIND "${X}" "-lsma" IDX) string(FIND "${X}" "-L" DIRIDX) string(FIND "${X}" "-Wl" SKIP) @@ -715,7 +715,9 @@ macro(hpx_setup_openshmem) endif() - if(OPENSHMEM_STATIC_CFLAGS) + if(OPENSHMEM_STATIC_CFLAGS + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) set(IS_PARAM "0") set(PARAM_FOUND "0") set(NEWPARAM "") @@ -753,7 +755,9 @@ macro(hpx_setup_openshmem) endforeach() endif() - if(OPENSHMEM_STATIC_CFLAGS_OTHER) + if(OPENSHMEM_STATIC_CFLAGS_OTHER + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) set(IS_PARAM "0") set(PARAM_FOUND "0") set(NEWPARAM "") @@ -791,7 +795,9 @@ macro(hpx_setup_openshmem) endforeach() endif() - if(OPENSHMEM_STATIC_LDFLAGS) + if(OPENSHMEM_STATIC_LDFLAGS + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) unset(FOUND_LIB) set(IS_PARAM "0") set(PARAM_FOUND "0") @@ -934,7 +940,9 @@ macro(hpx_setup_openshmem) endif() endif() - if(OPENSHMEM_STATIC_LDFLAGS_OTHER) + if(OPENSHMEM_STATIC_LDFLAGS_OTHER + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) unset(FOUND_LIB) set(IS_PARAM "0") set(PARAM_FOUND "0") @@ -1079,7 +1087,9 @@ macro(hpx_setup_openshmem) endif() endif() - if(OPENSHMEM_DIR) + if(OPENSHMEM_DIR AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL + "mpi")) + ) list(TRANSFORM OPENSHMEM_CFLAGS REPLACE "${OPENSHMEM_DIR}/install" "$" From 7c1cc66b7b4a32995a12986e37578b0324b036c6 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 20:38:37 -0500 Subject: [PATCH 064/154] additional fixes for openmpi-openshmem --- cmake/HPX_SetupOpenSHMEM.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 9c234fbf88b0..eb83512a1f12 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -136,7 +136,7 @@ macro(hpx_setup_openshmem) "-I${OSHMEM_INCDIR_PATH} -pthread -I${OSHMEM_LIBDIR_PATH}" ) set(OPENSHMEM_LDFLAGS - "-Wl,-rpath -Wl,${OSHMEM_LIBDIR_PATH} -Wl,--enable-new-dtags -loshmem -lmpi" + "-Wl,-rpath -Wl,${OSHMEM_LIBDIR_PATH} -loshmem -lmpi" ) set(OPENSHMEM_LIBRARY_DIRS "${OSHMEM_LIBDIR_PATH}") From 1ac506e616da8ad082c54819ec8dcd4218d2fbbf Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 20:40:17 -0500 Subject: [PATCH 065/154] additional fixes for openmpi-openshmem --- cmake/HPX_SetupOpenSHMEM.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index eb83512a1f12..0dbe830a15f0 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -136,7 +136,7 @@ macro(hpx_setup_openshmem) "-I${OSHMEM_INCDIR_PATH} -pthread -I${OSHMEM_LIBDIR_PATH}" ) set(OPENSHMEM_LDFLAGS - "-Wl,-rpath -Wl,${OSHMEM_LIBDIR_PATH} -loshmem -lmpi" + "-loshmem" ) set(OPENSHMEM_LIBRARY_DIRS "${OSHMEM_LIBDIR_PATH}") From 65894f0ba637553020075181e6583c12e114b255 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 6 Nov 2023 22:06:08 -0500 Subject: [PATCH 066/154] additional fixes for openmpi-openshmem --- cmake/HPX_SetupOpenSHMEM.cmake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 0dbe830a15f0..7bd2b931c3b4 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -135,9 +135,7 @@ macro(hpx_setup_openshmem) set(OPENSHMEM_CFLAGS "-I${OSHMEM_INCDIR_PATH} -pthread -I${OSHMEM_LIBDIR_PATH}" ) - set(OPENSHMEM_LDFLAGS - "-loshmem" - ) + set(OPENSHMEM_LDFLAGS "-loshmem") set(OPENSHMEM_LIBRARY_DIRS "${OSHMEM_LIBDIR_PATH}") add_library(PkgConfig::OPENSHMEM INTERFACE IMPORTED GLOBAL) From b6eb00653ed794bc876577822f4bbfd716b3ad2e Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 7 Nov 2023 10:04:57 -0500 Subject: [PATCH 067/154] fixing checks on oshmem_info --- cmake/HPX_SetupOpenSHMEM.cmake | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 7bd2b931c3b4..97d13aaa02f2 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -75,7 +75,13 @@ macro(hpx_setup_openshmem) file(READ ${OSHMEM_INFO_OUTPUT} OSHMEM_INFO_OUTPUT_CONTENT) - if(NOT OSHMEM_INFO_OUTPUT_CONTENT) + if(NOT DEFINED OSHMEM_INFO_OUTPUT_CONTENT) + message( + FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}" + ) + endif() + + if("${OSHMEM_INFO_OUTPUT_CONTENT}" STREQUAL "") message( FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}" ) @@ -119,7 +125,14 @@ macro(hpx_setup_openshmem) file(READ ${OSHMEM_INFO_INCOUTPUT} OSHMEM_INFO_OUTPUT_INCCONTENT) - if(NOT OSHMEM_INFO_OUTPUT_INCCONTENT) + if(NOT DEFINED OSHMEM_INFO_OUTPUT_INCCONTENT) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_INCERROR}" + ) + endif() + + if("${OSHMEM_INFO_OUTPUT_INCCONTENT}" STREQUAL "") message( FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_INCERROR}" From 1ab54a9679d89c502fe877cbffb7b05f26446643 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 7 Nov 2023 12:59:22 -0500 Subject: [PATCH 068/154] debugging issues with oshmem_info --- cmake/HPX_SetupOpenSHMEM.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 97d13aaa02f2..4b1a7b0c0274 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -77,13 +77,13 @@ macro(hpx_setup_openshmem) if(NOT DEFINED OSHMEM_INFO_OUTPUT_CONTENT) message( - FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}" + FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}\n${OSHMEM_INFO_OUTPUT_CONTENT}" ) endif() if("${OSHMEM_INFO_OUTPUT_CONTENT}" STREQUAL "") message( - FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}" + FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}\n${OSHMEM_INFO_OUTPUT_CONTENT}" ) endif() From 4cc40a442e874b3f91c24f024295b8f2f1881808 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 7 Nov 2023 14:44:36 -0500 Subject: [PATCH 069/154] cmake formatting change --- cmake/HPX_SetupOpenSHMEM.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 4b1a7b0c0274..16da4989b5d1 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -77,13 +77,15 @@ macro(hpx_setup_openshmem) if(NOT DEFINED OSHMEM_INFO_OUTPUT_CONTENT) message( - FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}\n${OSHMEM_INFO_OUTPUT_CONTENT}" + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}\n${OSHMEM_INFO_OUTPUT_CONTENT}" ) endif() if("${OSHMEM_INFO_OUTPUT_CONTENT}" STREQUAL "") message( - FATAL_ERROR "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}\n${OSHMEM_INFO_OUTPUT_CONTENT}" + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}\n${OSHMEM_INFO_OUTPUT_CONTENT}" ) endif() From 4a5769bfb2b85dd79d7e225aa6466fcb98149b62 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 7 Nov 2023 16:39:06 -0500 Subject: [PATCH 070/154] adding debugging information --- cmake/HPX_SetupOpenSHMEM.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 16da4989b5d1..43ffe7289336 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -137,7 +137,7 @@ macro(hpx_setup_openshmem) if("${OSHMEM_INFO_OUTPUT_INCCONTENT}" STREQUAL "") message( FATAL_ERROR - "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_INCERROR}" + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_INCERROR}\n${OSHMEM_INFO_OUTPUT_INCCONTENT}" ) endif() From b05f5fdf5d3f3321fa162e3c49c23992feeb3f9a Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 9 Nov 2023 13:19:44 -0500 Subject: [PATCH 071/154] several fixes for documentation and manual --- docs/sphinx/manual.rst | 1 + docs/sphinx/manual/using_the_openshmem_parcelport.rst | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/manual.rst b/docs/sphinx/manual.rst index 256456efd864..097257ed4f9c 100644 --- a/docs/sphinx/manual.rst +++ b/docs/sphinx/manual.rst @@ -32,6 +32,7 @@ information on how to build and use |hpx| in different scenarios. manual/debugging_hpx_applications manual/optimizing_hpx_applications manual/using_the_lci_parcelport + manual/using_the_openshmem_parcelport manual/hpx_runtime_and_resources manual/miscellaneous manual/troubleshooting diff --git a/docs/sphinx/manual/using_the_openshmem_parcelport.rst b/docs/sphinx/manual/using_the_openshmem_parcelport.rst index 4557bb92ea86..edf4dceb3980 100644 --- a/docs/sphinx/manual/using_the_openshmem_parcelport.rst +++ b/docs/sphinx/manual/using_the_openshmem_parcelport.rst @@ -7,9 +7,9 @@ .. _using_the_openshmem_parcelport: -======================== +============================== Using the OpenSHMEM parcelport -======================== +============================== .. _info_openshmem: From 9d9dd9fa32f9adaae4a19fdb3976dcfd9862bea4 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 20:52:39 -0500 Subject: [PATCH 072/154] updated merge --- libs/core/init_runtime_local/src/init_runtime_local.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/init_runtime_local/src/init_runtime_local.cpp b/libs/core/init_runtime_local/src/init_runtime_local.cpp index ad669d393721..71a771a4821e 100644 --- a/libs/core/init_runtime_local/src/init_runtime_local.cpp +++ b/libs/core/init_runtime_local/src/init_runtime_local.cpp @@ -444,7 +444,7 @@ namespace hpx { hpx::program_options::variables_map& vm)> const& f, int argc, char** argv, init_params const& params, bool blocking) { - int result; + int result = 0; try { if ((result = ensure_no_runtime_is_up()) != 0) From b59b4bf5aa15384dbc7835b76783422427cb2b9b Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 20:54:38 -0500 Subject: [PATCH 073/154] updated merge --- docs/sphinx/api/public_distributed_api.rst | 204 ++++++--------------- 1 file changed, 60 insertions(+), 144 deletions(-) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index bc5cbe322a73..28867af8af24 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -16,164 +16,80 @@ to harness the full potential of distributed computing. Here, you'll find a comp list of header files, classes and functions for various distributed computing features provided by |hpx|. -.. _public_distr_api_header_barrier: +.. _public_distr_api_header_all_gather: -``hpx/barrier.hpp`` -=================== +``hpx/collectives/all_gather.hpp`` +================================== -The header :hpx-header:`libs/full/include/include,hpx/barrier.hpp` includes -a distributed barrier implementation. For information regarding the C++ standard -library header :cppreference-header:`barrier`, see :ref:`public_api`. +The header :hpx-header:`libs/full/collectives/include,hpx/collectives/all_gather.hpp` +contains definitions and implementations related to the `all_gather` operation. -Classes -------- +Functions +--------- + +.. table:: `hpx` functions of header ``hpx/collectives/all_gather.hpp`` + + +------------------------------------------+ + | Function | + +==========================================+ + | :cpp:func:`hpx::collectives::all_gather` | + +------------------------------------------+ -.. table:: Distributed implementation of classes of header ``hpx/barrier.hpp`` +.. _public_distr_api_header_all_reduce: - +----------------------------------------+ - | Class | - +========================================+ - | :cpp:class:`hpx::distributed::barrier` | - +----------------------------------------+ +``hpx/collectives/all_reduce.hpp`` +================================== + +The header :hpx-header:`libs/full/collectives/include,hpx/collectives/all_reduce.hpp` +contains definitions and implementations related to the `all_reduce` operation. Functions --------- -.. table:: `hpx` functions of header ``hpx/barrier.hpp`` - - +-------------------------------------------+ - | Function | - +===========================================+ - | :cpp:func:`hpx::distributed::wait` | - +-------------------------------------------+ - | :cpp:func:`hpx::distributed::synchronize` | - +-------------------------------------------+ - -.. _public_distr_api_header_collectives: - -``hpx/collectives.hpp`` -======================= - -The header :hpx-header:`libs/full/include/include,hpx/collectives.hpp` -contains definitions and implementations related to the collectives operations. - -Classes -------- - -.. table:: `hpx` classes of header ``hpx/collectives.hpp`` - - +-----------------------------------------------------+ - | Class | - +=====================================================+ - | :cpp:struct:`hpx::collectives::num_sites_arg` | - +-----------------------------------------------------+ - | :cpp:struct:`hpx::collectives::this_site_arg` | - +-----------------------------------------------------+ - | :cpp:struct:`hpx::collectives::that_site_arg` | - +-----------------------------------------------------+ - | :cpp:struct:`hpx::collectives::generation_arg` | - +-----------------------------------------------------+ - | :cpp:struct:`hpx::collectives::root_site_arg` | - +-----------------------------------------------------+ - | :cpp:struct:`hpx::collectives::tag_arg` | - +-----------------------------------------------------+ - | :cpp:struct:`hpx::collectives::arity_arg` | - +-----------------------------------------------------+ - | :cpp:struct:`hpx::collectives::communicator` | - +-----------------------------------------------------+ - | :cpp:class:`hpx::collectives::channel_communicator` | - +-----------------------------------------------------+ +.. table:: `hpx` functions of header ``hpx/collectives/all_reduce.hpp`` + + +------------------------------------------+ + | Function | + +==========================================+ + | :cpp:func:`hpx::collectives::all_reduce` | + +------------------------------------------+ + +.. _public_distr_api_header_all_to_all: + +``hpx/collectives/all_to_all.hpp`` +================================== + +The header :hpx-header:`libs/full/collectives/include,hpx/collectives/all_to_all.hpp` +contains definitions and implementations related to the `all_to_all` operation. Functions --------- -.. table:: `hpx` functions of header ``hpx/collectives.hpp`` - - +-----------------------------------------------------------+ - | Function | - +===========================================================+ - | :cpp:func:`hpx::collectives::all_gather` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::all_reduce` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::all_to_all` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::broadcast_to` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::broadcast_from` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::create_channel_communicator` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::set` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::get` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::create_communication_set` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::create_communicator` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::create_local_communicator` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::communicator::set_info` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::communicator::get_info` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::communicator::is_root` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::exclusive_scan` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::gather_here` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::gather_there` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::inclusive_scan` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::reduce_here` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::reduce_there` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::scatter_from` | - +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::scatter_to` | - +-----------------------------------------------------------+ - -.. _public_distr_api_header_latch: - -``hpx/latch.hpp`` -================= - -The header :hpx-header:`libs/full/include/include,hpx/latch.hpp` includes -a distributed latch implementation. For information regarding the C++ standard -library header :cppreference-header:`latch`, see :ref:`public_api`. - -Classes -------- - -.. table:: Distributed implementation of classes of header ``hpx/latch.hpp`` - - +--------------------------------------+ - | Class | - +======================================+ - | :cpp:class:`hpx::distributed::latch` | - +--------------------------------------+ +.. table:: `hpx` functions of header ``hpx/collectives/all_to_all.hpp`` + + +------------------------------------------+ + | Function | + +==========================================+ + | :cpp:func:`hpx::collectives::all_to_all` | + +------------------------------------------+ + +.. _public_distr_api_header_argument_types: + +``hpx/collectives/argument_types.hpp`` +================================== + +The header :hpx-header:`libs/full/collectives/include,hpx/collectives/argument_types.hpp` +contains definitions and implementations related to the `argument_types` operation. Functions --------- -.. table:: `hpx` functions of header ``hpx/latch.hpp`` - - +---------------------------------------------------+ - | Function | - +===================================================+ - | :cpp:func:`hpx::distributed::count_down_and_wait` | - +---------------------------------------------------+ - | :cpp:func:`hpx::distributed::arrive_and_wait` | - +---------------------------------------------------+ - | :cpp:func:`hpx::distributed::count_down` | - +---------------------------------------------------+ - | :cpp:func:`hpx::distributed::is_ready` | - +---------------------------------------------------+ - | :cpp:func:`hpx::distributed::try_wait` | - +---------------------------------------------------+ - | :cpp:func:`hpx::distributed::wait` | - +---------------------------------------------------+ +.. table:: `hpx` functions of header ``hpx/collectives/argument_types.hpp`` + + +----------------------------------------------+ + | Function | + +==============================================+ + | :cpp:class:`hpx::collectives::argument_type` | + +----------------------------------------------+ + | :cpp:func:`hpx::collectives::argument_type` | + +----------------------------------------------+ From 19b6e73322189ce69a61648209a60afee632c2a0 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Sun, 1 Oct 2023 21:49:53 +0200 Subject: [PATCH 074/154] List collectives in a table --- docs/sphinx/api/public_distributed_api.rst | 125 ++++++++++----------- 1 file changed, 58 insertions(+), 67 deletions(-) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index 28867af8af24..2f9db939576c 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -16,80 +16,71 @@ to harness the full potential of distributed computing. Here, you'll find a comp list of header files, classes and functions for various distributed computing features provided by |hpx|. -.. _public_distr_api_header_all_gather: +.. _public_distr_api_header_collectives: -``hpx/collectives/all_gather.hpp`` -================================== +``hpx/collectives.hpp`` +======================= -The header :hpx-header:`libs/full/collectives/include,hpx/collectives/all_gather.hpp` -contains definitions and implementations related to the `all_gather` operation. +The header :hpx-header:`libs/full/include/include,hpx/collectives.hpp` +contains definitions and implementations related to the collectives operations. -Functions ---------- - -.. table:: `hpx` functions of header ``hpx/collectives/all_gather.hpp`` - - +------------------------------------------+ - | Function | - +==========================================+ - | :cpp:func:`hpx::collectives::all_gather` | - +------------------------------------------+ - -.. _public_distr_api_header_all_reduce: - -``hpx/collectives/all_reduce.hpp`` -================================== - -The header :hpx-header:`libs/full/collectives/include,hpx/collectives/all_reduce.hpp` -contains definitions and implementations related to the `all_reduce` operation. - -Functions ---------- - -.. table:: `hpx` functions of header ``hpx/collectives/all_reduce.hpp`` - - +------------------------------------------+ - | Function | - +==========================================+ - | :cpp:func:`hpx::collectives::all_reduce` | - +------------------------------------------+ - -.. _public_distr_api_header_all_to_all: - -``hpx/collectives/all_to_all.hpp`` -================================== - -The header :hpx-header:`libs/full/collectives/include,hpx/collectives/all_to_all.hpp` -contains definitions and implementations related to the `all_to_all` operation. - -Functions ---------- - -.. table:: `hpx` functions of header ``hpx/collectives/all_to_all.hpp`` - - +------------------------------------------+ - | Function | - +==========================================+ - | :cpp:func:`hpx::collectives::all_to_all` | - +------------------------------------------+ - -.. _public_distr_api_header_argument_types: +Classes +------- -``hpx/collectives/argument_types.hpp`` -================================== +.. table:: `hpx` classes of header ``hpx/collectives.hpp`` -The header :hpx-header:`libs/full/collectives/include,hpx/collectives/argument_types.hpp` -contains definitions and implementations related to the `argument_types` operation. + +-----------------------------------------------------+ + | Function | + +=====================================================+ + | :cpp:class:`hpx::collectives::barrier` | + +-----------------------------------------------------+ + | :cpp:class:`hpx::collectives::channel_communicator` | + +-----------------------------------------------------+ Functions --------- -.. table:: `hpx` functions of header ``hpx/collectives/argument_types.hpp`` +.. table:: `hpx` functions of header ``hpx/collectives.hpp`` + + +-----------------------------------------------------------+ + | Function | + +===========================================================+ + | :cpp:func:`hpx::collectives::all_gather` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::all_reduce` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::all_to_all` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::broadcast_to` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::broadcast_from` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::create_channel_communicator` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::set` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::get` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::create_communication_set` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::create_communicator` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::create_local_communicator` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::exclusive_scan` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::gather_here` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::gather_there` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::inclusive_scan` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::reduce_here` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::reduce_there` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::scatter_from` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::scatter_to` | + +-----------------------------------------------------------+ - +----------------------------------------------+ - | Function | - +==============================================+ - | :cpp:class:`hpx::collectives::argument_type` | - +----------------------------------------------+ - | :cpp:func:`hpx::collectives::argument_type` | - +----------------------------------------------+ From 3ca15cbdfdc184e499147a0f394387d05d4b1825 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Tue, 10 Oct 2023 19:48:10 +0200 Subject: [PATCH 075/154] Add distributed public API for collectives, latch, barrier --- docs/sphinx/api/public_distributed_api.rst | 99 +++++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index 2f9db939576c..f614189ca927 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -16,6 +16,39 @@ to harness the full potential of distributed computing. Here, you'll find a comp list of header files, classes and functions for various distributed computing features provided by |hpx|. +.. _public_distr_api_header_barrier: + +``hpx/barrier.hpp`` +=================== + +The header :hpx-header:`libs/full/include/include,hpx/barrier.hpp` includes +a distributed barrier implementation. For information regarding the C++ standard +library header :cppreference-header:`barrier`, see :ref:`public_api`. + +Classes +------- + +.. table:: Distributed implementation of classes of header ``hpx/barrier.hpp`` + + +----------------------------------------+ + | Class | + +========================================+ + | :cpp:class:`hpx::distributed::barrier` | + +----------------------------------------+ + +Functions +--------- + +.. table:: `hpx` functions of header ``hpx/barrier.hpp`` + + +-------------------------------------------+ + | Function | + +===========================================+ + | :cpp:func:`hpx::distributed::wait` | + +-------------------------------------------+ + | :cpp:func:`hpx::distributed::synchronize` | + +-------------------------------------------+ + .. _public_distr_api_header_collectives: ``hpx/collectives.hpp`` @@ -30,12 +63,28 @@ Classes .. table:: `hpx` classes of header ``hpx/collectives.hpp`` +-----------------------------------------------------+ - | Function | + | Class | +=====================================================+ - | :cpp:class:`hpx::collectives::barrier` | + | :cpp:struct:`hpx::collectives::num_sites_arg` | + +-----------------------------------------------------+ + | :cpp:struct:`hpx::collectives::this_site_arg` | + +-----------------------------------------------------+ + | :cpp:struct:`hpx::collectives::that_site_arg` | + +-----------------------------------------------------+ + | :cpp:struct:`hpx::collectives::generation_arg` | + +-----------------------------------------------------+ + | :cpp:struct:`hpx::collectives::root_site_arg` | + +-----------------------------------------------------+ + | :cpp:struct:`hpx::collectives::tag_arg` | + +-----------------------------------------------------+ + | :cpp:struct:`hpx::collectives::arity_arg` | + +-----------------------------------------------------+ + | :cpp:class:`hpx::collectives::communicator` | +-----------------------------------------------------+ | :cpp:class:`hpx::collectives::channel_communicator` | +-----------------------------------------------------+ + | :cpp:class:`hpx::collectives::communicator` | + +-----------------------------------------------------+ Functions --------- @@ -67,6 +116,12 @@ Functions +-----------------------------------------------------------+ | :cpp:func:`hpx::collectives::create_local_communicator` | +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::set_info` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::get_info` | + +-----------------------------------------------------------+ + | :cpp:func:`hpx::collectives::is_root` | + +-----------------------------------------------------------+ | :cpp:func:`hpx::collectives::exclusive_scan` | +-----------------------------------------------------------+ | :cpp:func:`hpx::collectives::gather_here` | @@ -84,3 +139,43 @@ Functions | :cpp:func:`hpx::collectives::scatter_to` | +-----------------------------------------------------------+ +.. _public_distr_api_header_latch: + +``hpx/latch.hpp`` +================= + +The header :hpx-header:`libs/full/include/include,hpx/latch.hpp` includes +a distributed latch implementation. For information regarding the C++ standard +library header :cppreference-header:`latch`, see :ref:`public_api`. + +Classes +------- + +.. table:: Distributed implementation of classes of header ``hpx/latch.hpp`` + + +--------------------------------------+ + | Class | + +======================================+ + | :cpp:class:`hpx::distributed::latch` | + +--------------------------------------+ + +Functions +--------- + +.. table:: `hpx` functions of header ``hpx/latch.hpp`` + + +---------------------------------------------------+ + | Function | + +===================================================+ + | :cpp:func:`hpx::distributed::count_down_and_wait` | + +---------------------------------------------------+ + | :cpp:func:`hpx::distributed::arrive_and_wait` | + +---------------------------------------------------+ + | :cpp:func:`hpx::distributed::count_down` | + +---------------------------------------------------+ + | :cpp:func:`hpx::distributed::is_ready` | + +---------------------------------------------------+ + | :cpp:func:`hpx::distributed::try_wait` | + +---------------------------------------------------+ + | :cpp:func:`hpx::distributed::wait` | + +---------------------------------------------------+ From 91c9569c337ee06410b04b5791ebb8c04bc906a3 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 20:57:38 -0500 Subject: [PATCH 076/154] updated merge --- .../include/hpx/collectives/channel_communicator.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp index ca011d3d17ef..47acb6d2340b 100644 --- a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp @@ -120,6 +120,8 @@ namespace hpx { namespace collectives { namespace hpx { namespace collectives { // forward declarations + /// a handle identifying the communication channel to use for get/set + /// operations class channel_communicator; template From dd932ef6a7c999eaa2258c9c23c4f618ddf4ece6 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 20:58:02 -0500 Subject: [PATCH 077/154] updated merge --- docs/sphinx/api/public_distributed_api.rst | 12 +++++------- .../include/hpx/collectives/channel_communicator.hpp | 2 -- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index f614189ca927..bc5cbe322a73 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -63,7 +63,7 @@ Classes .. table:: `hpx` classes of header ``hpx/collectives.hpp`` +-----------------------------------------------------+ - | Class | + | Class | +=====================================================+ | :cpp:struct:`hpx::collectives::num_sites_arg` | +-----------------------------------------------------+ @@ -79,12 +79,10 @@ Classes +-----------------------------------------------------+ | :cpp:struct:`hpx::collectives::arity_arg` | +-----------------------------------------------------+ - | :cpp:class:`hpx::collectives::communicator` | + | :cpp:struct:`hpx::collectives::communicator` | +-----------------------------------------------------+ | :cpp:class:`hpx::collectives::channel_communicator` | +-----------------------------------------------------+ - | :cpp:class:`hpx::collectives::communicator` | - +-----------------------------------------------------+ Functions --------- @@ -116,11 +114,11 @@ Functions +-----------------------------------------------------------+ | :cpp:func:`hpx::collectives::create_local_communicator` | +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::set_info` | + | :cpp:func:`hpx::collectives::communicator::set_info` | +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::get_info` | + | :cpp:func:`hpx::collectives::communicator::get_info` | +-----------------------------------------------------------+ - | :cpp:func:`hpx::collectives::is_root` | + | :cpp:func:`hpx::collectives::communicator::is_root` | +-----------------------------------------------------------+ | :cpp:func:`hpx::collectives::exclusive_scan` | +-----------------------------------------------------------+ diff --git a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp index 47acb6d2340b..ca011d3d17ef 100644 --- a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp @@ -120,8 +120,6 @@ namespace hpx { namespace collectives { namespace hpx { namespace collectives { // forward declarations - /// a handle identifying the communication channel to use for get/set - /// operations class channel_communicator; template From c4d1d9b5bb65c987e337ba4ef282c03a878ffaa5 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:07:30 -0500 Subject: [PATCH 078/154] updated merge --- cmake/HPX_AddTest.cmake | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index ba314546f4a6..85046e469acd 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -188,35 +188,6 @@ function(add_hpx_test category name) endif() endif() endif() - if(HPX_WITH_PARCELPORT_GASNET AND NOT ${name}_NO_PARCELPORT_GASNET) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" - "gasnet-smp" ${args} - ) - set_tests_properties( - "${_full_name}" - PROPERTIES - RUN_SERIAL TRUE ENVIRONMENT - "PATH=${PROJECT_BINARY_DIR}/_deps/gasnet-src/install/bin:$ENV{PATH}" - ) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() if(HPX_WITH_PARCELPORT_OPENSHMEM AND NOT ${name}_NO_PARCELPORT_OPENSHMEM) set(_add_test FALSE) if(DEFINED ${name}_PARCELPORTS) From b2cba86654bee5cd72c119013703c9b782ab11ad Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:15:18 -0500 Subject: [PATCH 079/154] updated merge --- .jenkins/lsu/env-clang-14.sh | 2 -- .jenkins/lsu/env-clang-15.sh | 3 --- CMakeLists.txt | 3 ++- cmake/HPX_SetupOpenSHMEM.cmake | 1 - libs/core/version/CMakeLists.txt | 7 +++++++ 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index f456d73e982c..144c17e7820d 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -25,8 +25,6 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 584045aead6f..1df2a9fd23b3 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -25,9 +25,6 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a6ff3b6c5d9..087a1248c51e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1233,7 +1233,8 @@ if(HPX_WITH_NETWORKING) OFF CATEGORY "Parcelport" ) hpx_option( - HPX_WITH_FETCH_OPENSHMEM BOOL + HPX_WITH_FETCH_OPENSHMEM + BOOL "Download an OpenSHMEM if one is not available" OFF CATEGORY "Build Targets" ADVANCED diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 43ffe7289336..244be1879a1b 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -1150,5 +1150,4 @@ macro(hpx_setup_openshmem) "${OPENSHMEM_LIBRARY_DIRS}" ) endif() - endmacro() diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index 08fe9e1b6a06..470fe90c852b 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -25,6 +25,13 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM) hpx_setup_openshmem() set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) endif() +if(HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM +) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT From b6f2f3cbe874eaa35f2544e49e0bd4296a24024b Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:18:52 -0500 Subject: [PATCH 080/154] updated merge --- .jenkins/lsu/env-clang-14.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 144c17e7820d..f456d73e982c 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" From 9a768bab32e3fb27396669d0f8047f431c33ff65 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:19:32 -0500 Subject: [PATCH 081/154] updated merge --- cmake/HPX_AddTest.cmake | 2 +- cmake/templates/hpxrun.py.in | 32 +++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index 85046e469acd..74e65eac2ad0 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -202,7 +202,7 @@ function(add_hpx_test category name) if(_add_test) set(_full_name "${category}.distributed.openshmem.${name}") add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "openshmem" "-r" - "openshmem" ${args} + "oshrun" ${args} ) set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) if(${name}_TIMEOUT) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 9aed278a204b..21a3b6e7e4f3 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -179,6 +179,20 @@ def run_none(cmd, localities, nodes, verbose): print('Executing command: ' + ' '.join(cmd)) subproc(cmd) + + def run_amudprun(cmd, localities, nodes, verbose): + assert len(nodes) == localities, "Number of localities must match number of nodes." + os.environ['GASNET_SPAWNFN'] = 'S' + os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) + os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] + os.environ['GASNET_ROUTE_OUTPUT'] = '0' + + exec_cmd = ['amudprun', '-np', len(nodes), cmd] + + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) + # Run with mpiexec # This is executing mpiexec with the "-np" option set to the number of localities def run_mpi(cmd, localities, verbose): @@ -197,7 +211,7 @@ def run_mpi(cmd, localities, verbose): # Run with oshrun # This is executing amudprun with the "-np" option set to the number of localities -def run_openshmem(cmd, localities, verbose): +def run_oshrun(cmd, localities, verbose): exec_cmd = ['oshrun', '-np', str(localities)] + cmd if verbose: print('Executing command: ' + ' '.join(exec_cmd)) @@ -226,9 +240,17 @@ def run(cmd, runwrapper, localities, nodes, verbose): if runwrapper == 'mpi': assert nodes is None, "nodes option only valid with tcp parcelport." run_mpi(cmd, localities, verbose) +<<<<<<< HEAD if runwrapper == 'openshmem': assert nodes is None, "nodes option only valid with tcp parcelport." run_openshmem(cmd, localities, verbose) +======= + if runwrapper == 'amudprun': + run_amudprun(cmd, localities, nodes, verbose) + if runwrapper == 'oshrun': + assert nodes is None, "nodes option only valid with tcp parcelport." + run_oshrun(cmd, localities, verbose) +>>>>>>> 8dcd6a445b (added amudprun and oshrun) if runwrapper == 'srun': assert nodes is None, "nodes option only valid with tcp parcelport." run_srun(cmd, localities, verbose) @@ -373,13 +395,21 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') +<<<<<<< HEAD , help='Which parcelport to use (Options are: mpi, lci, openshmem, tcp) ' +======= + , help='Which parcelport to use (Options are: mpi, lci, openshmem, gasnet, tcp) ' +>>>>>>> 8dcd6a445b (added amudprun and oshrun) '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' , action='store', type='string' , dest='runwrapper', default=default_env('HPXRUN_RUNWRAPPER', 'none') +<<<<<<< HEAD , help='Which runwrapper to use (Options are: none, mpi, openshmem, srun, jsrun) ' +======= + , help='Which runwrapper to use (Options are: none, mpi, oshrun, amudprun, srun, jsrun) ' +>>>>>>> 8dcd6a445b (added amudprun and oshrun) '(environment variable HPXRUN_RUNWRAPPER)') parser.add_option('-e', '--expected' From f1c92a8905c42b1d5e330038c6f65df92c9f1042 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 1 Dec 2023 22:58:35 -0500 Subject: [PATCH 082/154] fixing missing escaped quotes --- cmake/HPX_SetupGasnet.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 61690c08ddae..9c961dde464c 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -119,7 +119,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-udp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-udp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -129,7 +129,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-smp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-smp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -139,7 +139,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-ofi --with-ofi-home=${OFI_DIR} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -149,7 +149,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-ucx --with-ucx-home=${UCX_DIR} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -185,7 +185,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} From cff01d06c64a002c1b356d50988e9d057f50ad79 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 5 Dec 2023 11:37:38 -0500 Subject: [PATCH 083/154] fixed indentation error --- cmake/templates/hpxrun.py.in | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 21a3b6e7e4f3..3fe8c53e5548 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -180,18 +180,18 @@ def run_none(cmd, localities, nodes, verbose): subproc(cmd) - def run_amudprun(cmd, localities, nodes, verbose): - assert len(nodes) == localities, "Number of localities must match number of nodes." - os.environ['GASNET_SPAWNFN'] = 'S' - os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) - os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - - exec_cmd = ['amudprun', '-np', len(nodes), cmd] - - if verbose: - print('Executing command: ' + ' '.join(exec_cmd)) - subproc(exec_cmd) +def run_amudprun(cmd, localities, nodes, verbose): + assert len(nodes) == localities, "Number of localities must match number of nodes." + os.environ['GASNET_SPAWNFN'] = 'S' + os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) + os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] + os.environ['GASNET_ROUTE_OUTPUT'] = '0' + + exec_cmd = ['amudprun', '-np', len(nodes), cmd] + + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) # Run with mpiexec # This is executing mpiexec with the "-np" option set to the number of localities From 836ff64935f9bae8bd84c8ab0b41dcb8a59e5168 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 5 Dec 2023 14:19:20 -0600 Subject: [PATCH 084/154] fixing unindent issue --- cmake/templates/hpxrun.py.in | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 3fe8c53e5548..a7e73cb34817 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -179,20 +179,6 @@ def run_none(cmd, localities, nodes, verbose): print('Executing command: ' + ' '.join(cmd)) subproc(cmd) - -def run_amudprun(cmd, localities, nodes, verbose): - assert len(nodes) == localities, "Number of localities must match number of nodes." - os.environ['GASNET_SPAWNFN'] = 'S' - os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) - os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - - exec_cmd = ['amudprun', '-np', len(nodes), cmd] - - if verbose: - print('Executing command: ' + ' '.join(exec_cmd)) - subproc(exec_cmd) - # Run with mpiexec # This is executing mpiexec with the "-np" option set to the number of localities def run_mpi(cmd, localities, verbose): @@ -209,6 +195,19 @@ def run_mpi(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) +def run_amudprun(cmd, localities, nodes, verbose): + assert len(nodes) == localities, "Number of localities must match number of nodes." + os.environ['GASNET_SPAWNFN'] = 'S' + os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) + os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] + os.environ['GASNET_ROUTE_OUTPUT'] = '0' + + exec_cmd = ['amudprun', '-np', len(nodes), cmd] + + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) + # Run with oshrun # This is executing amudprun with the "-np" option set to the number of localities def run_oshrun(cmd, localities, verbose): From ce8103daa803b74c073268e3cfc3c6931503b2c2 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:22:27 -0500 Subject: [PATCH 085/154] updated merge --- .jenkins/lsu/env-clang-15.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 1df2a9fd23b3..7bc7148cec7f 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" From b68be506ced8ef1c9bb5acb856102d987e6f700c Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:26:26 -0500 Subject: [PATCH 086/154] updated merge --- cmake/HPX_SetupOpenSHMEM.cmake | 109 +++++++++++++++++++++++++++++++ libs/core/version/CMakeLists.txt | 7 ++ 2 files changed, 116 insertions(+) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 244be1879a1b..b10890ab4dc2 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -863,6 +863,7 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() +<<<<<<< HEAD endif() endforeach() @@ -978,6 +979,114 @@ macro(hpx_setup_openshmem) string(FIND "${X}" "-Wl" SKIP) if("${SKIP}" EQUAL "-1") +======= + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") +>>>>>>> b9f0e7ba56 (merge) if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index 470fe90c852b..86477685724a 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -32,6 +32,13 @@ if(HPX_WITH_NETWORKING hpx_setup_openshmem() set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) endif() +if(HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM +) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT From 8ac46b5c5314fab999c21e3f0a57b2ef685f6f0d Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 12 Jan 2024 14:35:23 -0500 Subject: [PATCH 087/154] merging openshmem changes --- docs/sphinx/api/public_distributed_api.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index bc5cbe322a73..019447b52d8d 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -177,3 +177,15 @@ Functions +---------------------------------------------------+ | :cpp:func:`hpx::distributed::wait` | +---------------------------------------------------+ +======= +.. table:: `hpx` functions of header ``hpx/collectives/all_to_all.hpp`` + + +------------------------------------------+ + | Function | + +==========================================+ + | :cpp:func:`hpx::collectives::all_to_all` | + +------------------------------------------+ + +.. _public_distr_api_header_argument_types: + +``hpx/collectives/argument_types.hpp`` From ee17f2b2cc70f49393f43b9769a043b334145375 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Tue, 10 Oct 2023 19:48:10 +0200 Subject: [PATCH 088/154] Add distributed public API for collectives, latch, barrier --- docs/sphinx/api/public_distributed_api.rst | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index 019447b52d8d..426e49e407a0 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -83,6 +83,8 @@ Classes +-----------------------------------------------------+ | :cpp:class:`hpx::collectives::channel_communicator` | +-----------------------------------------------------+ + | :cpp:class:`hpx::collectives::communicator` | + +-----------------------------------------------------+ Functions --------- @@ -177,15 +179,3 @@ Functions +---------------------------------------------------+ | :cpp:func:`hpx::distributed::wait` | +---------------------------------------------------+ -======= -.. table:: `hpx` functions of header ``hpx/collectives/all_to_all.hpp`` - - +------------------------------------------+ - | Function | - +==========================================+ - | :cpp:func:`hpx::collectives::all_to_all` | - +------------------------------------------+ - -.. _public_distr_api_header_argument_types: - -``hpx/collectives/argument_types.hpp`` From 0a08b422811aeeecf364a1a57d92b5de99d60126 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Fri, 13 Oct 2023 20:37:30 +0200 Subject: [PATCH 089/154] Add doxygen docs --- .../include/hpx/collectives/channel_communicator.hpp | 2 ++ .../include/hpx/collectives/create_communicator.hpp | 3 +++ 2 files changed, 5 insertions(+) diff --git a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp index ca011d3d17ef..47acb6d2340b 100644 --- a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp @@ -120,6 +120,8 @@ namespace hpx { namespace collectives { namespace hpx { namespace collectives { // forward declarations + /// a handle identifying the communication channel to use for get/set + /// operations class channel_communicator; template diff --git a/libs/full/collectives/include/hpx/collectives/create_communicator.hpp b/libs/full/collectives/include/hpx/collectives/create_communicator.hpp index d64b0bd7f255..878871197708 100644 --- a/libs/full/collectives/include/hpx/collectives/create_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/create_communicator.hpp @@ -140,6 +140,9 @@ struct hpx::util::extra_data_helper namespace hpx::collectives { + /////////////////////////////////////////////////////////////////////////// + /// a handle identifying the communication channel to use for a particular + /// collective operation struct communicator : hpx::components::client_base From b1609b6b34d6880c99ce5fc8005cf0f9252ada59 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Tue, 17 Oct 2023 23:46:49 +0200 Subject: [PATCH 090/154] Add documentation --- docs/sphinx/api/public_distributed_api.rst | 2 -- .../include/hpx/collectives/channel_communicator.hpp | 2 -- .../include/hpx/collectives/create_communicator.hpp | 3 --- 3 files changed, 7 deletions(-) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index 426e49e407a0..bc5cbe322a73 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -83,8 +83,6 @@ Classes +-----------------------------------------------------+ | :cpp:class:`hpx::collectives::channel_communicator` | +-----------------------------------------------------+ - | :cpp:class:`hpx::collectives::communicator` | - +-----------------------------------------------------+ Functions --------- diff --git a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp index 47acb6d2340b..ca011d3d17ef 100644 --- a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp @@ -120,8 +120,6 @@ namespace hpx { namespace collectives { namespace hpx { namespace collectives { // forward declarations - /// a handle identifying the communication channel to use for get/set - /// operations class channel_communicator; template diff --git a/libs/full/collectives/include/hpx/collectives/create_communicator.hpp b/libs/full/collectives/include/hpx/collectives/create_communicator.hpp index 878871197708..d64b0bd7f255 100644 --- a/libs/full/collectives/include/hpx/collectives/create_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/create_communicator.hpp @@ -140,9 +140,6 @@ struct hpx::util::extra_data_helper namespace hpx::collectives { - /////////////////////////////////////////////////////////////////////////// - /// a handle identifying the communication channel to use for a particular - /// collective operation struct communicator : hpx::components::client_base From 1bc97b20dceb0cfb84e2d295d873f4a9ab5a9a06 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 1 Dec 2023 21:57:37 -0500 Subject: [PATCH 091/154] added amudprun and oshrun --- cmake/templates/hpxrun.py.in | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index a7e73cb34817..9cba9bf40b96 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -179,6 +179,20 @@ def run_none(cmd, localities, nodes, verbose): print('Executing command: ' + ' '.join(cmd)) subproc(cmd) + + def run_amudprun(cmd, localities, nodes, verbose): + assert len(nodes) == localities, "Number of localities must match number of nodes." + os.environ['GASNET_SPAWNFN'] = 'S' + os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) + os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] + os.environ['GASNET_ROUTE_OUTPUT'] = '0' + + exec_cmd = ['amudprun', '-np', len(nodes), cmd] + + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) + # Run with mpiexec # This is executing mpiexec with the "-np" option set to the number of localities def run_mpi(cmd, localities, verbose): From 1218193fcf05f0a00cf544e52f2e23ebb465aa03 Mon Sep 17 00:00:00 2001 From: "ct.clmsn" Date: Sun, 10 Mar 2024 20:08:58 -0500 Subject: [PATCH 092/154] fixed broken script --- cmake/HPX_SetupOpenSHMEM.cmake | 109 --------------------------------- 1 file changed, 109 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index b10890ab4dc2..244be1879a1b 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -863,7 +863,6 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() -<<<<<<< HEAD endif() endforeach() @@ -979,114 +978,6 @@ macro(hpx_setup_openshmem) string(FIND "${X}" "-Wl" SKIP) if("${SKIP}" EQUAL "-1") -======= - endif() - endforeach() - - set(IDX 0) - list(LENGTH OPENSHMEM_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive - -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - endif() - - if(OPENSHMEM_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(SKIP 0) - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") ->>>>>>> b9f0e7ba56 (merge) if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") From 00b42ee572d231e9b661012a80588e65e1b7f36e Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 25 Mar 2024 11:52:32 -0400 Subject: [PATCH 093/154] several performance oriented updates --- .../openshmem_base/openshmem_environment.hpp | 8 ++--- .../src/openshmem_environment.cpp | 23 +++++++----- .../hpx/parcelport_openshmem/receiver.hpp | 5 +-- .../receiver_connection.hpp | 28 +++++++-------- .../hpx/parcelport_openshmem/sender.hpp | 2 +- .../sender_connection.hpp | 35 ++++++++++--------- 6 files changed, 55 insertions(+), 46 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index af57d81b0fe4..ae6f623ba3e5 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -32,8 +32,10 @@ namespace hpx { namespace util { struct openshmem_seginfo_t { - std::uint8_t* addr; - std::uint8_t* size; + std::uint8_t* beg_addr; + std::uint8_t* end_addr; + unsigned int * rcv; + unsigned int * xmt; }; struct HPX_CORE_EXPORT openshmem_environment @@ -100,8 +102,6 @@ namespace hpx { namespace util { static hpx::mutex* segment_mutex; static openshmem_seginfo_t* segments; static std::uint8_t* shmem_buffer; - static unsigned int rcv; - static unsigned int xmt; }; }} // namespace hpx::util diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 6345a722c5ad..7154c778c76b 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -102,8 +102,6 @@ namespace hpx::util { hpx::mutex* openshmem_environment::segment_mutex = nullptr; openshmem_seginfo_t* openshmem_environment::segments = nullptr; std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; - unsigned int openshmem_environment::rcv = 0; - unsigned int openshmem_environment::xmt = 0; /////////////////////////////////////////////////////////////////////////// int openshmem_environment::init([[maybe_unused]] int* argc, @@ -146,19 +144,28 @@ namespace hpx::util { // //segments.resize(hpx::threads::hardware_concurrency() * size()); // - openshmem_environment::segments = new openshmem_seginfo_t[size()]; - openshmem_environment::segment_mutex = new hpx::mutex[size()]; + const std::size_t page_count = size(); + const std::size_t signal_count = page_count*2; + openshmem_environment::segments = new openshmem_seginfo_t[page_count]; + openshmem_environment::segment_mutex = new hpx::mutex[page_count]; + + const std::size_t byte_count = ((sizeof(std::uint8_t)*OPENSHMEM_PER_RANK_PAGESIZE*page_count)+(sizeof(unsigned int)*signal_count)) / sizeof(std::uint8_t); hpx::util::openshmem_environment::shmem_buffer = static_cast(shmem_calloc( - OPENSHMEM_PER_RANK_PAGESIZE, sizeof(std::uint8_t))); + byte_count, sizeof(std::uint8_t))); - for (int i = 0; i < size(); ++i) + std::size_t beg_signal = (OPENSHMEM_PER_RANK_PAGESIZE*page_count); + for (std::size_t i = 0; i < page_count; ++i) { - segments[i].addr = hpx::util::openshmem_environment::shmem_buffer + + segments[i].beg_addr = hpx::util::openshmem_environment::shmem_buffer + (i * OPENSHMEM_PER_RANK_PAGESIZE); - segments[i].size = static_cast(segments[i].addr) + + segments[i].end_addr = static_cast(segments[i].beg_addr) + OPENSHMEM_PER_RANK_PAGESIZE; + segments[i].rcv = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + + beg_signal + signal_count); + segments[i].xmt = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + + beg_signal + signal_count + 1); } shmem_barrier_all(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 2c828f54e643..cdb1e25fca7e 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -143,13 +143,14 @@ namespace hpx::parcelset::policies::openshmem { { header h = rcv_header_; rcv_header_.reset(); + auto self_ = hpx::util::openshmem_environment::rank(); while (rcv_header_.data() == 0) { - bo(&hpx::util::openshmem_environment::rcv); + bo(hpx::util::openshmem_environment::segments[self_].rcv); } - hpx::util::openshmem_environment::rcv = 0; + (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; return h; } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index b2166090b437..908bffa9dc51 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -105,14 +105,14 @@ namespace hpx::parcelset::policies::openshmem { { hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::wait_until( - 1, &hpx::util::openshmem_environment::rcv); - hpx::util::openshmem_environment::rcv = 0; + 1, hpx::util::openshmem_environment::segments[self_].rcv); + (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast( buffer_.transmission_chunks_.data()), self_, - hpx::util::openshmem_environment::segments[self_].addr, + hpx::util::openshmem_environment::segments[self_].beg_addr, static_cast(buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type))); @@ -142,14 +142,14 @@ namespace hpx::parcelset::policies::openshmem { { auto self_ = hpx::util::openshmem_environment::rank(); hpx::util::openshmem_environment::scoped_lock l; - hpx::util::openshmem_environment::wait_until( - 1, &hpx::util::openshmem_environment::rcv); - hpx::util::openshmem_environment::rcv = 0; + hpx::util::openshmem_environment::wait_until( + 1, hpx::util::openshmem_environment::segments[self_].rcv); + (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(buffer_.data_.data()), self_, - hpx::util::openshmem_environment::segments[self_].addr, + hpx::util::openshmem_environment::segments[self_].beg_addr, buffer_.data_.size()); request_ptr_ = true; @@ -180,16 +180,16 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::wait_until( - 1, &hpx::util::openshmem_environment::rcv); - hpx::util::openshmem_environment::rcv = 0; + 1, hpx::util::openshmem_environment::segments[self_].rcv); + (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(c.data()), self_, - hpx::util::openshmem_environment::segments[self_].addr, + hpx::util::openshmem_environment::segments[self_].beg_addr, c.size()); hpx::util::openshmem_environment::put_signal(nullptr, src_, - nullptr, 0, &hpx::util::openshmem_environment::xmt); + nullptr, 0, hpx::util::openshmem_environment::segments[self_].xmt); request_ptr_ = true; } @@ -215,12 +215,12 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::wait_until( - 1, &hpx::util::openshmem_environment::rcv); - hpx::util::openshmem_environment::rcv = 0; + 1, hpx::util::openshmem_environment::segments[self_].rcv); + (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(&tag_), self_, - hpx::util::openshmem_environment::segments[self_].addr, + hpx::util::openshmem_environment::segments[self_].beg_addr, sizeof(int)); request_ptr_ = true; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp index fe75167a4208..98c87e0e9051 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -145,7 +145,7 @@ namespace hpx::parcelset::policies::openshmem { std::memcpy(&next_free, hpx::util::openshmem_environment::segments [hpx::util::openshmem_environment::rank()] - .addr, + .beg_addr, sizeof(int)); return next_free; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index eb1047405a61..0212ee0519b9 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -166,9 +166,8 @@ namespace hpx::parcelset::policies::openshmem { // hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks), dst_, - static_cast( - hpx::util::openshmem_environment::segments[dst_].addr), - sizeof_chunks, &hpx::util::openshmem_environment::rcv); + static_cast(hpx::util::openshmem_environment::segments[dst_].beg_addr), + sizeof_chunks, reinterpret_cast(hpx::util::openshmem_environment::segments[dst_].rcv)); } state_ = sent_header; @@ -187,16 +186,18 @@ namespace hpx::parcelset::policies::openshmem { chunks = buffer_.transmission_chunks_; if (!chunks.empty()) { + const std::size_t rank_ = + hpx::util::openshmem_environment::rank(); hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, - static_cast( - hpx::util::openshmem_environment::segments - [hpx::util::openshmem_environment::rank()] - .addr), + static_cast(hpx::util::openshmem_environment::segments + [rank_].beg_addr), static_cast(chunks.size() * sizeof(parcel_buffer_type::transmission_chunk_type)), - &hpx::util::openshmem_environment::rcv); + reinterpret_cast(hpx::util::openshmem_environment::segments + [rank_].rcv) + ); } state_ = sent_transmission_chunks; @@ -216,10 +217,10 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, - static_cast( - hpx::util::openshmem_environment::segments[dst_].addr), + static_cast(hpx::util::openshmem_environment::segments[dst_].beg_addr), buffer_.data_.size(), - &hpx::util::openshmem_environment::rcv); + reinterpret_cast(hpx::util::openshmem_environment::segments[dst_].rcv) + ); } state_ = sent_data; @@ -245,15 +246,15 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::put_signal( reinterpret_cast(c.data_.cpos_), dst_, - static_cast( - hpx::util::openshmem_environment::segments[dst_] - .addr), + static_cast(hpx::util::openshmem_environment::segments[dst_] + .beg_addr), static_cast(c.size_), - &hpx::util::openshmem_environment::rcv); + reinterpret_cast(hpx::util::openshmem_environment::segments[dst_].rcv) + ); hpx::util::openshmem_environment::wait_until( - 1, &hpx::util::openshmem_environment::xmt); - hpx::util::openshmem_environment::xmt = 0; + 1, hpx::util::openshmem_environment::segments[dst_].xmt); + (*(hpx::util::openshmem_environment::segments[dst_].xmt)) = 0; } ++chunks_idx_; From 5400e4e2a3c6b94caf8e783f4cf4822478ae5e07 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 25 Mar 2024 12:55:41 -0400 Subject: [PATCH 094/154] missing pkgconfig check --- cmake/HPX_SetupOpenSHMEM.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 244be1879a1b..2241dc21a2c4 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -8,6 +8,7 @@ include(HPX_Message) macro(hpx_setup_openshmem) + find_package(PkgConfig REQUIRED) if(NOT TARGET PkgConfig::OPENSHMEM) set(OPENSHMEM_PC "") From fff72b2377a48dc4f278262a9f25707696370d24 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 8 Apr 2024 16:13:15 -0400 Subject: [PATCH 095/154] several small fixes to improve performance; found some implementation errors --- .../openshmem_base/openshmem_environment.hpp | 6 +- .../src/openshmem_environment.cpp | 30 ++++++-- .../hpx/parcelport_openshmem/header.hpp | 19 +++-- .../receiver_connection.hpp | 74 +++++++++++++++---- .../sender_connection.hpp | 66 ++++++++++------- 5 files changed, 135 insertions(+), 60 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index ae6f623ba3e5..6d95f3039f36 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -23,11 +23,6 @@ #include -#if !defined(OPENSHMEM_PAGESIZE) -// GASNet default pagesize is 16384 -#define OPENSHMEM_PER_RANK_PAGESIZE 16384 -#endif - namespace hpx { namespace util { struct openshmem_seginfo_t @@ -99,6 +94,7 @@ namespace hpx { namespace util { static hpx::mutex dshm_mut; static int init_val_; + static std::size_t nthreads_; static hpx::mutex* segment_mutex; static openshmem_seginfo_t* segments; static std::uint8_t* shmem_buffer; diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 7154c778c76b..101820963aae 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include +#include #include #include #include @@ -102,6 +104,7 @@ namespace hpx::util { hpx::mutex* openshmem_environment::segment_mutex = nullptr; openshmem_seginfo_t* openshmem_environment::segments = nullptr; std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; + std::size_t openshmem_environment::nthreads_ = 0; /////////////////////////////////////////////////////////////////////////// int openshmem_environment::init([[maybe_unused]] int* argc, @@ -144,28 +147,39 @@ namespace hpx::util { // //segments.resize(hpx::threads::hardware_concurrency() * size()); // - const std::size_t page_count = size(); - const std::size_t signal_count = page_count*2; + + auto& rp = hpx::resource::get_partitioner(); + openshmem_environment::nthreads_ = + rp.get_num_threads(); + const std::size_t total_thread_num = + openshmem_environment::nthreads_; + const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); + const std::size_t page_count = size()*total_thread_num; openshmem_environment::segments = new openshmem_seginfo_t[page_count]; openshmem_environment::segment_mutex = new hpx::mutex[page_count]; - const std::size_t byte_count = ((sizeof(std::uint8_t)*OPENSHMEM_PER_RANK_PAGESIZE*page_count)+(sizeof(unsigned int)*signal_count)) / sizeof(std::uint8_t); + // symmetric allocation for number of pages total + number of signals + // + // (allocate page_size * number of PEs * number of threads) + (number of PEs * number of threads * 2 [for signaling]) + // + const std::size_t byte_count = (sys_pgsz*page_count)+(sizeof(unsigned int)*page_count*2); hpx::util::openshmem_environment::shmem_buffer = static_cast(shmem_calloc( byte_count, sizeof(std::uint8_t))); - std::size_t beg_signal = (OPENSHMEM_PER_RANK_PAGESIZE*page_count); + const std::size_t beg_signal = (sys_pgsz*page_count); + for (std::size_t i = 0; i < page_count; ++i) { segments[i].beg_addr = hpx::util::openshmem_environment::shmem_buffer + - (i * OPENSHMEM_PER_RANK_PAGESIZE); + (i * sys_pgsz); segments[i].end_addr = static_cast(segments[i].beg_addr) + - OPENSHMEM_PER_RANK_PAGESIZE; + sys_pgsz; segments[i].rcv = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + - beg_signal + signal_count); + beg_signal + i); segments[i].xmt = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + - beg_signal + signal_count + 1); + beg_signal + page_count + i); } shmem_barrier_all(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp index 3712581d08e2..c90d698f6c14 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp @@ -31,17 +31,18 @@ namespace hpx::parcelset::policies::openshmem { { pos_tag = 0 * sizeof(value_type), pos_size = 1 * sizeof(value_type), - pos_numbytes = 2 * sizeof(value_type), - pos_numchunks_first = 3 * sizeof(value_type), - pos_numchunks_second = 4 * sizeof(value_type), - pos_piggy_back_flag = 5 * sizeof(value_type), - pos_piggy_back_data = 5 * sizeof(value_type) + 1 + pos_sending_thread_id = 2 * sizeof(value_type), + pos_numbytes = 3 * sizeof(value_type), + pos_numchunks_first = 4 * sizeof(value_type), + pos_numchunks_second = 5 * sizeof(value_type), + pos_piggy_back_flag = 6 * sizeof(value_type), + pos_piggy_back_data = 7 * sizeof(value_type) + 1 }; static constexpr int data_size_ = 512; template - header(Buffer const& buffer, int tag) noexcept + header(Buffer const& buffer, int tag, const std::size_t sending_thread_id) noexcept { std::int64_t size = static_cast(buffer.size_); std::int64_t numbytes = @@ -52,6 +53,7 @@ namespace hpx::parcelset::policies::openshmem { set(tag); set(static_cast(size)); + set(static_cast(sending_thread_id)); set(static_cast(numbytes)); set( static_cast(buffer.num_chunks_.first)); @@ -110,6 +112,11 @@ namespace hpx::parcelset::policies::openshmem { return get(); } + value_type sending_thread_id() const noexcept + { + return get(); + } + value_type numbytes() const noexcept { return get(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index 908bffa9dc51..c9a9f5d5d89d 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -48,19 +48,43 @@ namespace hpx::parcelset::policies::openshmem { : state_(initialized) , src_(src) , tag_(h.tag()) + , self_(-1) + , sending_thd_id_(-1) , header_(h) , request_ptr_(false) + , num_bytes(0) + , need_recv_data(false) + , need_recv_tchunks(false) , chunks_idx_(0) , pp_(pp) { header_.assert_valid(); + + self_ = hpx::util::openshmem_environment::rank(); + sending_thd_id_ = header_.sending_thread_id(); + num_bytes = header_.numbytes(); + #if defined(HPX_HAVE_PARCELPORT_COUNTERS) parcelset::data_point& data = buffer_.data_point_; data.time_ = timer_.elapsed_nanoseconds(); data.bytes_ = static_cast(header_.numbytes()); #endif - buffer_.data_.resize(static_cast(header_.size())); + tag_ = header_.tag(); + // decode data buffer_.num_chunks_ = header_.num_chunks(); + buffer_.data_.resize(static_cast(header_.size())); + char* piggy_back_data = header_.piggy_back(); + if (piggy_back_data) + { + need_recv_data = false; + memcpy(buffer_.data_.data(), piggy_back_data, + buffer_.data_.size()); + } + else + { + need_recv_data = true; + } + need_recv_tchunks = false; } bool receive(std::size_t num_thread = -1) @@ -90,8 +114,6 @@ namespace hpx::parcelset::policies::openshmem { bool receive_transmission_chunks(std::size_t num_thread = -1) { - auto self_ = hpx::util::openshmem_environment::rank(); - // determine the size of the chunk buffer std::size_t num_zero_copy_chunks = static_cast( static_cast(buffer_.num_chunks_.first)); @@ -103,16 +125,21 @@ namespace hpx::parcelset::policies::openshmem { { buffer_.chunks_.resize(num_zero_copy_chunks); { + auto self_ = hpx::util::openshmem_environment::rank(); + auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + auto idx = (self_*nthreads_)+sending_thd_id_; + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[self_].rcv); - (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; + 1, hpx::util::openshmem_environment::segments[idx].rcv); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast( buffer_.transmission_chunks_.data()), self_, - hpx::util::openshmem_environment::segments[self_].beg_addr, + hpx::util::openshmem_environment::segments[idx].beg_addr, static_cast(buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type))); @@ -141,15 +168,18 @@ namespace hpx::parcelset::policies::openshmem { else { auto self_ = hpx::util::openshmem_environment::rank(); + auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + auto idx = (self_*nthreads_)+sending_thd_id_; + hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[self_].rcv); - (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; + 1, hpx::util::openshmem_environment::segments[idx].rcv); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(buffer_.data_.data()), self_, - hpx::util::openshmem_environment::segments[self_].beg_addr, + hpx::util::openshmem_environment::segments[idx].beg_addr, buffer_.data_.size()); request_ptr_ = true; @@ -177,19 +207,22 @@ namespace hpx::parcelset::policies::openshmem { c.resize(chunk_size); { auto self_ = hpx::util::openshmem_environment::rank(); + auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + auto idx = (self_*nthreads_)+sending_thd_id_; + hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[self_].rcv); - (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; + 1, hpx::util::openshmem_environment::segments[idx].rcv); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(c.data()), self_, - hpx::util::openshmem_environment::segments[self_].beg_addr, + hpx::util::openshmem_environment::segments[idx].beg_addr, c.size()); hpx::util::openshmem_environment::put_signal(nullptr, src_, - nullptr, 0, hpx::util::openshmem_environment::segments[self_].xmt); + nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); request_ptr_ = true; } @@ -212,15 +245,18 @@ namespace hpx::parcelset::policies::openshmem { #endif { auto self_ = hpx::util::openshmem_environment::rank(); + auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + auto idx = (self_*nthreads_)+sending_thd_id_; + hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[self_].rcv); - (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; + 1, hpx::util::openshmem_environment::segments[idx].rcv); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(&tag_), self_, - hpx::util::openshmem_environment::segments[self_].beg_addr, + hpx::util::openshmem_environment::segments[idx].beg_addr, sizeof(int)); request_ptr_ = true; @@ -256,10 +292,16 @@ namespace hpx::parcelset::policies::openshmem { int src_; int tag_; + int self_; + int sending_thd_id_; + header header_; buffer_type buffer_; bool request_ptr_; + std::size_t num_bytes; + bool need_recv_data; + bool need_recv_tchunks; std::size_t chunks_idx_; Parcelport& pp_; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 0212ee0519b9..36b9822ddb9a 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -69,11 +69,14 @@ namespace hpx::parcelset::policies::openshmem { , sender_(s) , tag_(-1) , dst_(dst) + , thd_id_(-1) , chunks_idx_(0) , ack_(0) , pp_(pp) , there_(parcelset::locality(locality(dst_))) { + thd_id_ = + hpx::get_worker_thread_num(); // current worker } parcelset::locality const& destination() const noexcept @@ -100,7 +103,7 @@ namespace hpx::parcelset::policies::openshmem { #endif chunks_idx_ = 0; tag_ = acquire_tag(sender_); - header_ = header(buffer_, tag_); + header_ = header(buffer_, tag_, thd_id_); header_.assert_valid(); state_ = initialized; @@ -149,25 +152,28 @@ namespace hpx::parcelset::policies::openshmem { bool send_header() { { - hpx::util::openshmem_environment::scoped_lock l; HPX_ASSERT(state_ == initialized); - // compute + send the number of OPENSHMEM_PAGEs to send and the - // remainder number of bytes to a OPENSHMEM_PAGE - // - std::size_t chunks[] = {static_cast(header_.data_size_ / - OPENSHMEM_PER_RANK_PAGESIZE), - static_cast( - header_.data_size_ % OPENSHMEM_PER_RANK_PAGESIZE)}; - const std::size_t sizeof_chunks = sizeof(chunks); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (dst_*nthreads_)+thd_id_; + + hpx::util::openshmem_environment::scoped_lock l; // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment // + hpx::util::openshmem_environment::put_signal( - reinterpret_cast(chunks), dst_, - static_cast(hpx::util::openshmem_environment::segments[dst_].beg_addr), - sizeof_chunks, reinterpret_cast(hpx::util::openshmem_environment::segments[dst_].rcv)); + reinterpret_cast(header_.data()), + dst_, + static_cast( + hpx::util::openshmem_environment::segments[idx].beg_addr + ), + header_.data_size_, + reinterpret_cast( + hpx::util::openshmem_environment::segments[idx].rcv + ) + ); } state_ = sent_header; @@ -184,19 +190,22 @@ namespace hpx::parcelset::policies::openshmem { std::vector& chunks = buffer_.transmission_chunks_; + if (!chunks.empty()) { - const std::size_t rank_ = - hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (dst_*nthreads_)+thd_id_; + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, static_cast(hpx::util::openshmem_environment::segments - [rank_].beg_addr), + [idx].beg_addr), static_cast(chunks.size() * sizeof(parcel_buffer_type::transmission_chunk_type)), reinterpret_cast(hpx::util::openshmem_environment::segments - [rank_].rcv) + [idx].rcv) ); } @@ -213,13 +222,16 @@ namespace hpx::parcelset::policies::openshmem { } if (!header_.piggy_back()) - { + { + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (dst_*nthreads_)+thd_id_; + hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, - static_cast(hpx::util::openshmem_environment::segments[dst_].beg_addr), + static_cast(hpx::util::openshmem_environment::segments[idx].beg_addr), buffer_.data_.size(), - reinterpret_cast(hpx::util::openshmem_environment::segments[dst_].rcv) + reinterpret_cast(hpx::util::openshmem_environment::segments[idx].rcv) ); } state_ = sent_data; @@ -242,19 +254,22 @@ namespace hpx::parcelset::policies::openshmem { return false; } + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (dst_*nthreads_)+thd_id_; + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( reinterpret_cast(c.data_.cpos_), dst_, - static_cast(hpx::util::openshmem_environment::segments[dst_] - .beg_addr), + static_cast(hpx::util::openshmem_environment::segments[idx].beg_addr), static_cast(c.size_), - reinterpret_cast(hpx::util::openshmem_environment::segments[dst_].rcv) + reinterpret_cast(hpx::util::openshmem_environment::segments[idx].rcv) ); hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[dst_].xmt); - (*(hpx::util::openshmem_environment::segments[dst_].xmt)) = 0; + 1, hpx::util::openshmem_environment::segments[idx].xmt); + (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; } ++chunks_idx_; @@ -303,6 +318,7 @@ namespace hpx::parcelset::policies::openshmem { sender_type* sender_; int tag_; int dst_; + int thd_id_; using handler_type = hpx::move_only_function; handler_type handler_; From 99504a65e4bc51e6c595e0b1eb4fdeb25720287e Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 12 Apr 2024 10:20:02 -0400 Subject: [PATCH 096/154] adding comments --- .../src/openshmem_environment.cpp | 33 +++++++++++++++++-- .../receiver_connection.hpp | 26 ++++++++------- .../sender_connection.hpp | 6 ++-- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 101820963aae..5fe771ac0e9d 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -149,13 +149,28 @@ namespace hpx::util { // auto& rp = hpx::resource::get_partitioner(); + + // get thread count + // openshmem_environment::nthreads_ = rp.get_num_threads(); - const std::size_t total_thread_num = - openshmem_environment::nthreads_; + + // get system page size + // const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); - const std::size_t page_count = size()*total_thread_num; + + // compute number of pages pages + // + // page_count = num_localities * number of threads + // + const std::size_t page_count = size()*openshmem_environment::nthreads_; + + // allocate the page cache + // openshmem_environment::segments = new openshmem_seginfo_t[page_count]; + + // allocate a mutex per-page + // openshmem_environment::segment_mutex = new hpx::mutex[page_count]; // symmetric allocation for number of pages total + number of signals @@ -164,20 +179,32 @@ namespace hpx::util { // const std::size_t byte_count = (sys_pgsz*page_count)+(sizeof(unsigned int)*page_count*2); + // allocate symmetric memory + // hpx::util::openshmem_environment::shmem_buffer = static_cast(shmem_calloc( byte_count, sizeof(std::uint8_t))); + // compute the base address for signals + // const std::size_t beg_signal = (sys_pgsz*page_count); + // initialize the page cache + // for (std::size_t i = 0; i < page_count; ++i) { segments[i].beg_addr = hpx::util::openshmem_environment::shmem_buffer + (i * sys_pgsz); segments[i].end_addr = static_cast(segments[i].beg_addr) + sys_pgsz; + + // all of the rcv signals are linearly arranged before the rcv signals + // segments[i].rcv = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + beg_signal + i); + + // all of the xmt signals are linearly arranged after the rcv signals + // segments[i].xmt = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + beg_signal + page_count + i); } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index c9a9f5d5d89d..b1cf0c49ad9f 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -121,13 +121,15 @@ namespace hpx::parcelset::policies::openshmem { static_cast(buffer_.num_chunks_.second)); buffer_.transmission_chunks_.resize( num_zero_copy_chunks + num_non_zero_copy_chunks); + + const auto idx = (self_*nthreads_)+sending_thd_id_; if (num_zero_copy_chunks != 0) { buffer_.chunks_.resize(num_zero_copy_chunks); { - auto self_ = hpx::util::openshmem_environment::rank(); - auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - auto idx = (self_*nthreads_)+sending_thd_id_; + const auto self_ = hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (self_*nthreads_)+sending_thd_id_; hpx::util::openshmem_environment::scoped_lock l; @@ -167,9 +169,9 @@ namespace hpx::parcelset::policies::openshmem { } else { - auto self_ = hpx::util::openshmem_environment::rank(); - auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - auto idx = (self_*nthreads_)+sending_thd_id_; + const auto self_ = hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (self_*nthreads_)+sending_thd_id_; hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::wait_until( @@ -206,9 +208,9 @@ namespace hpx::parcelset::policies::openshmem { data_type& c = buffer_.chunks_[idx]; c.resize(chunk_size); { - auto self_ = hpx::util::openshmem_environment::rank(); - auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - auto idx = (self_*nthreads_)+sending_thd_id_; + const auto self_ = hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (self_*nthreads_)+sending_thd_id_; hpx::util::openshmem_environment::scoped_lock l; @@ -244,9 +246,9 @@ namespace hpx::parcelset::policies::openshmem { data.time_ = timer_.elapsed_nanoseconds() - data.time_; #endif { - auto self_ = hpx::util::openshmem_environment::rank(); - auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - auto idx = (self_*nthreads_)+sending_thd_id_; + const auto self_ = hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (self_*nthreads_)+sending_thd_id_; hpx::util::openshmem_environment::scoped_lock l; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 36b9822ddb9a..a79591338a2d 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -243,6 +243,9 @@ namespace hpx::parcelset::policies::openshmem { { HPX_ASSERT(state_ == sent_data); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (dst_*nthreads_)+thd_id_; + while (chunks_idx_ < buffer_.chunks_.size()) { serialization::serialization_chunk& c = @@ -254,9 +257,6 @@ namespace hpx::parcelset::policies::openshmem { return false; } - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (dst_*nthreads_)+thd_id_; - hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::put_signal( From 0f249ce44019e43a0aec7d2d55120869d3f7de16 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:32:10 -0500 Subject: [PATCH 097/154] updated merge --- cmake/HPX_AddTest.cmake | 2 +- cmake/templates/hpxrun.py.in | 43 ------------------------------------ 2 files changed, 1 insertion(+), 44 deletions(-) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index 74e65eac2ad0..e67d100e8ff7 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -202,7 +202,7 @@ function(add_hpx_test category name) if(_add_test) set(_full_name "${category}.distributed.openshmem.${name}") add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "openshmem" "-r" - "oshrun" ${args} + "srun" ${args} ) set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) if(${name}_TIMEOUT) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 9cba9bf40b96..ee6315a8b2b8 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -180,19 +180,6 @@ def run_none(cmd, localities, nodes, verbose): subproc(cmd) - def run_amudprun(cmd, localities, nodes, verbose): - assert len(nodes) == localities, "Number of localities must match number of nodes." - os.environ['GASNET_SPAWNFN'] = 'S' - os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) - os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - - exec_cmd = ['amudprun', '-np', len(nodes), cmd] - - if verbose: - print('Executing command: ' + ' '.join(exec_cmd)) - subproc(exec_cmd) - # Run with mpiexec # This is executing mpiexec with the "-np" option set to the number of localities def run_mpi(cmd, localities, verbose): @@ -209,21 +196,7 @@ def run_mpi(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) -def run_amudprun(cmd, localities, nodes, verbose): - assert len(nodes) == localities, "Number of localities must match number of nodes." - os.environ['GASNET_SPAWNFN'] = 'S' - os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) - os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - - exec_cmd = ['amudprun', '-np', len(nodes), cmd] - - if verbose: - print('Executing command: ' + ' '.join(exec_cmd)) - subproc(exec_cmd) - # Run with oshrun -# This is executing amudprun with the "-np" option set to the number of localities def run_oshrun(cmd, localities, verbose): exec_cmd = ['oshrun', '-np', str(localities)] + cmd if verbose: @@ -253,17 +226,9 @@ def run(cmd, runwrapper, localities, nodes, verbose): if runwrapper == 'mpi': assert nodes is None, "nodes option only valid with tcp parcelport." run_mpi(cmd, localities, verbose) -<<<<<<< HEAD if runwrapper == 'openshmem': assert nodes is None, "nodes option only valid with tcp parcelport." run_openshmem(cmd, localities, verbose) -======= - if runwrapper == 'amudprun': - run_amudprun(cmd, localities, nodes, verbose) - if runwrapper == 'oshrun': - assert nodes is None, "nodes option only valid with tcp parcelport." - run_oshrun(cmd, localities, verbose) ->>>>>>> 8dcd6a445b (added amudprun and oshrun) if runwrapper == 'srun': assert nodes is None, "nodes option only valid with tcp parcelport." run_srun(cmd, localities, verbose) @@ -408,21 +373,13 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') -<<<<<<< HEAD , help='Which parcelport to use (Options are: mpi, lci, openshmem, tcp) ' -======= - , help='Which parcelport to use (Options are: mpi, lci, openshmem, gasnet, tcp) ' ->>>>>>> 8dcd6a445b (added amudprun and oshrun) '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' , action='store', type='string' , dest='runwrapper', default=default_env('HPXRUN_RUNWRAPPER', 'none') -<<<<<<< HEAD , help='Which runwrapper to use (Options are: none, mpi, openshmem, srun, jsrun) ' -======= - , help='Which runwrapper to use (Options are: none, mpi, oshrun, amudprun, srun, jsrun) ' ->>>>>>> 8dcd6a445b (added amudprun and oshrun) '(environment variable HPXRUN_RUNWRAPPER)') parser.add_option('-e', '--expected' From 561cc4d6de726f0e30bd2cdfb647b0fc5d7d81dd Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:33:58 -0500 Subject: [PATCH 098/154] updated merge --- cmake/templates/hpxrun.py.in | 7 ------- libs/core/openshmem_base/src/openshmem_environment.cpp | 2 +- libs/core/version/CMakeLists.txt | 7 +++++++ .../include/hpx/parcelport_openshmem/sender_connection.hpp | 1 - 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index ee6315a8b2b8..612dc3b82973 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -196,13 +196,6 @@ def run_mpi(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) -# Run with oshrun -def run_oshrun(cmd, localities, verbose): - exec_cmd = ['oshrun', '-np', str(localities)] + cmd - if verbose: - print('Executing command: ' + ' '.join(exec_cmd)) - subproc(exec_cmd) - # Run with srun # This is executing srun with the '-n' option set to the number of localities def run_srun(cmd, localities, verbose): diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 5fe771ac0e9d..82e7bbcef61c 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -306,10 +306,10 @@ namespace hpx::util { #if !defined(SHMEM_SIGNAL_SET) shmem_uint8_put(raddr, addr, size, node); - shmem_fence(); shmem_uint8_put(reinterpret_cast(sigaddr), reinterpret_cast(sigaddr), sizeof(sigaddr), node); + shmem_fence(); #else shmem_uint8_put_signal(raddr, addr, size, reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index 86477685724a..db9ca99d0956 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -39,6 +39,13 @@ if(HPX_WITH_NETWORKING hpx_setup_openshmem() set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) endif() +if(HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM +) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index a79591338a2d..2cac1f04ba9b 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -162,7 +162,6 @@ namespace hpx::parcelset::policies::openshmem { // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment // - hpx::util::openshmem_environment::put_signal( reinterpret_cast(header_.data()), dst_, From 137d15a139145d2eb72e998a58115a1435a26cab Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 25 Oct 2023 15:03:10 -0400 Subject: [PATCH 099/154] cmake formatting reapplied --- CMakeLists.txt | 3 +-- cmake/HPX_SetupOpenSHMEM.cmake | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 087a1248c51e..6a6ff3b6c5d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1233,8 +1233,7 @@ if(HPX_WITH_NETWORKING) OFF CATEGORY "Parcelport" ) hpx_option( - HPX_WITH_FETCH_OPENSHMEM - BOOL + HPX_WITH_FETCH_OPENSHMEM BOOL "Download an OpenSHMEM if one is not available" OFF CATEGORY "Build Targets" ADVANCED diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 2241dc21a2c4..ed181dfb28c7 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -900,7 +900,6 @@ macro(hpx_setup_openshmem) ) list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") string(APPEND NEWLINK "${FOUND_LIB}") set(FOUND_LIB "") From fe9d083470d4f5359d7ae70bed9a1a0ce42bd21b Mon Sep 17 00:00:00 2001 From: dimitraka Date: Tue, 10 Oct 2023 19:48:10 +0200 Subject: [PATCH 100/154] Add distributed public API for collectives, latch, barrier --- docs/sphinx/api/public_distributed_api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index bc5cbe322a73..426e49e407a0 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -83,6 +83,8 @@ Classes +-----------------------------------------------------+ | :cpp:class:`hpx::collectives::channel_communicator` | +-----------------------------------------------------+ + | :cpp:class:`hpx::collectives::communicator` | + +-----------------------------------------------------+ Functions --------- From a5e3f0bb593d9c1f64fa864827ad6871a3ba139c Mon Sep 17 00:00:00 2001 From: dimitraka Date: Fri, 13 Oct 2023 20:37:30 +0200 Subject: [PATCH 101/154] Add doxygen docs --- .../include/hpx/collectives/channel_communicator.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp index ca011d3d17ef..47acb6d2340b 100644 --- a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp @@ -120,6 +120,8 @@ namespace hpx { namespace collectives { namespace hpx { namespace collectives { // forward declarations + /// a handle identifying the communication channel to use for get/set + /// operations class channel_communicator; template From 005d30be1c6deef0b919366727a99d7f14a5ccd9 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Tue, 17 Oct 2023 23:46:49 +0200 Subject: [PATCH 102/154] Add documentation --- docs/sphinx/api/public_distributed_api.rst | 2 -- .../include/hpx/collectives/channel_communicator.hpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/docs/sphinx/api/public_distributed_api.rst b/docs/sphinx/api/public_distributed_api.rst index 426e49e407a0..bc5cbe322a73 100644 --- a/docs/sphinx/api/public_distributed_api.rst +++ b/docs/sphinx/api/public_distributed_api.rst @@ -83,8 +83,6 @@ Classes +-----------------------------------------------------+ | :cpp:class:`hpx::collectives::channel_communicator` | +-----------------------------------------------------+ - | :cpp:class:`hpx::collectives::communicator` | - +-----------------------------------------------------+ Functions --------- diff --git a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp index 47acb6d2340b..ca011d3d17ef 100644 --- a/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/channel_communicator.hpp @@ -120,8 +120,6 @@ namespace hpx { namespace collectives { namespace hpx { namespace collectives { // forward declarations - /// a handle identifying the communication channel to use for get/set - /// operations class channel_communicator; template From fb9b423f948fe4de85dc5a3f33302ba504dfb256 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:36:37 -0500 Subject: [PATCH 103/154] updated merge --- cmake/HPX_SetupGasnet.cmake | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 9c961dde464c..1ea2a68986d5 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -106,12 +106,11 @@ macro(hpx_setup_gasnet) if(CMAKE_BUILD_PARALLEL_LEVEL) set(GASNET_BUILD_PARALLEL_LEVEL ${CMAKE_BUILD_PARALLEL_LEVEL}) else() - cmake_host_system_information( - RESULT GASNET_BUILD_PARALLEL_LEVEL QUERY NUMBER_OF_PHYSICAL_CORES - ) + cmake_host_system_information(RESULT GASNET_BUILD_PARALLEL_LEVEL + QUERY NUMBER_OF_PHYSICAL_CORES) endif() - if(NOT ${GASNET_BUILD_PARALLEL_LEVEL}) + if (NOT ${GASNET_BUILD_PARALLEL_LEVEL}) set(GASNET_BUILD_PARALLEL_LEVEL 1) endif() From 3aa422a96e13fe5182ec96092cb2611512f912f2 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:37:11 -0500 Subject: [PATCH 104/154] updated merge --- cmake/HPX_AddTest.cmake | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index e67d100e8ff7..61784265be10 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -188,6 +188,35 @@ function(add_hpx_test category name) endif() endif() endif() + if(HPX_WITH_PARCELPORT_GASNET AND NOT ${name}_NO_PARCELPORT_GASNET) + set(_add_test FALSE) + if(DEFINED ${name}_PARCELPORTS) + set(PP_FOUND -1) + list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) + if(NOT PP_FOUND EQUAL -1) + set(_add_test TRUE) + endif() + else() + set(_add_test TRUE) + endif() + if(_add_test) + set(_full_name "${category}.distributed.gasnet.${name}") + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" + "srun" ${args} + ) + set_tests_properties( + "${_full_name}" + PROPERTIES + RUN_SERIAL TRUE ENVIRONMENT + "PATH=${PROJECT_BINARY_DIR}/_deps/gasnet-src/install/bin:$ENV{PATH}" + ) + if(${name}_TIMEOUT) + set_tests_properties( + "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} + ) + endif() + endif() + endif() if(HPX_WITH_PARCELPORT_OPENSHMEM AND NOT ${name}_NO_PARCELPORT_OPENSHMEM) set(_add_test FALSE) if(DEFINED ${name}_PARCELPORTS) From 62afa4894b0c02b5988b12ef2c3cab4cb224a3b9 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 1 Dec 2023 21:28:39 -0500 Subject: [PATCH 105/154] resolving rebase --- cmake/HPX_SetupGasnet.cmake | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 1ea2a68986d5..9c961dde464c 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -106,11 +106,12 @@ macro(hpx_setup_gasnet) if(CMAKE_BUILD_PARALLEL_LEVEL) set(GASNET_BUILD_PARALLEL_LEVEL ${CMAKE_BUILD_PARALLEL_LEVEL}) else() - cmake_host_system_information(RESULT GASNET_BUILD_PARALLEL_LEVEL - QUERY NUMBER_OF_PHYSICAL_CORES) + cmake_host_system_information( + RESULT GASNET_BUILD_PARALLEL_LEVEL QUERY NUMBER_OF_PHYSICAL_CORES + ) endif() - if (NOT ${GASNET_BUILD_PARALLEL_LEVEL}) + if(NOT ${GASNET_BUILD_PARALLEL_LEVEL}) set(GASNET_BUILD_PARALLEL_LEVEL 1) endif() From 32e4860071103277f4aae17ada38fb506e6814f9 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:37:42 -0500 Subject: [PATCH 106/154] updated merge --- cmake/templates/hpxrun.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 612dc3b82973..aca409c43479 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -366,7 +366,7 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') - , help='Which parcelport to use (Options are: mpi, lci, openshmem, tcp) ' + , help='Which parcelport to use (Options are: mpi, lci, openshmem, gasnet, tcp) ' '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' From 215adb9bb0a1d2a8c37c488d1c31e56c0d5e3c47 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 21:39:55 -0500 Subject: [PATCH 107/154] updated merge --- .jenkins/lsu/env-clang-14.sh | 2 -- .jenkins/lsu/env-clang-16.sh | 3 --- .jenkins/lsu/env-clang-17.sh | 3 --- 3 files changed, 8 deletions(-) diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index f456d73e982c..75e80b83dbf3 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -27,8 +27,6 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" -configure_extra_options+=" -DCMAKE_C_COMPILER=clang" -configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-16.sh b/.jenkins/lsu/env-clang-16.sh index fa32b317d27a..fe21d821a0a3 100644 --- a/.jenkins/lsu/env-clang-16.sh +++ b/.jenkins/lsu/env-clang-16.sh @@ -28,10 +28,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" -configure_extra_options+=" -DCMAKE_C_COMPILER=clang" -configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-17.sh b/.jenkins/lsu/env-clang-17.sh index 0e3dd4f9d663..9ef90c754a45 100644 --- a/.jenkins/lsu/env-clang-17.sh +++ b/.jenkins/lsu/env-clang-17.sh @@ -28,10 +28,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" -configure_extra_options+=" -DCMAKE_C_COMPILER=clang" -configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" From 793b745e5cf67df8e9925071c5e6212c8ec7b659 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 12 Apr 2024 20:03:13 -0400 Subject: [PATCH 108/154] correcting issues with gasnet and openshmem unit tests --- cmake/templates/hpxrun.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index aca409c43479..612dc3b82973 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -366,7 +366,7 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') - , help='Which parcelport to use (Options are: mpi, lci, openshmem, gasnet, tcp) ' + , help='Which parcelport to use (Options are: mpi, lci, openshmem, tcp) ' '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' From 63f915e33a41d5a74c275c0b2f074448d9a1b1d5 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Tue, 16 Apr 2024 11:38:45 -0400 Subject: [PATCH 109/154] redundant variable --- .../hpx/parcelport_openshmem/receiver_connection.hpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index b1cf0c49ad9f..de659401ee33 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -114,6 +114,10 @@ namespace hpx::parcelset::policies::openshmem { bool receive_transmission_chunks(std::size_t num_thread = -1) { + const auto self_ = hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (self_*nthreads_)+sending_thd_id_; + // determine the size of the chunk buffer std::size_t num_zero_copy_chunks = static_cast( static_cast(buffer_.num_chunks_.first)); @@ -122,15 +126,10 @@ namespace hpx::parcelset::policies::openshmem { buffer_.transmission_chunks_.resize( num_zero_copy_chunks + num_non_zero_copy_chunks); - const auto idx = (self_*nthreads_)+sending_thd_id_; if (num_zero_copy_chunks != 0) { buffer_.chunks_.resize(num_zero_copy_chunks); { - const auto self_ = hpx::util::openshmem_environment::rank(); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (self_*nthreads_)+sending_thd_id_; - hpx::util::openshmem_environment::scoped_lock l; hpx::util::openshmem_environment::wait_until( From 75b213b6e9a30f13d66121ce1acb1c335889a431 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Tue, 16 Apr 2024 11:51:20 -0400 Subject: [PATCH 110/154] fixed compilation issue w/num_threads --- libs/core/openshmem_base/src/openshmem_environment.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 82e7bbcef61c..6f52df7d29e5 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include #include @@ -147,13 +147,10 @@ namespace hpx::util { // //segments.resize(hpx::threads::hardware_concurrency() * size()); // - - auto& rp = hpx::resource::get_partitioner(); - // get thread count // openshmem_environment::nthreads_ = - rp.get_num_threads(); + hpx::get_worker_thread_num(); // get system page size // From 6d38b15fad1fc371c2a1367e119e785ecfad093e Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Wed, 17 Apr 2024 10:10:42 -0400 Subject: [PATCH 111/154] fixed locality scoped locking --- .../openshmem_base/openshmem_environment.hpp | 1 + .../src/openshmem_environment.cpp | 3 ++- .../receiver_connection.hpp | 22 +++++++++---------- .../sender_connection.hpp | 19 ++++++++-------- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index 6d95f3039f36..fa85e892909a 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -31,6 +31,7 @@ namespace hpx { namespace util { std::uint8_t* end_addr; unsigned int * rcv; unsigned int * xmt; + hpx::mutex * mut; }; struct HPX_CORE_EXPORT openshmem_environment diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 6f52df7d29e5..504676ef770f 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -204,6 +203,8 @@ namespace hpx::util { // segments[i].xmt = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + beg_signal + page_count + i); + + segments[i].mut = &(openshmem_environment::segment_mutex[i]); } shmem_barrier_all(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index de659401ee33..bdb87d49cc76 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -130,7 +130,7 @@ namespace hpx::parcelset::policies::openshmem { { buffer_.chunks_.resize(num_zero_copy_chunks); { - hpx::util::openshmem_environment::scoped_lock l; + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -172,8 +172,9 @@ namespace hpx::parcelset::policies::openshmem { const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; const auto idx = (self_*nthreads_)+sending_thd_id_; - hpx::util::openshmem_environment::scoped_lock l; - hpx::util::openshmem_environment::wait_until( + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + + hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; @@ -211,7 +212,7 @@ namespace hpx::parcelset::policies::openshmem { const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; const auto idx = (self_*nthreads_)+sending_thd_id_; - hpx::util::openshmem_environment::scoped_lock l; + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -249,7 +250,7 @@ namespace hpx::parcelset::policies::openshmem { const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; const auto idx = (self_*nthreads_)+sending_thd_id_; - hpx::util::openshmem_environment::scoped_lock l; + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -277,13 +278,12 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() noexcept { - hpx::util::openshmem_environment::scoped_try_lock l; - if (!l.locked) - { - return false; - } + const auto self_ = hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (self_*nthreads_)+sending_thd_id_; - return request_ptr_; + const bool l = hpx::util::openshmem_environment::segments[idx].mut->try_lock(); + return l; } #if defined(HPX_HAVE_PARCELPORT_COUNTERS) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 2cac1f04ba9b..b72145466555 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -157,7 +157,7 @@ namespace hpx::parcelset::policies::openshmem { const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; const auto idx = (dst_*nthreads_)+thd_id_; - hpx::util::openshmem_environment::scoped_lock l; + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment @@ -195,7 +195,7 @@ namespace hpx::parcelset::policies::openshmem { const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; const auto idx = (dst_*nthreads_)+thd_id_; - hpx::util::openshmem_environment::scoped_lock l; + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, @@ -225,7 +225,8 @@ namespace hpx::parcelset::policies::openshmem { const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; const auto idx = (dst_*nthreads_)+thd_id_; - hpx::util::openshmem_environment::scoped_lock l; + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, static_cast(hpx::util::openshmem_environment::segments[idx].beg_addr), @@ -256,7 +257,7 @@ namespace hpx::parcelset::policies::openshmem { return false; } - hpx::util::openshmem_environment::scoped_lock l; + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::put_signal( reinterpret_cast(c.data_.cpos_), @@ -304,11 +305,11 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() { - hpx::util::openshmem_environment::scoped_try_lock l; - if (!l.locked) - { - return false; - } + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (dst_*nthreads_)+thd_id_; + + const bool l = hpx::util::openshmem_environment::segments[idx].mut->try_lock(); + return l; return true; } From 1258a13534704f7b472f1c31f1a3368a65e34b55 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Thu, 18 Apr 2024 09:57:21 -0400 Subject: [PATCH 112/154] removed double lock --- libs/core/openshmem_base/src/openshmem_environment.cpp | 3 --- .../include/hpx/parcelport_openshmem/sender_connection.hpp | 6 ++---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 504676ef770f..e71628514f73 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -295,13 +295,10 @@ namespace hpx::util { { if (rank() == node) { - const std::lock_guard lk(segment_mutex[node]); std::memmove(raddr, addr, size); } else { - const std::lock_guard lk(segment_mutex[node]); - #if !defined(SHMEM_SIGNAL_SET) shmem_uint8_put(raddr, addr, size, node); shmem_uint8_put(reinterpret_cast(sigaddr), diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index b72145466555..5cc53467042f 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -163,8 +163,7 @@ namespace hpx::parcelset::policies::openshmem { // into the remote locality (dst_)'s shared memory segment // hpx::util::openshmem_environment::put_signal( - reinterpret_cast(header_.data()), - dst_, + reinterpret_cast(header_.data()), dst_, static_cast( hpx::util::openshmem_environment::segments[idx].beg_addr ), @@ -260,8 +259,7 @@ namespace hpx::parcelset::policies::openshmem { std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::put_signal( - reinterpret_cast(c.data_.cpos_), - dst_, + reinterpret_cast(c.data_.cpos_), dst_, static_cast(hpx::util::openshmem_environment::segments[idx].beg_addr), static_cast(c.size_), reinterpret_cast(hpx::util::openshmem_environment::segments[idx].rcv) From 3051bdd9f4074e43608f606dcda33f68dc02b8de Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Thu, 18 Apr 2024 16:18:26 -0400 Subject: [PATCH 113/154] moved locks around --- .../receiver_connection.hpp | 23 +++++++++++-------- .../sender_connection.hpp | 9 ++++---- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index bdb87d49cc76..2a7dd9752e7d 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -194,6 +194,15 @@ namespace hpx::parcelset::policies::openshmem { bool receive_chunks(std::size_t num_thread = -1) { + std::size_t cidx = 0; + std::size_t chunk_size = 0; + + const auto self_ = hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (self_*nthreads_)+sending_thd_id_; + + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + while (chunks_idx_ < buffer_.chunks_.size()) { if (!request_done()) @@ -201,19 +210,13 @@ namespace hpx::parcelset::policies::openshmem { return false; } - std::size_t idx = chunks_idx_++; - std::size_t chunk_size = - buffer_.transmission_chunks_[idx].second; + cidx = chunks_idx_++; + chunk_size = + buffer_.transmission_chunks_[cidx].second; - data_type& c = buffer_.chunks_[idx]; + data_type& c = buffer_.chunks_[cidx]; c.resize(chunk_size); { - const auto self_ = hpx::util::openshmem_environment::rank(); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (self_*nthreads_)+sending_thd_id_; - - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); - hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 5cc53467042f..381c5528ce1d 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -245,19 +245,20 @@ namespace hpx::parcelset::policies::openshmem { const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; const auto idx = (dst_*nthreads_)+thd_id_; + std::lock_guard l( + *(hpx::util::openshmem_environment::segments[idx].mut) + ); + while (chunks_idx_ < buffer_.chunks_.size()) { serialization::serialization_chunk& c = buffer_.chunks_[chunks_idx_]; if (c.type_ == serialization::chunk_type::chunk_type_pointer) { - if (!request_done()) - { + if (!request_done()) { return false; } - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); - hpx::util::openshmem_environment::put_signal( reinterpret_cast(c.data_.cpos_), dst_, static_cast(hpx::util::openshmem_environment::segments[idx].beg_addr), From 5a5d657a64e08d3853f5fe6273e0c69989992453 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 19 Apr 2024 12:17:33 -0400 Subject: [PATCH 114/154] rm'd tags --- .../hpx/parcelport_openshmem/header.hpp | 28 +++++++++++-------- .../hpx/parcelport_openshmem/receiver.hpp | 24 +++++++++------- .../receiver_connection.hpp | 19 +++++++------ .../hpx/parcelport_openshmem/sender.hpp | 22 +++++++++------ .../sender_connection.hpp | 11 ++++---- .../src/parcelport_openshmem.cpp | 3 +- 6 files changed, 61 insertions(+), 46 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp index c90d698f6c14..efeb82e7d39f 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp @@ -29,20 +29,23 @@ namespace hpx::parcelset::policies::openshmem { enum data_pos { - pos_tag = 0 * sizeof(value_type), - pos_size = 1 * sizeof(value_type), - pos_sending_thread_id = 2 * sizeof(value_type), - pos_numbytes = 3 * sizeof(value_type), - pos_numchunks_first = 4 * sizeof(value_type), - pos_numchunks_second = 5 * sizeof(value_type), - pos_piggy_back_flag = 6 * sizeof(value_type), - pos_piggy_back_data = 7 * sizeof(value_type) + 1 + //pos_tag = 0 * sizeof(value_type), + pos_size = 0 * sizeof(value_type), + pos_sending_thread_id = 1 * sizeof(value_type), + pos_numbytes = 2 * sizeof(value_type), + pos_numchunks_first = 3 * sizeof(value_type), + pos_numchunks_second = 4 * sizeof(value_type), + pos_piggy_back_flag = 5 * sizeof(value_type), + pos_piggy_back_data = 6 * sizeof(value_type) + 1 }; static constexpr int data_size_ = 512; + //template + //header(Buffer const& buffer, int tag, const std::size_t sending_thread_id) noexcept + template - header(Buffer const& buffer, int tag, const std::size_t sending_thread_id) noexcept + header(Buffer const& buffer, const std::size_t sending_thread_id) noexcept { std::int64_t size = static_cast(buffer.size_); std::int64_t numbytes = @@ -51,7 +54,7 @@ namespace hpx::parcelset::policies::openshmem { HPX_ASSERT(size <= (std::numeric_limits::max)()); HPX_ASSERT(numbytes <= (std::numeric_limits::max)()); - set(tag); + //set(tag); set(static_cast(size)); set(static_cast(sending_thread_id)); set(static_cast(numbytes)); @@ -90,7 +93,7 @@ namespace hpx::parcelset::policies::openshmem { void assert_valid() const noexcept { - HPX_ASSERT(tag() != -1); + //HPX_ASSERT(tag() != -1); HPX_ASSERT(size() != -1); HPX_ASSERT(numbytes() != -1); HPX_ASSERT(num_chunks().first != -1); @@ -102,11 +105,12 @@ namespace hpx::parcelset::policies::openshmem { return &data_[0]; } +/* value_type tag() const noexcept { return get(); } - +*/ value_type size() const noexcept { return get(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index cdb1e25fca7e..3b205b0fcd52 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -37,6 +37,7 @@ namespace hpx::parcelset::policies::openshmem { using connection_ptr = std::shared_ptr; using connection_list = std::deque; +/* struct exp_backoff { int numTries; @@ -59,17 +60,18 @@ namespace hpx::parcelset::policies::openshmem { } } }; +*/ explicit constexpr receiver(Parcelport& pp) noexcept : pp_(pp) - , bo() + //, bo() { } void run() noexcept { util::openshmem_environment::scoped_lock l; - new_header(); + new_header(-1); } bool background_work() noexcept @@ -125,7 +127,7 @@ namespace hpx::parcelset::policies::openshmem { if (l.locked) { - header h = new_header(); + header h = new_header(res->sending_thd_id_); l.unlock(); header_lock.unlock(); @@ -139,18 +141,20 @@ namespace hpx::parcelset::policies::openshmem { return res; } - header new_header() noexcept + header new_header(const auto sending_thd_id) noexcept { + const auto self_ = hpx::util::openshmem_environment::rank(); + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (self_*nthreads_)+sending_thd_id; header h = rcv_header_; rcv_header_.reset(); - auto self_ = hpx::util::openshmem_environment::rank(); - while (rcv_header_.data() == 0) - { - bo(hpx::util::openshmem_environment::segments[self_].rcv); + while (rcv_header_.data() == 0) { + hpx::util::openshmem_environment::wait_until( + 1, hpx::util::openshmem_environment::segments[idx].rcv); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; } - (*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0; return h; } @@ -164,7 +168,7 @@ namespace hpx::parcelset::policies::openshmem { hpx::spinlock connections_mtx_; connection_list connections_; - exp_backoff bo; + //exp_backoff bo; }; } // namespace hpx::parcelset::policies::openshmem diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index 2a7dd9752e7d..87452c6e9ca2 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -36,8 +36,8 @@ namespace hpx::parcelset::policies::openshmem { initialized, rcvd_transmission_chunks, rcvd_data, - rcvd_chunks, - sent_release_tag + rcvd_chunks + //,sent_release_tag }; using data_type = std::vector; @@ -47,7 +47,7 @@ namespace hpx::parcelset::policies::openshmem { receiver_connection(int src, header h, Parcelport& pp) noexcept : state_(initialized) , src_(src) - , tag_(h.tag()) + //, tag_(h.tag()) , self_(-1) , sending_thd_id_(-1) , header_(h) @@ -69,7 +69,7 @@ namespace hpx::parcelset::policies::openshmem { data.time_ = timer_.elapsed_nanoseconds(); data.bytes_ = static_cast(header_.numbytes()); #endif - tag_ = header_.tag(); + //tag_ = header_.tag(); // decode data buffer_.num_chunks_ = header_.num_chunks(); buffer_.data_.resize(static_cast(header_.size())); @@ -101,9 +101,9 @@ namespace hpx::parcelset::policies::openshmem { return receive_chunks(num_thread); case rcvd_chunks: - return send_release_tag(num_thread); + //return send_release_tag(num_thread); - case sent_release_tag: + //case sent_release_tag: return done(); default: @@ -235,9 +235,10 @@ namespace hpx::parcelset::policies::openshmem { state_ = rcvd_chunks; - return send_release_tag(num_thread); + return true; //send_release_tag(num_thread); } +/* bool send_release_tag(std::size_t num_thread = -1) { if (!request_done()) @@ -273,7 +274,7 @@ namespace hpx::parcelset::policies::openshmem { return done(); } - +*/ bool done() noexcept { return request_done(); @@ -295,7 +296,7 @@ namespace hpx::parcelset::policies::openshmem { connection_state state_; int src_; - int tag_; + //int tag_; int self_; int sending_thd_id_; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp index 98c87e0e9051..e209ea449b48 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -44,8 +44,8 @@ namespace hpx::parcelset::policies::openshmem { void run() noexcept { - hpx::util::openshmem_environment::scoped_lock l; - get_next_free_tag(); + //hpx::util::openshmem_environment::scoped_lock l; + //get_next_free_tag(); } connection_ptr create_connection(int dest, parcelset::parcelport* pp) @@ -59,10 +59,12 @@ namespace hpx::parcelset::policies::openshmem { connections_.push_back(ptr); } +/* int acquire_tag() noexcept { return tag_provider_.acquire(); } +*/ void send_messages(connection_ptr connection) { @@ -102,21 +104,21 @@ namespace hpx::parcelset::policies::openshmem { send_messages(HPX_MOVE(connection)); has_work = true; } - next_free_tag(); + //next_free_tag(connection->dst_, connection->thd_id_); return has_work; } private: tag_provider tag_provider_; - void next_free_tag() noexcept + void next_free_tag(const auto dst, const auto thd_id) noexcept { int next_free = -1; { std::unique_lock l(next_free_tag_mtx_, std::try_to_lock); if (l.owns_lock()) { - next_free = next_free_tag_locked(); + next_free = next_free_tag_locked(dst, thd_id); } } @@ -127,24 +129,26 @@ namespace hpx::parcelset::policies::openshmem { } } - int next_free_tag_locked() noexcept + int next_free_tag_locked(const auto dst, const auto thd_id) noexcept { hpx::util::openshmem_environment::scoped_try_lock l; if (l.locked) { - return get_next_free_tag(); + return get_next_free_tag(dst, thd_id); } return -1; } - int get_next_free_tag() noexcept + int get_next_free_tag(const auto dst, const auto thd_id) noexcept { int next_free = next_free_tag_; + const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; + const auto idx = (dst*nthreads_)+thd_id; hpx::util::openshmem_environment::scoped_lock l; std::memcpy(&next_free, hpx::util::openshmem_environment::segments - [hpx::util::openshmem_environment::rank()] + [idx] .beg_addr, sizeof(int)); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 381c5528ce1d..762fa297dcbd 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -37,7 +37,7 @@ namespace hpx::parcelset::policies::openshmem { struct sender; struct sender_connection; - int acquire_tag(sender*) noexcept; + //int acquire_tag(sender*) noexcept; void add_connection(sender*, std::shared_ptr const&); struct sender_connection @@ -67,7 +67,7 @@ namespace hpx::parcelset::policies::openshmem { sender_connection(sender_type* s, int dst, parcelset::parcelport* pp) : state_(initialized) , sender_(s) - , tag_(-1) + //, tag_(-1) , dst_(dst) , thd_id_(-1) , chunks_idx_(0) @@ -102,8 +102,9 @@ namespace hpx::parcelset::policies::openshmem { hpx::chrono::high_resolution_clock::now(); #endif chunks_idx_ = 0; - tag_ = acquire_tag(sender_); - header_ = header(buffer_, tag_, thd_id_); + //tag_ = acquire_tag(sender_); + //header_ = header(buffer_, tag_, thd_id_); + header_ = header(buffer_, thd_id_); header_.assert_valid(); state_ = initialized; @@ -315,7 +316,7 @@ namespace hpx::parcelset::policies::openshmem { connection_state state_; sender_type* sender_; - int tag_; + //int tag_; int dst_; int thd_id_; diff --git a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp index 7915e9d9bb3a..5129dff1f886 100644 --- a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp +++ b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp @@ -72,11 +72,12 @@ namespace hpx::parcelset { namespace policies::openshmem { +/* int acquire_tag(sender* s) noexcept { return s->acquire_tag(); } - +*/ void add_connection( sender* s, std::shared_ptr const& ptr) { From a0f5201a568ea6d886e0a2a9d8e3627b1018e31a Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 19 Apr 2024 15:46:12 -0400 Subject: [PATCH 115/154] updates --- .../include/hpx/parcelport_openshmem/receiver.hpp | 5 +++-- .../include/hpx/parcelport_openshmem/sender.hpp | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 3b205b0fcd52..3b029d4d0092 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -70,8 +70,8 @@ namespace hpx::parcelset::policies::openshmem { void run() noexcept { - util::openshmem_environment::scoped_lock l; - new_header(-1); + //util::openshmem_environment::scoped_lock l; + //new_header(-1); } bool background_work() noexcept @@ -149,6 +149,7 @@ namespace hpx::parcelset::policies::openshmem { header h = rcv_header_; rcv_header_.reset(); + // waiting for `sender_connection::send_header` invocation while (rcv_header_.data() == 0) { hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp index e209ea449b48..8e3c7cc0c2e4 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -109,6 +109,7 @@ namespace hpx::parcelset::policies::openshmem { } private: +/* tag_provider tag_provider_; void next_free_tag(const auto dst, const auto thd_id) noexcept @@ -154,7 +155,7 @@ namespace hpx::parcelset::policies::openshmem { return next_free; } - +*/ hpx::spinlock connections_mtx_; connection_list connections_; From 14cd57b92b679e3585aff40ba19672b0b77290d2 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 19 Apr 2024 16:20:04 -0400 Subject: [PATCH 116/154] added improved receive support --- .../openshmem_base/openshmem_environment.hpp | 1 + .../src/openshmem_environment.cpp | 13 ++++++++ .../hpx/parcelport_openshmem/receiver.hpp | 31 +++++++++++++++---- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index fa85e892909a..fa69a6a6e09c 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -56,6 +56,7 @@ namespace hpx { namespace util { std::uint8_t* raddr, const std::size_t size, unsigned int* sigaddr); static void wait_until(const unsigned int value, unsigned int* sigaddr); + static std::size_t wait_until_any(const unsigned int value, unsigned int* sigaddr, const std::size_t count); static void get(std::uint8_t* addr, const int rank, const std::uint8_t* raddr, const std::size_t size); diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index e71628514f73..3208c3fbedb2 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -319,6 +319,19 @@ namespace hpx::util { shmem_uint_wait_until(sigaddr, SHMEM_CMP_EQ, value); } + std::size_t wait_until_any(const unsigned int value, unsigned int* sigaddr, const std::size_t count) { + + const std::size_t sig_idx = shmem_wait_until_any( + sigaddr, + count, + nullptr, + SHMEM_CMP_EQ, + value + ); + + return sig_idx; + } + void openshmem_environment::get(std::uint8_t* addr, const int node, const std::uint8_t* raddr, const std::size_t size) { diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 3b029d4d0092..f802a2cfa4ce 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -145,17 +145,36 @@ namespace hpx::parcelset::policies::openshmem { { const auto self_ = hpx::util::openshmem_environment::rank(); const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (self_*nthreads_)+sending_thd_id; - header h = rcv_header_; - rcv_header_.reset(); + + const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); + const std::size_t page_count = + hpx::util::openshmem_environment::size() * hpx::util::openshmem_environment::nthreads_; + const std::size_t beg_rcv_signal = (sys_pgsz*page_count); // waiting for `sender_connection::send_header` invocation while (rcv_header_.data() == 0) { - hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[idx].rcv); - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + + const auto sig_idx = hpx::util::openshmem_environment::wait_until_any( + 1, + reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal), + page_count + ); + + const auto base_addr = hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal; + const auto sending_thd_id = static_cast((base_addr + sig_idx) - base_addr); + const auto idx = (self_*nthreads_)+sending_thd_id; + + hpx::util::openshmem_environment::get( + reinterpret_cast( + rcv_header_.data()), + self_, + hpx::util::openshmem_environment::segments[idx].beg_addr, + sizeof(rcv_header_) + ); } + header h = rcv_header_; + rcv_header_.reset(); return h; } From 4610458cf062d4b2d05ccb4ea95ed9805bee1778 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 19 Apr 2024 16:26:22 -0400 Subject: [PATCH 117/154] rm'd redundant index computations --- .../include/hpx/parcelport_openshmem/receiver.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index f802a2cfa4ce..e506fa720264 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -154,16 +154,12 @@ namespace hpx::parcelset::policies::openshmem { // waiting for `sender_connection::send_header` invocation while (rcv_header_.data() == 0) { - const auto sig_idx = hpx::util::openshmem_environment::wait_until_any( + const auto idx = hpx::util::openshmem_environment::wait_until_any( 1, reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal), page_count ); - const auto base_addr = hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal; - const auto sending_thd_id = static_cast((base_addr + sig_idx) - base_addr); - const auto idx = (self_*nthreads_)+sending_thd_id; - hpx::util::openshmem_environment::get( reinterpret_cast( rcv_header_.data()), From f0c6e9c088e697dddff5ff87a5c0ef2a2dcf5688 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 19 Apr 2024 16:29:54 -0400 Subject: [PATCH 118/154] corrected resetting rcv signal --- .../include/hpx/parcelport_openshmem/receiver.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index e506fa720264..2aa7f45baed8 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -167,6 +167,8 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::segments[idx].beg_addr, sizeof(rcv_header_) ); + + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; } header h = rcv_header_; From 455bfb2f18cb59214ce35b6241b43a02adeee0fc Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 19 Apr 2024 16:55:50 -0400 Subject: [PATCH 119/154] fixed some locking issues --- .../hpx/parcelport_openshmem/receiver.hpp | 21 ++++++++++++------- .../receiver_connection.hpp | 12 +++++++---- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 2aa7f45baed8..33d98d8ab793 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -151,8 +151,9 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::size() * hpx::util::openshmem_environment::nthreads_; const std::size_t beg_rcv_signal = (sys_pgsz*page_count); + // waiting for `sender_connection::send_header` invocation - while (rcv_header_.data() == 0) { + if (rcv_header_.data() == 0) { const auto idx = hpx::util::openshmem_environment::wait_until_any( 1, @@ -160,13 +161,17 @@ namespace hpx::parcelset::policies::openshmem { page_count ); - hpx::util::openshmem_environment::get( - reinterpret_cast( - rcv_header_.data()), - self_, - hpx::util::openshmem_environment::segments[idx].beg_addr, - sizeof(rcv_header_) - ); + { + std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + + hpx::util::openshmem_environment::get( + reinterpret_cast( + rcv_header_.data()), + self_, + hpx::util::openshmem_environment::segments[idx].beg_addr, + sizeof(rcv_header_) + ); + } (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index 87452c6e9ca2..78e796161d88 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -134,7 +134,6 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast( @@ -144,6 +143,8 @@ namespace hpx::parcelset::policies::openshmem { static_cast(buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type))); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + request_ptr_ = true; } } @@ -176,7 +177,6 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(buffer_.data_.data()), @@ -184,6 +184,8 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::segments[idx].beg_addr, buffer_.data_.size()); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + request_ptr_ = true; } @@ -219,13 +221,14 @@ namespace hpx::parcelset::policies::openshmem { { hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(c.data()), self_, hpx::util::openshmem_environment::segments[idx].beg_addr, c.size()); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + hpx::util::openshmem_environment::put_signal(nullptr, src_, nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); @@ -258,13 +261,14 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; hpx::util::openshmem_environment::get( reinterpret_cast(&tag_), self_, hpx::util::openshmem_environment::segments[idx].beg_addr, sizeof(int)); + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + request_ptr_ = true; } From cefee617f0fc9b25d61e676d343e5dbe408f1881 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 22 Apr 2024 09:40:22 -0400 Subject: [PATCH 120/154] implement wait loop in case MPI openshmem is installed --- libs/core/openshmem_base/src/openshmem_environment.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 3208c3fbedb2..89349cb4d93f 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -321,6 +321,13 @@ namespace hpx::util { std::size_t wait_until_any(const unsigned int value, unsigned int* sigaddr, const std::size_t count) { +#if defined(MPI_VERSION) + int rc = 0; + for(std::size_t i = 0; i < count; ++i) { + rc = shmem_test(sigaddr+i, SHMEM_CMP_EQ, 1); + if(rc) { return i; } + } +#else const std::size_t sig_idx = shmem_wait_until_any( sigaddr, count, @@ -328,6 +335,7 @@ namespace hpx::util { SHMEM_CMP_EQ, value ); +#endif return sig_idx; } From cc29b1fd4fab9755839c9752250587de076eebb8 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 22 Apr 2024 09:46:38 -0400 Subject: [PATCH 121/154] implement wait loop in case MPI openshmem is installed --- libs/core/openshmem_base/src/openshmem_environment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 89349cb4d93f..38b0db6e8c4c 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -324,7 +324,7 @@ namespace hpx::util { #if defined(MPI_VERSION) int rc = 0; for(std::size_t i = 0; i < count; ++i) { - rc = shmem_test(sigaddr+i, SHMEM_CMP_EQ, 1); + rc = shmem_test(sigaddr+i, SHMEM_CMP_EQ, value); if(rc) { return i; } } #else From 5078e17fd4b6e00c28bca279307733345ab5858a Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 22 Apr 2024 10:04:53 -0400 Subject: [PATCH 122/154] implement wait loop in case MPI openshmem is installed --- .../openshmem_base/openshmem_environment.hpp | 10 +++--- .../src/openshmem_environment.cpp | 31 ++++++++++++------- .../hpx/parcelport_openshmem/receiver.hpp | 2 +- .../sender_connection.hpp | 22 +++++-------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index fa69a6a6e09c..43ef3e03360a 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -29,8 +29,8 @@ namespace hpx { namespace util { { std::uint8_t* beg_addr; std::uint8_t* end_addr; - unsigned int * rcv; - unsigned int * xmt; + std::uint8_t * rcv; + std::uint8_t * xmt; hpx::mutex * mut; }; @@ -53,10 +53,10 @@ namespace hpx { namespace util { static std::string get_processor_name(); static void put_signal(const std::uint8_t* addr, const int rank, - std::uint8_t* raddr, const std::size_t size, unsigned int* sigaddr); + std::uint8_t* raddr, const std::size_t size, std::uint8_t * sigaddr); - static void wait_until(const unsigned int value, unsigned int* sigaddr); - static std::size_t wait_until_any(const unsigned int value, unsigned int* sigaddr, const std::size_t count); + static void wait_until(const std::uint8_t value, std::uint8_t * sigaddr); + static std::size_t wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count); static void get(std::uint8_t* addr, const int rank, const std::uint8_t* raddr, const std::size_t size); diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 38b0db6e8c4c..d0455f385707 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -196,13 +196,11 @@ namespace hpx::util { // all of the rcv signals are linearly arranged before the rcv signals // - segments[i].rcv = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + - beg_signal + i); + segments[i].rcv = hpx::util::openshmem_environment::shmem_buffer + beg_signal + i; // all of the xmt signals are linearly arranged after the rcv signals // - segments[i].xmt = reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + - beg_signal + page_count + i); + segments[i].xmt = hpx::util::openshmem_environment::shmem_buffer + beg_signal + page_count + i; segments[i].mut = &(openshmem_environment::segment_mutex[i]); } @@ -291,7 +289,7 @@ namespace hpx::util { void openshmem_environment::put_signal(const std::uint8_t* addr, const int node, std::uint8_t* raddr, const std::size_t size, - unsigned int* sigaddr) + std::uint8_t * sigaddr) { if (rank() == node) { @@ -314,22 +312,33 @@ namespace hpx::util { } void openshmem_environment::wait_until( - const unsigned int value, unsigned int* sigaddr) + const std::uint8_t value, std::uint8_t * sigaddr) { - shmem_uint_wait_until(sigaddr, SHMEM_CMP_EQ, value); + shmem_wait_until(reinterpret_cast(sigaddr), SHMEM_CMP_EQ, static_cast(value)); } - std::size_t wait_until_any(const unsigned int value, unsigned int* sigaddr, const std::size_t count) { + std::size_t wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { + + +#define SHMEM_MAJOR_VERSION 1 +#define SHMEM_MINOR_VERSION 4 +#define SHMEM_VENDOR_STRING "http://www.open-mpi.org/" +#define SHMEM_MAX_NAME_LEN 256 + +#if defined(SHMEM_MAJOR_VERSION) && defined(SHMEM_MINOR_VERSION) && \ + defined(SHMEM_VENDOR_STRING) && defined(SHMEM_MAX_NAME_LEN) && \ + SHMEM_MAJOR_VERSION == 1 && SHMEM_MINOR_VERSION == 4 && \ + SHMEM_MAX_NAME_LEN == 4 && SHMEM_VENDOR_STRING == "http://www.open-mpi.org/" -#if defined(MPI_VERSION) int rc = 0; for(std::size_t i = 0; i < count; ++i) { - rc = shmem_test(sigaddr+i, SHMEM_CMP_EQ, value); + rc = shmem_test(reinterpret_cast(sigaddr+i), SHMEM_CMP_EQ, value); if(rc) { return i; } } + #else const std::size_t sig_idx = shmem_wait_until_any( - sigaddr, + reinterpret_cast(sigaddr), count, nullptr, SHMEM_CMP_EQ, diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 33d98d8ab793..d495cb9db627 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -157,7 +157,7 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = hpx::util::openshmem_environment::wait_until_any( 1, - reinterpret_cast(hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal), + hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal, page_count ); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 762fa297dcbd..3b8971fde0f5 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -165,13 +165,9 @@ namespace hpx::parcelset::policies::openshmem { // hpx::util::openshmem_environment::put_signal( reinterpret_cast(header_.data()), dst_, - static_cast( - hpx::util::openshmem_environment::segments[idx].beg_addr - ), + hpx::util::openshmem_environment::segments[idx].beg_addr, header_.data_size_, - reinterpret_cast( - hpx::util::openshmem_environment::segments[idx].rcv - ) + hpx::util::openshmem_environment::segments[idx].rcv ); } @@ -199,12 +195,10 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, - static_cast(hpx::util::openshmem_environment::segments - [idx].beg_addr), + hpx::util::openshmem_environment::segments[idx].beg_addr, static_cast(chunks.size() * sizeof(parcel_buffer_type::transmission_chunk_type)), - reinterpret_cast(hpx::util::openshmem_environment::segments - [idx].rcv) + hpx::util::openshmem_environment::segments[idx].rcv ); } @@ -229,9 +223,9 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, - static_cast(hpx::util::openshmem_environment::segments[idx].beg_addr), + hpx::util::openshmem_environment::segments[idx].beg_addr, buffer_.data_.size(), - reinterpret_cast(hpx::util::openshmem_environment::segments[idx].rcv) + hpx::util::openshmem_environment::segments[idx].rcv ); } state_ = sent_data; @@ -262,9 +256,9 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::put_signal( reinterpret_cast(c.data_.cpos_), dst_, - static_cast(hpx::util::openshmem_environment::segments[idx].beg_addr), + hpx::util::openshmem_environment::segments[idx].beg_addr, static_cast(c.size_), - reinterpret_cast(hpx::util::openshmem_environment::segments[idx].rcv) + hpx::util::openshmem_environment::segments[idx].rcv ); hpx::util::openshmem_environment::wait_until( From f124a042263fd05d27410cd1236156563464d4ba Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 22 Apr 2024 10:07:13 -0400 Subject: [PATCH 123/154] implement wait loop in case MPI openshmem is installed --- libs/core/openshmem_base/src/openshmem_environment.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index d0455f385707..65888a120e47 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -319,16 +319,10 @@ namespace hpx::util { std::size_t wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { - -#define SHMEM_MAJOR_VERSION 1 -#define SHMEM_MINOR_VERSION 4 -#define SHMEM_VENDOR_STRING "http://www.open-mpi.org/" -#define SHMEM_MAX_NAME_LEN 256 - #if defined(SHMEM_MAJOR_VERSION) && defined(SHMEM_MINOR_VERSION) && \ defined(SHMEM_VENDOR_STRING) && defined(SHMEM_MAX_NAME_LEN) && \ SHMEM_MAJOR_VERSION == 1 && SHMEM_MINOR_VERSION == 4 && \ - SHMEM_MAX_NAME_LEN == 4 && SHMEM_VENDOR_STRING == "http://www.open-mpi.org/" + SHMEM_MAX_NAME_LEN == 256 int rc = 0; for(std::size_t i = 0; i < count; ++i) { From b4b1cf0d43105b73122029956651ef05ce906ffa Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 22 Apr 2024 11:25:36 -0400 Subject: [PATCH 124/154] implement wait loop in case MPI openshmem is installed --- libs/core/openshmem_base/src/openshmem_environment.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 65888a120e47..eafda3677e92 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -314,7 +314,7 @@ namespace hpx::util { void openshmem_environment::wait_until( const std::uint8_t value, std::uint8_t * sigaddr) { - shmem_wait_until(reinterpret_cast(sigaddr), SHMEM_CMP_EQ, static_cast(value)); + shmem_int_wait_until(reinterpret_cast(sigaddr), SHMEM_CMP_EQ, static_cast(value)); } std::size_t wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { @@ -330,6 +330,7 @@ namespace hpx::util { if(rc) { return i; } } + return -1; #else const std::size_t sig_idx = shmem_wait_until_any( reinterpret_cast(sigaddr), @@ -338,9 +339,9 @@ namespace hpx::util { SHMEM_CMP_EQ, value ); -#endif return sig_idx; +#endif } void openshmem_environment::get(std::uint8_t* addr, const int node, From c4d1107dd04f23646996b5b098c4010dbeca5d5f Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 22 Apr 2024 19:49:55 -0400 Subject: [PATCH 125/154] implement wait loop in case MPI openshmem is installed --- .../openshmem_base/include/hpx/openshmem_base/openshmem.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp index 60413db0dbdc..55ddb10b7f22 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -17,6 +17,10 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif +#ifndef OSHMEM_HAVE_C11 +#define OSHMEM_HAVE_C11 +#endif + #ifndef SHMEM_H #include #endif From d39b69a3bafd7f833b9df946db3c91466ea5f442 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Tue, 23 Apr 2024 09:56:26 -0400 Subject: [PATCH 126/154] implement wait loop in case MPI openshmem is installed --- .../include/hpx/openshmem_base/openshmem.hpp | 8 ++++---- libs/core/openshmem_base/src/openshmem_environment.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp index 55ddb10b7f22..50adf00d67a2 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -17,14 +17,14 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif -#ifndef OSHMEM_HAVE_C11 -#define OSHMEM_HAVE_C11 -#endif - #ifndef SHMEM_H #include #endif +#ifndef OSHMEM_HAVE_C11 +#define OSHMEM_HAVE_C11 1 +#endif + #if defined(__clang__) #pragma clang diagnostic pop #elif defined(__GNUC__) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index eafda3677e92..f430b44aa1c3 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -314,7 +314,7 @@ namespace hpx::util { void openshmem_environment::wait_until( const std::uint8_t value, std::uint8_t * sigaddr) { - shmem_int_wait_until(reinterpret_cast(sigaddr), SHMEM_CMP_EQ, static_cast(value)); + shmem_int_wait_until(reinterpret_cast(sigaddr), SHMEM_CMP_EQ, static_cast(value)); } std::size_t wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { From 56d135e96c2000b75a636bfdc94f84ac8875b494 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Tue, 23 Apr 2024 09:58:52 -0400 Subject: [PATCH 127/154] implement wait loop in case MPI openshmem is installed --- .../openshmem_base/include/hpx/openshmem_base/openshmem.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp index 50adf00d67a2..40a53c27a3ba 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -25,6 +25,11 @@ #define OSHMEM_HAVE_C11 1 #endif +#if defined(OSHMEM_HAVE_C11) && OSHMEM_HAVE_C11 == 0 +#undef OSHMEM_HAVE_C11 +#define OSHMEM_HAVE_C11 1 +#endif + #if defined(__clang__) #pragma clang diagnostic pop #elif defined(__GNUC__) From 6d6af074d3bed0277f46ac91e3dd8adf8f422f8b Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Tue, 23 Apr 2024 10:40:23 -0400 Subject: [PATCH 128/154] implement wait loop in case MPI openshmem is installed --- libs/core/openshmem_base/src/openshmem_environment.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index f430b44aa1c3..e12b1770febc 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -314,7 +314,13 @@ namespace hpx::util { void openshmem_environment::wait_until( const std::uint8_t value, std::uint8_t * sigaddr) { - shmem_int_wait_until(reinterpret_cast(sigaddr), SHMEM_CMP_EQ, static_cast(value)); + union { + std::uint8_t * uaddr; + volatile int * iaddr; + } tmp; + tmp.uaddr = sigaddr; + volatile int * ptr = tmp.iaddr; + shmem_int_wait_until(ptr, SHMEM_CMP_EQ, static_cast(value)); } std::size_t wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { From 0a61bb6474132505946517735e1d555b387082fe Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Tue, 23 Apr 2024 11:04:30 -0400 Subject: [PATCH 129/154] implement wait loop in case MPI openshmem is installed --- .../src/openshmem_environment.cpp | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index e12b1770febc..8154af5e7e16 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -311,16 +311,32 @@ namespace hpx::util { } } + template + struct signature; + + template + struct signature + { + using type = std::tuple; + }; + void openshmem_environment::wait_until( const std::uint8_t value, std::uint8_t * sigaddr) { + using arg_type = std::conditional< + std::is_same::type, + std::tuple>::value, + volatile int *, + int * + >::type; + union { std::uint8_t * uaddr; - volatile int * iaddr; + arg_type iaddr; } tmp; tmp.uaddr = sigaddr; - volatile int * ptr = tmp.iaddr; - shmem_int_wait_until(ptr, SHMEM_CMP_EQ, static_cast(value)); + + shmem_int_wait_until(tmp.iaddr, SHMEM_CMP_EQ, static_cast(value)); } std::size_t wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { From bb85e885ff01f4ab09658b559e7e8a2ee625e9e6 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Tue, 23 Apr 2024 20:47:11 -0400 Subject: [PATCH 130/154] updating openshmem_environment --- .../src/openshmem_environment.cpp | 85 ++++++++++++------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 8154af5e7e16..a1f883b3ddbf 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -27,41 +27,35 @@ /////////////////////////////////////////////////////////////////////////////// namespace hpx::util { - namespace detail { - - bool detect_openshmem_environment( - util::runtime_configuration const& cfg, char const* default_env) + bool openshmem_environment::check_openshmem_environment( + util::runtime_configuration const& cfg) + { +#if !defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) + return false; +#elif defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + const std::string default_env{"SHMEM_VERSION;SHMEM_INFO;SHMEM_SYMMETRIC_SIZE;MV2_COMM_WORLD_RANK;PMI_RANK;OMPI_COMM_WORLD_SIZE;ALPS_APP_PE;PMIX_RANK;PALS_NODEID"}; + std::string openshmem_environment_strings = + cfg.get_entry("hpx.parcel.openshmem.env", default_env); + + hpx::string_util::char_separator sep(";,: "); + hpx::string_util::tokenizer tokens(openshmem_environment_strings, sep); + for (auto const& tok : tokens) { - std::string openshmem_environment_strings = - cfg.get_entry("hpx.parcel.openshmem.env", default_env); - - hpx::string_util::char_separator sep(";,: "); - hpx::string_util::tokenizer tokens( - openshmem_environment_strings, sep); - for (auto const& tok : tokens) + char* env = std::getenv(tok.c_str()); + if (env) { - char* env = std::getenv(tok.c_str()); - if (env) - { - LBT_(debug) - << "Found OPENSHMEM environment variable: " << tok - << "=" << std::string(env) - << ", enabling OPENSHMEM support\n"; - return true; - } + LBT_(debug) + << "Found OpenSHMEM environment variable: " << tok << "=" + << std::string(env) << ", enabling OpenSHMEM support\n"; + return true; } - - LBT_(info) - << "No known OPENSHMEM environment variable found, disabling " - "OPENSHMEM support\n"; - return false; } - } // namespace detail - bool openshmem_environment::check_openshmem_environment( - [[maybe_unused]] util::runtime_configuration const& cfg) - { -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + LBT_(info) << "No known OpenSHMEM environment variable found, disabling " + "OpenSHMEM support\n"; + + return false; +#else // We disable the OPENSHMEM parcelport if any of these hold: // // - The parcelport is explicitly disabled @@ -78,11 +72,32 @@ namespace hpx::util { { LBT_(info) << "OpenSHMEM support disabled via configuration settings\n"; + + cfg.add_entry("hpx.parcel.openshmem.enable", "0"); return false; } - return true; -#else + const std::string default_env{"SHMEM_VERSION;SHMEM_INFO;SHMEM_SYMMETRIC_SIZE;MV2_COMM_WORLD_RANK;PMI_RANK;OMPI_COMM_WORLD_SIZE;ALPS_APP_PE;PMIX_RANK;PALS_NODEID"}; + std::string openshmem_environment_strings = + cfg.get_entry("hpx.parcel.openshmem.env", default_env); + + hpx::string_util::char_separator sep(";,: "); + hpx::string_util::tokenizer tokens(openshmem_environment_strings, sep); + for (auto const& tok : tokens) + { + char* env = std::getenv(tok.c_str()); + if (env) + { + LBT_(debug) + << "Found OpenSHMEM environment variable: " << tok << "=" + << std::string(env) << ", enabling OpenSHMEM support\n"; + return true; + } + } + + LBT_(info) << "No known OpenSHMEM environment variable found, disabling " + "OpenSHMEM support\n"; + return false; #endif } @@ -311,6 +326,8 @@ namespace hpx::util { } } + // template metaprogramming for `openshmem_environment::wait_until` + // template struct signature; @@ -323,6 +340,10 @@ namespace hpx::util { void openshmem_environment::wait_until( const std::uint8_t value, std::uint8_t * sigaddr) { + // some openshmem implementations place a `volatile` in the argument list + // this section compile-time section detects this situation and enables + // the right option + // using arg_type = std::conditional< std::is_same::type, std::tuple>::value, From b1529f7e9f7bb5dbf883fba00e4f3b4204037271 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Wed, 24 Apr 2024 09:58:41 -0400 Subject: [PATCH 131/154] reduced symmetric memory allocation --- .../src/openshmem_environment.cpp | 2 +- .../hpx/parcelport_openshmem/header.hpp | 23 ++---- .../hpx/parcelport_openshmem/receiver.hpp | 34 +------- .../receiver_connection.hpp | 82 ++++--------------- .../hpx/parcelport_openshmem/sender.hpp | 55 +------------ .../sender_connection.hpp | 21 ++--- 6 files changed, 31 insertions(+), 186 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index a1f883b3ddbf..edb8320f4903 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -174,7 +174,7 @@ namespace hpx::util { // // page_count = num_localities * number of threads // - const std::size_t page_count = size()*openshmem_environment::nthreads_; + const std::size_t page_count = size(); // allocate the page cache // diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp index efeb82e7d39f..50a4b2569eb5 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp @@ -29,23 +29,18 @@ namespace hpx::parcelset::policies::openshmem { enum data_pos { - //pos_tag = 0 * sizeof(value_type), pos_size = 0 * sizeof(value_type), - pos_sending_thread_id = 1 * sizeof(value_type), - pos_numbytes = 2 * sizeof(value_type), - pos_numchunks_first = 3 * sizeof(value_type), - pos_numchunks_second = 4 * sizeof(value_type), - pos_piggy_back_flag = 5 * sizeof(value_type), - pos_piggy_back_data = 6 * sizeof(value_type) + 1 + pos_numbytes = 1 * sizeof(value_type), + pos_numchunks_first = 2 * sizeof(value_type), + pos_numchunks_second = 3 * sizeof(value_type), + pos_piggy_back_flag = 4 * sizeof(value_type), + pos_piggy_back_data = 5 * sizeof(value_type) + 1 }; static constexpr int data_size_ = 512; - //template - //header(Buffer const& buffer, int tag, const std::size_t sending_thread_id) noexcept - template - header(Buffer const& buffer, const std::size_t sending_thread_id) noexcept + header(Buffer const& buffer) noexcept { std::int64_t size = static_cast(buffer.size_); std::int64_t numbytes = @@ -56,7 +51,6 @@ namespace hpx::parcelset::policies::openshmem { //set(tag); set(static_cast(size)); - set(static_cast(sending_thread_id)); set(static_cast(numbytes)); set( static_cast(buffer.num_chunks_.first)); @@ -116,11 +110,6 @@ namespace hpx::parcelset::policies::openshmem { return get(); } - value_type sending_thread_id() const noexcept - { - return get(); - } - value_type numbytes() const noexcept { return get(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index d495cb9db627..d420aae0961c 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -37,34 +37,8 @@ namespace hpx::parcelset::policies::openshmem { using connection_ptr = std::shared_ptr; using connection_list = std::deque; -/* - struct exp_backoff - { - int numTries; - const static int maxRetries = 10; - - void operator()(unsigned int* addr) - { - if (numTries <= maxRetries) - { - if (shmem_uint_test(addr, SHMEM_CMP_EQ, 1)) - { - return; - } - hpx::this_thread::suspend( - std::chrono::microseconds(1 << numTries)); - } - else - { - numTries = 0; - } - } - }; -*/ - explicit constexpr receiver(Parcelport& pp) noexcept : pp_(pp) - //, bo() { } @@ -127,7 +101,7 @@ namespace hpx::parcelset::policies::openshmem { if (l.locked) { - header h = new_header(res->sending_thd_id_); + header h = new_header(); l.unlock(); header_lock.unlock(); @@ -141,14 +115,13 @@ namespace hpx::parcelset::policies::openshmem { return res; } - header new_header(const auto sending_thd_id) noexcept + header new_header() noexcept { const auto self_ = hpx::util::openshmem_environment::rank(); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); const std::size_t page_count = - hpx::util::openshmem_environment::size() * hpx::util::openshmem_environment::nthreads_; + hpx::util::openshmem_environment::size(); const std::size_t beg_rcv_signal = (sys_pgsz*page_count); @@ -191,7 +164,6 @@ namespace hpx::parcelset::policies::openshmem { hpx::spinlock connections_mtx_; connection_list connections_; - //exp_backoff bo; }; } // namespace hpx::parcelset::policies::openshmem diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index 78e796161d88..82d7496e34c8 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -47,9 +47,7 @@ namespace hpx::parcelset::policies::openshmem { receiver_connection(int src, header h, Parcelport& pp) noexcept : state_(initialized) , src_(src) - //, tag_(h.tag()) , self_(-1) - , sending_thd_id_(-1) , header_(h) , request_ptr_(false) , num_bytes(0) @@ -61,7 +59,6 @@ namespace hpx::parcelset::policies::openshmem { header_.assert_valid(); self_ = hpx::util::openshmem_environment::rank(); - sending_thd_id_ = header_.sending_thread_id(); num_bytes = header_.numbytes(); #if defined(HPX_HAVE_PARCELPORT_COUNTERS) @@ -87,23 +84,20 @@ namespace hpx::parcelset::policies::openshmem { need_recv_tchunks = false; } - bool receive(std::size_t num_thread = -1) + bool receive() { switch (state_) { case initialized: - return receive_transmission_chunks(num_thread); + return receive_transmission_chunks(); case rcvd_transmission_chunks: - return receive_data(num_thread); + return receive_data(); case rcvd_data: - return receive_chunks(num_thread); + return receive_chunks(); case rcvd_chunks: - //return send_release_tag(num_thread); - - //case sent_release_tag: return done(); default: @@ -112,11 +106,9 @@ namespace hpx::parcelset::policies::openshmem { return false; } - bool receive_transmission_chunks(std::size_t num_thread = -1) + bool receive_transmission_chunks() { - const auto self_ = hpx::util::openshmem_environment::rank(); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (self_*nthreads_)+sending_thd_id_; + const auto idx = self_; // determine the size of the chunk buffer std::size_t num_zero_copy_chunks = static_cast( @@ -151,10 +143,10 @@ namespace hpx::parcelset::policies::openshmem { state_ = rcvd_transmission_chunks; - return receive_data(num_thread); + return receive_data(); } - bool receive_data(std::size_t num_thread = -1) + bool receive_data() { if (!request_done()) { @@ -169,9 +161,7 @@ namespace hpx::parcelset::policies::openshmem { } else { - const auto self_ = hpx::util::openshmem_environment::rank(); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (self_*nthreads_)+sending_thd_id_; + const auto idx = self_; std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); @@ -191,17 +181,15 @@ namespace hpx::parcelset::policies::openshmem { state_ = rcvd_data; - return receive_chunks(num_thread); + return receive_chunks(); } - bool receive_chunks(std::size_t num_thread = -1) + bool receive_chunks() { std::size_t cidx = 0; std::size_t chunk_size = 0; - const auto self_ = hpx::util::openshmem_environment::rank(); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (self_*nthreads_)+sending_thd_id_; + const auto idx = self_; std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); @@ -238,47 +226,9 @@ namespace hpx::parcelset::policies::openshmem { state_ = rcvd_chunks; - return true; //send_release_tag(num_thread); + return true; } -/* - bool send_release_tag(std::size_t num_thread = -1) - { - if (!request_done()) - { - return false; - } -#if defined(HPX_HAVE_PARCELPORT_COUNTERS) - parcelset::data_point& data = buffer_.data_point_; - data.time_ = timer_.elapsed_nanoseconds() - data.time_; -#endif - { - const auto self_ = hpx::util::openshmem_environment::rank(); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (self_*nthreads_)+sending_thd_id_; - - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); - - hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[idx].rcv); - - hpx::util::openshmem_environment::get( - reinterpret_cast(&tag_), self_, - hpx::util::openshmem_environment::segments[idx].beg_addr, - sizeof(int)); - - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - - request_ptr_ = true; - } - - decode_parcels(pp_, HPX_MOVE(buffer_), num_thread); - - state_ = sent_release_tag; - - return done(); - } -*/ bool done() noexcept { return request_done(); @@ -286,9 +236,7 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() noexcept { - const auto self_ = hpx::util::openshmem_environment::rank(); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (self_*nthreads_)+sending_thd_id_; + const auto idx = self_; const bool l = hpx::util::openshmem_environment::segments[idx].mut->try_lock(); return l; @@ -300,9 +248,7 @@ namespace hpx::parcelset::policies::openshmem { connection_state state_; int src_; - //int tag_; int self_; - int sending_thd_id_; header header_; buffer_type buffer_; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp index 8e3c7cc0c2e4..daa16267ea99 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -59,13 +59,6 @@ namespace hpx::parcelset::policies::openshmem { connections_.push_back(ptr); } -/* - int acquire_tag() noexcept - { - return tag_provider_.acquire(); - } -*/ - void send_messages(connection_ptr connection) { // Check if sending has been completed.... @@ -104,58 +97,12 @@ namespace hpx::parcelset::policies::openshmem { send_messages(HPX_MOVE(connection)); has_work = true; } - //next_free_tag(connection->dst_, connection->thd_id_); + return has_work; } private: -/* - tag_provider tag_provider_; - void next_free_tag(const auto dst, const auto thd_id) noexcept - { - int next_free = -1; - { - std::unique_lock l(next_free_tag_mtx_, std::try_to_lock); - if (l.owns_lock()) - { - next_free = next_free_tag_locked(dst, thd_id); - } - } - - if (next_free != -1) - { - HPX_ASSERT(next_free > 1); - tag_provider_.release(next_free); - } - } - - int next_free_tag_locked(const auto dst, const auto thd_id) noexcept - { - hpx::util::openshmem_environment::scoped_try_lock l; - if (l.locked) - { - return get_next_free_tag(dst, thd_id); - } - return -1; - } - - int get_next_free_tag(const auto dst, const auto thd_id) noexcept - { - int next_free = next_free_tag_; - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (dst*nthreads_)+thd_id; - - hpx::util::openshmem_environment::scoped_lock l; - std::memcpy(&next_free, - hpx::util::openshmem_environment::segments - [idx] - .beg_addr, - sizeof(int)); - - return next_free; - } -*/ hpx::spinlock connections_mtx_; connection_list connections_; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 3b8971fde0f5..2be0b13f3bfb 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -67,7 +67,6 @@ namespace hpx::parcelset::policies::openshmem { sender_connection(sender_type* s, int dst, parcelset::parcelport* pp) : state_(initialized) , sender_(s) - //, tag_(-1) , dst_(dst) , thd_id_(-1) , chunks_idx_(0) @@ -102,9 +101,7 @@ namespace hpx::parcelset::policies::openshmem { hpx::chrono::high_resolution_clock::now(); #endif chunks_idx_ = 0; - //tag_ = acquire_tag(sender_); - //header_ = header(buffer_, tag_, thd_id_); - header_ = header(buffer_, thd_id_); + header_ = header(buffer_); header_.assert_valid(); state_ = initialized; @@ -154,9 +151,7 @@ namespace hpx::parcelset::policies::openshmem { { { HPX_ASSERT(state_ == initialized); - - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (dst_*nthreads_)+thd_id_; + const auto idx = dst_; std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); @@ -188,8 +183,7 @@ namespace hpx::parcelset::policies::openshmem { if (!chunks.empty()) { - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (dst_*nthreads_)+thd_id_; + const auto idx = dst_; std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); @@ -216,8 +210,7 @@ namespace hpx::parcelset::policies::openshmem { if (!header_.piggy_back()) { - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (dst_*nthreads_)+thd_id_; + const auto idx = dst_; std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); @@ -237,8 +230,7 @@ namespace hpx::parcelset::policies::openshmem { { HPX_ASSERT(state_ == sent_data); - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (dst_*nthreads_)+thd_id_; + const auto idx = dst_; std::lock_guard l( *(hpx::util::openshmem_environment::segments[idx].mut) @@ -299,8 +291,7 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() { - const auto nthreads_ = hpx::util::openshmem_environment::nthreads_; - const auto idx = (dst_*nthreads_)+thd_id_; + const auto idx = dst_; const bool l = hpx::util::openshmem_environment::segments[idx].mut->try_lock(); return l; From aa5040f6b72f935d26892f7cec3c9acd77d61b9f Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Wed, 24 Apr 2024 10:51:26 -0400 Subject: [PATCH 132/154] fixing lock issue --- .../hpx/openshmem_base/openshmem_environment.hpp | 4 ++-- libs/core/openshmem_base/src/openshmem_environment.cpp | 9 +++++---- .../include/hpx/parcelport_openshmem/receiver.hpp | 2 +- .../hpx/parcelport_openshmem/receiver_connection.hpp | 8 ++++---- .../hpx/parcelport_openshmem/sender_connection.hpp | 10 +++++----- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index 43ef3e03360a..b41b4f9f174f 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -31,7 +31,7 @@ namespace hpx { namespace util { std::uint8_t* end_addr; std::uint8_t * rcv; std::uint8_t * xmt; - hpx::mutex * mut; + std::shared_ptr * mut; }; struct HPX_CORE_EXPORT openshmem_environment @@ -97,7 +97,7 @@ namespace hpx { namespace util { static hpx::mutex dshm_mut; static int init_val_; static std::size_t nthreads_; - static hpx::mutex* segment_mutex; + static std::vector> segment_mutex; static openshmem_seginfo_t* segments; static std::uint8_t* shmem_buffer; }; diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index edb8320f4903..7b9a82ef651e 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -115,7 +115,7 @@ namespace hpx::util { int openshmem_environment::provided_threading_flag_ = 0; int openshmem_environment::is_initialized_ = -1; int openshmem_environment::init_val_ = 0; - hpx::mutex* openshmem_environment::segment_mutex = nullptr; + std::vector> openshmem_environment::segment_mutex{}; openshmem_seginfo_t* openshmem_environment::segments = nullptr; std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; std::size_t openshmem_environment::nthreads_ = 0; @@ -182,7 +182,10 @@ namespace hpx::util { // allocate a mutex per-page // - openshmem_environment::segment_mutex = new hpx::mutex[page_count]; + openshmem_environment::segment_mutex.reserve(page_count); + for(std::size_t i = 0; i < page_count; ++i) { + openshmem_environment::segment_mutex.emplace_back(std::make_shared()); + } // symmetric allocation for number of pages total + number of signals // @@ -413,10 +416,8 @@ namespace hpx::util { shmem_finalize(); delete segments; - delete segment_mutex; shmem_free(hpx::util::openshmem_environment::shmem_buffer); segments = nullptr; - segment_mutex = nullptr; shmem_buffer = nullptr; } } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index d420aae0961c..08c0726df12b 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -135,7 +135,7 @@ namespace hpx::parcelset::policies::openshmem { ); { - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::get( reinterpret_cast( diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index 82d7496e34c8..6a4fbf2d5fd3 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -122,7 +122,7 @@ namespace hpx::parcelset::policies::openshmem { { buffer_.chunks_.resize(num_zero_copy_chunks); { - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + std::lock_guard l(*(*hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -163,7 +163,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = self_; - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -191,7 +191,7 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = self_; - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); while (chunks_idx_ < buffer_.chunks_.size()) { @@ -238,7 +238,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = self_; - const bool l = hpx::util::openshmem_environment::segments[idx].mut->try_lock(); + const bool l = (*hpx::util::openshmem_environment::segments[idx].mut)->try_lock(); return l; } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 2be0b13f3bfb..039fad197a54 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -153,7 +153,7 @@ namespace hpx::parcelset::policies::openshmem { HPX_ASSERT(state_ == initialized); const auto idx = dst_; - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment @@ -185,7 +185,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, @@ -212,7 +212,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - std::lock_guard l(*(hpx::util::openshmem_environment::segments[idx].mut)); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, @@ -233,7 +233,7 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; std::lock_guard l( - *(hpx::util::openshmem_environment::segments[idx].mut) + *(*(hpx::util::openshmem_environment::segments[idx].mut)) ); while (chunks_idx_ < buffer_.chunks_.size()) @@ -293,7 +293,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - const bool l = hpx::util::openshmem_environment::segments[idx].mut->try_lock(); + const bool l = (*(hpx::util::openshmem_environment::segments[idx].mut))->try_lock(); return l; return true; From db9a62318cc5bb9e1e01c7315c09786817edae0d Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Wed, 24 Apr 2024 15:53:59 -0400 Subject: [PATCH 133/154] switching to spinlocks: --- .../include/hpx/openshmem_base/openshmem_environment.hpp | 4 ++-- libs/core/openshmem_base/src/openshmem_environment.cpp | 4 ++-- .../include/hpx/parcelport_openshmem/receiver.hpp | 8 ++++---- .../hpx/parcelport_openshmem/receiver_connection.hpp | 6 +++--- .../include/hpx/parcelport_openshmem/sender.hpp | 4 +--- .../hpx/parcelport_openshmem/sender_connection.hpp | 8 ++++---- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index b41b4f9f174f..ac82893c40b3 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -31,7 +31,7 @@ namespace hpx { namespace util { std::uint8_t* end_addr; std::uint8_t * rcv; std::uint8_t * xmt; - std::shared_ptr * mut; + std::shared_ptr * mut; }; struct HPX_CORE_EXPORT openshmem_environment @@ -97,7 +97,7 @@ namespace hpx { namespace util { static hpx::mutex dshm_mut; static int init_val_; static std::size_t nthreads_; - static std::vector> segment_mutex; + static std::vector> segment_mutex; static openshmem_seginfo_t* segments; static std::uint8_t* shmem_buffer; }; diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 7b9a82ef651e..5efb54ddf7cc 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -115,7 +115,7 @@ namespace hpx::util { int openshmem_environment::provided_threading_flag_ = 0; int openshmem_environment::is_initialized_ = -1; int openshmem_environment::init_val_ = 0; - std::vector> openshmem_environment::segment_mutex{}; + std::vector> openshmem_environment::segment_mutex{}; openshmem_seginfo_t* openshmem_environment::segments = nullptr; std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; std::size_t openshmem_environment::nthreads_ = 0; @@ -184,7 +184,7 @@ namespace hpx::util { // openshmem_environment::segment_mutex.reserve(page_count); for(std::size_t i = 0; i < page_count; ++i) { - openshmem_environment::segment_mutex.emplace_back(std::make_shared()); + openshmem_environment::segment_mutex.emplace_back(std::make_shared()); } // symmetric allocation for number of pages total + number of signals diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 08c0726df12b..07d0e0eba77a 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -57,7 +57,7 @@ namespace hpx::parcelset::policies::openshmem { // already accepted ones. if (!connection) { - std::unique_lock l(connections_mtx_, std::try_to_lock); + std::unique_lock l(connections_mtx_, std::try_to_lock); if (l.owns_lock() && !connections_.empty()) { connection = HPX_MOVE(connections_.front()); @@ -78,14 +78,14 @@ namespace hpx::parcelset::policies::openshmem { { if (!connection->receive()) { - std::unique_lock l(connections_mtx_); + std::unique_lock l(connections_mtx_); connections_.push_back(HPX_MOVE(connection)); } } connection_ptr accept() noexcept { - std::unique_lock l(headers_mtx_, std::try_to_lock); + std::unique_lock l(headers_mtx_, std::try_to_lock); if (l.owns_lock()) { return accept_locked(l); @@ -135,7 +135,7 @@ namespace hpx::parcelset::policies::openshmem { ); { - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::get( reinterpret_cast( diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index 6a4fbf2d5fd3..135ef4363441 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -122,7 +122,7 @@ namespace hpx::parcelset::policies::openshmem { { buffer_.chunks_.resize(num_zero_copy_chunks); { - std::lock_guard l(*(*hpx::util::openshmem_environment::segments[idx].mut)); + std::lock_guard l(*(*hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -163,7 +163,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = self_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -191,7 +191,7 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = self_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); while (chunks_idx_ < buffer_.chunks_.size()) { diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp index daa16267ea99..2850846aa9a9 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -16,7 +16,6 @@ #include #include -#include #include #include @@ -37,7 +36,7 @@ namespace hpx::parcelset::policies::openshmem { // different versions of clang-format disagree // clang-format off sender() noexcept - : next_free_tag_(-1) + : connections_mtx_(), connections_(), next_free_tag_mtx_() { } // clang-format on @@ -107,7 +106,6 @@ namespace hpx::parcelset::policies::openshmem { connection_list connections_; hpx::spinlock next_free_tag_mtx_; - int next_free_tag_; }; } // namespace hpx::parcelset::policies::openshmem diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 039fad197a54..1c9b0abe0207 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -153,7 +153,7 @@ namespace hpx::parcelset::policies::openshmem { HPX_ASSERT(state_ == initialized); const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment @@ -185,7 +185,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, @@ -212,7 +212,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, @@ -232,7 +232,7 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; - std::lock_guard l( + std::lock_guard l( *(*(hpx::util::openshmem_environment::segments[idx].mut)) ); From 0f6818eaa2d458e2a39b8f88fa5d2c93c6513841 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Wed, 24 Apr 2024 21:49:36 -0400 Subject: [PATCH 134/154] updated data types --- .../openshmem_base/openshmem_environment.hpp | 7 ++--- .../src/openshmem_environment.cpp | 28 ++++--------------- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index ac82893c40b3..4242b6486a14 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -32,6 +32,8 @@ namespace hpx { namespace util { std::uint8_t * rcv; std::uint8_t * xmt; std::shared_ptr * mut; + + openshmem_seginfo_t() : beg_addr(nullptr), end_addr(nullptr), rcv(nullptr), xmt(nullptr), mut(nullptr) {} }; struct HPX_CORE_EXPORT openshmem_environment @@ -85,7 +87,6 @@ namespace hpx { namespace util { typedef hpx::spinlock mutex_type; public: - static hpx::spinlock pollingLock; static hpx::mutex mtx_; static bool enabled_; @@ -94,11 +95,9 @@ namespace hpx { namespace util { static int is_initialized_; - static hpx::mutex dshm_mut; static int init_val_; - static std::size_t nthreads_; static std::vector> segment_mutex; - static openshmem_seginfo_t* segments; + static std::vector segments; static std::uint8_t* shmem_buffer; }; }} // namespace hpx::util diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 5efb54ddf7cc..1650a6248934 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -107,8 +107,6 @@ namespace hpx::util { namespace hpx::util { - hpx::spinlock openshmem_environment::pollingLock{}; - hpx::mutex openshmem_environment::dshm_mut{}; hpx::mutex openshmem_environment::mtx_{}; bool openshmem_environment::enabled_ = false; bool openshmem_environment::has_called_init_ = false; @@ -116,9 +114,8 @@ namespace hpx::util { int openshmem_environment::is_initialized_ = -1; int openshmem_environment::init_val_ = 0; std::vector> openshmem_environment::segment_mutex{}; - openshmem_seginfo_t* openshmem_environment::segments = nullptr; + std::vector openshmem_environment::segments{}; std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; - std::size_t openshmem_environment::nthreads_ = 0; /////////////////////////////////////////////////////////////////////////// int openshmem_environment::init([[maybe_unused]] int* argc, @@ -156,16 +153,6 @@ namespace hpx::util { "OPENSHMEM not ready error"); } - // create a number of segments equal to the number of hardware - // threads per machine (locality) - // - //segments.resize(hpx::threads::hardware_concurrency() * size()); - // - // get thread count - // - openshmem_environment::nthreads_ = - hpx::get_worker_thread_num(); - // get system page size // const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); @@ -176,10 +163,6 @@ namespace hpx::util { // const std::size_t page_count = size(); - // allocate the page cache - // - openshmem_environment::segments = new openshmem_seginfo_t[page_count]; - // allocate a mutex per-page // openshmem_environment::segment_mutex.reserve(page_count); @@ -203,6 +186,10 @@ namespace hpx::util { // const std::size_t beg_signal = (sys_pgsz*page_count); + // allocate the page cache + // + openshmem_environment::segments.resize(page_count); + // initialize the page cache // for (std::size_t i = 0; i < page_count; ++i) @@ -413,12 +400,9 @@ namespace hpx::util { { if (enabled() && has_called_init()) { - shmem_finalize(); - - delete segments; shmem_free(hpx::util::openshmem_environment::shmem_buffer); - segments = nullptr; shmem_buffer = nullptr; + shmem_finalize(); } } From 5ffb5fd49b2e1ebcf78e655c883cd19e1f68ba9d Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Thu, 25 Apr 2024 09:54:33 -0400 Subject: [PATCH 135/154] rm'd locks --- .../include/hpx/parcelport_openshmem/receiver.hpp | 2 +- .../hpx/parcelport_openshmem/receiver_connection.hpp | 6 +++--- .../hpx/parcelport_openshmem/sender_connection.hpp | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 07d0e0eba77a..d8b358f19057 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -135,7 +135,7 @@ namespace hpx::parcelset::policies::openshmem { ); { - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::get( reinterpret_cast( diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index 135ef4363441..ec916d0bb781 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -122,7 +122,7 @@ namespace hpx::parcelset::policies::openshmem { { buffer_.chunks_.resize(num_zero_copy_chunks); { - std::lock_guard l(*(*hpx::util::openshmem_environment::segments[idx].mut)); + //std::lock_guard l(*(*hpx::util::openshmem_environment::segments[idx].mut)); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -163,7 +163,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = self_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); @@ -191,7 +191,7 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = self_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); while (chunks_idx_ < buffer_.chunks_.size()) { diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 1c9b0abe0207..c283957e1f5b 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -153,7 +153,7 @@ namespace hpx::parcelset::policies::openshmem { HPX_ASSERT(state_ == initialized); const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment @@ -185,7 +185,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, @@ -212,7 +212,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, @@ -232,9 +232,9 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; - std::lock_guard l( - *(*(hpx::util::openshmem_environment::segments[idx].mut)) - ); + //std::lock_guard l( + // *(*(hpx::util::openshmem_environment::segments[idx].mut)) + //); while (chunks_idx_ < buffer_.chunks_.size()) { From 6a385daacf7fe210bd01480f34af3f977ad244d9 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Thu, 25 Apr 2024 13:16:32 -0400 Subject: [PATCH 136/154] working to remove locks --- .../hpx/parcelport_openshmem/receiver_connection.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index ec916d0bb781..ceaadc78fc48 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -236,10 +236,10 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() noexcept { - const auto idx = self_; + //const auto idx = self_; - const bool l = (*hpx::util::openshmem_environment::segments[idx].mut)->try_lock(); - return l; + //const bool l = (*hpx::util::openshmem_environment::segments[idx].mut)->try_lock(); + return true; } #if defined(HPX_HAVE_PARCELPORT_COUNTERS) From cbcd9d83c08fe6d755fe86f9c4747a9a6e67eb5a Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Thu, 25 Apr 2024 14:36:18 -0400 Subject: [PATCH 137/154] adding improvement to locality support --- .../include/hpx/parcelport_openshmem/locality.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp index d0fe8b8eecf4..4366dfc5a6fb 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp @@ -24,6 +24,9 @@ namespace hpx::parcelset::policies::openshmem { constexpr locality() noexcept : rank_(-1) { + if(openshmem_environment::enabled()) { + rank_ = openshmem_environment::rank(); + } } explicit constexpr locality(std::int32_t rank) noexcept From a0be4bc32791cda94bf487b2bcdf594be48732b8 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 26 Apr 2024 09:42:37 -0400 Subject: [PATCH 138/154] small fixes --- .../openshmem_base/src/openshmem_environment.cpp | 12 +++++++++--- .../include/hpx/parcelport_openshmem/locality.hpp | 3 --- .../include/hpx/parcelport_openshmem/receiver.hpp | 14 +++++++++----- .../hpx/parcelport_openshmem/sender_connection.hpp | 12 ++++++------ 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 1650a6248934..21ecb94a83e6 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -279,6 +279,8 @@ namespace hpx::util { std::string openshmem_environment::get_processor_name() { + scoped_lock l; + char name[1024 + 1] = {'\0'}; const std::string rnkstr = std::to_string(rank()); const int len = rnkstr.size(); @@ -400,8 +402,8 @@ namespace hpx::util { { if (enabled() && has_called_init()) { + scoped_lock l; shmem_free(hpx::util::openshmem_environment::shmem_buffer); - shmem_buffer = nullptr; shmem_finalize(); } } @@ -424,16 +426,20 @@ namespace hpx::util { int openshmem_environment::size() { int res(-1); - if (enabled()) + if (enabled()) { + scoped_lock l; res = static_cast(shmem_n_pes()); + } return res; } int openshmem_environment::rank() { int res(-1); - if (enabled()) + if (enabled()) { + scoped_lock l; res = static_cast(shmem_my_pe()); + } return res; } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp index 4366dfc5a6fb..d0fe8b8eecf4 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp @@ -24,9 +24,6 @@ namespace hpx::parcelset::policies::openshmem { constexpr locality() noexcept : rank_(-1) { - if(openshmem_environment::enabled()) { - rank_ = openshmem_environment::rank(); - } } explicit constexpr locality(std::int32_t rank) noexcept diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index d8b358f19057..0230b5c9f37d 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -96,20 +96,25 @@ namespace hpx::parcelset::policies::openshmem { template connection_ptr accept_locked(Lock& header_lock) noexcept { - connection_ptr res; + HPX_ASSERT_OWNS_LOCK(header_lock); + util::openshmem_environment::scoped_try_lock l; + connection_ptr res; if (l.locked) { header h = new_header(); + + const auto rank = + hpx::util::openshmem_environment::rank(); + l.unlock(); header_lock.unlock(); // remote localities 'put' into the openshmem shared // memory segment on this machine // - res.reset(new connection_type( - hpx::util::openshmem_environment::rank(), h, pp_)); + res.reset(new connection_type(rank, h, pp_)); return res; } return res; @@ -124,7 +129,6 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::size(); const std::size_t beg_rcv_signal = (sys_pgsz*page_count); - // waiting for `sender_connection::send_header` invocation if (rcv_header_.data() == 0) { @@ -135,7 +139,7 @@ namespace hpx::parcelset::policies::openshmem { ); { - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::get( reinterpret_cast( diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index c283957e1f5b..1c9b0abe0207 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -153,7 +153,7 @@ namespace hpx::parcelset::policies::openshmem { HPX_ASSERT(state_ == initialized); const auto idx = dst_; - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment @@ -185,7 +185,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, @@ -212,7 +212,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, @@ -232,9 +232,9 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; - //std::lock_guard l( - // *(*(hpx::util::openshmem_environment::segments[idx].mut)) - //); + std::lock_guard l( + *(*(hpx::util::openshmem_environment::segments[idx].mut)) + ); while (chunks_idx_ < buffer_.chunks_.size()) { From caaad175227f1dd81e32f6fc2f58567435fefdc3 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 26 Apr 2024 13:01:18 -0400 Subject: [PATCH 139/154] additional fixes --- .../openshmem_base/openshmem_environment.hpp | 4 +-- .../src/openshmem_environment.cpp | 5 +-- .../hpx/parcelport_openshmem/receiver.hpp | 19 +++++----- .../sender_connection.hpp | 35 +++++++++++-------- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index 4242b6486a14..8864bd96b32b 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -31,7 +31,7 @@ namespace hpx { namespace util { std::uint8_t* end_addr; std::uint8_t * rcv; std::uint8_t * xmt; - std::shared_ptr * mut; +// std::shared_ptr * mut; openshmem_seginfo_t() : beg_addr(nullptr), end_addr(nullptr), rcv(nullptr), xmt(nullptr), mut(nullptr) {} }; @@ -96,7 +96,7 @@ namespace hpx { namespace util { static int is_initialized_; static int init_val_; - static std::vector> segment_mutex; + //static std::vector> segment_mutex; static std::vector segments; static std::uint8_t* shmem_buffer; }; diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 21ecb94a83e6..646e15c4dcc1 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -165,11 +165,12 @@ namespace hpx::util { // allocate a mutex per-page // +/* openshmem_environment::segment_mutex.reserve(page_count); for(std::size_t i = 0; i < page_count; ++i) { openshmem_environment::segment_mutex.emplace_back(std::make_shared()); } - +*/ // symmetric allocation for number of pages total + number of signals // // (allocate page_size * number of PEs * number of threads) + (number of PEs * number of threads * 2 [for signaling]) @@ -207,7 +208,7 @@ namespace hpx::util { // segments[i].xmt = hpx::util::openshmem_environment::shmem_buffer + beg_signal + page_count + i; - segments[i].mut = &(openshmem_environment::segment_mutex[i]); + //segments[i].mut = &(openshmem_environment::segment_mutex[i]); } shmem_barrier_all(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 0230b5c9f37d..c072faff61b5 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -131,15 +131,16 @@ namespace hpx::parcelset::policies::openshmem { // waiting for `sender_connection::send_header` invocation if (rcv_header_.data() == 0) { + { - const auto idx = hpx::util::openshmem_environment::wait_until_any( - 1, - hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal, - page_count - ); + util::openshmem_environment::scoped_lock l; + const auto idx = hpx::util::openshmem_environment::wait_until_any( + 1, + hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal, + page_count + ); - { - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::get( reinterpret_cast( @@ -148,9 +149,9 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::segments[idx].beg_addr, sizeof(rcv_header_) ); - } - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + } } header h = rcv_header_; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 1c9b0abe0207..e0d75524f133 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -153,7 +153,8 @@ namespace hpx::parcelset::policies::openshmem { HPX_ASSERT(state_ == initialized); const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + util::openshmem_environment::scoped_lock l; + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment @@ -185,7 +186,8 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + util::openshmem_environment::scoped_lock l; + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, @@ -212,7 +214,8 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + util::openshmem_environment::scoped_lock l; + //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, @@ -232,10 +235,11 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; +/* std::lock_guard l( *(*(hpx::util::openshmem_environment::segments[idx].mut)) ); - +*/ while (chunks_idx_ < buffer_.chunks_.size()) { serialization::serialization_chunk& c = @@ -246,16 +250,19 @@ namespace hpx::parcelset::policies::openshmem { return false; } - hpx::util::openshmem_environment::put_signal( - reinterpret_cast(c.data_.cpos_), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, - static_cast(c.size_), - hpx::util::openshmem_environment::segments[idx].rcv - ); - - hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[idx].xmt); - (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + { + util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(c.data_.cpos_), dst_, + hpx::util::openshmem_environment::segments[idx].beg_addr, + static_cast(c.size_), + hpx::util::openshmem_environment::segments[idx].rcv + ); + + hpx::util::openshmem_environment::wait_until( + 1, hpx::util::openshmem_environment::segments[idx].xmt); + (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + } } ++chunks_idx_; From a03504d866e22618991f0799c2302b4596c1ed9d Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Fri, 26 Apr 2024 13:18:37 -0400 Subject: [PATCH 140/154] additional fixes --- .../openshmem_base/openshmem_environment.hpp | 27 ++++++++++++++++++- .../src/openshmem_environment.cpp | 2 +- .../sender_connection.hpp | 7 ++--- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index 8864bd96b32b..4cde8354f310 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -33,7 +33,7 @@ namespace hpx { namespace util { std::uint8_t * xmt; // std::shared_ptr * mut; - openshmem_seginfo_t() : beg_addr(nullptr), end_addr(nullptr), rcv(nullptr), xmt(nullptr), mut(nullptr) {} + openshmem_seginfo_t() : beg_addr(nullptr), end_addr(nullptr), rcv(nullptr), xmt(nullptr) {} //, mut(nullptr) {} }; struct HPX_CORE_EXPORT openshmem_environment @@ -63,23 +63,48 @@ namespace hpx { namespace util { static void get(std::uint8_t* addr, const int rank, const std::uint8_t* raddr, const std::size_t size); + static void quiet() { shmem_quiet(); } + static void global_barrier(); struct HPX_CORE_EXPORT scoped_lock { scoped_lock(); + scoped_lock(scoped_lock const&) = delete; + scoped_lock(scoped_lock&&) = delete; + scoped_lock& operator=(scoped_lock const&) = delete; + scoped_lock& operator=(scoped_lock&&) = delete; + ~scoped_lock(); + + constexpr bool owns_lock() const noexcept + { + return locked; + } + void unlock(); + bool locked; }; struct HPX_CORE_EXPORT scoped_try_lock { scoped_try_lock(); + scoped_try_lock(scoped_try_lock const&) = delete; + scoped_try_lock(scoped_try_lock&&) = delete; + scoped_try_lock& operator=(scoped_try_lock const&) = delete; + scoped_try_lock& operator=(scoped_try_lock&&) = delete; + ~scoped_try_lock(); + + constexpr bool owns_lock() const noexcept + { + return locked; + } + void unlock(); bool locked; }; diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 646e15c4dcc1..f7d8c376857a 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -113,7 +113,7 @@ namespace hpx::util { int openshmem_environment::provided_threading_flag_ = 0; int openshmem_environment::is_initialized_ = -1; int openshmem_environment::init_val_ = 0; - std::vector> openshmem_environment::segment_mutex{}; + //std::vector> openshmem_environment::segment_mutex{}; std::vector openshmem_environment::segments{}; std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index e0d75524f133..6517e207c52b 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -252,6 +252,7 @@ namespace hpx::parcelset::policies::openshmem { { util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( reinterpret_cast(c.data_.cpos_), dst_, hpx::util::openshmem_environment::segments[idx].beg_addr, @@ -298,10 +299,10 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() { - const auto idx = dst_; + util::openshmem_environment::scoped_lock l; + if(!l.locked) { return false; } - const bool l = (*(hpx::util::openshmem_environment::segments[idx].mut))->try_lock(); - return l; + hpx::util::openshmem_environment::quiet(); return true; } From 7d472dabfcc07ef85a6e566ec6ed9122a70bfcbb Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Sun, 28 Apr 2024 22:26:14 -0400 Subject: [PATCH 141/154] rm'd double-lock --- .../include/hpx/parcelport_openshmem/receiver.hpp | 2 +- .../include/hpx/parcelport_openshmem/sender_connection.hpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index c072faff61b5..f2a4129ac145 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -133,7 +133,7 @@ namespace hpx::parcelset::policies::openshmem { if (rcv_header_.data() == 0) { { - util::openshmem_environment::scoped_lock l; + //util::openshmem_environment::scoped_lock l; const auto idx = hpx::util::openshmem_environment::wait_until_any( 1, hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal, diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 6517e207c52b..ecb2773d7686 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -37,7 +37,6 @@ namespace hpx::parcelset::policies::openshmem { struct sender; struct sender_connection; - //int acquire_tag(sender*) noexcept; void add_connection(sender*, std::shared_ptr const&); struct sender_connection @@ -309,7 +308,6 @@ namespace hpx::parcelset::policies::openshmem { connection_state state_; sender_type* sender_; - //int tag_; int dst_; int thd_id_; From 59f8f14bd202bf9ed49bc8b1b1f754e9016c2783 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 29 Apr 2024 11:25:01 -0400 Subject: [PATCH 142/154] working on locks --- .../src/openshmem_environment.cpp | 2 +- .../hpx/parcelport_openshmem/receiver.hpp | 84 +++++++++---------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index f7d8c376857a..b9eae7add7c7 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -353,7 +353,7 @@ namespace hpx::util { shmem_int_wait_until(tmp.iaddr, SHMEM_CMP_EQ, static_cast(value)); } - std::size_t wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { + std::size_t openshmem_environment::wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { #if defined(SHMEM_MAJOR_VERSION) && defined(SHMEM_MINOR_VERSION) && \ defined(SHMEM_VENDOR_STRING) && defined(SHMEM_MAX_NAME_LEN) && \ diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index f2a4129ac145..3e26c3656758 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -38,14 +38,15 @@ namespace hpx::parcelset::policies::openshmem { using connection_list = std::deque; explicit constexpr receiver(Parcelport& pp) noexcept - : pp_(pp) + : pp_(pp), rcv_header_(hpx::util::openshmem_environment::size()), headers_mtx_{}, handles_header_mtx_{}, + handles_header_{}, connections_mtx_{}, connections_{} { } void run() noexcept { - //util::openshmem_environment::scoped_lock l; - //new_header(-1); + util::openshmem_environment::scoped_lock l; + post_new_header(l); } bool background_work() noexcept @@ -94,34 +95,43 @@ namespace hpx::parcelset::policies::openshmem { } template - connection_ptr accept_locked(Lock& header_lock) noexcept + connection_ptr accept_locked(Lock& header_lock) { HPX_ASSERT_OWNS_LOCK(header_lock); util::openshmem_environment::scoped_try_lock l; - connection_ptr res; + + // Caller failing to hold lock 'header_lock' before calling function +#if defined(HPX_MSVC) +#pragma warning(push) +#pragma warning(disable : 26110) +#endif if (l.locked) { - header h = new_header(); - - const auto rank = - hpx::util::openshmem_environment::rank(); + const auto idx = post_new_header(l); + header h = rcv_header_[idx]; + rcv_header_[idx].reset(); l.unlock(); header_lock.unlock(); - // remote localities 'put' into the openshmem shared - // memory segment on this machine - // - res.reset(new connection_type(rank, h, pp_)); - return res; + return std::make_shared( + idx, HPX_MOVE(h), pp_); } - return res; + +#if defined(HPX_MSVC) +#pragma warning(pop) +#endif + + return {}; } - header new_header() noexcept + template + std::size_t post_new_header([[maybe_unused]] Lock& l) noexcept { + HPX_ASSERT_OWNS_LOCK(l); + const auto self_ = hpx::util::openshmem_environment::rank(); const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); @@ -129,40 +139,30 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::size(); const std::size_t beg_rcv_signal = (sys_pgsz*page_count); - // waiting for `sender_connection::send_header` invocation - if (rcv_header_.data() == 0) { - { - - //util::openshmem_environment::scoped_lock l; - const auto idx = hpx::util::openshmem_environment::wait_until_any( - 1, - hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal, - page_count - ); + const auto idx = hpx::util::openshmem_environment::wait_until_any( + 1, + hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal, + page_count + ); - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + hpx::util::openshmem_environment::get( + reinterpret_cast( + rcv_header_[idx].data()), + self_, + hpx::util::openshmem_environment::segments[idx].beg_addr, + sizeof(header) + ); - hpx::util::openshmem_environment::get( - reinterpret_cast( - rcv_header_.data()), - self_, - hpx::util::openshmem_environment::segments[idx].beg_addr, - sizeof(rcv_header_) - ); - - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - } - } + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - header h = rcv_header_; - rcv_header_.reset(); - return h; + HPX_ASSERT_LOCKED(l, idx < 0); + return idx; } Parcelport& pp_; + std::vector
rcv_header_; hpx::spinlock headers_mtx_; - header rcv_header_; hpx::spinlock handles_header_mtx_; handles_header_type handles_header_; From c16d93dd1ac37f22dbf3f3309044ce261bc86825 Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Mon, 29 Apr 2024 21:12:38 -0400 Subject: [PATCH 143/154] several changes to improve locking --- .../openshmem_base/openshmem_environment.hpp | 2 +- .../src/openshmem_environment.cpp | 24 +++++------------ .../receiver_connection.hpp | 26 +++++++------------ .../hpx/parcelport_openshmem/sender.hpp | 3 +-- .../sender_connection.hpp | 10 +------ 5 files changed, 20 insertions(+), 45 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index 4cde8354f310..4193048f38ee 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -121,9 +121,9 @@ namespace hpx { namespace util { static int is_initialized_; static int init_val_; - //static std::vector> segment_mutex; static std::vector segments; static std::uint8_t* shmem_buffer; + static std::size_t this_rank; }; }} // namespace hpx::util diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index b9eae7add7c7..b87a252b79a8 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -113,6 +113,7 @@ namespace hpx::util { int openshmem_environment::provided_threading_flag_ = 0; int openshmem_environment::is_initialized_ = -1; int openshmem_environment::init_val_ = 0; + std::size_t openshmem_environment::this_rank = -1; //std::vector> openshmem_environment::segment_mutex{}; std::vector openshmem_environment::segments{}; std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; @@ -163,14 +164,6 @@ namespace hpx::util { // const std::size_t page_count = size(); - // allocate a mutex per-page - // -/* - openshmem_environment::segment_mutex.reserve(page_count); - for(std::size_t i = 0; i < page_count; ++i) { - openshmem_environment::segment_mutex.emplace_back(std::make_shared()); - } -*/ // symmetric allocation for number of pages total + number of signals // // (allocate page_size * number of PEs * number of threads) + (number of PEs * number of threads * 2 [for signaling]) @@ -224,7 +217,11 @@ namespace hpx::util { if (enabled_) return; // don't call twice - int this_rank = -1; + if (enabled()) { + scoped_lock l; + this_rank = static_cast(shmem_my_pe()); + } + has_called_init_ = false; // We assume to use the OpenSHMEM parcelport if it is not explicitly disabled @@ -256,8 +253,6 @@ namespace hpx::util { rtcfg.add_entry("hpx.parcel.openshmem.multithreaded", "0"); } - this_rank = rank(); - #if defined(HPX_HAVE_NETWORKING) if (this_rank == 0) { @@ -436,12 +431,7 @@ namespace hpx::util { int openshmem_environment::rank() { - int res(-1); - if (enabled()) { - scoped_lock l; - res = static_cast(shmem_my_pe()); - } - return res; + return this_rank; } openshmem_environment::scoped_lock::scoped_lock() diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index ceaadc78fc48..a54b393289fe 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -47,7 +47,6 @@ namespace hpx::parcelset::policies::openshmem { receiver_connection(int src, header h, Parcelport& pp) noexcept : state_(initialized) , src_(src) - , self_(-1) , header_(h) , request_ptr_(false) , num_bytes(0) @@ -58,7 +57,6 @@ namespace hpx::parcelset::policies::openshmem { { header_.assert_valid(); - self_ = hpx::util::openshmem_environment::rank(); num_bytes = header_.numbytes(); #if defined(HPX_HAVE_PARCELPORT_COUNTERS) @@ -108,7 +106,7 @@ namespace hpx::parcelset::policies::openshmem { bool receive_transmission_chunks() { - const auto idx = self_; + const auto idx = hpx::util::openshmem_environment::rank(); // determine the size of the chunk buffer std::size_t num_zero_copy_chunks = static_cast( @@ -122,15 +120,13 @@ namespace hpx::parcelset::policies::openshmem { { buffer_.chunks_.resize(num_zero_copy_chunks); { - //std::lock_guard l(*(*hpx::util::openshmem_environment::segments[idx].mut)); - hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); hpx::util::openshmem_environment::get( reinterpret_cast( buffer_.transmission_chunks_.data()), - self_, + idx, hpx::util::openshmem_environment::segments[idx].beg_addr, static_cast(buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type))); @@ -161,16 +157,14 @@ namespace hpx::parcelset::policies::openshmem { } else { - const auto idx = self_; - - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + const auto idx = hpx::util::openshmem_environment::rank(); hpx::util::openshmem_environment::wait_until( 1, hpx::util::openshmem_environment::segments[idx].rcv); hpx::util::openshmem_environment::get( reinterpret_cast(buffer_.data_.data()), - self_, + idx, hpx::util::openshmem_environment::segments[idx].beg_addr, buffer_.data_.size()); @@ -189,9 +183,7 @@ namespace hpx::parcelset::policies::openshmem { std::size_t cidx = 0; std::size_t chunk_size = 0; - const auto idx = self_; - - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); + const auto idx = hpx::util::openshmem_environment::rank(); while (chunks_idx_ < buffer_.chunks_.size()) { @@ -211,7 +203,7 @@ namespace hpx::parcelset::policies::openshmem { 1, hpx::util::openshmem_environment::segments[idx].rcv); hpx::util::openshmem_environment::get( - reinterpret_cast(c.data()), self_, + reinterpret_cast(c.data()), idx, hpx::util::openshmem_environment::segments[idx].beg_addr, c.size()); @@ -236,9 +228,11 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() noexcept { - //const auto idx = self_; + util::openshmem_environment::scoped_try_lock const l; + if(!l.locked) { return false; } + + hpx::util::openshmem_environment::quiet(); - //const bool l = (*hpx::util::openshmem_environment::segments[idx].mut)->try_lock(); return true; } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp index 2850846aa9a9..a5ec2d0f9502 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -36,7 +36,7 @@ namespace hpx::parcelset::policies::openshmem { // different versions of clang-format disagree // clang-format off sender() noexcept - : connections_mtx_(), connections_(), next_free_tag_mtx_() + : connections_mtx_(), connections_() { } // clang-format on @@ -105,7 +105,6 @@ namespace hpx::parcelset::policies::openshmem { hpx::spinlock connections_mtx_; connection_list connections_; - hpx::spinlock next_free_tag_mtx_; }; } // namespace hpx::parcelset::policies::openshmem diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index ecb2773d7686..e0c923e2811a 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -153,7 +153,6 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; util::openshmem_environment::scoped_lock l; - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment @@ -186,7 +185,6 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; util::openshmem_environment::scoped_lock l; - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(chunks.data()), dst_, @@ -214,7 +212,6 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; util::openshmem_environment::scoped_lock l; - //std::lock_guard l(*(*(hpx::util::openshmem_environment::segments[idx].mut))); hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, @@ -234,11 +231,6 @@ namespace hpx::parcelset::policies::openshmem { const auto idx = dst_; -/* - std::lock_guard l( - *(*(hpx::util::openshmem_environment::segments[idx].mut)) - ); -*/ while (chunks_idx_ < buffer_.chunks_.size()) { serialization::serialization_chunk& c = @@ -298,7 +290,7 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() { - util::openshmem_environment::scoped_lock l; + util::openshmem_environment::scoped_try_lock const l; if(!l.locked) { return false; } hpx::util::openshmem_environment::quiet(); From da7219a110d255e5a983df0ee556f66614362c9b Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 07:55:55 -0500 Subject: [PATCH 144/154] compile fixes --- cmake/HPX_SetupOpenSHMEM.cmake | 21 +++++++++++++------ .../src/openshmem_environment.cpp | 3 +-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index ed181dfb28c7..bc1d84aac88a 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -1120,34 +1120,43 @@ macro(hpx_setup_openshmem) message(STATUS "OPENSHMEM_LDFLAGS:\t${OPENSHMEM_LDFLAGS}") message(STATUS "OPENSHMEM_LIBRARY_DIRS:\t${OPENSHMEM_LIBRARY_DIRS}") +separate_arguments(OPENSHMEM_CFLAGS UNIX_COMMAND "${OPENSHMEM_CFLAGS}") +separate_arguments(OPENSHMEM_LDFLAGS UNIX_COMMAND "${OPENSHMEM_LDFLAGS}") +separate_arguments(OPENSHMEM_LIBRARY_DIRS UNIX_COMMAND "${OPENSHMEM_LIBRARY_DIRS}") + set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS - "${OPENSHMEM_CFLAGS}" + ${OPENSHMEM_CFLAGS} ) set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS - "${OPENSHMEM_LDFLAGS}" + ${OPENSHMEM_LDFLAGS} ) set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES - "${OPENSHMEM_LIBRARY_DIRS}" + ${OPENSHMEM_LIBRARY_DIRS} ) else() message(STATUS "OPENSHMEM_CFLAGS:\t${OPENSHMEM_CFLAGS}") message(STATUS "OPENSHMEM_LDFLAGS:\t${OPENSHMEM_LDFLAGS}") message(STATUS "OPENSHMEM_LIBRARY_DIRS:\t${OPENSHMEM_LIBRARY_DIRS}") +separate_arguments(OPENSHMEM_CFLAGS UNIX_COMMAND "${OPENSHMEM_CFLAGS}") +separate_arguments(OPENSHMEM_LDFLAGS UNIX_COMMAND "${OPENSHMEM_LDFLAGS}") +separate_arguments(OPENSHMEM_LIBRARY_DIRS UNIX_COMMAND "${OPENSHMEM_LIBRARY_DIRS}") + + set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS - "${OPENSHMEM_CFLAGS}" + ${OPENSHMEM_CFLAGS} ) set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS - "${OPENSHMEM_LDFLAGS}" + ${OPENSHMEM_LDFLAGS} ) set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES - "${OPENSHMEM_LIBRARY_DIRS}" + ${OPENSHMEM_LIBRARY_DIRS} ) endif() endmacro() diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index b87a252b79a8..273b97572208 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -352,12 +352,11 @@ namespace hpx::util { #if defined(SHMEM_MAJOR_VERSION) && defined(SHMEM_MINOR_VERSION) && \ defined(SHMEM_VENDOR_STRING) && defined(SHMEM_MAX_NAME_LEN) && \ - SHMEM_MAJOR_VERSION == 1 && SHMEM_MINOR_VERSION == 4 && \ SHMEM_MAX_NAME_LEN == 256 int rc = 0; for(std::size_t i = 0; i < count; ++i) { - rc = shmem_test(reinterpret_cast(sigaddr+i), SHMEM_CMP_EQ, value); + rc = shmem_uint_test(reinterpret_cast(sigaddr+i), SHMEM_CMP_EQ, value); if(rc) { return i; } } From d26b61dfbd78816e79df119075e7214ff050227c Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sat, 4 May 2024 22:34:36 -0500 Subject: [PATCH 145/154] updated merge --- libs/core/openshmem_base/src/openshmem_environment.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 273b97572208..fadfaeb429c7 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -163,6 +163,11 @@ namespace hpx::util { // page_count = num_localities * number of threads // const std::size_t page_count = size(); + if(page_count < 1) { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM not ready error"); + } // symmetric allocation for number of pages total + number of signals // @@ -420,7 +425,7 @@ namespace hpx::util { int openshmem_environment::size() { - int res(-1); + int res(0); if (enabled()) { scoped_lock l; res = static_cast(shmem_n_pes()); From ab462c3eeeaa35f99d3aa5bb662f2857ad282951 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Tue, 7 May 2024 21:11:32 -0500 Subject: [PATCH 146/154] several fixes for mpi-openshmem and ucx-openshmem --- cmake/HPX_SetupOpenSHMEM.cmake | 1 - .../src/openshmem_environment.cpp | 86 ++++++++++++++----- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index bc1d84aac88a..3dc82285fb06 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -1145,7 +1145,6 @@ separate_arguments(OPENSHMEM_CFLAGS UNIX_COMMAND "${OPENSHMEM_CFLAGS}") separate_arguments(OPENSHMEM_LDFLAGS UNIX_COMMAND "${OPENSHMEM_LDFLAGS}") separate_arguments(OPENSHMEM_LIBRARY_DIRS UNIX_COMMAND "${OPENSHMEM_LIBRARY_DIRS}") - set_target_properties( PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS ${OPENSHMEM_CFLAGS} diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index fadfaeb429c7..a869b13b547b 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -16,6 +16,8 @@ #include #include +#include + #include #include #include @@ -340,8 +342,8 @@ namespace hpx::util { using arg_type = std::conditional< std::is_same::type, std::tuple>::value, - volatile int *, - int * + volatile unsigned int *, + unsigned int * >::type; union { @@ -350,33 +352,73 @@ namespace hpx::util { } tmp; tmp.uaddr = sigaddr; - shmem_int_wait_until(tmp.iaddr, SHMEM_CMP_EQ, static_cast(value)); + shmem_uint_wait_until(tmp.iaddr, SHMEM_CMP_EQ, static_cast(value)); } - std::size_t openshmem_environment::wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { + constexpr bool vendor_strings_equal(char const * a, char const * b) { + return *a == *b && (*a == '\0' || vendor_strings_equal(a + 1, b + 1)); + } -#if defined(SHMEM_MAJOR_VERSION) && defined(SHMEM_MINOR_VERSION) && \ - defined(SHMEM_VENDOR_STRING) && defined(SHMEM_MAX_NAME_LEN) && \ - SHMEM_MAX_NAME_LEN == 256 + struct ucx {}; + struct mpi {}; + struct err {}; - int rc = 0; - for(std::size_t i = 0; i < count; ++i) { - rc = shmem_uint_test(reinterpret_cast(sigaddr+i), SHMEM_CMP_EQ, value); - if(rc) { return i; } - } + template + struct wait_until_any_wrapper { + std::size_t operator()(unsigned int * addr, const std::size_t count, const std::uint8_t value) { return -1; } + }; + +#if defined(OSHMEM_SHMEMX_H) + template<> + struct wait_until_any_wrapper { + std::size_t operator()(unsigned int * addr, const std::size_t count, const std::uint8_t value) { + int rc = 0; + for(std::size_t i = 0; i < count; ++i) { + rc = shmem_uint_test(addr+i, SHMEM_CMP_EQ, value); + if(rc) { return i; } + } + + return -1; + } + }; +#endif + +#if defined(_SHMEM_H) + template<> + struct wait_until_any_wrapper { + std::size_t operator()(unsigned int * addr, const std::size_t count, const std::uint8_t value) { + const std::size_t sig_idx = +#if defined(_SHMEMX_H) && _SHMEMX_H == 1 + shmemx_uint_wait_until_any +#else + shmem_uint_wait_until_any +#endif + ( + addr, + count, + nullptr, + SHMEM_CMP_EQ, + value + ); + + return sig_idx; + } + }; +#endif - return -1; + std::size_t openshmem_environment::wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { +#if defined(SHMEM_VENDOR_STRING) + using tag = std::conditional< std::integral_constant::value, ucx, + std::conditional< std::integral_constant::value, mpi, err>::type + >::type; + using wait_until_any_type = wait_until_any_wrapper; #else - const std::size_t sig_idx = shmem_wait_until_any( - reinterpret_cast(sigaddr), - count, - nullptr, - SHMEM_CMP_EQ, - value - ); - - return sig_idx; + #define SHMEM_VENDOR_STRING "SHMEM_VENDOR_STRING not defined" + using tag = err; + using wait_until_any_type = wait_until_any_wrapper; #endif + wait_until_any_type t{}; + return t(reinterpret_cast(sigaddr), count, value); } void openshmem_environment::get(std::uint8_t* addr, const int node, From 7b9e2ca71e22fc94e4f8c50f7c83bf4f466e1bcc Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Tue, 7 May 2024 21:57:44 -0500 Subject: [PATCH 147/154] fixed a small issue with state machine calls --- .../include/hpx/parcelport_openshmem/receiver_connection.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index a54b393289fe..aef52a637a28 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -218,7 +218,7 @@ namespace hpx::parcelset::policies::openshmem { state_ = rcvd_chunks; - return true; + return done(); } bool done() noexcept @@ -242,7 +242,6 @@ namespace hpx::parcelset::policies::openshmem { connection_state state_; int src_; - int self_; header header_; buffer_type buffer_; From e9ed800fe91e4f27b94b74de140c250ac478e9e7 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Thu, 9 May 2024 22:43:39 -0500 Subject: [PATCH 148/154] added sos support --- .../src/openshmem_environment.cpp | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index a869b13b547b..22fad8180a28 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -360,6 +360,7 @@ namespace hpx::util { } struct ucx {}; + struct sos {}; struct mpi {}; struct err {}; @@ -404,13 +405,35 @@ namespace hpx::util { return sig_idx; } }; + +#endif + +#if defined(SHMEM_H) + template<> + struct wait_until_any_wrapper { + std::size_t operator()(unsigned int * addr, const std::size_t count, const std::uint8_t value) { + const std::size_t sig_idx = + shmem_uint_wait_until_any + ( + addr, + count, + nullptr, + SHMEM_CMP_EQ, + value + ); + + return sig_idx; + } + }; #endif std::size_t openshmem_environment::wait_until_any(const std::uint8_t value, std::uint8_t * sigaddr, const std::size_t count) { #if defined(SHMEM_VENDOR_STRING) using tag = std::conditional< std::integral_constant::value, ucx, - std::conditional< std::integral_constant::value, mpi, err>::type - >::type; + std::conditional< std::integral_constant::value, mpi, + std::conditional< std::integral_constant::value, sos, err>::type + >::type + >::type; using wait_until_any_type = wait_until_any_wrapper; #else #define SHMEM_VENDOR_STRING "SHMEM_VENDOR_STRING not defined" From 34b362804576a51fb342a7e5b70b93e6cacfd3f6 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Fri, 17 May 2024 09:05:43 -0500 Subject: [PATCH 149/154] updates --- .../hpx/openshmem_base/openshmem_environment.hpp | 3 +-- .../openshmem_base/src/openshmem_environment.cpp | 14 ++++---------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index 4193048f38ee..54862d33361d 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -112,7 +112,7 @@ namespace hpx { namespace util { typedef hpx::spinlock mutex_type; public: - static hpx::mutex mtx_; + static hpx::spinlock mtx_; static bool enabled_; static bool has_called_init_; @@ -123,7 +123,6 @@ namespace hpx { namespace util { static int init_val_; static std::vector segments; static std::uint8_t* shmem_buffer; - static std::size_t this_rank; }; }} // namespace hpx::util diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 22fad8180a28..002b8e066ee0 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -109,13 +109,12 @@ namespace hpx::util { namespace hpx::util { - hpx::mutex openshmem_environment::mtx_{}; + hpx::spinlock openshmem_environment::mtx_; bool openshmem_environment::enabled_ = false; bool openshmem_environment::has_called_init_ = false; int openshmem_environment::provided_threading_flag_ = 0; int openshmem_environment::is_initialized_ = -1; int openshmem_environment::init_val_ = 0; - std::size_t openshmem_environment::this_rank = -1; //std::vector> openshmem_environment::segment_mutex{}; std::vector openshmem_environment::segments{}; std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; @@ -224,11 +223,6 @@ namespace hpx::util { if (enabled_) return; // don't call twice - if (enabled()) { - scoped_lock l; - this_rank = static_cast(shmem_my_pe()); - } - has_called_init_ = false; // We assume to use the OpenSHMEM parcelport if it is not explicitly disabled @@ -261,7 +255,7 @@ namespace hpx::util { } #if defined(HPX_HAVE_NETWORKING) - if (this_rank == 0) + if (rank() == 0) { rtcfg.mode_ = hpx::runtime_mode::console; } @@ -275,7 +269,7 @@ namespace hpx::util { rtcfg.mode_ = hpx::runtime_mode::local; #endif - rtcfg.add_entry("hpx.parcel.openshmem.rank", std::to_string(this_rank)); + rtcfg.add_entry("hpx.parcel.openshmem.rank", std::to_string(rank())); rtcfg.add_entry( "hpx.parcel.openshmem.processorname", get_processor_name()); } @@ -500,7 +494,7 @@ namespace hpx::util { int openshmem_environment::rank() { - return this_rank; + return shmem_my_pe(); } openshmem_environment::scoped_lock::scoped_lock() From 6af97e8bfcf5bda005e2ef9fdf0135e92fa61923 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Thu, 23 May 2024 19:15:31 -0500 Subject: [PATCH 150/154] reworking communications --- .../openshmem_base/openshmem_environment.hpp | 3 ++ .../src/openshmem_environment.cpp | 20 ++++++--- .../hpx/parcelport_openshmem/receiver.hpp | 22 ++++------ .../receiver_connection.hpp | 41 ++++++++----------- .../sender_connection.hpp | 7 ++-- 5 files changed, 45 insertions(+), 48 deletions(-) diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index 54862d33361d..df96d159838a 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -54,6 +54,9 @@ namespace hpx { namespace util { static std::string get_processor_name(); + static void fence(); + static void quiet(); + static void put_signal(const std::uint8_t* addr, const int rank, std::uint8_t* raddr, const std::size_t size, std::uint8_t * sigaddr); diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 002b8e066ee0..9f0de6beb6ca 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -291,6 +291,14 @@ namespace hpx::util { return name; } + void openshmem_environment::quiet() { + shmem_quiet(); + } + + void openshmem_environment::fence() { + shmem_fence(); + } + void openshmem_environment::put_signal(const std::uint8_t* addr, const int node, std::uint8_t* raddr, const std::size_t size, std::uint8_t * sigaddr) @@ -303,14 +311,10 @@ namespace hpx::util { { #if !defined(SHMEM_SIGNAL_SET) shmem_uint8_put(raddr, addr, size, node); - shmem_uint8_put(reinterpret_cast(sigaddr), - reinterpret_cast(sigaddr), sizeof(sigaddr), - node); shmem_fence(); + shmem_uint8_put(sigaddr, sigaddr, 1, node); #else - shmem_uint8_put_signal(raddr, addr, size, - reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, - node); + shmem_uint8_put_signal(raddr, addr, size, sigaddr, 1, SHMEM_SIGNAL_SET, node); #endif } } @@ -346,7 +350,11 @@ namespace hpx::util { } tmp; tmp.uaddr = sigaddr; +#if defined(OSHMEM_SHMEMX_H) + while(shmem_uint_test(tmp.iaddr, SHMEM_CMP_EQ, value)) {} +#else shmem_uint_wait_until(tmp.iaddr, SHMEM_CMP_EQ, static_cast(value)); +#endif } constexpr bool vendor_strings_equal(char const * a, char const * b) { diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 3e26c3656758..1312fb5f061c 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -139,21 +139,17 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::size(); const std::size_t beg_rcv_signal = (sys_pgsz*page_count); - const auto idx = hpx::util::openshmem_environment::wait_until_any( - 1, - hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal, - page_count - ); - - hpx::util::openshmem_environment::get( - reinterpret_cast( - rcv_header_[idx].data()), - self_, + int idx = 0; + for(idx = 0; i < page_count; ++i) { + if(shmem_test(hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal + i, SHMEM_CMP_EQ, 1)) { + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + } + } + + std::memcpy(reinterpret_cast(rcv_header_[idx].data()), hpx::util::openshmem_environment::segments[idx].beg_addr, sizeof(header) - ); - - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + ); HPX_ASSERT_LOCKED(l, idx < 0); return idx; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index aef52a637a28..e2f0cedd7783 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -120,18 +120,13 @@ namespace hpx::parcelset::policies::openshmem { { buffer_.chunks_.resize(num_zero_copy_chunks); { - hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[idx].rcv); + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - hpx::util::openshmem_environment::get( - reinterpret_cast( - buffer_.transmission_chunks_.data()), - idx, + std::memcpy(reinterpret_cast(buffer_.transmission_chunks_.data()), hpx::util::openshmem_environment::segments[idx].beg_addr, - static_cast(buffer_.transmission_chunks_.size() * - sizeof(buffer_type::transmission_chunk_type))); - - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type) + ); request_ptr_ = true; } @@ -159,16 +154,13 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = hpx::util::openshmem_environment::rank(); - hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[idx].rcv); + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - hpx::util::openshmem_environment::get( - reinterpret_cast(buffer_.data_.data()), - idx, + std::memcpy(reinterpret_cast(buffer_.transmission_chunks_.data()), hpx::util::openshmem_environment::segments[idx].beg_addr, - buffer_.data_.size()); - - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + buffer_.data_.size() * sizeof(decltype(buffer_.data_)) + ); request_ptr_ = true; } @@ -199,16 +191,15 @@ namespace hpx::parcelset::policies::openshmem { data_type& c = buffer_.chunks_[cidx]; c.resize(chunk_size); { - hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[idx].rcv); - - hpx::util::openshmem_environment::get( - reinterpret_cast(c.data()), idx, - hpx::util::openshmem_environment::segments[idx].beg_addr, - c.size()); + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + std::memcpy(reinterpret_cast(c.data()), + hpx::util::openshmem_environment::segments[idx].beg_addr, + c.size() * sizeof(decltype(c.data())) + ); + hpx::util::openshmem_environment::put_signal(nullptr, src_, nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index e0c923e2811a..8eb287dafb00 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -160,7 +160,7 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::put_signal( reinterpret_cast(header_.data()), dst_, hpx::util::openshmem_environment::segments[idx].beg_addr, - header_.data_size_, + header_.data_size_*sizeof(decltype(header_.data)), hpx::util::openshmem_environment::segments[idx].rcv ); } @@ -216,7 +216,7 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::put_signal( reinterpret_cast(buffer_.data_.data()), dst_, hpx::util::openshmem_environment::segments[idx].beg_addr, - buffer_.data_.size(), + buffer_.data_.size() * sizeof(decltype(buffer_.data_.data)), hpx::util::openshmem_environment::segments[idx].rcv ); } @@ -251,8 +251,7 @@ namespace hpx::parcelset::policies::openshmem { hpx::util::openshmem_environment::segments[idx].rcv ); - hpx::util::openshmem_environment::wait_until( - 1, hpx::util::openshmem_environment::segments[idx].xmt); + while(shmem_test(hpx::util::openshmem_environment::segments[idx].xmt, SHMEM_CMP_EQ, 1)) {} (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; } } From 8ee4370cdbe7a3a3b2eca36f1ca80ffee87f26bb Mon Sep 17 00:00:00 2001 From: Christoper Taylor Date: Wed, 26 Jun 2024 10:34:51 -0400 Subject: [PATCH 151/154] updates to parcelport --- .../hpx/parcelport_openshmem/receiver.hpp | 48 ++++- .../receiver_connection.hpp | 117 ++++++++---- .../sender_connection.hpp | 168 +++++++++++++++--- 3 files changed, 265 insertions(+), 68 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 1312fb5f061c..0f89a4421352 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -134,22 +134,52 @@ namespace hpx::parcelset::policies::openshmem { const auto self_ = hpx::util::openshmem_environment::rank(); - const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); const std::size_t page_count = hpx::util::openshmem_environment::size(); - const std::size_t beg_rcv_signal = (sys_pgsz*page_count); - int idx = 0; - for(idx = 0; i < page_count; ++i) { - if(shmem_test(hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal + i, SHMEM_CMP_EQ, 1)) { - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - } + const std::size_t sys_pgsz = + sysconf(_SC_PAGESIZE); + + const std::size_t beg_rcv_signal = + (sys_pgsz*page_count); + + const std::size_t num_bytes = + sizeof(header); + const std::size_t rcv_numitrs = + (num_bytes + sys_pgsz - 1) / sys_pgsz; + + const std::size_t rcv_numitrs_term = rcv_numitrs - 1; + std::size_t data_seg [2] = { sys_pgsz, num_bytes % sys_pgsz }; + + int idx = 0; + for(idx = 0; idx < page_count; ++idx) { + if(shmem_test(hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal + idx, SHMEM_CMP_EQ, 1)) { + break; + } } std::memcpy(reinterpret_cast(rcv_header_[idx].data()), hpx::util::openshmem_environment::segments[idx].beg_addr, - sizeof(header) - ); + data_seg[0] + ); + + auto chunk_beg = sys_pgsz; + + for(std::size_t itr = 1; itr < header_numitrs; ++itr) { + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + + std::memcpy(reinterpret_cast(rcv_header_[idx].data())+chunk_beg, + hpx::util::openshmem_environment::segments[idx].beg_addr, + data_seg[(itr == rcv_numitrs_term)] + ); + + if(i != rcv_numitrs_term) { + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + chunk_beg = i * sys_pgsz; + hpx::util::openshmem_environment::put_signal(nullptr, src_, + nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); + } + } HPX_ASSERT_LOCKED(l, idx < 0); return idx; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index e2f0cedd7783..77de7c1d2f52 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -72,7 +72,7 @@ namespace hpx::parcelset::policies::openshmem { if (piggy_back_data) { need_recv_data = false; - memcpy(buffer_.data_.data(), piggy_back_data, + std::memcpy(buffer_.data_.data(), piggy_back_data, buffer_.data_.size()); } else @@ -108,6 +108,9 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = hpx::util::openshmem_environment::rank(); + const std::size_t sys_pgsz = + sysconf(_SC_PAGESIZE); + // determine the size of the chunk buffer std::size_t num_zero_copy_chunks = static_cast( static_cast(buffer_.num_chunks_.first)); @@ -119,14 +122,32 @@ namespace hpx::parcelset::policies::openshmem { if (num_zero_copy_chunks != 0) { buffer_.chunks_.resize(num_zero_copy_chunks); + const std::size_t num_bytes = + buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type); + const std::size_t rcv_numitrs = + (num_bytes + sys_pgsz - 1) / sys_pgsz; + + const std::size_t rcv_numitrs_term = rcv_numitrs - 1; + { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + std::size_t data_seg [2] = { sys_pgsz, num_bytes % sys_pgsz }; - std::memcpy(reinterpret_cast(buffer_.transmission_chunks_.data()), - hpx::util::openshmem_environment::segments[idx].beg_addr, - buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type) - ); + auto chunk_beg = 0; + for(std::size_t i = 0; i < rcv_numitrs; ++i) { + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + + std::memcpy(reinterpret_cast(buffer_.transmission_chunks_.data())+chunk_beg, + hpx::util::openshmem_environment::segments[idx].beg_addr, + data_seg[(i == rcv_numitrs_term)] + ); + + if(i != rcv_numitrs_term) { + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + chunk_beg = i * sys_pgsz; + hpx::util::openshmem_environment::put_signal(nullptr, src_, + nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); + } + } request_ptr_ = true; } @@ -154,13 +175,36 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = hpx::util::openshmem_environment::rank(); - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + const std::size_t sys_pgsz = + sysconf(_SC_PAGESIZE); + + const std::size_t num_bytes = + buffer_.data_.size() * sizeof(decltype(buffer_.data_); + + const std::size_t rcv_numitrs = + (num_bytes + sys_pgsz - 1) / sys_pgsz; + + const std::size_t rcv_numitrs_term = rcv_numitrs - 1; - std::memcpy(reinterpret_cast(buffer_.transmission_chunks_.data()), - hpx::util::openshmem_environment::segments[idx].beg_addr, - buffer_.data_.size() * sizeof(decltype(buffer_.data_)) - ); + std::size_t data_seg [2] = { sys_pgsz, num_bytes % sys_pgsz }; + + auto chunk_beg = 0; + + for(std::size_t i = 0; i < rcv_numitrs; ++i) { + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + + std::memcpy(reinterpret_cast(buffer_.transmission_chunks_.data())+chunk_beg, + hpx::util::openshmem_environment::segments[idx].beg_addr, + data_seg[(i == rcv_numitrs_term)] + ); + + if(i != rcv_numitrs_term) { + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + chunk_beg = i * sys_pgsz; + hpx::util::openshmem_environment::put_signal(nullptr, src_, + nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); + } + } request_ptr_ = true; } @@ -172,41 +216,48 @@ namespace hpx::parcelset::policies::openshmem { bool receive_chunks() { - std::size_t cidx = 0; - std::size_t chunk_size = 0; + const std::size_t sys_pgsz = + sysconf(_SC_PAGESIZE); const auto idx = hpx::util::openshmem_environment::rank(); - while (chunks_idx_ < buffer_.chunks_.size()) - { - if (!request_done()) - { - return false; - } + for(auto i = 0; i < buffer_.chunks_.size(); ++i) { + buffer_.chunks_[i].resize(buffer_.transmission_chunks_[i].second); + } - cidx = chunks_idx_++; - chunk_size = - buffer_.transmission_chunks_[cidx].second; + for(auto i = 0; i < buffer_.chunks_.size(); ++i) { + data_type& c = buffer_.chunks_[i]; - data_type& c = buffer_.chunks_[cidx]; - c.resize(chunk_size); - { + const std::size_t num_bytes = c.size() * sizeof(decltype(c.data())); + std::size_t data_seg [2] = { sys_pgsz, num_bytes % sys_pgsz }; + + const std::size_t rcv_numitrs = + (num_bytes + sys_pgsz - 1) / sys_pgsz; + + const std::size_t rcv_numitrs_term = rcv_numitrs - 1; + + auto chunk_beg = 0; + for(std::size_t i = 0; i < rcv_numitrs; ++i) { while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - std::memcpy(reinterpret_cast(c.data()), + std::memcpy(reinterpret_cast(c.data())+chunk_beg, hpx::util::openshmem_environment::segments[idx].beg_addr, - c.size() * sizeof(decltype(c.data())) + data_seg[(i == rcv_numitrs_term)] ); - hpx::util::openshmem_environment::put_signal(nullptr, src_, - nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); - - request_ptr_ = true; + if(i != rcv_numitrs_term) { + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + chunk_beg = i * sys_pgsz; + hpx::util::openshmem_environment::put_signal(nullptr, src_, + nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); + } } } + request_ptr_ = true; + state_ = rcvd_chunks; return done(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 8eb287dafb00..5a73480b6cab 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -150,6 +150,23 @@ namespace hpx::parcelset::policies::openshmem { { { HPX_ASSERT(state_ == initialized); + + const std::size_t sys_pgsz = + sysconf(_SC_PAGESIZE); + const std::size_t page_count = + hpx::util::openshmem_environment::size(); + const std::size_t beg_rcv_signal = + (sys_pgsz*page_count); + + // header_.data_size_*sizeof(decltype(header_.data)), + const std::size_t data_amt = sizeof(header); + std::size_t data_seg [2] = { sys_pgsz, data_amt % sys_pgsz }; + + const std::size_t header_numitrs = + (sizeof(header) + sys_pgsz - 1) / sys_pgsz; + + const std::size_t header_numitrs_term = header_numitrs - 1; + const auto idx = dst_; util::openshmem_environment::scoped_lock l; @@ -157,12 +174,20 @@ namespace hpx::parcelset::policies::openshmem { // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment // - hpx::util::openshmem_environment::put_signal( - reinterpret_cast(header_.data()), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, - header_.data_size_*sizeof(decltype(header_.data)), - hpx::util::openshmem_environment::segments[idx].rcv - ); + + for(std::size_t itr = 0; itr < header_numitrs; ++i) { + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(header_.data()) + (itr * sys_pgsz), dst_, + hpx::util::openshmem_environment::segments[idx].beg_addr, + data_seg[(itr == header_numitrs_term)], + hpx::util::openshmem_environment::segments[idx].rcv + ); + + if(itr != header_numitrs_term) { + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + } + } } state_ = sent_header; @@ -186,13 +211,45 @@ namespace hpx::parcelset::policies::openshmem { util::openshmem_environment::scoped_lock l; - hpx::util::openshmem_environment::put_signal( - reinterpret_cast(chunks.data()), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, + const std::size_t sys_pgsz = + sysconf(_SC_PAGESIZE); + const std::size_t page_count = + hpx::util::openshmem_environment::size(); + const std::size_t beg_rcv_signal = + (sys_pgsz*page_count); + + const std::size_t data_amt = static_cast(chunks.size() * - sizeof(parcel_buffer_type::transmission_chunk_type)), - hpx::util::openshmem_environment::segments[idx].rcv - ); + sizeof(parcel_buffer_type::transmission_chunk_type)); + + std::size_t data_seg [2] = { sys_pgsz, data_amt % sys_pgsz }; + + const std::size_t header_numitrs = + ((data_amt + sys_pgsz - 1) / sys_pgsz) + data_seg[1] ? 0 : - 1; + + const std::size_t header_numitrs_term = header_numitrs - 1; + + const auto idx = dst_; + + util::openshmem_environment::scoped_lock l; + + // put from this localities openshmem shared memory segment + // into the remote locality (dst_)'s shared memory segment + // + + for(std::size_t itr = 0; itr < header_numitrs; ++i) { + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(chunks.data()) + (itr * sys_pgsz), dst_, + hpx::util::openshmem_environment::segments[idx].beg_addr, + data_seg[(itr == header_numitrs_term)], + hpx::util::openshmem_environment::segments[idx].rcv + ); + + if(itr != header_numitrs_term) { + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + } + } } state_ = sent_transmission_chunks; @@ -213,12 +270,44 @@ namespace hpx::parcelset::policies::openshmem { util::openshmem_environment::scoped_lock l; - hpx::util::openshmem_environment::put_signal( - reinterpret_cast(buffer_.data_.data()), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, - buffer_.data_.size() * sizeof(decltype(buffer_.data_.data)), - hpx::util::openshmem_environment::segments[idx].rcv - ); + const std::size_t sys_pgsz = + sysconf(_SC_PAGESIZE); + const std::size_t page_count = + hpx::util::openshmem_environment::size(); + const std::size_t beg_rcv_signal = + (sys_pgsz*page_count); + + const std::size_t data_amt = + buffer_.data_.size() * sizeof(decltype(buffer_.data_.data)); + + std::size_t data_seg [2] = { sys_pgsz, data_amt % sys_pgsz }; + + const std::size_t header_numitrs = + ((data_amt + sys_pgsz - 1) / sys_pgsz) + data_seg[1] ? 0 : - 1; + + const std::size_t header_numitrs_term = header_numitrs - 1; + + const auto idx = dst_; + + util::openshmem_environment::scoped_lock l; + + // put from this localities openshmem shared memory segment + // into the remote locality (dst_)'s shared memory segment + // + + for(std::size_t itr = 0; itr < header_numitrs; ++i) { + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(buffer_.data_.data() + (itr * sys_pgsz)), dst_, + hpx::util::openshmem_environment::segments[idx].beg_addr, + data_seg[(itr == header_numitrs_term)], + hpx::util::openshmem_environment::segments[idx].rcv + ); + + if(itr != header_numitrs_term) { + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + } + } } state_ = sent_data; @@ -244,15 +333,42 @@ namespace hpx::parcelset::policies::openshmem { { util::openshmem_environment::scoped_lock l; - hpx::util::openshmem_environment::put_signal( - reinterpret_cast(c.data_.cpos_), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, - static_cast(c.size_), - hpx::util::openshmem_environment::segments[idx].rcv - ); + const std::size_t sys_pgsz = + sysconf(_SC_PAGESIZE); + const std::size_t page_count = + hpx::util::openshmem_environment::size(); + const std::size_t beg_rcv_signal = + (sys_pgsz*page_count); - while(shmem_test(hpx::util::openshmem_environment::segments[idx].xmt, SHMEM_CMP_EQ, 1)) {} - (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + const std::size_t data_amt = + static_cast(c.size_); + + std::size_t data_seg [2] = { sys_pgsz, data_amt % sys_pgsz }; + + const std::size_t header_numitrs = + ((data_amt + sys_pgsz - 1) / sys_pgsz) + data_seg[1] ? 0 : - 1; + + const std::size_t header_numitrs_term = header_numitrs - 1; + + const auto idx = dst_; + + // put from this localities openshmem shared memory segment + // into the remote locality (dst_)'s shared memory segment + // + + for(std::size_t itr = 0; itr < header_numitrs; ++i) { + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(c.data_.cpos_ + (itr * sys_pgsz)), dst_, + hpx::util::openshmem_environment::segments[idx].beg_addr, + data_seg[(itr == header_numitrs_term)], + hpx::util::openshmem_environment::segments[idx].rcv + ); + + if(itr != header_numitrs_term) { + while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + } + } } } From 345286ce3c102c42ea2de2d0c5db54862d8071f4 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Mon, 15 Jul 2024 21:25:59 -0500 Subject: [PATCH 152/154] resolves merge conflicts --- libs/core/config/include/hpx/config.hpp | 12 +++++ .../openshmem_base/openshmem_environment.hpp | 8 +-- .../src/openshmem_environment.cpp | 53 +++++++++++++------ .../hpx/parcelport_openshmem/receiver.hpp | 27 ++++++---- .../receiver_connection.hpp | 17 ++++-- .../hpx/parcelport_openshmem/sender.hpp | 2 - .../sender_connection.hpp | 45 +++++++++++----- .../src/parcelport_openshmem.cpp | 28 ++++++---- 8 files changed, 135 insertions(+), 57 deletions(-) diff --git a/libs/core/config/include/hpx/config.hpp b/libs/core/config/include/hpx/config.hpp index c3b3425d4bad..b7954b5a349b 100644 --- a/libs/core/config/include/hpx/config.hpp +++ b/libs/core/config/include/hpx/config.hpp @@ -118,6 +118,18 @@ # define HPX_HAVE_PARCELPORT_MPI_BACKGROUND_THREADS std::size_t(-1) #endif +/// This defines the number of cores that perform background work for the MPI +/// parcelport +/// This value can be changed at runtime by setting the configuration parameter: +/// +/// hpx.parcel.openshmem.background_threads = ... +/// +/// (or by setting the corresponding environment variable +/// HPX_HAVE_PARCELPORT_OPENSHMEM_BACKGROUND_THREADS). +#if !defined(HPX_HAVE_PARCELPORT_OPENSHMEM_BACKGROUND_THREADS) +# define HPX_HAVE_PARCELPORT_OPENSHMEM_BACKGROUND_THREADS std::size_t(-1) +#endif + /////////////////////////////////////////////////////////////////////////////// /// This defines the number of outgoing (parcel-) connections kept alive (to /// each of the other localities). This value can be changed at runtime by diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp index df96d159838a..778d98769c76 100644 --- a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -57,6 +57,8 @@ namespace hpx { namespace util { static void fence(); static void quiet(); + static int test(std::uint8_t * addr, const std::uint8_t value); + static void put_signal(const std::uint8_t* addr, const int rank, std::uint8_t* raddr, const std::size_t size, std::uint8_t * sigaddr); @@ -66,8 +68,6 @@ namespace hpx { namespace util { static void get(std::uint8_t* addr, const int rank, const std::uint8_t* raddr, const std::size_t size); - static void quiet() { shmem_quiet(); } - static void global_barrier(); struct HPX_CORE_EXPORT scoped_lock @@ -112,10 +112,10 @@ namespace hpx { namespace util { bool locked; }; - typedef hpx::spinlock mutex_type; + using mutex_type = hpx::spinlock; public: - static hpx::spinlock mtx_; + static mutex_type mtx_; static bool enabled_; static bool has_called_init_; diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 9f0de6beb6ca..8ffd0aa6446d 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -109,7 +109,7 @@ namespace hpx::util { namespace hpx::util { - hpx::spinlock openshmem_environment::mtx_; + openshmem_environment::mutex_type openshmem_environment::mtx_; bool openshmem_environment::enabled_ = false; bool openshmem_environment::has_called_init_ = false; int openshmem_environment::provided_threading_flag_ = 0; @@ -123,13 +123,19 @@ namespace hpx::util { int openshmem_environment::init([[maybe_unused]] int* argc, [[maybe_unused]] char*** argv, [[maybe_unused]] int& provided) { + int provided_ = 0; if (!has_called_init_) { - shmem_init(); +std::cout << "SHMEM_INIT" << std::endl; + shmem_init_thread(SHMEM_THREAD_MULTIPLE, &provided_); openshmem_environment::init_val_ = 1; has_called_init_ = true; } - + if(provided_ != 0) { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM initialization error"); + } if (openshmem_environment::init_val_ == 0) { HPX_THROW_EXCEPTION(error::invalid_status, @@ -206,12 +212,11 @@ namespace hpx::util { // all of the xmt signals are linearly arranged after the rcv signals // segments[i].xmt = hpx::util::openshmem_environment::shmem_buffer + beg_signal + page_count + i; - - //segments[i].mut = &(openshmem_environment::segment_mutex[i]); } shmem_barrier_all(); +std::cout << "1 parcelport constructor" << std::endl << std::flush; return openshmem_environment::init_val_; } @@ -235,7 +240,9 @@ namespace hpx::util { rtcfg.add_entry("hpx.parcel.bootstrap", "openshmem"); +std::cout << "2 parcelport constructor" << std::endl << std::flush; int retval = init(argc, argv, provided_threading_flag_); +std::cout << "3 parcelport constructor" << std::endl << std::flush; if (1 != retval) { // explicitly disable openshmem if not run by openshmemrun @@ -255,7 +262,8 @@ namespace hpx::util { } #if defined(HPX_HAVE_NETWORKING) - if (rank() == 0) + const auto rnk = rank(); + if (rnk == 0) { rtcfg.mode_ = hpx::runtime_mode::console; } @@ -269,15 +277,15 @@ namespace hpx::util { rtcfg.mode_ = hpx::runtime_mode::local; #endif - rtcfg.add_entry("hpx.parcel.openshmem.rank", std::to_string(rank())); + rtcfg.add_entry("hpx.parcel.openshmem.rank", std::to_string(rnk)); rtcfg.add_entry( "hpx.parcel.openshmem.processorname", get_processor_name()); +std::cout << "4 parcelport constructor" << std::endl << std::flush; } std::string openshmem_environment::get_processor_name() { - scoped_lock l; - +std::cout << "5 parcelport constructor" << std::endl << std::flush; char name[1024 + 1] = {'\0'}; const std::string rnkstr = std::to_string(rank()); const int len = rnkstr.size(); @@ -291,18 +299,24 @@ namespace hpx::util { return name; } - void openshmem_environment::quiet() { - shmem_quiet(); - } - - void openshmem_environment::fence() { - shmem_fence(); + void openshmem_environment::fence() { shmem_fence(); } + void openshmem_environment::quiet() { shmem_quiet(); } + + int openshmem_environment::test(std::uint8_t * addr, const std::uint8_t value) { +std::cout << "6 parcelport constructor" << std::endl << std::flush; + union { + std::uint8_t * oaddr; + volatile unsigned int* iaddr; + } tmp; + tmp.oaddr = addr; + return shmem_uint_test(tmp.iaddr, SHMEM_CMP_EQ, value); } void openshmem_environment::put_signal(const std::uint8_t* addr, const int node, std::uint8_t* raddr, const std::size_t size, std::uint8_t * sigaddr) { +std::cout << "7 parcelport constructor" << std::endl << std::flush; if (rank() == node) { std::memmove(raddr, addr, size); @@ -344,6 +358,8 @@ namespace hpx::util { unsigned int * >::type; +std::cout << "8 parcelport constructor" << std::endl << std::flush; + union { std::uint8_t * uaddr; arg_type iaddr; @@ -437,6 +453,7 @@ namespace hpx::util { >::type >::type; using wait_until_any_type = wait_until_any_wrapper; +std::cout << "9 parcelport constructor" << std::endl << std::flush; #else #define SHMEM_VENDOR_STRING "SHMEM_VENDOR_STRING not defined" using tag = err; @@ -449,6 +466,7 @@ namespace hpx::util { void openshmem_environment::get(std::uint8_t* addr, const int node, const std::uint8_t* raddr, const std::size_t size) { +std::cout << "10 parcelport constructor" << std::endl << std::flush; if (rank() == node) { std::memmove(addr, raddr, size); @@ -462,11 +480,13 @@ namespace hpx::util { void openshmem_environment::global_barrier() { +std::cout << "11 parcelport constructor" << std::endl << std::flush; shmem_barrier_all(); } void openshmem_environment::finalize() { +std::cout << "12 parcelport constructor" << std::endl << std::flush; if (enabled() && has_called_init()) { scoped_lock l; @@ -492,9 +512,9 @@ namespace hpx::util { int openshmem_environment::size() { +std::cout << "13 parcelport constructor" << std::endl << std::flush; int res(0); if (enabled()) { - scoped_lock l; res = static_cast(shmem_n_pes()); } return res; @@ -502,6 +522,7 @@ namespace hpx::util { int openshmem_environment::rank() { +std::cout << "14 parcelport constructor" << std::endl << std::flush; return shmem_my_pe(); } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index 0f89a4421352..eb929160f46e 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace hpx::parcelset::policies::openshmem { @@ -45,8 +46,6 @@ namespace hpx::parcelset::policies::openshmem { void run() noexcept { - util::openshmem_environment::scoped_lock l; - post_new_header(l); } bool background_work() noexcept @@ -99,7 +98,7 @@ namespace hpx::parcelset::policies::openshmem { { HPX_ASSERT_OWNS_LOCK(header_lock); - util::openshmem_environment::scoped_try_lock l; + //util::openshmem_environment::scoped_try_lock l; // Caller failing to hold lock 'header_lock' before calling function #if defined(HPX_MSVC) @@ -107,13 +106,18 @@ namespace hpx::parcelset::policies::openshmem { #pragma warning(disable : 26110) #endif - if (l.locked) + //if (l.locked) { - const auto idx = post_new_header(l); + std::size_t idx = post_new_header(); + + while(idx != -1UL) { + idx = post_new_header(); //l); + } + header h = rcv_header_[idx]; rcv_header_[idx].reset(); - l.unlock(); + //l.unlock(); header_lock.unlock(); return std::make_shared( @@ -127,11 +131,10 @@ namespace hpx::parcelset::policies::openshmem { return {}; } - template - std::size_t post_new_header([[maybe_unused]] Lock& l) noexcept + //template + std::size_t post_new_header() noexcept //[[maybe_unused]] Lock& l) noexcept { - HPX_ASSERT_OWNS_LOCK(l); - + //HPX_ASSERT_OWNS_LOCK(l); const auto self_ = hpx::util::openshmem_environment::rank(); const std::size_t page_count = @@ -158,6 +161,8 @@ namespace hpx::parcelset::policies::openshmem { } } + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + std::memcpy(reinterpret_cast(rcv_header_[idx].data()), hpx::util::openshmem_environment::segments[idx].beg_addr, data_seg[0] @@ -181,7 +186,7 @@ namespace hpx::parcelset::policies::openshmem { } } - HPX_ASSERT_LOCKED(l, idx < 0); + //HPX_ASSERT_LOCKED(l, idx < 0); return idx; } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index 77de7c1d2f52..a6cc05e58669 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -54,7 +54,8 @@ namespace hpx::parcelset::policies::openshmem { , need_recv_tchunks(false) , chunks_idx_(0) , pp_(pp) - { + { +std::cout << "receiver_connection" << std::endl; header_.assert_valid(); num_bytes = header_.numbytes(); @@ -84,18 +85,23 @@ namespace hpx::parcelset::policies::openshmem { bool receive() { +std::cout << "recieve" << std::endl; switch (state_) { case initialized: +std::cout << "rcvd_init" << std::endl; return receive_transmission_chunks(); case rcvd_transmission_chunks: +std::cout << "rcvd_transmission_chunks" << std::endl; return receive_data(); case rcvd_data: +std::cout << "rcvd_data" << std::endl; return receive_chunks(); case rcvd_chunks: +std::cout << "rcvd_chunks" << std::endl; return done(); default: @@ -106,6 +112,7 @@ namespace hpx::parcelset::policies::openshmem { bool receive_transmission_chunks() { +std::cout << "receive_transmission_chunks" << std::endl; const auto idx = hpx::util::openshmem_environment::rank(); const std::size_t sys_pgsz = @@ -122,6 +129,7 @@ namespace hpx::parcelset::policies::openshmem { if (num_zero_copy_chunks != 0) { buffer_.chunks_.resize(num_zero_copy_chunks); + const std::size_t num_bytes = buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type); const std::size_t rcv_numitrs = @@ -160,6 +168,7 @@ namespace hpx::parcelset::policies::openshmem { bool receive_data() { +std::cout << "receive_data" << std::endl; if (!request_done()) { return false; @@ -265,13 +274,15 @@ namespace hpx::parcelset::policies::openshmem { bool done() noexcept { +std::cout << "done" << std::endl; return request_done(); } bool request_done() noexcept { - util::openshmem_environment::scoped_try_lock const l; - if(!l.locked) { return false; } +std::cout << "request_done" << std::endl; + //util::openshmem_environment::scoped_try_lock const l; + //if(!l.locked) { return false; } hpx::util::openshmem_environment::quiet(); diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp index a5ec2d0f9502..41cd86df675a 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -43,8 +43,6 @@ namespace hpx::parcelset::policies::openshmem { void run() noexcept { - //hpx::util::openshmem_environment::scoped_lock l; - //get_next_free_tag(); } connection_ptr create_connection(int dest, parcelset::parcelport* pp) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 5a73480b6cab..16188ed531e9 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -67,14 +67,12 @@ namespace hpx::parcelset::policies::openshmem { : state_(initialized) , sender_(s) , dst_(dst) - , thd_id_(-1) , chunks_idx_(0) , ack_(0) , pp_(pp) , there_(parcelset::locality(locality(dst_))) { - thd_id_ = - hpx::get_worker_thread_num(); // current worker +std::cout << "sender_connection_instantiated" << std::endl; } parcelset::locality const& destination() const noexcept @@ -123,21 +121,27 @@ namespace hpx::parcelset::policies::openshmem { bool send() { +std::cout << "send" << std::endl; switch (state_) { case initialized: +std::cout << "sent_init" << std::endl; return send_header(); case sent_header: +std::cout << "sent_header" << std::endl; return send_transmission_chunks(); case sent_transmission_chunks: +std::cout << "sent_transmission_chunks" << std::endl; return send_data(); case sent_data: +std::cout << "sent_data" << std::endl; return send_chunks(); case sent_chunks: +std::cout << "sent_chunks" << std::endl; return done(); default: @@ -168,8 +172,11 @@ namespace hpx::parcelset::policies::openshmem { const std::size_t header_numitrs_term = header_numitrs - 1; const auto idx = dst_; - - util::openshmem_environment::scoped_lock l; +std::cout << "sendHeader0\t" << dst_ << ' ' << std::endl << std::flush; + //util::openshmem_environment::scoped_lock l; + //HPX_ASSERT_OWNS_LOCK(header_lock); + HPX_ASSERT(state_ == initialized); +std::cout << "sendHeader1" << std::endl << std::flush; // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment @@ -196,6 +203,7 @@ namespace hpx::parcelset::policies::openshmem { bool send_transmission_chunks() { +std::cout << "send_transmission_chunks" << std::endl; HPX_ASSERT(state_ == sent_header); if (!request_done()) { @@ -209,7 +217,7 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - util::openshmem_environment::scoped_lock l; + //util::openshmem_environment::scoped_lock l; const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); @@ -258,6 +266,7 @@ namespace hpx::parcelset::policies::openshmem { bool send_data() { +std::cout << "send_data" << std::endl; HPX_ASSERT(state_ == sent_transmission_chunks); if (!request_done()) { @@ -268,7 +277,13 @@ namespace hpx::parcelset::policies::openshmem { { const auto idx = dst_; - util::openshmem_environment::scoped_lock l; + //util::openshmem_environment::scoped_lock l; + std::cout << "is null" << idx << ' ' << (hpx::util::openshmem_environment::segments[idx].beg_addr == nullptr) << ' ' << (hpx::util::openshmem_environment::segments[idx].rcv == nullptr) << std::endl; + + const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); +std::cout << "1pagsz\t" << sys_pgsz << ' ' << (buffer_.data_.size() * sizeof(std::uint8_t)) << std::endl; + +// issue here is page size < data size const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); @@ -316,6 +331,7 @@ namespace hpx::parcelset::policies::openshmem { bool send_chunks() { +std::cout << "send_chunks" << std::endl; HPX_ASSERT(state_ == sent_data); const auto idx = dst_; @@ -331,7 +347,10 @@ namespace hpx::parcelset::policies::openshmem { } { - util::openshmem_environment::scoped_lock l; + //util::openshmem_environment::scoped_lock l; + + const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); +std::cout << "2pagsz\t" << sys_pgsz << ' ' << (buffer_.data_.size() * sizeof(std::uint8_t)) << std::endl; const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); @@ -382,6 +401,7 @@ namespace hpx::parcelset::policies::openshmem { bool done() { +std::cout << "done" << std::endl; if (!request_done()) { return false; @@ -405,10 +425,12 @@ namespace hpx::parcelset::policies::openshmem { bool request_done() { - util::openshmem_environment::scoped_try_lock const l; - if(!l.locked) { return false; } +std::cout << "request_done1" << std::endl; + //util::openshmem_environment::scoped_try_lock const l; + //if(!l.locked) { return false; } - hpx::util::openshmem_environment::quiet(); + hpx::util::openshmem_environment::fence(); +std::cout << "request_done2" << std::endl; return true; } @@ -416,7 +438,6 @@ namespace hpx::parcelset::policies::openshmem { connection_state state_; sender_type* sender_; int dst_; - int thd_id_; using handler_type = hpx::move_only_function; handler_type handler_; diff --git a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp index 5129dff1f886..705808ceaad1 100644 --- a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp +++ b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp @@ -72,12 +72,6 @@ namespace hpx::parcelset { namespace policies::openshmem { -/* - int acquire_tag(sender* s) noexcept - { - return s->acquire_tag(); - } -*/ void add_connection( sender* s, std::shared_ptr const& ptr) { @@ -107,12 +101,26 @@ namespace hpx::parcelset { static std::size_t background_threads( [[maybe_unused]] util::runtime_configuration const& ini) { - /* + // limit the number of cores accessing MPI to one if + // multi-threading in MPI is disabled + if (!multi_threaded_openshmem(ini)) + { + return 1; + } return hpx::util::get_entry_as(ini, "hpx.parcel.openshmem.background_threads", HPX_HAVE_PARCELPORT_OPENSHMEM_BACKGROUND_THREADS); - */ - return 1UL; + } + + static bool multi_threaded_openshmem( + util::runtime_configuration const& ini) + { + if (hpx::util::get_entry_as( + ini, "hpx.parcel.openshmem.multithreaded", 1) != 0) + { + return true; + } + return false; } public: @@ -122,6 +130,7 @@ namespace hpx::parcelset { , stopped_(false) , receiver_(*this) , background_threads_(background_threads(ini)) + , multi_threaded_openshmem_(multi_threaded_openshmem(ini)) { } @@ -244,6 +253,7 @@ namespace hpx::parcelset { } std::size_t background_threads_; + bool multi_threaded_openshmem_; void early_write_handler(std::error_code const& ec, parcel const& p) { From 50318e1e6b7d170842e220e231138fce1894b39a Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Tue, 16 Jul 2024 21:36:15 -0500 Subject: [PATCH 153/154] several fixes to protocol --- .../hpx/parcelport_openshmem/receiver.hpp | 34 ++++--- .../receiver_connection.hpp | 34 ++++--- .../sender_connection.hpp | 93 +++++++++---------- 3 files changed, 80 insertions(+), 81 deletions(-) diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp index eb929160f46e..d9bc70606e6c 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -154,35 +154,45 @@ namespace hpx::parcelset::policies::openshmem { const std::size_t rcv_numitrs_term = rcv_numitrs - 1; std::size_t data_seg [2] = { sys_pgsz, num_bytes % sys_pgsz }; - int idx = 0; - for(idx = 0; idx < page_count; ++idx) { - if(shmem_test(hpx::util::openshmem_environment::shmem_buffer + beg_rcv_signal + idx, SHMEM_CMP_EQ, 1)) { - break; + std::size_t idx = 0; + { + bool term = false; + while(!term) { + for(idx = 0; idx < page_count; ++idx) { + if(hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[idx].rcv, 1)) { + term = true; + break; + } + } } } (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - std::memcpy(reinterpret_cast(rcv_header_[idx].data()), + std::memcpy(reinterpret_cast(rcv_header_.data()+(sizeof(header)*idx)), hpx::util::openshmem_environment::segments[idx].beg_addr, data_seg[0] ); + hpx::util::openshmem_environment::put_signal(nullptr, idx, + nullptr, 1, hpx::util::openshmem_environment::segments[self_].xmt); + auto chunk_beg = sys_pgsz; - for(std::size_t itr = 1; itr < header_numitrs; ++itr) { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + for(std::size_t itr = 1; itr < rcv_numitrs; ++itr) { + while(hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[idx].rcv, 1)) {} + (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; std::memcpy(reinterpret_cast(rcv_header_[idx].data())+chunk_beg, hpx::util::openshmem_environment::segments[idx].beg_addr, data_seg[(itr == rcv_numitrs_term)] ); - if(i != rcv_numitrs_term) { - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; - chunk_beg = i * sys_pgsz; - hpx::util::openshmem_environment::put_signal(nullptr, src_, - nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); + if(itr != rcv_numitrs_term) { + chunk_beg = itr * sys_pgsz; + + hpx::util::openshmem_environment::put_signal(nullptr, idx, + nullptr, 1, hpx::util::openshmem_environment::segments[self_].xmt); } } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp index a6cc05e58669..54b51b52dc48 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -47,6 +47,7 @@ namespace hpx::parcelset::policies::openshmem { receiver_connection(int src, header h, Parcelport& pp) noexcept : state_(initialized) , src_(src) + , idx(0) , header_(h) , request_ptr_(false) , num_bytes(0) @@ -57,6 +58,7 @@ namespace hpx::parcelset::policies::openshmem { { std::cout << "receiver_connection" << std::endl; header_.assert_valid(); + idx = hpx::util::openshmem_environment::rank(); num_bytes = header_.numbytes(); @@ -113,7 +115,6 @@ std::cout << "rcvd_chunks" << std::endl; bool receive_transmission_chunks() { std::cout << "receive_transmission_chunks" << std::endl; - const auto idx = hpx::util::openshmem_environment::rank(); const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); @@ -142,16 +143,17 @@ std::cout << "receive_transmission_chunks" << std::endl; auto chunk_beg = 0; for(std::size_t i = 0; i < rcv_numitrs; ++i) { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + while(!hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[src_].rcv, 1)) {} + (*(hpx::util::openshmem_environment::segments[src_].rcv)) = 0; std::memcpy(reinterpret_cast(buffer_.transmission_chunks_.data())+chunk_beg, - hpx::util::openshmem_environment::segments[idx].beg_addr, + hpx::util::openshmem_environment::segments[src_].beg_addr, data_seg[(i == rcv_numitrs_term)] ); if(i != rcv_numitrs_term) { - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; chunk_beg = i * sys_pgsz; + hpx::util::openshmem_environment::put_signal(nullptr, src_, nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); } @@ -182,13 +184,11 @@ std::cout << "receive_data" << std::endl; } else { - const auto idx = hpx::util::openshmem_environment::rank(); - const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); const std::size_t num_bytes = - buffer_.data_.size() * sizeof(decltype(buffer_.data_); + buffer_.data_.size() * sizeof(decltype(buffer_.data_)); const std::size_t rcv_numitrs = (num_bytes + sys_pgsz - 1) / sys_pgsz; @@ -200,15 +200,15 @@ std::cout << "receive_data" << std::endl; auto chunk_beg = 0; for(std::size_t i = 0; i < rcv_numitrs; ++i) { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} + while(!hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[src_].rcv, 1)) {} + (*(hpx::util::openshmem_environment::segments[src_].rcv)) = 0; std::memcpy(reinterpret_cast(buffer_.transmission_chunks_.data())+chunk_beg, - hpx::util::openshmem_environment::segments[idx].beg_addr, + hpx::util::openshmem_environment::segments[src_].beg_addr, data_seg[(i == rcv_numitrs_term)] ); if(i != rcv_numitrs_term) { - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; chunk_beg = i * sys_pgsz; hpx::util::openshmem_environment::put_signal(nullptr, src_, nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); @@ -228,13 +228,11 @@ std::cout << "receive_data" << std::endl; const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); - const auto idx = hpx::util::openshmem_environment::rank(); - - for(auto i = 0; i < buffer_.chunks_.size(); ++i) { + for(std::size_t i = 0; i < buffer_.chunks_.size(); ++i) { buffer_.chunks_[i].resize(buffer_.transmission_chunks_[i].second); } - for(auto i = 0; i < buffer_.chunks_.size(); ++i) { + for(std::size_t i = 0; i < buffer_.chunks_.size(); ++i) { data_type& c = buffer_.chunks_[i]; const std::size_t num_bytes = c.size() * sizeof(decltype(c.data())); @@ -248,16 +246,15 @@ std::cout << "receive_data" << std::endl; auto chunk_beg = 0; for(std::size_t i = 0; i < rcv_numitrs; ++i) { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; + while(!hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[src_].rcv, 1)) {} + (*(hpx::util::openshmem_environment::segments[src_].rcv)) = 0; std::memcpy(reinterpret_cast(c.data())+chunk_beg, - hpx::util::openshmem_environment::segments[idx].beg_addr, + hpx::util::openshmem_environment::segments[src_].beg_addr, data_seg[(i == rcv_numitrs_term)] ); if(i != rcv_numitrs_term) { - (*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0; chunk_beg = i * sys_pgsz; hpx::util::openshmem_environment::put_signal(nullptr, src_, nullptr, 0, hpx::util::openshmem_environment::segments[idx].xmt); @@ -295,6 +292,7 @@ std::cout << "request_done" << std::endl; connection_state state_; int src_; + int idx; header header_; buffer_type buffer_; diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 16188ed531e9..455e0235d805 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -43,6 +43,8 @@ namespace hpx::parcelset::policies::openshmem { : parcelset::parcelport_connection> { private: + using buffer_type = std::vector; + using buffer_value_type = std::vector::value_type; using sender_type = sender; using write_handler_type = @@ -67,12 +69,15 @@ namespace hpx::parcelset::policies::openshmem { : state_(initialized) , sender_(s) , dst_(dst) + , src_(0) , chunks_idx_(0) , ack_(0) , pp_(pp) , there_(parcelset::locality(locality(dst_))) { std::cout << "sender_connection_instantiated" << std::endl; + + src_ = hpx::util::openshmem_environment::rank(); } parcelset::locality const& destination() const noexcept @@ -171,7 +176,6 @@ std::cout << "sent_chunks" << std::endl; const std::size_t header_numitrs_term = header_numitrs - 1; - const auto idx = dst_; std::cout << "sendHeader0\t" << dst_ << ' ' << std::endl << std::flush; //util::openshmem_environment::scoped_lock l; //HPX_ASSERT_OWNS_LOCK(header_lock); @@ -181,18 +185,19 @@ std::cout << "sendHeader1" << std::endl << std::flush; // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment // - - for(std::size_t itr = 0; itr < header_numitrs; ++i) { + auto amt = 0; + for(std::size_t itr = 0; itr < header_numitrs; ++itr) { hpx::util::openshmem_environment::put_signal( - reinterpret_cast(header_.data()) + (itr * sys_pgsz), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, + reinterpret_cast(header_.data()) + amt, dst_, + hpx::util::openshmem_environment::segments[src_].beg_addr, data_seg[(itr == header_numitrs_term)], - hpx::util::openshmem_environment::segments[idx].rcv + hpx::util::openshmem_environment::segments[src_].rcv ); if(itr != header_numitrs_term) { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} - (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + amt = data_seg[(itr == header_numitrs_term)] * itr; + while(hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[dst_].xmt, 1)) {} + (*(hpx::util::openshmem_environment::segments[dst_].xmt)) = 0; } } } @@ -215,8 +220,6 @@ std::cout << "send_transmission_chunks" << std::endl; if (!chunks.empty()) { - const auto idx = dst_; - //util::openshmem_environment::scoped_lock l; const std::size_t sys_pgsz = @@ -237,25 +240,24 @@ std::cout << "send_transmission_chunks" << std::endl; const std::size_t header_numitrs_term = header_numitrs - 1; - const auto idx = dst_; - - util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::scoped_lock l; // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment // - - for(std::size_t itr = 0; itr < header_numitrs; ++i) { + auto amt = 0; + for(std::size_t itr = 0; itr < header_numitrs; ++itr) { hpx::util::openshmem_environment::put_signal( - reinterpret_cast(chunks.data()) + (itr * sys_pgsz), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, + reinterpret_cast(chunks.data()) + amt, dst_, + hpx::util::openshmem_environment::segments[src_].beg_addr, data_seg[(itr == header_numitrs_term)], - hpx::util::openshmem_environment::segments[idx].rcv + hpx::util::openshmem_environment::segments[src_].rcv ); if(itr != header_numitrs_term) { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} - (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + amt = data_seg[(itr == header_numitrs_term)] * itr; + while(hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[dst_].xmt, 1)) {} + (*(hpx::util::openshmem_environment::segments[dst_].xmt)) = 0; } } } @@ -275,13 +277,8 @@ std::cout << "send_data" << std::endl; if (!header_.piggy_back()) { - const auto idx = dst_; - //util::openshmem_environment::scoped_lock l; - std::cout << "is null" << idx << ' ' << (hpx::util::openshmem_environment::segments[idx].beg_addr == nullptr) << ' ' << (hpx::util::openshmem_environment::segments[idx].rcv == nullptr) << std::endl; - - const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); -std::cout << "1pagsz\t" << sys_pgsz << ' ' << (buffer_.data_.size() * sizeof(std::uint8_t)) << std::endl; + std::cout << "is null" << dst_ << ' ' << (hpx::util::openshmem_environment::segments[src_].beg_addr == nullptr) << ' ' << (hpx::util::openshmem_environment::segments[src_].rcv == nullptr) << std::endl; // issue here is page size < data size @@ -293,7 +290,7 @@ std::cout << "1pagsz\t" << sys_pgsz << ' ' << (buffer_.data_.size() * sizeof(std (sys_pgsz*page_count); const std::size_t data_amt = - buffer_.data_.size() * sizeof(decltype(buffer_.data_.data)); + buffer_.data_.size() * sizeof(buffer_value_type); std::size_t data_seg [2] = { sys_pgsz, data_amt % sys_pgsz }; @@ -302,25 +299,24 @@ std::cout << "1pagsz\t" << sys_pgsz << ' ' << (buffer_.data_.size() * sizeof(std const std::size_t header_numitrs_term = header_numitrs - 1; - const auto idx = dst_; - - util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::scoped_lock l; // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment // - - for(std::size_t itr = 0; itr < header_numitrs; ++i) { + auto amt = 0; + for(std::size_t itr = 0; itr < header_numitrs; ++itr) { hpx::util::openshmem_environment::put_signal( - reinterpret_cast(buffer_.data_.data() + (itr * sys_pgsz)), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, + reinterpret_cast(buffer_.data_.data() + amt), dst_, + hpx::util::openshmem_environment::segments[src_].beg_addr, data_seg[(itr == header_numitrs_term)], - hpx::util::openshmem_environment::segments[idx].rcv + hpx::util::openshmem_environment::segments[src_].rcv ); if(itr != header_numitrs_term) { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} - (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + amt = data_seg[(itr == header_numitrs_term)] * itr; + while(hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[dst_].xmt, 1)) {} + (*(hpx::util::openshmem_environment::segments[dst_].xmt)) = 0; } } } @@ -334,8 +330,6 @@ std::cout << "1pagsz\t" << sys_pgsz << ' ' << (buffer_.data_.size() * sizeof(std std::cout << "send_chunks" << std::endl; HPX_ASSERT(state_ == sent_data); - const auto idx = dst_; - while (chunks_idx_ < buffer_.chunks_.size()) { serialization::serialization_chunk& c = @@ -349,9 +343,6 @@ std::cout << "send_chunks" << std::endl; { //util::openshmem_environment::scoped_lock l; - const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); -std::cout << "2pagsz\t" << sys_pgsz << ' ' << (buffer_.data_.size() * sizeof(std::uint8_t)) << std::endl; - const std::size_t sys_pgsz = sysconf(_SC_PAGESIZE); const std::size_t page_count = @@ -369,23 +360,22 @@ std::cout << "2pagsz\t" << sys_pgsz << ' ' << (buffer_.data_.size() * sizeof(std const std::size_t header_numitrs_term = header_numitrs - 1; - const auto idx = dst_; - // put from this localities openshmem shared memory segment // into the remote locality (dst_)'s shared memory segment // - - for(std::size_t itr = 0; itr < header_numitrs; ++i) { + auto amt = 0; + for(std::size_t itr = 0; itr < header_numitrs; ++itr) { hpx::util::openshmem_environment::put_signal( - reinterpret_cast(c.data_.cpos_ + (itr * sys_pgsz)), dst_, - hpx::util::openshmem_environment::segments[idx].beg_addr, + reinterpret_cast(c.data_.cpos_ + amt), dst_, + hpx::util::openshmem_environment::segments[src_].beg_addr, data_seg[(itr == header_numitrs_term)], - hpx::util::openshmem_environment::segments[idx].rcv + hpx::util::openshmem_environment::segments[src_].rcv ); if(itr != header_numitrs_term) { - while(shmem_test(hpx::util::openshmem_environment::segments[idx].rcv, SHMEM_CMP_EQ, 1)) {} - (*(hpx::util::openshmem_environment::segments[idx].xmt)) = 0; + amt = data_seg[(itr == header_numitrs_term)] * itr; + while(hpx::util::openshmem_environment::test(hpx::util::openshmem_environment::segments[dst_].xmt, 1)) {} + (*(hpx::util::openshmem_environment::segments[dst_].xmt)) = 0; } } } @@ -438,6 +428,7 @@ std::cout << "request_done2" << std::endl; connection_state state_; sender_type* sender_; int dst_; + int src_; using handler_type = hpx::move_only_function; handler_type handler_; From d0f5ca02a0ee7c23bd46bcc3194785e580e85fce Mon Sep 17 00:00:00 2001 From: Alexander Strack Date: Sat, 27 Jul 2024 15:17:57 -0500 Subject: [PATCH 154/154] Fix last issues --- libs/core/openshmem_base/src/openshmem_environment.cpp | 2 +- .../include/hpx/parcelport_openshmem/sender_connection.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp index 8ffd0aa6446d..b0647022099f 100644 --- a/libs/core/openshmem_base/src/openshmem_environment.cpp +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -328,7 +328,7 @@ std::cout << "7 parcelport constructor" << std::endl << std::flush; shmem_fence(); shmem_uint8_put(sigaddr, sigaddr, 1, node); #else - shmem_uint8_put_signal(raddr, addr, size, sigaddr, 1, SHMEM_SIGNAL_SET, node); + shmem_uint8_put_signal(raddr, addr, size, reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, node); #endif } } diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp index 455e0235d805..57bcb2ededc4 100644 --- a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -366,7 +366,7 @@ std::cout << "send_chunks" << std::endl; auto amt = 0; for(std::size_t itr = 0; itr < header_numitrs; ++itr) { hpx::util::openshmem_environment::put_signal( - reinterpret_cast(c.data_.cpos_ + amt), dst_, + reinterpret_cast(const_cast(c.data_.cpos_)) + amt, dst_, hpx::util::openshmem_environment::segments[src_].beg_addr, data_seg[(itr == header_numitrs_term)], hpx::util::openshmem_environment::segments[src_].rcv