openshmem parcelport

.jenkins/lsu/env-clang-12.sh

@@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DCMAKE_C_COMPILER=clang"
 configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
 configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"

.jenkins/lsu/env-clang-13.sh

@@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DCMAKE_C_COMPILER=clang"
 configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
 configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"

.jenkins/lsu/env-clang-14.sh

@@ -25,8 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
-configure_extra_options+=" -DCMAKE_C_COMPILER=clang"
-configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"
 configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE"
 configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON"

.jenkins/lsu/env-clang-15.sh

@@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DCMAKE_C_COMPILER=clang"
 configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
 configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"

.jenkins/lsu/env-clang-16.sh

@@ -27,6 +27,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
 configure_extra_options+=" -DCMAKE_C_COMPILER=clang"
 configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"
 configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE"
 configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON"

.jenkins/lsu/env-clang-17.sh

@@ -27,6 +27,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
 configure_extra_options+=" -DCMAKE_C_COMPILER=clang"
 configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"
 configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE"
 configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON"

.jenkins/lsu/env-gcc-10.sh

@@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DCMAKE_C_COMPILER=gcc"
 configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
 

.jenkins/lsu/env-gcc-12.sh

@@ -24,6 +24,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DCMAKE_C_COMPILER=gcc"
 configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
 configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE"

.jenkins/lsu/env-gcc-13.sh

@@ -24,6 +24,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON"
+configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi"
 configure_extra_options+=" -DCMAKE_C_COMPILER=gcc"
 configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC"
 configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE"

CMakeLists.txt

@@ -1228,6 +1228,25 @@ if(HPX_WITH_NETWORKING)
     )
   endif()
 
+  hpx_option(
+    HPX_WITH_PARCELPORT_OPENSHMEM BOOL "Enable the OPENSHMEM based parcelport."
+    OFF CATEGORY "Parcelport"
+  )
+  hpx_option(
+    HPX_WITH_FETCH_OPENSHMEM BOOL
+    "Download an OpenSHMEM if one is not available" OFF
+    CATEGORY "Build Targets"
+    ADVANCED
+  )
+  if(HPX_WITH_PARCELPORT_OPENSHMEM)
+    hpx_add_config_define(HPX_HAVE_PARCELPORT_OPENSHMEM)
+    hpx_option(
+      HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT STRING
+      "Define which conduit to use for the OpenSHMEM parcelport" "sos"
+      STRINGS "sos;ucx;mpi"
+    )
+  endif()
+
   hpx_option(
     HPX_WITH_PARCELPORT_TCP BOOL "Enable the TCP based parcelport." ON
     CATEGORY "Parcelport"
@@ -2238,6 +2257,10 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET)
   include(HPX_SetupGasnet)
   hpx_setup_gasnet()
 endif()
+if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM)
+  include(HPX_SetupOpenSHMEM)
+  hpx_setup_openshmem()
+endif()
 
 # Setup packages and subprojects
 include(HPX_SetupSYCL)

cmake/FindOpenShmemPmi.cmake

@@ -0,0 +1,65 @@
+# Copyright (c)      2023 Christopher Taylor
+#
+# SPDX-License-Identifier: BSL-1.0
+# Distributed under the Boost Software License, Version 1.0. (See accompanying
+# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+find_package(PkgConfig QUIET)
+# look for cray pmi...
+pkg_check_modules(PC_PMI_CRAY QUIET cray-pmi)
+# look for the rest if we couldn't find the cray package
+if(NOT PC_PMI_CRAY_FOUND)
+  pkg_check_modules(PC_PMI QUIET pmi)
+endif()
+
+find_path(
+  PMI_INCLUDE_DIR pmi2.h
+  HINTS ${PMI_ROOT}
+        ENV
+        PMI_ROOT
+        ${PMI_DIR}
+        ENV
+        PMI_DIR
+        ${PC_PMI_CRAY_INCLUDEDIR}
+        ${PC_PMI_CRAY_INCLUDE_DIRS}
+        ${PC_PMI_INCLUDEDIR}
+        ${PC_PMI_INCLUDE_DIRS}
+  PATH_SUFFIXES include
+)
+
+find_library(
+  PMI_LIBRARY
+  NAMES pmi
+  HINTS ${PMI_ROOT}
+        ENV
+        PMI_ROOT
+        ${PC_PMI_CRAY_LIBDIR}
+        ${PC_PMI_CRAY_LIBRARY_DIRS}
+        ${PC_PMI_LIBDIR}
+        ${PC_PMI_LIBRARY_DIRS}
+  PATH_SUFFIXES lib lib64
+)
+
+# Set PMI_ROOT in case the other hints are used
+if(PMI_ROOT)
+  # The call to file is for compatibility with windows paths
+  file(TO_CMAKE_PATH ${PMI_ROOT} PMI_ROOT)
+elseif("$ENV{PMI_ROOT}")
+  file(TO_CMAKE_PATH $ENV{PMI_ROOT} PMI_ROOT)
+else()
+  file(TO_CMAKE_PATH "${PMI_INCLUDE_DIR}" PMI_INCLUDE_DIR)
+  string(REPLACE "/include" "" PMI_ROOT "${PMI_INCLUDE_DIR}")
+endif()
+
+if(NOT PMI_LIBRARY OR NOT PMI_INCLUDE_DIR)
+  set(PMI_FOUND=OFF)
+  return()
+endif()
+
+# hpx_error( "PMI_LIBRARY OR PMI_INCLUDE_DIR not found, please install PMI or
+# set \ the right PMI_ROOT path" )
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(PMI DEFAULT_MSG PMI_LIBRARY PMI_INCLUDE_DIR)
+
+mark_as_advanced(PMI_ROOT PMI_LIBRARY PMI_INCLUDE_DIR)

cmake/HPX_AddTest.cmake

@@ -5,8 +5,14 @@
 # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
 function(add_hpx_test category name)
-  set(options FAILURE_EXPECTED RUN_SERIAL NO_PARCELPORT_TCP NO_PARCELPORT_MPI
-              NO_PARCELPORT_LCI NO_PARCELPORT_GASNET
+  set(options
+      FAILURE_EXPECTED
+      RUN_SERIAL
+      NO_PARCELPORT_TCP
+      NO_PARCELPORT_MPI
+      NO_PARCELPORT_LCI
+      NO_PARCELPORT_GASNET
+      NO_PARCELPORT_OPENSHMEM
   )
   set(one_value_args EXECUTABLE LOCALITIES THREADS_PER_LOCALITY TIMEOUT
                      RUNWRAPPER
@@ -196,7 +202,7 @@ function(add_hpx_test category name)
       if(_add_test)
         set(_full_name "${category}.distributed.gasnet.${name}")
         add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r"
-                                              "gasnet-smp" ${args}
+                                              "srun" ${args}
         )
         set_tests_properties(
           "${_full_name}"
@@ -211,6 +217,30 @@ function(add_hpx_test category name)
         endif()
       endif()
     endif()
+    if(HPX_WITH_PARCELPORT_OPENSHMEM AND NOT ${name}_NO_PARCELPORT_OPENSHMEM)
+      set(_add_test FALSE)
+      if(DEFINED ${name}_PARCELPORTS)
+        set(PP_FOUND -1)
+        list(FIND ${name}_PARCELPORTS "openshmem" PP_FOUND)
+        if(NOT PP_FOUND EQUAL -1)
+          set(_add_test TRUE)
+        endif()
+      else()
+        set(_add_test TRUE)
+      endif()
+      if(_add_test)
+        set(_full_name "${category}.distributed.openshmem.${name}")
+        add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "openshmem" "-r"
+                                              "srun" ${args}
+        )
+        set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE)
+        if(${name}_TIMEOUT)
+          set_tests_properties(
+            "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT}
+          )
+        endif()
+      endif()
+    endif()
     if(HPX_WITH_PARCELPORT_TCP AND NOT ${${name}_NO_PARCELPORT_TCP})
       set(_add_test FALSE)
       if(DEFINED ${name}_PARCELPORTS)

cmake/HPX_SetupGasnet.cmake

@@ -119,7 +119,7 @@ macro(hpx_setup_gasnet)
         execute_process(
           COMMAND
             bash -c
-            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-udp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
+            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-udp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
           WORKING_DIRECTORY ${GASNET_DIR}
           RESULT_VARIABLE GASNET_BUILD_STATUS
           OUTPUT_FILE ${GASNET_BUILD_OUTPUT}
@@ -129,7 +129,7 @@ macro(hpx_setup_gasnet)
         execute_process(
           COMMAND
             bash -c
-            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-smp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
+            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-smp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
           WORKING_DIRECTORY ${GASNET_DIR}
           RESULT_VARIABLE GASNET_BUILD_STATUS
           OUTPUT_FILE ${GASNET_BUILD_OUTPUT}
@@ -139,7 +139,7 @@ macro(hpx_setup_gasnet)
         execute_process(
           COMMAND
             bash -c
-            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
+            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-ofi --with-ofi-home=${OFI_DIR} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
           WORKING_DIRECTORY ${GASNET_DIR}
           RESULT_VARIABLE GASNET_BUILD_STATUS
           OUTPUT_FILE ${GASNET_BUILD_OUTPUT}
@@ -149,7 +149,7 @@ macro(hpx_setup_gasnet)
         execute_process(
           COMMAND
             bash -c
-            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
+            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-ucx --with-ucx-home=${UCX_DIR} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
           WORKING_DIRECTORY ${GASNET_DIR}
           RESULT_VARIABLE GASNET_BUILD_STATUS
           OUTPUT_FILE ${GASNET_BUILD_OUTPUT}
@@ -185,7 +185,7 @@ macro(hpx_setup_gasnet)
         execute_process(
           COMMAND
             bash -c
-            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
+            "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install"
           WORKING_DIRECTORY ${GASNET_DIR}
           RESULT_VARIABLE GASNET_BUILD_STATUS
           OUTPUT_FILE ${GASNET_BUILD_OUTPUT}