Merge CTest Resources

This uses CTest resource spec files to run GPU and OpenMP tests in a constrained parallel setting for CI. Related PR: #1373
ginkgo-project · Sep 21, 2023 · 96d01cc · 96d01cc
2 parents 1100cbd + 95240dc
commit 96d01cc
Show file tree

Hide file tree

Showing 54 changed files with 824 additions and 222 deletions.
diff --git a/.gitlab/image.yml b/.gitlab/image.yml
@@ -72,15 +72,13 @@
   image: ginkgohub/rocm:45-mvapich2-gnu8-llvm8
   tags:
     - private_ci
-    - amdci
-    - gpu
+    - amd-gpu
 
 .use_gko-rocm502-nompi-gnu11-llvm11:
   image: ginkgohub/rocm:502-openmpi-gnu11-llvm11
   tags:
     - private_ci
-    - amdci
-    - gpu
+    - amd-gpu
 
 .use_gko-oneapi-cpu:
   image: ginkgohub/oneapi:2022.1

diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml
@@ -100,7 +100,7 @@
     - awk '!/^#/ { print ($2 - $1)/1000 " " $4 }' .ninja_log | sort -nr
     - |
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
-    - ctest -V --timeout 6000
+    - ctest --output-on-failure --timeout 6000 ${CTEST_EXTRA_ARGS}
     - ninja test_install
     - pushd test/test_install
     - ninja install
@@ -152,7 +152,7 @@
     - cd ${CI_JOB_NAME/test/build}
     - |
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
-    - ctest -V --timeout 6000
+    - ctest --output-on-failure --timeout 6000 ${CTEST_EXTRA_ARGS}
     - ninja test_install
     - pushd test/test_install
     - ninja install

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -89,6 +89,8 @@ option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS)
 option(GINKGO_INSTALL_RPATH_DEPENDENCIES "Add dependencies to the installation RPATH." OFF)
 option(GINKGO_FORCE_GPU_AWARE_MPI "Assert that the MPI library is GPU aware. This forces Ginkgo to assume that GPU aware functionality is available (OFF (default) or ON), but may fail
      catastrophically in case the MPI implementation is not GPU Aware, and GPU aware functionality has been forced" OFF)
+set(GINKGO_CI_TEST_OMP_PARALLELISM "4" CACHE STRING
+    "The number of OpenMP threads to use for a test binary during CTest resource file-constrained test.")
 
 # load executor-specific configuration
 if(GINKGO_BUILD_CUDA)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
@@ -1,26 +1,19 @@
-set(gko_test_single_args "MPI_SIZE")
+set(gko_test_resource_args "RESOURCE_LOCAL_CORES;RESOURCE_TYPE")
+set(gko_test_single_args "MPI_SIZE;${gko_test_resource_args}")
 set(gko_test_multi_args "DISABLE_EXECUTORS;ADDITIONAL_LIBRARIES;ADDITIONAL_INCLUDES")
+set(gko_test_option_args "NO_RESOURCES")
 
 ## Replaces / by _ to create valid target names from relative paths
 function(ginkgo_build_test_name test_name target_name)
     file(RELATIVE_PATH REL_BINARY_DIR
-        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+         ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
     string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
     set(${target_name} ${TEST_TARGET_NAME} PARENT_SCOPE)
-endfunction(ginkgo_build_test_name)
-
-function(ginkgo_create_gtest_mpi_main)
-    add_library(gtest_mpi_main "")
-    target_sources(gtest_mpi_main
-      PRIVATE
-      ${PROJECT_SOURCE_DIR}/core/test/mpi/gtest/mpi_listener.cpp)
-    find_package(MPI 3.1 COMPONENTS CXX REQUIRED)
-    target_link_libraries(gtest_mpi_main PRIVATE GTest::GTest MPI::MPI_CXX)
-endfunction(ginkgo_create_gtest_mpi_main)
+endfunction()
 
 ## Set up shared target properties and handle ADDITIONAL_LIBRARIES/ADDITIONAL_INCLUDES
 ## `MPI_SIZE size` causes the tests to be run with `size` MPI processes.
-function(ginkgo_set_test_target_properties test_target_name)
+function(ginkgo_set_test_target_properties test_target_name test_library_suffix)
     cmake_parse_arguments(PARSE_ARGV 1 set_properties "" "${gko_test_single_args}" "${gko_test_multi_args}")
     if (GINKGO_FAST_TESTS)
         target_compile_definitions(${test_target_name} PRIVATE GINKGO_FAST_TESTS)
@@ -31,26 +24,56 @@ function(ginkgo_set_test_target_properties test_target_name)
     if (GINKGO_COMPILING_DPCPP_TEST AND GINKGO_DPCPP_SINGLE_MODE)
         target_compile_definitions(${test_target_name} PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
     endif()
-    if (GINKGO_CHECK_CIRCULAR_DEPS)
+    if(GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${test_target_name} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    if (set_properties_MPI_SIZE)
-        if(NOT TARGET gtest_mpi_main)
-            ginkgo_create_gtest_mpi_main()
-        endif()
-        set(gtest_main gtest_mpi_main MPI::MPI_CXX)
+    if(set_properties_MPI_SIZE)
+        target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main_mpi${test_library_suffix})
     else()
-        set(gtest_main GTest::Main)
+        target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main${test_library_suffix})
     endif()
     target_compile_features(${test_target_name} PUBLIC cxx_std_14)
     target_compile_options(${test_target_name} PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${GINKGO_COMPILER_FLAGS}>)
     target_include_directories(${test_target_name} PRIVATE ${Ginkgo_BINARY_DIR} ${set_properties_ADDITIONAL_INCLUDES})
-    target_link_libraries(${test_target_name} PRIVATE ginkgo ${gtest_main} GTest::GTest ${set_properties_ADDITIONAL_LIBRARIES})
+    target_link_libraries(${test_target_name} PRIVATE ginkgo GTest::GTest ${set_properties_ADDITIONAL_LIBRARIES})
+endfunction()
+
+function(ginkgo_add_resource_requirement test_name)
+    cmake_parse_arguments(PARSE_ARGV 1 add_rr "${gko_test_option_args}" "${gko_test_single_args}" "")
+    if(add_rr_NO_RESOURCES OR (NOT add_rr_RESOURCE_TYPE))
+        return()
+    endif ()
+
+    if(add_rr_RESOURCE_TYPE STREQUAL "cpu")
+        if(NOT add_rr_RESOURCE_LOCAL_CORES)
+            set(add_rr_RESOURCE_LOCAL_CORES ${GINKGO_CI_TEST_OMP_PARALLELISM})
+        endif()
+        if(NOT add_rr_RESOURCE_LOCAL_CORES MATCHES "^[0-9]+")
+            message(FATAL_ERROR "Resource specification is invalid: RESOURCE_LOCAL_CORES=${add_rr_RESOURCE_LOCAL_CORES}")
+        endif()
+
+        set(single_resource "cpu:${add_rr_RESOURCE_LOCAL_CORES}")
+    elseif(add_rr_RESOURCE_TYPE MATCHES "^(cudagpu|hipgpu|sycl)$")
+        set(single_resource "${add_rr_RESOURCE_TYPE}:1")
+    else()
+        message(FATAL_ERROR "Unrecognized resource type ${add_rr_RESOURCE_TYPE}, allowed are: cpu, cudagpu, hipgpu, sycl.")
+    endif()
+
+    if(NOT add_rr_MPI_SIZE)
+        set(add_rr_MPI_SIZE 1)
+    endif()
+    set_property(TEST ${test_name}
+                 PROPERTY
+                 RESOURCE_GROUPS "${add_rr_MPI_SIZE},${single_resource}")
 endfunction()
 
+
 ## Adds a test to the list executed by ctest and sets its output binary name
 ## Possible additional arguments:
 ## - `MPI_SIZE size` causes the tests to be run with `size` MPI processes.
+## - `RESOURCE_LOCAL_CORES` the number of threads used by a test, default is
+##    $GINKGO_CI_TEST_OMP_PARALLELISM
+## - `RESOURCE_TYPE` the resource type, can be cpu, cudagpu, hipgpu, sycl
 ## - `DISABLE_EXECUTORS exec1 exec2` disables the test for certain backends (if built for multiple)
 ## - `ADDITIONAL_LIBRARIES lib1 lib2` adds additional target link dependencies
 ## - `ADDITIONAL_INCLUDES path1 path2` adds additional target include paths
@@ -71,6 +94,9 @@ function(ginkgo_add_test test_name test_target_name)
                  COMMAND ${test_target_name}
                  WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
     endif()
+
+    ginkgo_add_resource_requirement(${REL_BINARY_DIR}/${test_name} ${ARGN})
+
     set(test_preload)
     if (GINKGO_TEST_NONDEFAULT_STREAM AND GINKGO_BUILD_CUDA)
         set(test_preload $<TARGET_FILE:identify_stream_usage_cuda>:${test_preload})
@@ -87,8 +113,8 @@ endfunction()
 function(ginkgo_create_test test_name)
     ginkgo_build_test_name(${test_name} test_target_name)
     add_executable(${test_target_name} ${test_name}.cpp)
-    target_link_libraries(${test_target_name} PRIVATE ${create_test_ADDITIONAL_LIBRARIES})
-    ginkgo_set_test_target_properties(${test_target_name} ${ARGN})
+    target_link_libraries(${test_target_name})
+    ginkgo_set_test_target_properties(${test_target_name} "_cpu" ${ARGN})
     ginkgo_add_test(${test_name} ${test_target_name} ${ARGN})
 endfunction(ginkgo_create_test)
 
@@ -99,8 +125,8 @@ function(ginkgo_create_dpcpp_test test_name)
     target_compile_features(${test_target_name} PUBLIC cxx_std_17)
     target_compile_options(${test_target_name} PRIVATE ${GINKGO_DPCPP_FLAGS})
     target_link_options(${test_target_name} PRIVATE -fsycl-device-code-split=per_kernel)
-    ginkgo_set_test_target_properties(${test_target_name} ${ARGN})
-    ginkgo_add_test(${test_name} ${test_target_name} ${ARGN})
+    ginkgo_set_test_target_properties(${test_target_name} "_dpcpp" ${ARGN})
+    ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE sycl)
     # Note: MKL_ENV is empty on linux. Maybe need to apply MKL_ENV to all test.
     if (MKL_ENV)
         set_tests_properties(${test_target_name} PROPERTIES ENVIRONMENT "${MKL_ENV}")
@@ -133,8 +159,8 @@ function(ginkgo_create_cuda_test_internal test_name filename test_target_name)
     if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
         set_target_properties(${test_target_name} PROPERTIES CUDA_ARCHITECTURES OFF)
     endif()
-    ginkgo_set_test_target_properties(${test_target_name} ${ARGN})
-    ginkgo_add_test(${test_name} ${test_target_name} ${ARGN})
+    ginkgo_set_test_target_properties(${test_target_name} "_cuda" ${ARGN})
+    ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE cudagpu)
 endfunction(ginkgo_create_cuda_test_internal)
 
 ## Test compiled with HIP
@@ -189,10 +215,26 @@ function(ginkgo_create_hip_test_internal test_name filename test_target_name add
         ${hiprand_INCLUDE_DIRS}
         ${HIPSPARSE_INCLUDE_DIRS}
         )
-    ginkgo_set_test_target_properties(${test_target_name} ${ARGN})
-    ginkgo_add_test(${test_name} ${test_target_name} ${ARGN})
+    ginkgo_set_test_target_properties(${test_target_name} "_hip" ${ARGN})
+    ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE hipgpu)
 endfunction(ginkgo_create_hip_test_internal)
 
+
+## Test compiled with OpenMP
+function(ginkgo_create_omp_test test_name)
+    ginkgo_build_test_name(${test_name} test_target_name)
+    ginkgo_create_omp_test_internal(${test_name} ${test_name}.cpp ${test_target_name} "" ${ARGN})
+endfunction()
+
+function(ginkgo_create_omp_test_internal test_name filename test_target_name)
+    ginkgo_build_test_name(${test_name} test_target_name)
+    add_executable(${test_target_name} ${test_name}.cpp)
+    target_compile_definitions(${test_target_name} PRIVATE GKO_COMPILING_OMP)
+    target_link_libraries(${test_target_name} PRIVATE OpenMP::OpenMP_CXX)
+    ginkgo_set_test_target_properties(${test_target_name} "_omp" ${ARGN})
+    ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE cpu)
+endfunction()
+
 ## Common test compiled with the host compiler, one target for each enabled backend
 function(ginkgo_create_common_test test_name)
     if(GINKGO_BUILD_OMP)
@@ -214,20 +256,38 @@ function(ginkgo_create_common_test_internal test_name exec_type exec)
     if(exec IN_LIST common_test_DISABLE_EXECUTORS)
         return()
     endif()
+    if (exec STREQUAL reference)
+        set(test_resource_type "")
+    elseif (exec STREQUAL omp)
+        set(test_resource_type cpu)
+    elseif (exec STREQUAL cuda)
+        set(test_resource_type cudagpu)
+    elseif (exec STREQUAL hip)
+        set(test_resource_type hipgpu)
+    else ()
+        set(test_resource_type sycl)
+    endif ()
     ginkgo_build_test_name(${test_name} test_target_name)
     string(TOUPPER ${exec} exec_upper)
+
     # set up actual test
     set(test_target_name ${test_target_name}_${exec})
     add_executable(${test_target_name} ${test_name}.cpp)
+
+    # also need to add runtime libraries for other backends
+    if (exec STREQUAL omp)
+        target_link_libraries(${test_target_name} PRIVATE OpenMP::OpenMP_CXX)
+    endif ()
+
     target_compile_definitions(${test_target_name} PRIVATE EXEC_TYPE=${exec_type} EXEC_NAMESPACE=${exec} GKO_COMPILING_${exec_upper})
     target_link_libraries(${test_target_name} PRIVATE ${common_test_ADDITIONAL_LIBRARIES})
     # use float for DPC++ if necessary
     if((exec STREQUAL "dpcpp") AND GINKGO_DPCPP_SINGLE_MODE)
         target_compile_definitions(${test_target_name} PRIVATE GINKGO_COMMON_SINGLE_MODE=1)
         target_compile_definitions(${test_target_name} PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
     endif()
-    ginkgo_set_test_target_properties(${test_target_name} ${ARGN})
-    ginkgo_add_test(${test_name}_${exec} ${test_target_name} ${ARGN})
+    ginkgo_set_test_target_properties(${test_target_name} "_${exec}" ${ARGN})
+    ginkgo_add_test(${test_name}_${exec} ${test_target_name} ${ARGN} RESOURCE_TYPE ${test_resource_type})
 endfunction(ginkgo_create_common_test_internal)
 
 ## Common test compiled with the device compiler, one target for each enabled backend

diff --git a/core/device_hooks/omp_hooks.cpp b/core/device_hooks/omp_hooks.cpp
@@ -31,6 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/scoped_device_id_guard.hpp>
 #include <ginkgo/core/base/version.hpp>
 
@@ -51,6 +52,9 @@ scoped_device_id_guard::scoped_device_id_guard(const OmpExecutor* exec,
     GKO_NOT_COMPILED(omp);
 
 
+int OmpExecutor::get_num_omp_threads() { return 1; }
+
+
 }  // namespace gko
 
 

diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt
@@ -1,5 +1,6 @@
 include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
+add_subdirectory(gtest)
 add_subdirectory(accessor)
 add_subdirectory(base)
 add_subdirectory(components)

diff --git a/core/test/gtest/CMakeLists.txt b/core/test/gtest/CMakeLists.txt
@@ -0,0 +1,32 @@
+function(add_gtest_main suffix definitions)
+    add_library(ginkgo_gtest_main${suffix} STATIC ginkgo_main.cpp resources.cpp)
+    target_link_libraries(ginkgo_gtest_main${suffix} PUBLIC Ginkgo::ginkgo GTest::GTest)
+    target_compile_definitions(ginkgo_gtest_main${suffix} PRIVATE ${definitions})
+    ginkgo_compile_features(ginkgo_gtest_main${suffix})
+    if (GINKGO_BUILD_MPI)
+        add_library(ginkgo_gtest_main_mpi${suffix} STATIC ginkgo_mpi_main.cpp resources.cpp)
+        target_link_libraries(ginkgo_gtest_main_mpi${suffix} PUBLIC Ginkgo::ginkgo GTest::GTest MPI::MPI_CXX)
+        target_compile_definitions(ginkgo_gtest_main_mpi${suffix} PRIVATE ${definitions})
+        ginkgo_compile_features(ginkgo_gtest_main_mpi${suffix})
+    endif()
+endfunction()
+
+add_gtest_main("" "")
+add_library(ginkgo_gtest_main_reference ALIAS ginkgo_gtest_main)
+add_library(ginkgo_gtest_main_cpu ALIAS ginkgo_gtest_main)
+if (GINKGO_BUILD_MPI)
+    add_library(ginkgo_gtest_main_mpi_reference ALIAS ginkgo_gtest_main_mpi)
+    add_library(ginkgo_gtest_main_mpi_cpu ALIAS ginkgo_gtest_main_mpi)
+endif()
+if (GINKGO_BUILD_OMP)
+    add_gtest_main("_omp" "GKO_COMPILING_OMP")
+endif()
+if (GINKGO_BUILD_CUDA)
+    add_gtest_main("_cuda" "GKO_COMPILING_CUDA")
+endif()
+if (GINKGO_BUILD_HIP)
+    add_gtest_main("_hip" "GKO_COMPILING_HIP")
+endif()
+if (GINKGO_BUILD_DPCPP)
+    add_gtest_main("_dpcpp" "GKO_COMPILING_DPCPP")
+endif()