Merge fixes for CUDA 12.4 and METIS 5

This works around a bug in CUDA 12.4 Thrust, adds support for version 5 to FindMETIS,
and finally fixes some CI issues.

Related PR: #1569

.gitlab-ci.yml

@@ -261,8 +261,6 @@ test/cuda110/mvapich2/gcc/cuda/debug/shared:
     SLURM_GRES: "gpu:4"
     SLURM_TIME: "02:00:00"
   dependencies: null
-  # FIXME: current slurm always reports failure even if all tests are passed.
-  allow_failure: yes
   needs: [ "build/cuda110/mvapich2/gcc/cuda/debug/shared" ]
 
 
@@ -295,8 +293,6 @@ test/cuda110/nompi/clang/cuda/release/static:
     SLURM_GRES: "gpu:4"
     SLURM_TIME: "01:30:00"
   dependencies: null
-  # FIXME: current slurm always reports failure even if all tests are passed.
-  allow_failure: yes
   needs: [ "build/cuda110/nompi/clang/cuda/release/static" ]
 
 
@@ -331,8 +327,6 @@ test/cuda110/nompi/intel/cuda/debug/static:
     SLURM_GRES: "gpu:4"
     SLURM_TIME: "02:00:00"
   dependencies: null
-  # FIXME: current slurm always reports failure even if all tests are passed.
-  allow_failure: yes
   needs: [ "build/cuda110/nompi/intel/cuda/debug/static" ]
 
 

.gitlab/scripts.yml

@@ -127,6 +127,7 @@
     # build to test
     - cd ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
     - cd ${CI_JOB_NAME/test/build}
+    - export INSTALL_PREFIX=`pwd`/install_prefix
     - |
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
     - ctest --output-on-failure --timeout 6000 ${CTEST_EXTRA_ARGS}
@@ -135,7 +136,7 @@
     - pushd test/test_install
     - ninja install
     - popd
-    - LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH ninja test_pkgconfig
+    - PKG_CONFIG_PATH=${INSTALL_PREFIX}/lib/pkgconfig:$PKG_CONFIG_PATH LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:$LD_LIBRARY_PATH ninja test_pkgconfig
   cache: []
 
 

INSTALL.md

@@ -220,6 +220,11 @@ packages can be turned off by disabling the relevant options.
 + GINKGO_BUILD_DOC=ON:
   [doxygen](https://www.doxygen.nl/) is required to build the documentation and
   additionally [graphviz](https://graphviz.org/) is required to build the class hierarchy graphs.
++ [METIS](http://glaros.dtc.umn.edu/gkhome/metis/metis/overview) is required
+  when using the `NestedDissection` reordering functionality.
+  If METIS is not found, the functionality is disabled.
++ [PAPI](https://icl.utk.edu/papi/) (>= 7.1.0) is required when using the `Papi` logger.
+  If PAPI is not found, the functionality is disabled.
 
 Ginkgo attempts to use pre-installed versions of these package if they match
 version requirements using `find_package`. Otherwise, the configuration step

cmake/Modules/FindMETIS.cmake

@@ -45,16 +45,24 @@ if (METIS_INCLUDE_DIR)
     string(REGEX REPLACE "^#define[\t ]+METIS_VER_MINOR[\t ]+([0-9]+).*" "\\1" METIS_VERSION_MINOR "${metis_version_str_minor}")
     set(METIS_VERSION ${METIS_VERSION_MAJOR}.${METIS_VERSION_MINOR})
     find_library(METIS_LIBRARY ${METIS_LIB_NAME} HINTS ${METIS_DIR} ENV METIS_DIR PATH_SUFFIXES lib lib64)
+    if (METIS_VERSION VERSION_GREATER_EQUAL 5.0)
+        find_library(METIS_GKLIB_LIBRARY GKlib HINTS ${METIS_DIR} ENV METIS_DIR PATH_SUFFIXES lib lib64)
+    else()
+        # set it to a non-empty value to satisfy find_package_handle_standard_args
+        set(METIS_GKLIB_LIBRARY libGKlib.so)
+    endif()
 endif()
 
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(METIS REQUIRED_VARS METIS_LIBRARY METIS_INCLUDE_DIR VERSION_VAR METIS_VERSION)
+find_package_handle_standard_args(METIS REQUIRED_VARS METIS_LIBRARY METIS_GKLIB_LIBRARY METIS_INCLUDE_DIR VERSION_VAR METIS_VERSION)
 
 if(METIS_FOUND)
     set(METIS_LIBRARIES ${METIS_LIBRARY})
+    set(METIS_GKLIB_LIBRARIES ${METIS_GKLIB_LIBRARY})
     set(METIS_INCLUDE_DIRS ${METIS_INCLUDE_DIR})
     unset(METIS_LIBRARY)
     unset(METIS_INCLUDE_DIR)
+    unset(METIS_GKLIB_LIBRARY)
 
     if(NOT TARGET METIS::METIS)
         add_library(METIS::METIS UNKNOWN IMPORTED)
@@ -63,5 +71,13 @@ if(METIS_FOUND)
         set_target_properties(METIS::METIS PROPERTIES
             IMPORTED_LINK_INTERFACE_LANGUAGES "C"
             IMPORTED_LOCATION "${METIS_LIBRARIES}")
+        if (METIS_VERSION VERSION_GREATER_EQUAL 5.0)
+            add_library(METIS::GKlib UNKNOWN IMPORTED)
+            set_target_properties(METIS::GKlib PROPERTIES
+                IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                IMPORTED_LOCATION "${METIS_GKLIB_LIBRARIES}")
+            set_target_properties(METIS::METIS PROPERTIES
+                IMPORTED_LINK_INTERFACE_LIBRARIES METIS::GKlib)
+        endif()
     endif()
 endif()

common/cuda_hip/base/device_matrix_data_kernels.hpp.inc

@@ -48,8 +48,9 @@ void sum_duplicates(std::shared_ptr<const DefaultExecutor> exec, size_type,
                     array<IndexType>& col_idxs)
 {
     const auto size = values.get_size();
-    const auto rows = row_idxs.get_const_data();
-    const auto cols = col_idxs.get_const_data();
+    // CUDA 12.4 has a bug that requires these pointers to be non-const
+    const auto rows = row_idxs.get_data();
+    const auto cols = col_idxs.get_data();
     auto iota = thrust::make_counting_iterator(size_type{});
     const auto new_size = static_cast<size_type>(thrust::count_if(
         thrust_policy(exec), iota, iota + size,

common/cuda_hip/multigrid/pgm_kernels.hpp.inc

@@ -36,8 +36,11 @@ void compute_coarse_coo(std::shared_ptr<const DefaultExecutor> exec,
                         matrix::Coo<ValueType, IndexType>* coarse_coo)
 {
     auto vals_it = as_device_type(vals);
-    auto key_it =
-        thrust::make_zip_iterator(thrust::make_tuple(row_idxs, col_idxs));
+    // this const_cast is necessary as a workaround for CCCL bug
+    // https://github.com/NVIDIA/cccl/issues/1527
+    // shipped with CUDA 12.4
+    auto key_it = thrust::make_zip_iterator(thrust::make_tuple(
+        const_cast<IndexType*>(row_idxs), const_cast<IndexType*>(col_idxs)));
 
     auto coarse_vals_it = as_device_type(coarse_coo->get_values());
     auto coarse_key_it = thrust::make_zip_iterator(thrust::make_tuple(

reference/test/reorder/CMakeLists.txt

@@ -1,5 +1,5 @@
 if(GINKGO_HAVE_METIS)
-    ginkgo_create_test(nested_dissection)
+    ginkgo_create_test(nested_dissection ADDITIONAL_LIBRARIES METIS::METIS)
 endif()
 ginkgo_create_test(rcm)
 ginkgo_create_test(rcm_kernels)