Feature/corona pipeline tpls (#276)

Adds another GitLab CI pipeline job for Corona testing Sundials with Ginkgo, Kokkos, and Kokkos-kernels with Rocm 5.2.3. Notes - The Ginkgo dense tests are commented out. - The sunlinsol kokkosdense test occasionally fails due to the error being just outside the tolerance. --------- Co-authored-by: David J. Gardner <[email protected]> Co-authored-by: Cody J. Balos <[email protected]>
LLNL · Jul 18, 2023 · d4f8a12 · d4f8a12
1 parent 3247363
commit d4f8a12
Show file tree

Hide file tree

Showing 22 changed files with 90 additions and 209 deletions.
diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh
@@ -48,14 +48,15 @@ BUILD_JOBS=${BUILD_JOBS:-"1"}
 
 # load newer python to try the clingo concretizer
 # machine specific loads
-if [[ "${hostname}" == "corona" ]]; then
+if [[ "${hostname}" == "lassen" ]]; then
+    echo "module load python/3.8.2"
+    module load python/3.8.2
+elif [[ "${hostname}" == "corona" ]]; then
     echo "module load python/3.9.12"
     module load python/3.9.12
-    echo "module load rocm/5.4.1"
-    module load rocm/5.4.1
 else
-    echo "module load python/3.8.2"
-    module load python/3.8.2
+    echo "module load python"
+    module load python
 fi
 
 if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]]

diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml
@@ -14,11 +14,11 @@
 # HIP
 # ------------------------------------------------------------------------------
 
-# Builds with Hip
-corona_clang_hip:
+# Builds with HIP
+corona_rocmcc_550:
   parallel:
     matrix:
-      - COMPILER_SPEC: rocmcc@5.4.1
+      - COMPILER_SPEC: rocmcc@5.5.0
         AMDGPU_TARGET: [gfx906]
   variables:
     SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi"
@@ -27,11 +27,22 @@ corona_clang_hip:
 # ------------------------------------------------------------------------------
 # HIP + TPLs
 # ------------------------------------------------------------------------------
-corona_clang_hip_tpls:
+corona_rocmcc_550_tpls:
   parallel:
     matrix:
-      - COMPILER_SPEC: rocmcc@5.4.1
+      - COMPILER_SPEC: rocmcc@5.5.0
         AMDGPU_TARGET: [gfx906]
+        ROCM_VERSION: 5.5.0
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}"
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja+kokkos+kokkos-kernels~ginkgo ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm~profiling amdgpu_target=${AMDGPU_TARGET} ^hipblas@${ROCM_VERSION} ^hipsparse@${ROCM_VERSION} ^hip@${ROCM_VERSION} ^hsa-rocr-dev@${ROCM_VERSION} ^llvm-amdgpu@${ROCM_VERSION}"
+  extends: .corona_build_and_test
+
+corona_rocmcc_523_tpls:
+  parallel:
+    matrix:
+      - COMPILER_SPEC: [email protected]
+        AMDGPU_TARGET: [gfx906]
+        ROCM_VERSION: 5.2.3
+  variables:
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja+kokkos+kokkos-kernels+ginkgo ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm~profiling amdgpu_target=${AMDGPU_TARGET} ^ginkgo+rocm amdgpu_target=${AMDGPU_TARGET} ^hipblas@${ROCM_VERSION} ^hipsparse@${ROCM_VERSION} ^rocrand@${ROCM_VERSION} ^rocthrust@${ROCM_VERSION} ^hip@${ROCM_VERSION} ^hsa-rocr-dev@${ROCM_VERSION} ^llvm-amdgpu@${ROCM_VERSION} ^rocprim@${ROCM_VERSION}"
   extends: .corona_build_and_test
diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml
@@ -10,75 +10,23 @@
 # SUNDIALS Copyright End
 # ------------------------------------------------------------------------------
 
-# ------------------------------------------------------------------------------
-# CPU ONLY
-# ------------------------------------------------------------------------------
-
-# Builds without tpls
-# lassen_gcc:
-#   parallel:
-#     matrix:
-#       - COMPILER_SPEC: [email protected]
-#         INDEX_SPEC: [~int64, +int64]
-#         PRECISION_SPEC: [double]
-#   variables:
-#     SPEC: "%${COMPILER_SPEC} ${INDEX_SPEC} precision=${PRECISION_SPEC} ~cuda"
-#   extends: .lassen_build_and_test
-
-# lassen_xl:
-#   parallel:
-#     matrix:
-#       - COMPILER_SPEC: [email protected]
-#         INDEX_SPEC: [~int64, +int64]
-#         PRECISION_SPEC: [double]
-#   variables:
-#     SPEC: "%${COMPILER_SPEC} ${INDEX_SPEC} precision=${PRECISION_SPEC} ~cuda"
-#   extends: .lassen_build_and_test
-
-# ------------------------------------------------------------------------------
-# CUDA
-# ------------------------------------------------------------------------------
-
-# # Builds with CUDA, RAJA and other TPLs
-# lassen_xl_cuda:
-#   parallel:
-#     matrix:
-#       - COMPILER_SPEC: [email protected]
-#         CUDA_SPEC: [[email protected], [email protected]]
-#   variables:
-#     SPEC: "%${COMPILER_SPEC} ~int64 precision=double +cuda cuda_arch=70 ^${CUDA_SPEC}"
-#   extends: .lassen_build_and_test
-
-# lassen_gcc_cuda:
-#   parallel:
-#     matrix:
-#       - COMPILER_SPEC: [email protected]
-#         CUDA_SPEC: [[email protected], [email protected]]
-#   variables:
-#     SPEC: "%${COMPILER_SPEC} ~int64 precision=double +cuda cuda_arch=70 ^${CUDA_SPEC}"
-#   extends: .lassen_build_and_test
-
-# ------------------------------------------------------------------------------
-# CUDA + Extras
-# ------------------------------------------------------------------------------
-
 # Builds with CUDA, RAJA and other TPLs
-lassen_xl_cuda_tpls:
+lassen_cuda_no_tpls:
   parallel:
     matrix:
-      - COMPILER_SPEC: [email protected].10
+      - COMPILER_SPEC: [[email protected].14]
         CUDA_SPEC: [[email protected]]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}"
+      SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja~magma~superlu-dist~petsc~hypre~ginkgo cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers"
   extends: .lassen_build_and_test
 
 lassen_gcc_cuda_tpls:
   parallel:
     matrix:
       - COMPILER_SPEC: [email protected]
-        CUDA_SPEC: [cuda@11.5.0]
+        CUDA_SPEC: [cuda@11.8.0]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist+petsc+hypre+ginkgo cuda_arch=70 ^ginkgo+cuda cuda_arch=70 ^hypre ^petsc+cuda cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}"
+      SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist+petsc+hypre+ginkgo cuda_arch=70 ^ginkgo+cuda cuda_arch=70 ^hypre~cuda ^petsc~cuda ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers"
   extends: .lassen_build_and_test
 
 # ------------------------------------------------------------------------------
@@ -89,7 +37,7 @@ lassen_gcc_cuda_bench:
   parallel:
     matrix:
       - COMPILER_SPEC: [email protected]
-        CUDA_SPEC: [cuda@11.5.0]
+        CUDA_SPEC: [cuda@11.8.0]
   variables:
     SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double ~int64 +benchmarks+profiling+caliper +mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+cuda cuda_arch=70 ^${CUDA_SPEC}"
   extends: .lassen_build_and_bench

diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml
@@ -29,7 +29,7 @@
 # quartz_gcc:
 #   parallel:
 #     matrix:
-#       - COMPILER_SPEC: gcc@10.2.1
+#       - COMPILER_SPEC: gcc@10.3.1
 #         INDEX_SPEC: [~int64, +int64]
 #         PRECISION_SPEC: [double]
 #   variables:
@@ -39,7 +39,7 @@
 # quartz_intel:
 #   parallel:
 #     matrix:
-#       - COMPILER_SPEC: intel@19.0.4
+#       - COMPILER_SPEC: intel@19.1.2
 #         INDEX_SPEC: [~int64, +int64]
 #         PRECISION_SPEC: [double]
 #   variables:
@@ -55,7 +55,7 @@
 quartz_clang_tpls:
   parallel:
     matrix:
-      - COMPILER_SPEC: clang@12.0.1
+      - COMPILER_SPEC: clang@14.0.6
         INDEX_SPEC: [~int64]
         PRECISION_SPEC: [double]
   variables:
@@ -65,19 +65,21 @@ quartz_clang_tpls:
 quartz_gcc_tpls:
   parallel:
     matrix:
-      - COMPILER_SPEC: gcc@10.2.1
+      - COMPILER_SPEC: gcc@10.3.1
         INDEX_SPEC: [~int64]
         PRECISION_SPEC: [double]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^[email protected]"
+    # For some reason nvhpc gets picked up for lapack w/o ^openblas
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^[email protected] ^openblas"
   extends: .quartz_build_and_test
 
 quartz_intel_tpls:
   parallel:
     matrix:
-      - COMPILER_SPEC: intel@19.0.4
+      - COMPILER_SPEC: intel@19.1.2
         INDEX_SPEC: [~int64]
         PRECISION_SPEC: [double]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu ^[email protected]"
+    # [email protected] needed until spack v0.20.0: https://github.com/spack/spack/pull/35666 
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu ^[email protected] ^[email protected]"
   extends: .quartz_build_and_test
diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs
diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py
@@ -76,7 +76,7 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage):
         "cxxstd",
         default="14",
         description="C++ language standard",
-        values=("99", "11", "14", "17"),
+        values=("14", "17"),
     )
 
     # Logging
@@ -181,7 +181,7 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage):
     )
 
     # Scheduler
-    variant("scheduler", default="slurm", description="Specify which scheduler the system runs on.", values=("flux", "lsf", "slurm"))
+    variant("scheduler", default="default", description="Specify which scheduler the system runs on.", values=("flux", "lsf", "slurm", "default"))
 
     # ==========================================================================
     # Dependencies
@@ -764,7 +764,9 @@ def initconfig_package_entries(self):
                 self.cache_option_from_variant("RAJA_ENABLE", "raja"),
                 self.cache_option_from_variant("SUPERLUDIST_ENABLE", "superlu-dist"),
                 self.cache_option_from_variant("SUPERLUMT_ENABLE", "superlu-mt"),
-                self.cache_option_from_variant("Trilinos_ENABLE", "trilinos")
+                self.cache_option_from_variant("Trilinos_ENABLE", "trilinos"),
+                self.cache_option_from_variant("ENABLE_KOKKOS", "kokkos"),
+                self.cache_option_from_variant("ENABLE_KOKKOS_KERNELS", "kokkos-kernels")
             ]
         )
 
@@ -805,9 +807,9 @@ def initconfig_package_entries(self):
 
         # Building with Kokkos and KokkosKernels
         if "+kokkos" in spec:
-            entries.extend([self.cache_option_from_variant("Kokkos_DIR", spec["kokkos"].prefix)])
+            entries.extend([cmake_cache_path("Kokkos_DIR", spec["kokkos"].prefix)])
         if "+kokkos-kernels" in spec:
-            entries.extend([self.cache_option_from_variant("KokkosKernels_DIR", spec["kokkos-kernels"].prefix)])
+            entries.extend([cmake_cache_path("KokkosKernels_DIR", spec["kokkos-kernels"].prefix)])
 
         # Building with KLU
         if "+klu" in spec:
@@ -837,6 +839,11 @@ def initconfig_package_entries(self):
         if "+petsc" in spec:
             if spec.version >= Version("5"):
                 entries.append(cmake_cache_path("PETSC_DIR", spec["petsc"].prefix))
+                if "+kokkos" in spec["petsc"]:
+                    entries.extend([
+                        cmake_cache_path("Kokkos_DIR", spec["kokkos"].prefix),
+                        cmake_cache_path("KokkosKernels_DIR", spec["kokkos-kernels"].prefix)
+                    ])
             else:
                 entries.extend(
                     [

diff --git a/cmake/SundialsSetupCXX.cmake b/cmake/SundialsSetupCXX.cmake
@@ -24,6 +24,11 @@ set(CXX_FOUND TRUE)
 # compiler and related options.
 # ---------------------------------------------------------------
 
+# Do not allow decaying to previous standards -- generates error if the standard
+# is not supported
+sundials_option(CMAKE_CXX_STANDARD_REQUIRED BOOL
+  "Require C++ standard version" ON)
+
 if(ENABLE_SYCL)
   set(DOCSTR "The C++ standard to use if C++ is enabled (17, 20)")
   sundials_option(CMAKE_CXX_STANDARD STRING "${DOCSTR}" "17"

diff --git a/cmake/SundialsSetupCuda.cmake b/cmake/SundialsSetupCuda.cmake
@@ -33,6 +33,11 @@ endif()
 # Configure the CUDA flags
 # ===============================================================
 
+# Do not allow decaying to previous standards -- generates error if the standard
+# is not supported
+sundials_option(CMAKE_CUDA_STANDARD_REQUIRED BOOL
+  "Require C++ standard version" ON)
+
 set(DOCSTR "The CUDA standard to use if CUDA is enabled (14, 17, 20)")
 sundials_option(CMAKE_CUDA_STANDARD STRING "${DOCSTR}" "${CMAKE_CXX_STANDARD}"
                 OPTIONS "14;17;20")

diff --git a/examples/arkode/CXX_parhyp/CMakeLists.txt b/examples/arkode/CXX_parhyp/CMakeLists.txt
@@ -30,17 +30,6 @@ set(ARKODE_extras
   plot_heat2D_p.py
   )
 
-if(MPI_CXX_COMPILER)
-  # use MPI wrapper as the compiler
-  set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
-
-  # disable C++ extensions (for known wrappers)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -DLAM_BUILDING")
-else()
-  # add MPI_INCLUDE_PATH to include directories
-  include_directories(${MPI_INCLUDE_PATH})
-endif()
-
 # Specify libraries to link against
 set(ARKODE_LIB sundials_arkode)
 set(NVECP_LIB sundials_nvecparallel)
@@ -69,11 +58,8 @@ foreach(example_tuple ${ARKODE_examples})
     set_target_properties(${example} PROPERTIES FOLDER "Examples")
 
     # libraries to link against
-    target_link_libraries(${example} ${SUNDIALS_LIBS} SUNDIALS::HYPRE)
-
-    if(NOT MPI_CXX_COMPILER)
-      target_link_libraries(${example} ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARIES})
-    endif()
+    target_link_libraries(${example} PRIVATE 
+      MPI::MPI_CXX ${SUNDIALS_LIBS} SUNDIALS::HYPRE)
   endif()
 
   # check if example args are provided and set the test name

diff --git a/examples/arkode/CXX_serial/CMakeLists.txt b/examples/arkode/CXX_serial/CMakeLists.txt
@@ -69,8 +69,6 @@ foreach(example_tuple ${ARKODE_examples})
 
     set_target_properties(${example_target} PROPERTIES FOLDER "Examples")
 
-    target_compile_features(${example_target} PRIVATE cxx_std_14)
-
     # directories to include
     target_include_directories(${example_target}
       PRIVATE

diff --git a/examples/arkode/CXX_superludist/CMakeLists.txt b/examples/arkode/CXX_superludist/CMakeLists.txt
@@ -23,16 +23,6 @@ list(APPEND ARKODE_examples "ark_brusselator1D_FEM_sludist.cpp\;1\;1\;develop")
 set(ARKODE_extras
   )
 
-if(MPI_CXX_COMPILER)
-  # use MPI wrapper as the compiler
-  set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
-  # disable C++ extensions (for known wrappers)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -DLAM_BUILDING")
-else()
-  # add MPI_INCLUDE_PATH to include directories
-  include_directories(${MPI_INCLUDE_PATH})
-endif()
-
 # Add the build and install targets for each example
 foreach(example_tuple ${ARKODE_examples})
 
@@ -59,6 +49,7 @@ foreach(example_tuple ${ARKODE_examples})
 
   # libraries to link against
   target_link_libraries(${example_target} PRIVATE
+    MPI::MPI_CXX
     sundials_arkode
     sundials_sunlinsolsuperludist)
 

diff --git a/examples/arkode/CXX_xbraid/CMakeLists.txt b/examples/arkode/CXX_xbraid/CMakeLists.txt
@@ -118,13 +118,8 @@ if(ENABLE_HYPRE AND HYPRE_FOUND)
       set_target_properties(${example} PROPERTIES FOLDER "Examples")
 
       # libraries to link against
-      target_link_libraries(${example} ${SUNDIALS_LIBS})
-      target_link_libraries(${example} SUNDIALS::HYPRE)
-      target_link_libraries(${example} SUNDIALS::XBRAID)
-
-      if(NOT MPI_CXX_COMPILER)
-        target_link_libraries(${example} ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARIES})
-      endif()
+      target_link_libraries(${example} PRIVATE
+        MPI::MPI_CXX ${SUNDIALS_LIBS}  SUNDIALS::HYPRE SUNDIALS::XBRAID)
 
     endif()
 

diff --git a/examples/cvode/CXX_parallel/CMakeLists.txt b/examples/cvode/CXX_parallel/CMakeLists.txt
@@ -27,17 +27,6 @@ set(CVODE_extras
   plot_heat2D_p.py
   )
 
-if(MPI_CXX_COMPILER)
-  # use MPI wrapper as the compiler
-  set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
-
-  # disable C++ extensions (for known wrappers)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -DLAM_BUILDING")
-else()
-  # add MPI_INCLUDE_PATH to include directories
-  include_directories(${MPI_INCLUDE_PATH})
-endif()
-
 # Specify libraries to link against
 set(CVODE_LIB sundials_cvode)
 set(NVECP_LIB sundials_nvecparallel)
@@ -69,11 +58,7 @@ foreach(example_tuple ${CVODE_examples})
     set_target_properties(${example} PROPERTIES FOLDER "Examples")
 
     # libraries to link against
-    target_link_libraries(${example} ${SUNDIALS_LIBS})
-
-    if(NOT MPI_CXX_COMPILER)
-      target_link_libraries(${example} ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARIES})
-    endif()
+    target_link_libraries(${example} PRIVATE MPI::MPI_CXX ${SUNDIALS_LIBS})
   endif()
 
   # check if example args are provided and set the test name