ROCm Build

.gitignore

@@ -15,3 +15,4 @@ __pycache__/
 view
 /_serac_build_and_test*
 build-linux-*-*-*
+*.core

.gitlab-ci.yml

@@ -70,5 +70,6 @@ variables:
 include:
   - local: .gitlab/build_blueos.yml
   - local: .gitlab/build_toss4.yml
+  - local: .gitlab/build_toss4_cray.yml
   - project: 'lc-templates/id_tokens'
     file: 'id_tokens.yml'

.gitlab/build_toss4_cray.yml

@@ -0,0 +1,49 @@
+####
+# This is the shared configuration of jobs for toss4_cray
+.on_toss4_cray:
+  variables:
+    SCHEDULER_PARAMETERS: "--queue pci --exclusive --time-limit=${ALLOC_TIME}m --nodes=${ALLOC_NODES}"
+  tags:
+    - batch
+    - tioga
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_TOSS4_CRAY == "OFF"' #run except if ...
+      when: never
+  before_script:
+    - module load cmake/3.23.1
+
+####
+# Templates
+.src_build_on_toss4_cray:
+  extends: [.src_build_script, .on_toss4_cray, .src_workflow]
+  needs: []
+
+.full_build_on_toss4_cray:
+  extends: [.full_build_script, .on_toss4_cray, .full_workflow]
+  needs: []
+  before_script:
+    # LC version of pip is ancient
+    - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi
+
+
+####
+# Build jobs
+
+# Only run integration tests on one spec
+toss4_cray-clang_17_0_0-src:
+  variables:
+    COMPILER: "[email protected]"
+    HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
+    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: .src_build_on_toss4_cray
+
+toss4_cray-clang_17_0_0-full:
+  variables:
+    COMPILER: "[email protected]"
+    SPEC: "--spec=%${COMPILER}"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "45"
+    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
+  extends: .full_build_on_toss4_cray

.uberenv_config.json

@@ -4,7 +4,7 @@
 "package_final_phase" : "initconfig",
 "package_source_dir" : "../..",
 "spack_url": "https://github.com/spack/spack.git",
-"spack_commit": "6d244b3f6776b3697861c6b5cebbee376c30bff3",
+"spack_commit": "cfee88a5bb56a1c8ec892879e04cb6a17c4f9404",
 "spack_configs_path": "scripts/spack/configs",
 "spack_packages_path": ["scripts/spack/radiuss-spack-configs/packages", "scripts/spack/packages"],
 "spack_concretizer": "clingo"

CMakeLists.txt

@@ -47,7 +47,9 @@ if(NOT DEFINED SERAC_SOURCE_DIR)
 endif()
 message(STATUS "Serac Source Dir: ${SERAC_SOURCE_DIR}")
 
-if(SERAC_ENABLE_CODEVELOP)
+# For HIP case, this flag prevents the following Strumpack linker error:
+# undefined symbol: mpi_abort_
+if(SERAC_ENABLE_CODEVELOP OR (ENABLE_HIP AND STRUMPACK_DIR))
     set(ENABLE_FORTRAN ON CACHE BOOL "")
 endif()
 

cmake/SeracBasics.cmake

@@ -28,6 +28,10 @@ endif()
 option(SERAC_ENABLE_CODE_CHECKS "Enable Serac's code checks" ${_enable_serac_code_checks})
 
 cmake_dependent_option(SERAC_ENABLE_TESTS "Enables Serac Tests" ON "ENABLE_TESTS" OFF)
+cmake_dependent_option(SERAC_ENABLE_CUDA "Enables Serac with CUDA support" ON "ENABLE_CUDA" OFF)
+cmake_dependent_option(SERAC_ENABLE_HIP "Enables Serac with HIP support" ON "ENABLE_HIP" OFF)
+cmake_dependent_option(SERAC_ENABLE_MPI "Enables Serac with MPI support" ON "ENABLE_MPI" OFF)
+cmake_dependent_option(SERAC_ENABLE_OPENMP "Enables Serac with OPENMP support" ON "ENABLE_OPENMP" OFF)
 
 #------------------------------------------------------------------------------
 # Profiling options

cmake/SeracConfigHeader.cmake

@@ -36,7 +36,7 @@ message(STATUS "Configuring Serac version ${SERAC_VERSION_FULL}")
 #------------------------------------------------------------------------------
 # Create variable for every TPL
 #------------------------------------------------------------------------------
-set(TPL_DEPS ADIAK AXOM CALIPER CAMP CONDUIT CUDA FMT HDF5 LUA MFEM MPI PETSC RAJA SLEPC STRUMPACK SUNDIALS TRIBOL UMPIRE)
+set(TPL_DEPS ADIAK AXOM CALIPER CAMP CONDUIT CUDA FMT HDF5 HIP LUA MFEM MPI PETSC RAJA SLEPC STRUMPACK SUNDIALS TRIBOL UMPIRE)
 foreach(dep ${TPL_DEPS})
     if( ${dep}_FOUND OR ENABLE_${dep} )
         set(SERAC_USE_${dep} TRUE)
@@ -60,7 +60,7 @@ endif()
 # General Build Info
 #------------------------------------------------------------------------------
 serac_convert_to_native_escaped_file_path(${PROJECT_SOURCE_DIR} SERAC_REPO_DIR)
-serac_convert_to_native_escaped_file_path(${CMAKE_BINARY_DIR}   SERAC_BIN_DIR)
+serac_convert_to_native_escaped_file_path(${CMAKE_BINARY_DIR}   SERAC_BINARY_DIR)
 
 #------------------------------------------------------------------------------
 # Create Config Header

cmake/serac-config.cmake.in

@@ -27,7 +27,10 @@ if(NOT SERAC_FOUND)
   #----------------------------------------------------------------------------
 
   set(SERAC_ENABLE_CODEVELOP   @SERAC_ENABLE_CODEVELOP@)
-  set(SERAC_ENABLE_CUDA        @ENABLE_CUDA@)
+  set(SERAC_ENABLE_CUDA        @SERAC_ENABLE_CUDA@)
+  set(SERAC_ENABLE_HIP         @SERAC_ENABLE_HIP@)
+  set(SERAC_ENABLE_MPI         @SERAC_ENABLE_MPI@)
+  set(SERAC_ENABLE_OPENMP      @SERAC_ENABLE_OPENMP@)
 
   set(SERAC_USE_ADIAK          @SERAC_USE_ADIAK@)
   set(SERAC_USE_AXOM           @SERAC_USE_AXOM@)

cmake/thirdparty/FindMFEM.cmake

@@ -116,7 +116,7 @@ else()
     list(APPEND MFEM_LIBRARIES ${mfem_tpl_lnk_flags})
 
     if(mfem_cfg_file_txt MATCHES "MFEM_USE_CUDA += YES")
-        if(NOT ENABLE_CUDA)
+        if(NOT SERAC_ENABLE_CUDA)
             message(WARNING "MFEM was built with CUDA but CUDA is not enabled")
         endif()
         list(APPEND MFEM_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS})

cmake/thirdparty/SetupSeracThirdParty.cmake

@@ -19,7 +19,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
     #------------------------------------------------------------------------------
     # CUDA
     #------------------------------------------------------------------------------
-    if(ENABLE_CUDA)
+    if(SERAC_ENABLE_CUDA)
         # Manually set includes as system includes
         foreach(_target cuda_runtime cuda)
             get_target_property(_dirs ${_target} INTERFACE_INCLUDE_DIRECTORIES)
@@ -37,6 +37,17 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
 
     include(CMakeFindDependencyMacro)
 
+    #------------------------------------------------------------------------------
+    # Create global variable to toggle between GPU targets
+    #------------------------------------------------------------------------------
+    if(SERAC_ENABLE_CUDA)
+        set(serac_device_depends blt::cuda CACHE STRING "" FORCE)
+    elseif(SERAC_ENABLE_HIP)
+        set(serac_device_depends blt::hip CACHE STRING "" FORCE)
+    else()
+        set(serac_device_depends "" CACHE STRING "" FORCE)
+    endif()
+
     #------------------------------------------------------------------------------
     # Camp
     #------------------------------------------------------------------------------
@@ -206,27 +217,27 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         #### MFEM "Use" Options
 
         # Assumes that we have AMGX if we have CUDA
-        set(MFEM_USE_AMGX ${ENABLE_CUDA} CACHE BOOL "")
+        set(MFEM_USE_AMGX ${SERAC_ENABLE_CUDA} CACHE BOOL "")
         set(MFEM_USE_CALIPER ${CALIPER_FOUND} CACHE BOOL "")
         # We don't use MFEM's Conduit/Axom support
         set(MFEM_USE_CONDUIT OFF CACHE BOOL "")
-        set(MFEM_USE_CUDA ${ENABLE_CUDA} CACHE BOOL "")
+        set(MFEM_USE_CUDA ${SERAC_ENABLE_CUDA} CACHE BOOL "")
         set(MFEM_USE_LAPACK ON CACHE BOOL "")
         # mfem+mpi requires metis
-        set(MFEM_USE_METIS ${ENABLE_MPI} CACHE BOOL "")
-        set(MFEM_USE_METIS_5 ${ENABLE_MPI} CACHE BOOL "")
-        set(MFEM_USE_MPI ${ENABLE_MPI} CACHE BOOL "")
+        set(MFEM_USE_METIS ${SERAC_ENABLE_MPI} CACHE BOOL "")
+        set(MFEM_USE_METIS_5 ${SERAC_ENABLE_MPI} CACHE BOOL "")
+        set(MFEM_USE_MPI ${SERAC_ENABLE_MPI} CACHE BOOL "")
         if(NETCDF_DIR)
             serac_assert_is_directory(DIR_VARIABLE NETCDF_DIR)
             set(MFEM_USE_NETCDF ON CACHE BOOL "")
         endif()
         # mfem+mpi also needs parmetis
-        if(ENABLE_MPI)
+        if(SERAC_ENABLE_MPI)
             serac_assert_is_directory(DIR_VARIABLE PARMETIS_DIR)
             # Slightly different naming convention
             set(ParMETIS_DIR ${PARMETIS_DIR} CACHE PATH "")
         endif()
-        set(MFEM_USE_OPENMP ${ENABLE_OPENMP} CACHE BOOL "")
+        set(MFEM_USE_OPENMP ${SERAC_ENABLE_OPENMP} CACHE BOOL "")
 
         if(PETSC_DIR)
             set(MFEM_USE_PETSC ON CACHE BOOL "")
@@ -248,7 +259,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
             serac_assert_is_directory(DIR_VARIABLE SUPERLUDIST_DIR)
             # MFEM uses a slightly different naming convention
             set(SuperLUDist_DIR ${SUPERLUDIST_DIR} CACHE PATH "")
-            set(MFEM_USE_SUPERLU ${ENABLE_MPI} CACHE BOOL "")
+            set(MFEM_USE_SUPERLU ${SERAC_ENABLE_MPI} CACHE BOOL "")
         endif()
         if(STRUMPACK_DIR)
             serac_assert_is_directory(DIR_VARIABLE STRUMPACK_DIR)
@@ -429,7 +440,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
             target_link_libraries(sidre PUBLIC STRUMPACK::strumpack)
         endif()
 
-        if(ENABLE_OPENMP)
+        if(SERAC_ENABLE_OPENMP)
             target_link_libraries(core INTERFACE blt::openmp)
         endif()
 
@@ -461,8 +472,8 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE)
         # Otherwise we use the submodule
         message(STATUS "Using Tribol submodule")
-        set(BUILD_REDECOMP ${ENABLE_MPI} CACHE BOOL "")
-        set(TRIBOL_USE_MPI ${ENABLE_MPI} CACHE BOOL "")
+        set(BUILD_REDECOMP ${SERAC_ENABLE_MPI} CACHE BOOL "")
+        set(TRIBOL_USE_MPI ${SERAC_ENABLE_MPI} CACHE BOOL "")
         set(TRIBOL_ENABLE_TESTS OFF CACHE BOOL "")
         set(TRIBOL_ENABLE_EXAMPLES OFF CACHE BOOL "")
         set(TRIBOL_ENABLE_DOCS OFF CACHE BOOL "")
@@ -594,7 +605,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         # Should this logic be in the Caliper CMake package?
         # If CMake version doesn't support CUDAToolkit the libraries
         # are just "baked in"
-        if(ENABLE_CUDA)
+        if(SERAC_ENABLE_CUDA)
             if(CMAKE_VERSION VERSION_LESS 3.17)
                 message(FATAL_ERROR "Serac+Caliper+CUDA requires CMake > 3.17.")
             else()