Merge pull request #285 from LLNL/feature/chai-submodule

CHAI Submodule

.gitlab-ci.yml

@@ -16,6 +16,7 @@ variables:
     - SPHERAL_REV_STR="$SPHERAL_REV"
 
 stages:
+  - tpls
   - build_and_install
   - run_ats
   - update_tpls

.gitlab/jobs-mpi.yml

@@ -1,22 +1,34 @@
 # ------------------------------------------------------------------------------
 # BUILD JOBS
 
+toss_gcc_mvapich2_cxxonly_tpls:
+  extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls]
+
 toss_gcc_mvapich2_cxxonly_build:
   extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build]
+  needs: [toss_gcc_mvapich2_cxxonly_tpls]
+
 
 
+toss_gcc_mvapich2_tpls:
+  extends: [.toss_resource2, .gcc_mvapich2, .tpls]
 
 toss_gcc_mvapich2_build:
   extends: [.toss_resource2, .gcc_mvapich2, .build_and_test]
+  needs: [toss_gcc_mvapich2_tpls]
 
 toss_gcc_mvapich2_test:
   extends: [.toss_resource2, .gcc_mvapich2, .run_ats]
   needs: [toss_gcc_mvapich2_build]
 
 
 
+toss_clang_mvapich2_tpls:
+  extends: [.toss_resource1, .clang_mvapich2, .tpls]
+
 toss_clang_mvapich2_build:
   extends: [.toss_resource1, .clang_mvapich2, .build_and_test]
+  needs: [toss_clang_mvapich2_tpls]
 
 toss_clang_mvapich2_test:
   extends: [.toss_resource2, .clang_mvapich2, .run_ats]
@@ -29,17 +41,25 @@ toss_clang_mvapich2_test:
 
 
 
+blueos_gcc_spectrum_tpls:
+  extends: [.blueos_resource1, .gcc_spectrum, .tpls]
+
 blueos_gcc_spectrum_build:
   extends: [.blueos_resource1, .gcc_spectrum, .build_and_test]
+  needs: [blueos_gcc_spectrum_tpls]
 
 blueos_gcc_spectrum_test:
   extends: [.blueos_resource1, .gcc_spectrum, .run_ats]
   needs: [blueos_gcc_spectrum_build]
 
 
 
+blueos_cuda_11_gcc_spectrum_tpls:
+  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .tpls]
+
 blueos_cuda_11_gcc_spectrum_build:
   extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .build_and_test]
+  needs: [blueos_cuda_11_gcc_spectrum_tpls]
 
 blueos_cuda_11_gcc_spectrum_test:
   extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats]

.gitlab/jobs-seq.yml

@@ -1,24 +1,36 @@
 # ------------------------------------------------------------------------------
 # BUILD JOBS
 
+toss_gcc_~mpi_tpls:
+  extends: [.gcc_~mpi, .tpls, .toss_resource1]
+
 toss_gcc_~mpi_build:
   extends: [.gcc_~mpi, .build_and_test, .toss_resource1]
+  needs: [toss_gcc_~mpi_tpls]
 
 toss_gcc_~mpi_test:
   extends: [.gcc_~mpi, .run_ats, .toss_resource1]
   needs: [toss_gcc_~mpi_build]
 
 
+blueos_cuda_11_gcc_~mpi_tpls:
+  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .tpls]
+
 blueos_cuda_11_gcc_~mpi_build:
   extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .build_and_test]
+  needs: [blueos_cuda_11_gcc_~mpi_tpls]
 
 blueos_cuda_11_gcc_~mpi_test:
   extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .run_ats]
   needs: [blueos_cuda_11_gcc_~mpi_build]
 
 
+blueos_gcc_~mpi_Debug_tpls:
+  extends: [.blueos_resource1, .gcc_~mpi_Debug, .tpls]
+
 blueos_gcc_~mpi_Debug_build:
   extends: [.blueos_resource1, .gcc_~mpi_Debug, .build_and_test]
+  needs: [blueos_gcc_~mpi_Debug_tpls]
 
 blueos_gcc_~mpi_Debug_test:
   extends: [.blueos_resource1, .gcc_~mpi_Debug, .run_ats]

.gitlab/scripts.yml

@@ -2,8 +2,9 @@
 # CI Job Scripts
 # ------------------------------------------------------------------------------
 
-.build:
-  stage: build_and_install
+
+.tpls:
+  stage: tpls
   script:
     - CI_BUILD_DIR=$SPHERAL_BUILDS_DIR/$CI_JOB_ID/project
     - echo $CI_BUILD_DIR &> ci-dir.txt && echo $CI_JOB_NAME &> job-name.txt
@@ -12,7 +13,18 @@
     - cd $CI_BUILD_DIR
     - echo $SPEC
 
-    - $BUILD_ALLOC ./$SCRIPT_DIR/gitlab/build_and_install.py --spec="$SPEC" --extra-cmake-args="$EXTRA_CMAKE_ARGS"
+    - $BUILD_ALLOC ./$SCRIPT_DIR/gitlab/build_and_install.py --spec="$SPEC" --tpls-only
+  artifacts:
+    paths:
+      - ci-dir.txt
+      - job-name.txt
+
+.build:
+  stage: build_and_install
+  script:
+    - CI_BUILD_DIR=$(cat ci-dir.txt)
+    - cd $CI_BUILD_DIR && cat job-name.txt
+    - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/host-config-build.py --host-config gitlab.cmake --build $EXTRA_CMAKE_ARGS
 
 .build_and_test:
   extends: [.build]

.gitlab/specs.yml

@@ -24,17 +24,14 @@
 .gcc_spectrum:
   variables:
     SPEC: 'gcc@$GCC_VERSION^spectrum-mpi'
-    EXTRA_CMAKE_ARGS: ''
 
 .clang_mvapich2:
   variables:
     SPEC: 'clang@$CLANG_VERSION^mvapich2'
-    EXTRA_CMAKE_ARGS: ''
 
 .cuda_11_gcc_~mpi:
   variables:
     SPEC: 'gcc@$GCC_VERSION+cuda~mpi cuda_arch=70'
-    EXTRA_CMAKE_ARGS: ''
 
 .cuda_11_gcc_spectrum:
   variables:
@@ -44,6 +41,4 @@
 .oneapi_2022_1_mvapich2:
   variables:
     SPEC: '[email protected]^mvapich2'
-    EXTRA_CMAKE_ARGS: ''
-
 

.gitmodules

@@ -12,3 +12,7 @@
 [submodule "extern/PolyClipper"]
 	path = extern/PolyClipper
 	url = https://github.com/LLNL/PolyClipper
+[submodule "extern/chai"]
+	path = extern/chai
+	url = https://github.com/llnl/chai
+	branch = feature/ManagedSharedPtr

CMakeLists.txt

@@ -1,6 +1,6 @@
 # CMakeLists to build the Spheral library.
 
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.14)
 include(cmake/SpheralVersion.cmake)
 project(spheral LANGUAGES C CXX Fortran VERSION ${SPHERAL_VERSION})
 

RELEASE_NOTES.md

@@ -4,19 +4,22 @@ Version vYYYY.MM.p -- Release date YYYY-MM-DD
 
 Notable changes include:
 
-  * New features/ API changes:
+  * New features / API changes:
     * MPI variables are now wrapped as
       ```
       SPHERAL_OP_SUM, SPHERAL_OP_MAX, SPHERAL_OP_MIN
       ```
+    * CHAI added as a submodule of Spheral for co-developing features necessary for GPU port.
+    * RAJA & Umpire added as first level dependencies.
+    * Axom updated to v0.9.0.
+    * TPL builds have been split off into a separate Gitlab CI stage to help with timeouts on allocations.
 
   * Build changes / improvements:
     * Distributed source directory must always be built now
 
   * Bug Fixes / improvements:
     * Wrappers for MPI calls are simplified and improved
 
-
 Version v2024.06.1 -- Release date 2024-07-09
 ==============================================
 
@@ -261,7 +264,7 @@ Version vYYYY.MM.p -- Release date YYYY-MM-DD
 
 Notable changes include:
 
-  * New features/ API changes:
+  * New features / API changes:
 
   * Build changes / improvements:
 

cmake/InstallTPLs.cmake

@@ -20,7 +20,7 @@ if (NOT ENABLE_CXXONLY)
   find_package(Python3 COMPONENTS Interpreter Development)
   set(PYTHON_EXE ${Python3_EXECUTABLE})
   set(SPHERAL_SITE_PACKAGES_PATH "lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/site-packages" )
-  list(APPEND SPHERAL_BLT_DEPENDS Python3::Python)
+  list(APPEND SPHERAL_CXX_DEPENDS Python3::Python)
 
   # Set the PYB11Generator path
   if (NOT PYB11GENERATOR_ROOT_DIR)
@@ -31,7 +31,7 @@ if (NOT ENABLE_CXXONLY)
     set(PYBIND11_ROOT_DIR "${PYB11GENERATOR_ROOT_DIR}/extern/pybind11" CACHE PATH "")
   endif()
   include(${PYB11GENERATOR_ROOT_DIR}/cmake/PYB11Generator.cmake)
-  list(APPEND SPHERAL_BLT_DEPENDS pybind11_headers)
+  list(APPEND SPHERAL_CXX_DEPENDS pybind11_headers)
   install(TARGETS pybind11_headers
     EXPORT spheral_cxx-targets
     DESTINATION lib/cmake)
@@ -67,27 +67,46 @@ endif()
 #-----------------------------------------------------------------------------------
 
 # Use find_package to get axom (which brings in fmt) and patch fmt
-find_package(axom REQUIRED QUIET NO_DEFAULT_PATH PATHS ${axom_DIR}/lib/cmake)
-if(axom_FOUND)
-  list(APPEND SPHERAL_BLT_DEPENDS axom)
-  # Add fmt library to external library list
-  set(fmt_name fmt)
-  # Newer Axom versions call fmt target axom::fmt
-  if(NOT TARGET fmt)
-    set(fmt_name axom::fmt)
-  endif()
-  list(APPEND SPHERAL_BLT_DEPENDS ${fmt_name})
-  # BLT Macro for doing this
-  blt_convert_to_system_includes(TARGET ${fmt_name})
-endif()
+find_package(axom REQUIRED NO_DEFAULT_PATH PATHS ${axom_DIR}/lib/cmake)
+list(APPEND SPHERAL_BLT_DEPENDS axom )
+
 # This is a hack to handle transitive issues that come
 # from using object libraries with newer version of axom
 foreach(_comp ${AXOM_COMPONENTS_ENABLED})
-  list(APPEND SPHERAL_BLT_DEPENDS axom::${_comp})
   get_target_property(axom_deps axom::${_comp} INTERFACE_LINK_LIBRARIES)
+  blt_convert_to_system_includes(TARGET ${axom_deps})
   list(APPEND SPHERAL_BLT_DEPENDS ${axom_deps})
 endforeach()
 
+message("-----------------------------------------------------------------------------")
+find_package(RAJA REQUIRED NO_DEFAULT_PATH PATHS ${raja_DIR})
+if (RAJA_FOUND) 
+  message("Found RAJA External Package.")
+endif()
+message("-----------------------------------------------------------------------------")
+find_package(umpire REQUIRED NO_DEFAULT_PATH PATHS ${umpire_DIR})
+if (umpire_FOUND) 
+  message("Found umpire External Package.")
+endif()
+message("-----------------------------------------------------------------------------")
+
+# Chai
+if(chai_DIR AND USE_EXTERNAL_CHAI)
+  find_package(chai REQUIRED NO_DEFAULT_PATH PATHS ${chai_DIR})
+  if (chai_FOUND) 
+    message("Found chai External Package.")
+  endif()
+else()
+  message("Using chai Submodule.")
+  set(chai_DIR "${SPHERAL_ROOT_DIR}/extern/chai")
+  set(CHAI_ENABLE_RAJA_PLUGIN On CACHE BOOL "")
+  add_subdirectory(${chai_DIR})
+endif()
+
+list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire)
+
+message("-----------------------------------------------------------------------------")
+
 # TPLs that must be imported
 list(APPEND SPHERAL_EXTERN_LIBS boost eigen qhull silo hdf5 polytope)
 
@@ -99,6 +118,7 @@ blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS caliper IF ENABLE_TIMER)
 foreach(lib ${SPHERAL_EXTERN_LIBS})
   if(NOT TARGET ${lib})
     Spheral_Handle_TPL(${lib} ${TPL_SPHERAL_CMAKE_DIR})
+    blt_convert_to_system_includes(TARGET ${lib})
   endif()
   list(APPEND SPHERAL_BLT_DEPENDS ${lib})
 endforeach()
@@ -108,25 +128,3 @@ endforeach()
 if (EXISTS ${EXTERNAL_SPHERAL_TPL_CMAKE})
   include(${EXTERNAL_SPHERAL_TPL_CMAKE})
 endif()
-
-# Copied from serac, needed to bypass generator expression issue during export
-set(_props)
-if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0" )
-  list(APPEND _props INTERFACE_LINK_OPTIONS)
-endif()
-list(APPEND _props INTERFACE_COMPILE_OPTIONS)
-foreach(_target axom axom::openmp)
-  if(TARGET ${_target})
-    message(STATUS "Removing OpenMP Flags from target[${_target}]")
-    foreach(_prop ${_props})
-      get_target_property(_flags ${_target} ${_prop})
-      if ( _flags )
-	string( REPLACE "${OpenMP_CXX_FLAGS}" ""
-	  correct_flags "${_flags}" )
-	string( REPLACE "${OpenMP_Fortran_FLAGS}" ""
-	  correct_flags "${correct_flags}" )
-	set_target_properties( ${_target} PROPERTIES ${_prop} "${correct_flags}" )
-      endif()
-    endforeach()
-  endif()
-endforeach()

cmake/SetupSpheral.cmake

@@ -70,15 +70,16 @@ endif()
 
 if(ENABLE_MPI)
   set(BLT_MPI_COMPILE_FLAGS -DUSE_MPI -DMPICH_SKIP_MPICXX -ULAM_WANT_MPI2CPP -DOMPI_SKIP_MPICXX)
-  list(APPEND SPHERAL_BLT_DEPENDS mpi)
+  list(APPEND SPHERAL_CXX_DEPENDS mpi)
 endif()
 
 if(ENABLE_OPENMP)
-  list(APPEND SPHERAL_BLT_DEPENDS openmp)
+  list(APPEND SPHERAL_CXX_DEPENDS openmp)
 endif()
 
 if(ENABLE_CUDA)
-  #set(CMAKE_CUDA_FLAGS  "${CMAKE_CUDA_FLAGS} -arch=${CUDA_ARCH} --extended-lambda -Xcudafe --display_error_number")
+  set(CMAKE_CUDA_FLAGS  "${CMAKE_CUDA_FLAGS} -arch=${CUDA_ARCH} --extended-lambda -Xcudafe --display_error_number")
+  #set(CMAKE_CUDA_FLAGS  "${CMAKE_CUDA_FLAGS} -arch=${CUDA_ARCH} --expt-relaxed-constexpr --extended-lambda -Xcudafe --display_error_number")
   set(CMAKE_CUDA_STANDARD 17)
   list(APPEND SPHERAL_CXX_DEPENDS cuda)
 endif()