GPU Evaluate Derivatives

RELEASE_NOTES.md

@@ -65,6 +65,7 @@ Notable changes include:
     * ATS bumped to version 7.0.9 for blueos smpi option support.
     * Eigen bumped to 3.4.0 for NVCC compatiblity.
     * C++ flag suppression is gaurded with build time CMake generators to only apply to C++ compilers.
+    * Adding RAJA, CAMP, Umpire and CHAI as dependencies of Spheral.
     * Python runtime libraries are now managed through Spack / tpl-manager.
     * Added ENABLE_NAN_EXCEPTIONS (default OFF) Cmake flag to raise an exception when a NAN occurs (Gnu only).
     * Byte-compiling python installed in virtual spheral environment.

cmake/CMakeDefinitions.cmake

@@ -9,7 +9,7 @@ option(ENABLE_NAN_EXCEPTIONS "Raise an excpetion when a NAN occurs (Gnu only)" O
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
   message("-- building Debug")
   add_definitions("-DDEBUG=1")
-  if (NOT DBC_MODE)
+  if (NOT DEFINED DBC_MODE)
     set(DBC_MODE "All")
   endif()
 else()

cmake/Compilers.cmake

@@ -10,6 +10,10 @@ option(ENABLE_UNUSED_VARIABLE_WARNINGS "show unused variable compiler warnings"
 option(ENABLE_UNUSED_PARAMETER_WARNINGS "show unused parameter warnings" OFF)
 option(ENABLE_MISSING_INCLUDE_DIR_WARNINGS "show unused parameter warnings" ON)
 
+if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
+endif()
 
 set(CXX_WARNING_FLAGS "")
 if (ENABLE_WARNINGS)
@@ -48,7 +52,11 @@ if (NOT ENABLE_MISSING_INCLUDE_DIR_WARNINGS)
 endif()
 message("-- Compiler missing include dir warnings ${ENABLE_MISSING_INCLUDE_DIR_WARNINGS}")
 
-set(CUDA_WARNING_FLAGS -Xcudafe=\"--diag_suppress=esa_on_defaulted_function_ignored\")
+set(CUDA_WARNING_FLAGS )
+# Suppress Eigen warnings.
+list(APPEND CUDA_WARNING_FLAGS -Xcudafe=\"--diag_suppress=esa_on_defaulted_function_ignored\")
+# Suppress: warning: a __device__ function("Spheral::X") redeclared with __host__ __device__, hence treated as a __host__ __device__ function
+list(APPEND CUDA_WARNING_FLAGS -Xcudafe=\"--diag_suppress=3148\")
 
 add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:${CXX_WARNING_FLAGS}>")
 add_compile_options("$<$<COMPILE_LANGUAGE:CUDA>:${CUDA_WARNING_FLAGS}>")

cmake/InstallTPLs.cmake

@@ -53,6 +53,12 @@ Spheral_Handle_TPL(boost spheral_depends cxx)
 Spheral_Handle_TPL(eigen spheral_depends cxx)
 Spheral_Handle_TPL(qhull spheral_depends cxx)
 Spheral_Handle_TPL(silo spheral_depends cxx)
+Spheral_Handle_TPL(conduit spheral_depends cxx)
+Spheral_Handle_TPL(camp spheral_depends cxx)
+Spheral_Handle_TPL(raja spheral_depends cxx)
+Spheral_Handle_TPL(chai spheral_depends cxx)
+Spheral_Handle_TPL(umpire spheral_depends cxx)
+Spheral_Handle_TPL(lvarray spheral_depends cxx)
 
 # AXOM PUlls in HDF5 and Conduit for us
 #Spheral_Handle_TPL(conduit spheral_depends cxx)

cmake/SetupSpheral.cmake

@@ -12,6 +12,21 @@ if (NOT SPHERAL_CMAKE_MODULE_PATH)
 endif()
 list(APPEND CMAKE_MODULE_PATH "${SPHERAL_CMAKE_MODULE_PATH}")
 
+
+#-------------------------------------------------------------------------------#
+# Set a default build type if none was specified
+#-------------------------------------------------------------------------------#
+set(default_build_type "Release")
+
+if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+  message(STATUS "Setting build type to '${default_build_type}' as none was specified.")
+  set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING "Choose the type of build (debug, release, etc)." FORCE)
+
+  # Set the possible values of build type for cmake-gui
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
+    "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+endif()
+
 #-------------------------------------------------------------------------------
 # Set Compiler Flags / Options
 #-------------------------------------------------------------------------------
@@ -67,27 +82,13 @@ if(ENABLE_OPENMP)
 endif()
 
 if(ENABLE_CUDA)
-  #set(CMAKE_CUDA_FLAGS  "${CMAKE_CUDA_FLAGS} -arch=${CUDA_ARCH} --extended-lambda -Xcudafe --display_error_number")
+  set(CMAKE_CUDA_FLAGS  "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr --extended-lambda -Xcudafe --display_error_number")
   set(CMAKE_CUDA_STANDARD 14)
   list(APPEND SPHERAL_CXX_DEPENDS cuda)
 endif()
 
 option(BOOST_HEADER_ONLY "only use the header only components of Boost" OFF)
 
-#-------------------------------------------------------------------------------#
-# Set a default build type if none was specified
-#-------------------------------------------------------------------------------#
-set(default_build_type "Release")
-
-if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
-  message(STATUS "Setting build type to '${default_build_type}' as none was specified.")
-  set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING "Choose the type of build (debug, release, etc)." FORCE)
-
-  # Set the possible values of build type for cmake-gui
-  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
-    "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
-endif()
-
 #-------------------------------------------------------------------------------
 # Should we build sphinx documentation
 #-------------------------------------------------------------------------------

cmake/spheral/SpheralAddLibs.cmake

@@ -45,7 +45,7 @@ function(spheral_add_obj_library
           )
 
   if(ENABLE_CUDA)
-    set_target_properties(Spheral_${package_name} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+    set_target_properties(Spheral_${package_name} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
   endif()
 
   # Add this to the obj_libs_list list
@@ -224,7 +224,7 @@ function(spheral_add_pybind11_library package_name)
 
   # Set the r-path of the C++ lib such that it is independent of the build dir when installed
   set_target_properties(${package_name} PROPERTIES
-                        INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${conduit_DIR}/lib;${axom_DIR}/lib;${boost_DIR}/lib;${python_DIR}/lib;${hdf5_DIR}/lib;${zlib_DIR}/lib;${SPHERAL_ADDITIONAL_RPATHS}"
+                        INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${conduit_DIR}/lib;${axom_DIR}/lib;${boost_DIR}/lib;${python_DIR}/lib;${lvarray_DIR}/lib;${hdf5_DIR}/lib;${zlib_DIR}/lib;${SPHERAL_ADDITIONAL_RPATHS}"
                         )
 
 endfunction()

cmake/tpl/camp.cmake

@@ -0,0 +1 @@
+set(${lib_name}_libs libcamp.a)

cmake/tpl/chai.cmake

@@ -0,0 +1 @@
+set(${lib_name}_libs libchai.a)

cmake/tpl/lvarray.cmake

@@ -0,0 +1 @@
+set(${lib_name}_libs liblvarray.so)

cmake/tpl/raja.cmake

@@ -0,0 +1 @@
+set(${lib_name}_libs libRAJA.a libdesul_atomics.a)

cmake/tpl/umpire.cmake

@@ -0,0 +1 @@
+set(${lib_name}_libs libumpire.a)

scripts/spack/configs/toss_3_x86_64_ib/compilers.yaml

@@ -4,9 +4,11 @@ compilers:
     paths:
       cc: /usr/tce/packages/clang/clang-9.0.0/bin/clang
       cxx: /usr/tce/packages/clang/clang-9.0.0/bin/clang++
-      f77: /usr/tce/packages/gcc/gcc-8.1.0/bin/gfortran
-      fc: /usr/tce/packages/gcc/gcc-8.1.0/bin/gfortran
-    flags: {}
+      f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran
+      fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran
+    flags:
+      cflags: --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1
+      cxxflags: --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1
     operating_system: rhel7
     target: x86_64
     modules: []

scripts/spack/packages/blt/package.py

@@ -0,0 +1,40 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Blt(Package):
+    """BLT is a streamlined CMake-based foundation for Building, Linking and
+    Testing large-scale high performance computing (HPC) applications."""
+
+    homepage = "https://github.com/LLNL/blt"
+    url = "https://github.com/LLNL/blt/archive/v0.4.0.tar.gz"
+    git = "https://github.com/LLNL/blt.git"
+    tags = ["radiuss"]
+
+    maintainers = ["white238", "davidbeckingsale"]
+
+    version("develop", branch="develop")
+    version("main", branch="main")
+    # Note: 0.4.0+ contains a breaking change to BLT created targets
+    #  if you export targets this could cause problems in downstream
+    #  projects if not handled properly. More info here:
+    #  https://llnl-blt.readthedocs.io/en/develop/tutorial/exporting_targets.html
+    version("0.5.2", sha256="95b924cfbb2bddd9b1a92e96603b2fd485a19721d59ddf8ff50baefc1714d7ea")
+    version("0.5.1", sha256="ff7e87eefc48704a0721b66174612b945955adaa0a56aa69dd0473074fa4badf")
+    version("0.5.0", sha256="5f680ef922d0e0a7ff1b1a5fc8aa107cd4f543ad888cbc9b12639bea72a6ab1f")
+    version("0.4.1", sha256="16cc3e067ddcf48b99358107e5035a17549f52dcc701a35cd18a9d9f536826c1")
+    version("0.4.0", sha256="f3bc45d28b9b2eb6df43b75d4f6f89a1557d73d012da7b75bac1be0574767193")
+    version("0.3.6", sha256="6276317c29e7ff8524fbea47d9288ddb40ac06e9f9da5e878bf9011e2c99bf71")
+    version("0.3.5", sha256="68a1c224bb9203461ae6f5ab0ff3c50b4a58dcce6c2d2799489a1811f425fb84")
+    version("0.3.0", sha256="bb917a67cb7335d6721c997ba9c5dca70506006d7bba5e0e50033dd0836481a5")
+    version("0.2.5", sha256="3a000f60194e47b3e5623cc528cbcaf88f7fea4d9620b3c7446ff6658dc582a5")
+    version("0.2.0", sha256="c0cadf1269c2feb189e398a356e3c49170bc832df95e5564e32bdbb1eb0fa1b3")
+
+    depends_on("cmake", type="run")
+
+    def install(self, spec, prefix):
+        install_tree(".", prefix)

scripts/spack/packages/camp/package.py

@@ -0,0 +1,84 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import glob
+
+from spack.package import *
+
+
+def hip_repair_options(options, spec):
+    # there is only one dir like this, but the version component is unknown
+    options.append(
+        "-DHIP_CLANG_INCLUDE_PATH="
+        + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0]
+    )
+
+
+class Camp(CMakePackage, CudaPackage, ROCmPackage):
+    """
+    Compiler agnostic metaprogramming library providing concepts,
+    type operations and tuples for C++ and cuda
+    """
+
+    homepage = "https://github.com/LLNL/camp"
+    git = "https://github.com/LLNL/camp.git"
+    url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz"
+
+    maintainers = ["trws"]
+
+    version("main", branch="main", submodules="True")
+    version("2022.10.0", sha256="3561c3ef00bbcb61fe3183c53d49b110e54910f47e7fc689ad9ccce57e55d6b8")
+    version("2022.03.2", sha256="bc4aaeacfe8f2912e28f7a36fc731ab9e481bee15f2c6daf0cb208eed3f201eb")
+    version("2022.03.0", sha256="e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721")
+    version("0.3.0", sha256="129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb")
+    version("0.2.3", sha256="58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7")
+    version("0.2.2", sha256="194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819")
+    version("0.1.0", sha256="fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc")
+
+    # TODO: figure out gtest dependency and then set this default True.
+    variant("tests", default=False, description="Build tests")
+    variant("openmp", default=False, description="Build OpenMP support")
+
+    depends_on("cub", when="+cuda")
+
+    depends_on("blt")
+
+    def cmake_args(self):
+        spec = self.spec
+
+        options = []
+
+        options.append("-DBLT_SOURCE_DIR={0}".format(spec["blt"].prefix))
+
+        options.append("-DENABLE_OPENMP=" + ("On" if "+openmp" in spec else "Off"))
+        if "+cuda" in spec:
+            options.extend(
+                ["-DENABLE_CUDA=ON", "-DCUDA_TOOLKIT_ROOT_DIR=%s" % (spec["cuda"].prefix)]
+            )
+
+            if not spec.satisfies("cuda_arch=none"):
+                cuda_arch = spec.variants["cuda_arch"].value
+                options.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0]))
+                options.append("-DCUDA_ARCH=sm_{0}".format(cuda_arch[0]))
+                flag = "-arch sm_{0}".format(cuda_arch[0])
+                options.append("-DCMAKE_CUDA_FLAGS:STRING={0}".format(flag))
+        else:
+            options.append("-DENABLE_CUDA=OFF")
+
+        if "+rocm" in spec:
+            options.extend(["-DENABLE_HIP=ON", "-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)])
+
+            hip_repair_options(options, spec)
+
+            archs = self.spec.variants["amdgpu_target"].value
+            if archs != "none":
+                arch_str = ",".join(archs)
+                options.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(arch_str))
+        else:
+            options.append("-DENABLE_HIP=OFF")
+
+        options.append(self.define_from_variant("ENABLE_TESTS", "tests"))
+
+        return options