Merge branch 'develop' into feautre/adapt-opt

CHANGELOG.md

@@ -31,6 +31,13 @@ Volta GPUs while the automatically selected value will vary across compilers and
 compiler versions. As such, users are encouraged to override this value with the
 architecture for their system.
 
+The Trilinos Teptra NVector interface has been updated to utilize CMake
+imported targets added in Trilinos 14 to improve support for different Kokkos
+backends with Trilinos.  As such, Trilinos 14 or newer is required and the
+`Trilinos_INTERFACE_*` CMake options have been removed.
+
+Example programs using *hypre* have been updated to support v2.20 and newer.
+
 ### Bug Fixes
 
 Removed error floors from the SUNAdaptController implementations which could
@@ -44,6 +51,9 @@ Fixed the loading of ARKStep's default first order explicit method.
 Fixed a CMake bug regarding usage of missing "print_warning" macro
 that was only triggered when the deprecated `CUDA_ARCH` option was used.
 
+Fixed compilation errors when building the Trilinos Teptra NVector with CUDA
+support.
+
 ### Deprecation Notices
 
 ## Changes to SUNDIALS in release 7.1.1

benchmarks/CMakeLists.txt

@@ -24,8 +24,10 @@ sundials_option(BENCHMARK_NVECTOR BOOL "NVector benchmarks are on" ON)
 
 # Disable some warnings for benchmarks
 if(ENABLE_ALL_WARNINGS)
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-parameter")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
+  set(CMAKE_C_FLAGS
+      "${CMAKE_C_FLAGS} -Wno-unused-parameter -Wno-unused-function")
+  set(CMAKE_CXX_FLAGS
+      "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-unused-function")
 endif()
 
 # ----------------------------------------

benchmarks/advection_reaction_3D/raja/backends.hpp

@@ -114,12 +114,23 @@ using XYZ_KERNEL_POL =
 
 #else
 #define NVECTOR_ID_STRING "Serial"
-using EXEC_POLICY           = RAJA::seq_exec;
-using XYZ_KERNEL_POL        = RAJA::KernelPolicy<RAJA::statement::For<
+
+#if RAJA_VERSION_MAJOR < 2024
+using EXEC_POLICY    = RAJA::seq_exec;
+using XYZ_KERNEL_POL = RAJA::KernelPolicy<RAJA::statement::For<
   2, RAJA::loop_exec,
   RAJA::statement::For<
     1, RAJA::loop_exec,
     RAJA::statement::For<0, RAJA::loop_exec, RAJA::statement::Lambda<0>>>>>;
+#else
+using EXEC_POLICY    = RAJA::seq_exec;
+using XYZ_KERNEL_POL = RAJA::KernelPolicy<RAJA::statement::For<
+  2, RAJA::seq_exec,
+  RAJA::statement::For<
+    1, RAJA::seq_exec,
+    RAJA::statement::For<0, RAJA::seq_exec, RAJA::statement::Lambda<0>>>>>;
+#endif
+
 constexpr auto LocalNvector = N_VNew_Serial;
 #define CopyVecFromDevice(v)
 

cmake/SUNDIALSConfig.cmake.in

@@ -161,10 +161,8 @@ if("@ENABLE_RAJA@" AND NOT TARGET RAJA)
   find_dependency(RAJA PATHS "@RAJA_DIR@")
 endif()
 
-if("@ENABLE_TRILINOS@" AND NOT TARGET SUNDIALS::TRILINOS)
-  add_library(SUNDIALS::TRILINOS INTERFACE IMPORTED)
-  target_link_libraries(SUNDIALS::TRILINOS INTERFACE "@Trilinos_LIBRARIES@")
-  set_target_properties(SUNDIALS::TRILINOS PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "@Trilinos_INCLUDE_DIRS@")
+if("@ENABLE_TRILINOS@" AND NOT TARGET Tpetra::all_libs)
+  find_dependency(Trilinos COMPONENTS Tpetra PATHS "@Trilinos_DIR@")
 endif()
 
 if("@ENABLE_XBRAID@" AND NOT TARGET SUNDIALS::XBRAID)

cmake/SundialsBuildOptionsPost.cmake

@@ -123,7 +123,7 @@ list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_RAJA")
 sundials_option(
   BUILD_NVECTOR_TRILINOS BOOL
   "Build the NVECTOR_TRILINOS module (requires Trilinos)" ON
-  DEPENDS_ON ENABLE_TRILINOS Trilinos_WORKS
+  DEPENDS_ON ENABLE_TRILINOS
   ADVANCED)
 list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_TRILINOS")
 

cmake/SundialsSetupConfig.cmake

@@ -65,7 +65,7 @@ foreach(tpl ${SUNDIALS_TPL_LIST})
 endforeach()
 
 # prepare substitution variable SUNDIALS_TRILINOS_HAVE_MPI for sundials_config.h
-if(Trilinos_MPI)
+if(ENABLE_MPI)
   set(SUNDIALS_TRILINOS_HAVE_MPI TRUE)
 endif()
 

cmake/SundialsTPLOptions.cmake

@@ -269,42 +269,6 @@ sundials_option(ENABLE_TRILINOS BOOL "Enable Trilinos support" OFF)
 sundials_option(Trilinos_DIR PATH "Path to root of Trilinos installation"
                 "${Trilinos_DIR}" DEPENDS_ON ENABLE_TRILINOS)
 
-sundials_option(
-  Trilinos_INTERFACE_CXX_COMPILER STRING "C++ compiler for Trilinos interface"
-  "${Trilinos_CXX_COMPILER}"
-  DEPENDS_ON ENABLE_TRILINOS
-  ADVANCED)
-
-sundials_option(
-  Trilinos_INTERFACE_C_COMPILER STRING "C compiler for Trilinos interface"
-  "${Trilinos_C_COMPILER}"
-  DEPENDS_ON ENABLE_TRILINOS
-  ADVANCED)
-
-sundials_option(
-  Trilinos_INTERFACE_CXX_COMPILER_FLAGS STRING
-  "C++ compiler flags for Trilinos interface" "${Trilinos_CXX_COMPILER_FLAGS}"
-  DEPENDS_ON ENABLE_TRILINOS
-  ADVANCED)
-
-sundials_option(
-  Trilinos_INTERFACE_C_COMPILER_FLAGS STRING
-  "C compiler flags for Trilinos interface" "${Trilinos_C_COMPILER_FLAGS}"
-  DEPENDS_ON ENABLE_TRILINOS
-  ADVANCED)
-
-sundials_option(
-  Trilinos_INTERFACE_MPIEXEC STRING "MPI executable for Trilinos interface"
-  "${Trilinos_MPI_EXEC}"
-  DEPENDS_ON ENABLE_TRILINOS
-  ADVANCED)
-
-sundials_option(
-  Trilinos_WORKS BOOL
-  "Set to ON to force CMake to accept a given Trilinos configuration" OFF
-  DEPENDS_ON ENABLE_TRILINOS
-  ADVANCED)
-
 # ---------------------------------------------------------------
 # Enable XBraid support?
 # ---------------------------------------------------------------

cmake/tpl/FindHYPRE.cmake

@@ -59,8 +59,7 @@ else()
     HYPRE_LIBRARY
     NAMES ${HYPRE_LIBRARY_NAMES}
     HINTS "${HYPRE_DIR}" "${HYPRE_DIR}/lib" "${HYPRE_DIR}/lib64"
-          "${HYPRE_LIBRARY_DIR}"
-    NO_DEFAULT_PATH)
+          "${HYPRE_LIBRARY_DIR}")
 endif()
 mark_as_advanced(HYPRE_LIBRARY)
 
@@ -71,6 +70,23 @@ if(_idx EQUAL -1)
       CACHE STRING "" FORCE)
 endif()
 
+# Extract version parts from the version information
+if(HYPRE_INCLUDE_DIR)
+
+  # HYPRE_config.h file added in at least v2.11.0 (possibly sooner)
+  find_file(HYPRE_CONFIG_FILE HYPRE_config.h PATHS "${HYPRE_INCLUDE_DIR}")
+
+  file(STRINGS "${HYPRE_CONFIG_FILE}" _hypre_release_version
+       REGEX "HYPRE_RELEASE_VERSION")
+  string(REGEX MATCH "[0-9]+\.[0-9]+\.[0-9]+" HYPRE_VERSION
+               "${_hypre_release_version}")
+  string(REGEX MATCHALL "[0-9]+" _hypre_version_numbers "${HYPRE_VERSION}")
+  list(GET _hypre_version_numbers 0 HYPRE_VERSION_MAJOR)
+  list(GET _hypre_version_numbers 1 HYPRE_VERSION_MINOR)
+  list(GET _hypre_version_numbers 2 HYPRE_VERSION_PATCH)
+
+endif()
+
 # set a more informative error message in case the library was not found
 set(HYPRE_NOT_FOUND_MESSAGE
     "\
@@ -83,6 +99,7 @@ ERROR: Could not find HYPRE. Please check the variables:\n\
 find_package_handle_standard_args(
   HYPRE
   REQUIRED_VARS HYPRE_LIBRARY HYPRE_LIBRARIES HYPRE_INCLUDE_DIR
+  VERSION_VAR HYPRE_VERSION
   FAIL_MESSAGE "${HYPRE_NOT_FOUND_MESSAGE}")
 
 # Create target for HYPRE

cmake/tpl/SundialsTrilinos.cmake

@@ -26,11 +26,7 @@
 # Section 1: Include guard
 # -----------------------------------------------------------------------------
 
-if(NOT DEFINED SUNDIALS_TRILINOS_INCLUDED)
-  set(SUNDIALS_TRILINOS_INCLUDED)
-else()
-  return()
-endif()
+include_guard(GLOBAL)
 
 # -----------------------------------------------------------------------------
 # Section 2: Check to make sure options are compatible
@@ -41,144 +37,20 @@ endif()
 # -----------------------------------------------------------------------------
 
 # Find Trilinos
-find_package(Trilinos REQUIRED)
+find_package(
+  Trilinos REQUIRED
+  COMPONENTS Tpetra HINTS "${Trilinos_DIR}/lib/cmake/Trilinos"
+             "${Trilinos_DIR}")
 
-# Check if Trilinos was built with MPI Starting with TriBITS 2022-10-16
-# <Project_TPL_LIST> is no longer defined so we base MPI support on ENABLE_MPI
-if(Trilinos_TPL_LIST)
-  if(";${Trilinos_TPL_LIST};" MATCHES ";MPI;")
-    set(Trilinos_MPI TRUE)
-  else()
-    set(Trilinos_MPI FALSE)
-  endif()
-else()
-  if(ENABLE_MPI)
-    set(Trilinos_MPI TRUE)
-  else()
-    set(Trilinos_MPI FALSE)
-  endif()
-endif()
-
-# For XSDK compatibility, only use the user/spack provided compiler and flags to
-# build SUNDIALS modules that use Trilinos. If we are not in XSDK mode, we can
-# use the imported Trilinos compiler and flags by default, but allow the user to
-# change it through CMake the Trilinos_INTERFACE_* options.
-
-if(USE_XSDK_DEFAULTS)
-  if(Trilinos_MPI AND MPI_CXX_FOUND)
-    force_variable(Trilinos_INTERFACE_CXX_COMPILER STRING
-                   "C++ compiler for Trilinos interface" "${MPI_CXX_COMPILER}")
-    set(Trilinos_INTERFACE_MPI_CXX_FOUND
-        ${Trilinos_MPI}
-        CACHE INTERNAL "Is Trilinos interface C++ compiler MPI")
-  else()
-    force_variable(Trilinos_INTERFACE_CXX_COMPILER STRING
-                   "C compiler for Trilinos interface" "${CMAKE_CXX_COMPILER}")
-    set(Trilinos_INTERFACE_MPI_CXX_FOUND
-        FALSE
-        CACHE INTERNAL "Is Trilinos interface C++ compiler MPI")
-  endif()
-  if(Trilinos_MPI AND MPI_C_FOUND)
-    force_variable(Trilinos_INTERFACE_C_COMPILER STRING
-                   "C compiler for Trilinos interface" "${MPI_C_COMPILER}")
-    set(Trilinos_INTERFACE_MPI_C_FOUND
-        ${Trilinos_MPI}
-        CACHE INTERNAL "Is Trilinos interface C compiler MPI")
-  else()
-    force_variable(Trilinos_INTERFACE_C_COMPILER STRING
-                   "C compiler for Trilinos interface" "${CMAKE_C_COMPILER}")
-    set(Trilinos_INTERFACE_MPI_C_FOUND
-        FALSE
-        CACHE INTERNAL "Is Trilinos interface C compiler MPI")
-  endif()
-  force_variable(Trilinos_INTERFACE_CXX_COMPILER_FLAGS STRING
-                 "C++ compiler flags specific to Trilinos interface" "")
-  force_variable(Trilinos_INTERFACE_C_COMPILER_FLAGS STRING
-                 "C compiler flags specific to Trilinos interface" "")
-  force_variable(
-    Trilinos_INTERFACE_MPIEXEC STRING "MPI executable for Trilinos interface"
-    "${MPIEXEC_EXECUTABLE}")
-else()
-  force_variable(
-    Trilinos_INTERFACE_CXX_COMPILER STRING
-    "C++ compiler for Trilinos interface" "${Trilinos_CXX_COMPILER}")
-  force_variable(Trilinos_INTERFACE_C_COMPILER STRING
-                 "C compiler for Trilinos interface" "${Trilinos_C_COMPILER}")
-  force_variable(
-    Trilinos_INTERFACE_CXX_COMPILER_FLAGS STRING
-    "C++ compiler flags for Trilinos interface"
-    "${Trilinos_CXX_COMPILER_FLAGS}")
-  force_variable(
-    Trilinos_INTERFACE_C_COMPILER_FLAGS STRING
-    "C compiler flags for Trilinos interface" "${Trilinos_C_COMPILER_FLAGS}")
-  force_variable(Trilinos_INTERFACE_MPIEXEC STRING
-                 "MPI executable for Trilinos interface" "${Trilinos_MPI_EXEC}")
-  set(Trilinos_INTERFACE_MPI_CXX_FOUND
-      ${Trilinos_MPI}
-      CACHE INTERNAL "Is Trilinos interface C++ compiler MPI")
-  set(Trilinos_INTERFACE_MPI_C_FOUND
-      ${Trilinos_MPI}
-      CACHE INTERNAL "Is Trilinos interface C compiler MPI")
-endif()
-
-message(STATUS "Trilinos_MPI:          ${Trilinos_MPI}")
-message(STATUS "Trilinos_LIBRARIES:    ${Trilinos_LIBRARIES}")
-message(STATUS "Trilinos_INCLUDE_DIRS: ${Trilinos_INCLUDE_DIRS}")
+message(STATUS "Trilinos Libraries: ${Trilinos_LIBRARIES}")
+message(STATUS "Trilinos Includes: ${Trilinos_INCLUDE_DIRS}")
+message(STATUS "Trilinos Devices: ${Kokkos_DEVICES}")
 
 # -----------------------------------------------------------------------------
 # Section 4: Test the TPL
 # -----------------------------------------------------------------------------
 
-if(Trilinos_FOUND AND (NOT Trilinos_WORKS))
-  # Do any checks which don't require compilation first.
-
-  # Create the Trilinos_TEST directory
-  set(Trilinos_TEST_DIR ${PROJECT_BINARY_DIR}/Trilinos_TEST)
-  file(MAKE_DIRECTORY ${Trilinos_TEST_DIR})
-
-  # Create a CMakeLists.txt file
-  file(
-    WRITE ${Trilinos_TEST_DIR}/CMakeLists.txt
-    "CMAKE_MINIMUM_REQUIRED(VERSION ${CMAKE_VERSION})\n"
-    "PROJECT(ltest CXX)\n"
-    "SET(CMAKE_VERBOSE_MAKEFILE ON)\n"
-    "SET(CMAKE_BUILD_TYPE \"${CMAKE_BUILD_TYPE}\")\n"
-    "SET(CMAKE_CXX_COMPILER \"${Trilinos_INTERFACE_CXX_COMPILER}\")\n"
-    "SET(CMAKE_CXX_STANDARD \"${CMAKE_CXX_STANDARD}\")\n"
-    "SET(CMAKE_CXX_FLAGS \"${Trilinos_INTERFACE_CXX_COMPILER_FLAGS}\")\n"
-    "SET(Trilinos_DIR \"${Trilinos_DIR}\")\n"
-    "INCLUDE(FindPackageHandleStandardArgs)\n"
-    "INCLUDE(${PROJECT_SOURCE_DIR}/cmake/tpl/FindTrilinos.cmake)\n"
-    "ADD_EXECUTABLE(ltest ltest.cpp)\n"
-    "TARGET_LINK_LIBRARIES(ltest SUNDIALS::TRILINOS)\n")
-
-  # Create a C++ source file which calls a Trilinos function
-  file(WRITE ${Trilinos_TEST_DIR}/ltest.cpp
-       "#include <Tpetra_Version.hpp>\n" "int main(void) {\n"
-       "std::cout << Tpetra::version() << std::endl;\n" "return(0);\n" "}\n")
-
-  # Attempt to build and link the "ltest" executable
-  try_compile(
-    COMPILE_OK ${Trilinos_TEST_DIR}
-    ${Trilinos_TEST_DIR} ltest
-    OUTPUT_VARIABLE COMPILE_OUTPUT)
-
-  # Process test result
-  if(COMPILE_OK)
-    message(STATUS "Checking if Trilinos works with SUNDIALS... OK")
-    set(Trilinos_WORKS
-        TRUE
-        CACHE BOOL "Trilinos works with SUNDIALS as configured" FORCE)
-  else()
-    message(STATUS "Checking if Trilinos works with SUNDIALS... FAILED")
-    message(STATUS "Check output: ")
-    message("${COMPILE_OUTPUT}")
-    message(FATAL_ERROR "SUNDIALS interface to Trilinos is not functional.")
-  endif()
-
-elseif(Trilinos_FOUND AND Trilinos_WORKS)
-  message(
-    STATUS
-      "Skipped Trilinos tests, assuming Trilinos works with SUNDIALS. Set Trilinos_WORKS=FALSE to (re)run compatibility test."
-  )
-endif()
+# Does not currently work with Trilinos imported targets due to an error from
+# evaluating generator expression: $<LINK_LANGUAGE:CXX> may only be used with
+# binary targets to specify link libraries, link directories, link options and
+# link depends.