From d74c0026c03a90714c2bbc5f05c9ec74cd1ebbd6 Mon Sep 17 00:00:00 2001
From: "Balos, Cody, J" <balos1@llnl.gov>
Date: Sun, 5 Nov 2023 22:45:10 -0800
Subject: [PATCH] create a mpi addon oject lib for mpi errors

---
 src/nvector/manyvector/CMakeLists.txt |  1 +
 src/nvector/mpiplusx/CMakeLists.txt   |  1 +
 src/nvector/parallel/CMakeLists.txt   |  1 +
 src/nvector/parhyp/CMakeLists.txt     |  1 +
 src/nvector/petsc/CMakeLists.txt      |  1 +
 src/nvector/trilinos/CMakeLists.txt   |  1 +
 src/sundials/CMakeLists.txt           | 29 ++++++++++++++++++++++-----
 7 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/src/nvector/manyvector/CMakeLists.txt b/src/nvector/manyvector/CMakeLists.txt
index 96c18b94cc..50a996a368 100644
--- a/src/nvector/manyvector/CMakeLists.txt
+++ b/src/nvector/manyvector/CMakeLists.txt
@@ -28,6 +28,7 @@ if(BUILD_NVECTOR_MANYVECTOR)
       nvector
     OBJECT_LIBRARIES
       sundials_generic_obj
+      sundials_core_mpi_addons_obj
     OUTPUT_NAME
       sundials_nvecmanyvector
     VERSION
diff --git a/src/nvector/mpiplusx/CMakeLists.txt b/src/nvector/mpiplusx/CMakeLists.txt
index b6ca271360..713c6e4a8a 100644
--- a/src/nvector/mpiplusx/CMakeLists.txt
+++ b/src/nvector/mpiplusx/CMakeLists.txt
@@ -34,6 +34,7 @@ sundials_add_library(sundials_nvecmpiplusx
     nvector
   OBJECT_LIBRARIES
     sundials_generic_obj
+    sundials_core_mpi_addons_obj
     sundials_nvecmpimanyvector_obj
   OUTPUT_NAME
     sundials_nvecmpiplusx
diff --git a/src/nvector/parallel/CMakeLists.txt b/src/nvector/parallel/CMakeLists.txt
index d5d2e28a59..021d3b29ba 100644
--- a/src/nvector/parallel/CMakeLists.txt
+++ b/src/nvector/parallel/CMakeLists.txt
@@ -34,6 +34,7 @@ sundials_add_library(sundials_nvecparallel
     nvector
   OBJECT_LIBRARIES
     sundials_generic_obj
+    sundials_core_mpi_addons_obj
   OUTPUT_NAME
     sundials_nvecparallel
   VERSION
diff --git a/src/nvector/parhyp/CMakeLists.txt b/src/nvector/parhyp/CMakeLists.txt
index 0cf104551a..068d019d3c 100644
--- a/src/nvector/parhyp/CMakeLists.txt
+++ b/src/nvector/parhyp/CMakeLists.txt
@@ -26,6 +26,7 @@ sundials_add_library(sundials_nvecparhyp
     nvector
   OBJECT_LIBRARIES
     sundials_generic_obj
+    sundials_core_mpi_addons_obj
   LINK_LIBRARIES
     PUBLIC
     SUNDIALS::HYPRE
diff --git a/src/nvector/petsc/CMakeLists.txt b/src/nvector/petsc/CMakeLists.txt
index f585597319..d7227baae9 100644
--- a/src/nvector/petsc/CMakeLists.txt
+++ b/src/nvector/petsc/CMakeLists.txt
@@ -34,6 +34,7 @@ sundials_add_library(sundials_nvecpetsc
     nvector
   OBJECT_LIBRARIES
     sundials_generic_obj
+    sundials_core_mpi_addons_obj
   LINK_LIBRARIES
     PUBLIC SUNDIALS::PETSC
   OUTPUT_NAME
diff --git a/src/nvector/trilinos/CMakeLists.txt b/src/nvector/trilinos/CMakeLists.txt
index 204d50d5e2..47f2b635cb 100644
--- a/src/nvector/trilinos/CMakeLists.txt
+++ b/src/nvector/trilinos/CMakeLists.txt
@@ -32,6 +32,7 @@ sundials_add_library(sundials_nvectrilinos
     nvector
   OBJECT_LIBRARIES
     sundials_generic_obj
+    sundials_core_mpi_addons_obj
   LINK_LIBRARIES
     PUBLIC SUNDIALS::TRILINOS
   OUTPUT_NAME
diff --git a/src/sundials/CMakeLists.txt b/src/sundials/CMakeLists.txt
index 9258bfbebe..b26766f090 100644
--- a/src/sundials/CMakeLists.txt
+++ b/src/sundials/CMakeLists.txt
@@ -107,12 +107,24 @@ endif()
 # Add prefix with complete path to the source files
 add_prefix(${SUNDIALS_SOURCE_DIR}/src/sundials/ sundials_SOURCES)
 
+if(ENABLE_MPI AND MPI_C_FOUND)
+  # Create a library out of the generic sundials modules
+  sundials_add_library(sundials_core_mpi_addons
+    SOURCES
+      sundials_mpi_errors.c
+    HEADERS
+      ${SUNDIALS_SOURCE_DIR}/include/sundials/sundials_mpi_errors.h
+    INCLUDE_SUBDIR
+      sundials
+    INCLUDE_DIRECTORIES
+      PUBLIC ${MPI_C_INCLUDE_DIRS}
+    LINK_LIBRARIES
+      PUBLIC ${MPI_C_LIBRARIES}
+    OBJECT_LIB_ONLY
+  )
+endif()
+
 if(SUNDIALS_BUILD_WITH_PROFILING OR SUNDIALS_LOGGING_ENABLE_MPI)
-  if(ENABLE_MPI AND MPI_C_FOUND)
-    # Workaround bug in CMake < 3.17.3 when using MPI::MPI_CXX and CUDA
-    set(_include_mpi_if_needed PUBLIC ${MPI_C_INCLUDE_DIRS})
-    set(_link_mpi_if_needed PUBLIC ${MPI_C_LIBRARIES})
-  endif()
   if(ENABLE_CALIPER)
     set(_link_caliper_if_needed PUBLIC caliper)
   endif()
@@ -121,6 +133,13 @@ if(SUNDIALS_BUILD_WITH_PROFILING OR SUNDIALS_LOGGING_ENABLE_MPI)
   endif()
 endif()
 
+if(SUNDIALS_LOGGING_ENABLE_MPI)
+  if(ENABLE_MPI AND MPI_C_FOUND)
+    set(_include_mpi_if_needed PUBLIC ${MPI_C_INCLUDE_DIRS})
+    set(_link_mpi_if_needed PUBLIC ${MPI_C_LIBRARIES})
+  endif()
+endif()
+
 # Create a library out of the generic sundials modules
 sundials_add_library(sundials_generic
   SOURCES