Bugfix: Remove MPI wtime in profiler (#317)

Updated `SUNProfiler` to note rely on `MPI_WTime`. Fixes #312 --------- Co-authored-by: David Gardner <[email protected]>
LLNL · Nov 2, 2023 · 26317e2 · 26317e2
1 parent 6c60a56
commit 26317e2
Show file tree

Hide file tree

Showing 25 changed files with 686 additions and 213 deletions.
diff --git a/.github/workflows/windows-latest-mingw.yml b/.github/workflows/windows-latest-mingw.yml
@@ -21,7 +21,7 @@ jobs:
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -G "MinGW Makefiles" -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -G "MinGW Makefiles" -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DSUNDIALS_BUILD_WITH_PROFILING=ON -DSUNDIALS_LOGGING_LEVEL=2 -DSUNDIALS_TEST_UNITTESTS=OFF -DEXAMPLES_ENABLE_CXX=ON
 
     - name: Build
       # Build your program with the given configuration

diff --git a/.github/workflows/windows-latest.yml b/.github/workflows/windows-latest.yml
@@ -17,7 +17,7 @@ jobs:
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBUILD_STATIC_LIBS=OFF
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBUILD_STATIC_LIBS=OFF -DSUNDIALS_BUILD_WITH_PROFILING=ON -DSUNDIALS_LOGGING_LEVEL=2 -DSUNDIALS_TEST_UNITTESTS=ON -DEXAMPLES_ENABLE_CXX=ON
 
     - name: Build
       # Build your program with the given configuration

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -27,7 +27,7 @@ Fixed scaling bug in `SUNMatScaleAddI_Sparse` for non-square matrices.
 Fixed missing soversions in some `SUNLinearSolver` and `SUNNonlinearSolver`
 CMake targets.
 
-Added Fortran support for the LAPACK dense `SUNLinearSolver` implementations.
+Added Fortran support for the LAPACK dense `SUNLinearSolver` implementation.
 
 Fixed the build system support for MAGMA when using a NVIDIA HPC SDK installation of CUDA
 and fixed the targets used for rocBLAS and rocSPARSE.
@@ -38,6 +38,8 @@ ERK method `ARKODE_SOFRONIOU_SPALETTA_5_3_4`, the sixth order ERK method
 the eighth order ERK method `ARKODE_VERNER_13_7_8`, and the ninth order ERK
 method `ARKODE_VERNER_16_8_9`.
 
+Changed the `SUNProfiler` so that it does not rely on `MPI_WTime` in any case.
+This fixes https://github.com/LLNL/sundials/issues/312. 
 
 ## Changes to SUNDIALS in release 6.6.1
 

diff --git a/cmake/SundialsSetupCompilers.cmake b/cmake/SundialsSetupCompilers.cmake
@@ -223,13 +223,6 @@ if(SUNDIALS_POSIX_TIMERS AND POSIX_TIMERS_NEED_POSIX_C_SOURCE)
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_POSIX_C_SOURCE=${SUNDIALS_POSIX_C_SOURCE}")
 endif()
 
-# Check if profiling is being built with no timers.
-if(SUNDIALS_BUILD_WITH_PROFILING AND
-   (NOT ENABLE_CALIPER) AND
-   (NOT ENABLE_MPI) AND
-   (NOT SUNDIALS_POSIX_TIMERS))
-  message(SEND_ERROR "The SUNDIALS native profiler requires POSIX timers or MPI_Wtime, but neither were found.")
-endif()
 
 # ---------------------------------------------------------------
 # Check for deprecated attribute with message

diff --git a/cmake/macros/SundialsAddLibrary.cmake b/cmake/macros/SundialsAddLibrary.cmake
@@ -152,6 +152,8 @@ macro(sundials_add_library target)
     # create the target for the object library
     add_library(${obj_target} OBJECT ${sources})
 
+    set_target_properties(${obj_target} PROPERTIES FOLDER "obj")
+
     # add all object libraries to object library
     if(sundials_add_library_OBJECT_LIBRARIES)
       target_link_libraries(${obj_target}
@@ -229,6 +231,8 @@ macro(sundials_add_library target)
 
       add_library(${_actual_target_name} ${_libtype} $<TARGET_OBJECTS:${obj_target}>)
 
+      set_target_properties(${_actual_target_name} PROPERTIES FOLDER "src")
+
       # add any object library dependencies
       if(sundials_add_library_OBJECT_LIBRARIES)
         if(${_libtype} MATCHES "STATIC")

diff --git a/doc/arkode/guide/source/Introduction.rst b/doc/arkode/guide/source/Introduction.rst
@@ -162,6 +162,9 @@ ERK method ``ARKODE_SOFRONIOU_SPALETTA_5_3_4``, the sixth order ERK method
 the eighth order ERK method ``ARKODE_VERNER_13_7_8``, and the ninth order ERK
 method ``ARKODE_VERNER_16_8_9``.
 
+Changed the ``SUNProfiler`` so that it does not rely on ``MPI_WTime`` in any case.
+This fixes `GitHub Issue #312 <https://github.com/LLNL/sundials/issues/312>`_. 
+
 Added Fortran support for the LAPACK  dense ``SUNLinearSolver`` implementation.
 
 Changes in v5.6.1
@@ -993,7 +996,6 @@ utilize a zero initial guess.
 A bug was fixed in the ARKODE stepper modules where the stop time may be passed
 after resetting the integrator.
 
-
 Changes in v4.7.0
 -----------------
 

diff --git a/doc/cvode/guide/source/Introduction.rst b/doc/cvode/guide/source/Introduction.rst
@@ -129,6 +129,9 @@ Fixed scaling bug in ``SUNMatScaleAddI_Sparse`` for non-square matrices.
 Fixed missing soversions in some ``SUNLinearSolver`` and ``SUNNonlinearSolver``
 CMake targets.
 
+Changed the ``SUNProfiler`` so that it does not rely on ``MPI_WTime`` in any case.
+This fixes `GitHub Issue #312 <https://github.com/LLNL/sundials/issues/312>`_. 
+
 Added Fortran support for the LAPACK  dense ``SUNLinearSolver`` implementation.
 
 Changes in v6.6.1

diff --git a/doc/cvodes/guide/source/Introduction.rst b/doc/cvodes/guide/source/Introduction.rst
@@ -132,6 +132,9 @@ Fixed scaling bug in ``SUNMatScaleAddI_Sparse`` for non-square matrices.
 Fixed missing soversions in some ``SUNLinearSolver`` and ``SUNNonlinearSolver``
 CMake targets.
 
+Changed the ``SUNProfiler`` so that it does not rely on ``MPI_WTime`` in any case.
+This fixes `GitHub Issue #312 <https://github.com/LLNL/sundials/issues/312>`_. 
+
 Added Fortran support for the LAPACK  dense ``SUNLinearSolver`` implementation.
 
 Changes in v6.6.1

diff --git a/doc/ida/guide/source/Introduction.rst b/doc/ida/guide/source/Introduction.rst
@@ -90,6 +90,9 @@ Fixed scaling bug in ``SUNMatScaleAddI_Sparse`` for non-square matrices.
 Fixed missing soversions in some ``SUNLinearSolver`` and ``SUNNonlinearSolver``
 CMake targets.
 
+Changed the ``SUNProfiler`` so that it does not rely on ``MPI_WTime`` in any case.
+This fixes `GitHub Issue #312 <https://github.com/LLNL/sundials/issues/312>`_. 
+
 Added Fortran support for the LAPACK  dense ``SUNLinearSolver`` implementation.
 
 Changes in v6.6.1

diff --git a/doc/idas/guide/source/Introduction.rst b/doc/idas/guide/source/Introduction.rst
@@ -107,6 +107,9 @@ Fixed scaling bug in ``SUNMatScaleAddI_Sparse`` for non-square matrices.
 Fixed missing soversions in some ``SUNLinearSolver`` and ``SUNNonlinearSolver``
 CMake targets.
 
+Changed the ``SUNProfiler`` so that it does not rely on ``MPI_WTime`` in any case.
+This fixes `GitHub Issue #312 <https://github.com/LLNL/sundials/issues/312>`_. 
+
 Added Fortran support for the LAPACK  dense ``SUNLinearSolver`` implementation.
 
 Changes in v5.6.1

diff --git a/doc/kinsol/guide/source/Introduction.rst b/doc/kinsol/guide/source/Introduction.rst
@@ -112,6 +112,9 @@ Updated the Tpetra NVector interface to support Trilinos 14.
 Fixed a memory leak when destroying a CUDA, HIP, SYCL, or system SUNMemoryHelper
 object.
 
+Changed the ``SUNProfiler`` so that it does not rely on ``MPI_WTime`` in any case.
+This fixes `GitHub Issue #312 <https://github.com/LLNL/sundials/issues/312>`_. 
+
 Changes in v6.6.0
 -----------------
 

diff --git a/doc/shared/Install.rst b/doc/shared/Install.rst
@@ -1022,6 +1022,7 @@ illustration only.
 .. cmakeoption:: SUNDIALS_BUILD_WITH_PROFILING
 
    Build SUNDIALS with capabilties for fine-grained profiling.
+   This requires POSIX timers or the Windows ``profileapi.h`` timers.
 
    Default: OFF
 

diff --git a/doc/shared/sundials/Profiling.rst b/doc/shared/sundials/Profiling.rst
@@ -48,6 +48,11 @@ If Caliper is enabled, then users should refer to the `Caliper documentation <ht
 for information on getting profiler output. In most cases, this involves
 setting the ``CALI_CONFIG`` environment variable.
 
+
+.. note:: 
+
+   The SUNDIALS profiler requires POSIX timers or the Windows ``profileapi.h`` timers.
+
 .. warning::
 
    While the SUNDIALS profiling scheme is relatively lightweight, enabling
@@ -149,6 +154,31 @@ are available.
       * Returns zero if successful, or non-zero if an error occurred
 
 
+.. c:function:: int SUNProfiler_GetElapsedTime(SUNProfiler p, const char* name, double* time)
+
+   Get the elapsed time for the timer "name" in seconds.
+
+   **Arguments:**
+      * ``p`` -- a ``SUNProfiler`` object
+      * ``name`` -- the name for the profiling region of interest
+      * ``time`` -- upon return, the elapsed time for the timer
+
+   **Returns:**
+      * Returns zero if successful, or non-zero if an error occurred
+
+
+.. c:function:: int SUNProfiler_GetTimerResolution(SUNProfiler p, double* resolution)
+
+   Get the timer resolution in seconds.
+
+   **Arguments:**
+      * ``p`` -- a ``SUNProfiler`` object
+      * ``resolution`` -- upon return, the resolution for the timer
+
+   **Returns:**
+      * Returns zero if successful, or non-zero if an error occurred
+
+
 .. c:function:: int SUNProfiler_Print(SUNProfiler p, FILE* fp)
 
    Prints out a profiling summary. When constructed with an MPI comm the summary

diff --git a/include/sundials/sundials_profiler.h b/include/sundials/sundials_profiler.h
@@ -23,16 +23,21 @@
 #include "caliper/cali.h"
 #endif
 
-#ifdef __cplusplus  /* wrapper to enable C++ usage */
+#ifdef __cplusplus /* wrapper to enable C++ usage */
 extern "C" {
 #endif
 
-typedef struct _SUNProfiler *SUNProfiler;
+typedef struct _SUNProfiler* SUNProfiler;
 
-SUNDIALS_EXPORT int SUNProfiler_Create(void* comm, const char* title, SUNProfiler* p);
+SUNDIALS_EXPORT int SUNProfiler_Create(void* comm, const char* title,
+                                       SUNProfiler* p);
 SUNDIALS_EXPORT int SUNProfiler_Free(SUNProfiler* p);
 SUNDIALS_EXPORT int SUNProfiler_Begin(SUNProfiler p, const char* name);
 SUNDIALS_EXPORT int SUNProfiler_End(SUNProfiler p, const char* name);
+SUNDIALS_EXPORT int SUNProfiler_GetTimerResolution(SUNProfiler p,
+                                                   double* resolution);
+SUNDIALS_EXPORT int SUNProfiler_GetElapsedTime(SUNProfiler p, const char* name,
+                                               double* time);
 SUNDIALS_EXPORT int SUNProfiler_Print(SUNProfiler p, FILE* fp);
 SUNDIALS_EXPORT int SUNProfiler_Reset(SUNProfiler p);
 
@@ -42,35 +47,29 @@ SUNDIALS_EXPORT int SUNProfiler_Reset(SUNProfiler p);
 
 #define SUNDIALS_MARK_FUNCTION_END(profobj) CALI_MARK_FUNCTION_END
 
-#define SUNDIALS_WRAP_STATEMENT(profobj, name, stmt) CALI_WRAP_STATEMENT(name, stmt)
+#define SUNDIALS_WRAP_STATEMENT(profobj, name, stmt) \
+  CALI_WRAP_STATEMENT(name, stmt)
 
 #define SUNDIALS_MARK_BEGIN(profobj, name) CALI_MARK_BEGIN(name)
 
 #define SUNDIALS_MARK_END(profobj, name) CALI_MARK_END(name)
 
-#ifdef __cplusplus
-#define SUNDIALS_CXX_MARK_FUNCTION(projobj) CALI_CXX_MARK_FUNCTION
-#endif
-
 #elif defined(SUNDIALS_BUILD_WITH_PROFILING)
 
-#define SUNDIALS_MARK_FUNCTION_BEGIN(profobj) SUNProfiler_Begin(profobj, __func__)
+#define SUNDIALS_MARK_FUNCTION_BEGIN(profobj) \
+  SUNProfiler_Begin(profobj, __func__)
 
 #define SUNDIALS_MARK_FUNCTION_END(profobj) SUNProfiler_End(profobj, __func__)
 
 #define SUNDIALS_WRAP_STATEMENT(profobj, name, stmt) \
-    SUNProfiler_Begin(profobj, (name)); \
-    stmt; \
-    SUNProfiler_End(profobj, (name));
+  SUNProfiler_Begin(profobj, (name));                \
+  stmt;                                              \
+  SUNProfiler_End(profobj, (name));
 
 #define SUNDIALS_MARK_BEGIN(profobj, name) SUNProfiler_Begin(profobj, (name))
 
 #define SUNDIALS_MARK_END(profobj, name) SUNProfiler_End(profobj, (name))
 
-#ifdef __cplusplus
-#define SUNDIALS_CXX_MARK_FUNCTION(profobj) sundials::ProfilerMarkScope __ProfilerMarkScope(profobj, __func__)
-#endif
-
 #else
 
 #define SUNDIALS_MARK_FUNCTION_BEGIN(profobj)
@@ -83,36 +82,14 @@ SUNDIALS_EXPORT int SUNProfiler_Reset(SUNProfiler p);
 
 #define SUNDIALS_MARK_END(profobj, name)
 
-#ifdef __cplusplus
-#define SUNDIALS_CXX_MARK_FUNCTION(profobj)
-#endif
-
 #endif
 
 #ifdef __cplusplus
 }
 
-namespace sundials
-{
-/* Convenience class for C++ codes.
-   Allows for simpler profiler statements using C++ scoping rules. */
-class ProfilerMarkScope
-{
-public:
-  ProfilerMarkScope(SUNProfiler prof, const char* name) {
-    prof_ = prof;
-    name_ = name;
-    SUNProfiler_Begin(prof_, name_);
-  }
-
-  ~ProfilerMarkScope() {
-    SUNProfiler_End(prof_, name_);
-  }
-private:
-  SUNProfiler prof_;
-  const char* name_;
-};
-}
+/* We include this here for backwards compatibility
+   (the contents used to be defined here directly) */
+#include <sundials/sundials_profiler.hpp>
 
 #endif
-#endif /* SUNDIALS_PROFILER_H_ */
+#endif /* SUNDIALS_PROFILER_H */
diff --git a/include/sundials/sundials_profiler.hpp b/include/sundials/sundials_profiler.hpp
@@ -0,0 +1,52 @@
+/* -----------------------------------------------------------------
+ * Programmer: Cody J. Balos @ LLNL
+ * -----------------------------------------------------------------
+ * SUNDIALS Copyright Start
+ * Copyright (c) 2002-2023, Lawrence Livermore National Security
+ * and Southern Methodist University.
+ * All rights reserved.
+ *
+ * See the top-level LICENSE and NOTICE files for details.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SUNDIALS Copyright End
+ * -----------------------------------------------------------------*/
+
+#ifndef _SUNDIALS_PROFILER_HPP
+#define _SUNDIALS_PROFILER_HPP
+
+#include <cstring>
+#include <sundials/sundials_config.h>
+#include <sundials/sundials_profiler.h>
+
+#if defined(SUNDIALS_BUILD_WITH_PROFILING) && defined(SUNDIALS_CALIPER_ENABLED)
+#define SUNDIALS_CXX_MARK_FUNCTION(projobj) CALI_CXX_MARK_FUNCTION
+#elif defined(SUNDIALS_BUILD_WITH_PROFILING)
+#define SUNDIALS_CXX_MARK_FUNCTION(profobj) \
+  sundials::ProfilerMarkScope __ProfilerMarkScope(profobj, __func__)
+#else
+#define SUNDIALS_CXX_MARK_FUNCTION(profobj)
+#endif
+
+namespace sundials {
+/* Convenience class for C++ codes.
+   Allows for simpler profiler statements using C++ scoping rules. */
+class ProfilerMarkScope
+{
+public:
+  ProfilerMarkScope(SUNProfiler prof, const char* name)
+  {
+    prof_ = prof;
+    name_ = name;
+    SUNProfiler_Begin(prof_, name_);
+  }
+
+  ~ProfilerMarkScope() { SUNProfiler_End(prof_, name_); }
+
+private:
+  SUNProfiler prof_;
+  const char* name_;
+};
+} // namespace sundials
+
+#endif /* SUNDIALS_PROFILER_HPP */
diff --git a/scripts/shared b/scripts/shared
@@ -152,6 +152,7 @@ $tar $tarfile $distrobase/include/sundials/sundials_nonlinearsolver.hpp
 $tar $tarfile $distrobase/include/sundials/sundials_nvector_senswrapper.h
 $tar $tarfile $distrobase/include/sundials/sundials_nvector.h
 $tar $tarfile $distrobase/include/sundials/sundials_profiler.h
+$tar $tarfile $distrobase/include/sundials/sundials_profiler.hpp
 $tar $tarfile $distrobase/include/sundials/sundials_sycl_policies.hpp
 $tar $tarfile $distrobase/include/sundials/sundials_types.h
 $tar $tarfile $distrobase/include/sundials/sundials_version.h
@@ -842,3 +843,4 @@ echo "   --- Add unit tests files to $tarfile"
 $tar $tarfile $distrobase/test/unit_tests/CMakeLists.txt
 $tar $tarfile $distrobase/test/unit_tests/reductions
 $tar $tarfile $distrobase/test/unit_tests/sunmemory
+$tar $tarfile $distrobase/test/unit_tests/profiling
diff --git a/src/sundials/CMakeLists.txt b/src/sundials/CMakeLists.txt
@@ -42,6 +42,7 @@ set(sundials_HEADERS
   sundials_nvector.h
   sundials_nvector.hpp
   sundials_profiler.h
+  sundials_profiler.hpp
   sundials_logger.h
   sundials_types.h
   sundials_version.h