Merge branch 'dev' into dev-paramfiles

src/main.cpp

@@ -14,7 +14,9 @@
 #include "global/global.h"
 #include "grid/grid3D.h"
 #include "io/io.h"
+#include "utils/cuda_utilities.h"
 #include "utils/error_handling.h"
+
 #ifdef SUPERNOVA
   #include "particles/supernova.h"
   #ifdef ANALYSIS
@@ -297,6 +299,7 @@ int main(int argc, char *argv[])
 #endif
 
 #ifdef CPU_TIME
+    cuda_utilities::Print_GPU_Memory_Usage();
     G.Timer.Total.End();
 #endif  // CPU_TIME
 

src/mpi/mpi_routines.cpp

@@ -716,6 +716,29 @@ Real ReduceRealAvg(Real x)
   return y;
 }
 
+size_t Reduce_size_t_Max(size_t in)
+{
+  // Get the right MPI type
+  #if SIZE_MAX == UCHAR_MAX
+    #define my_MPI_SIZE_T MPI_UNSIGNED_CHAR
+  #elif SIZE_MAX == USHRT_MAX
+    #define my_MPI_SIZE_T MPI_UNSIGNED_SHORT
+  #elif SIZE_MAX == UINT_MAX
+    #define my_MPI_SIZE_T MPI_UNSIGNED
+  #elif SIZE_MAX == ULONG_MAX
+    #define my_MPI_SIZE_T MPI_UNSIGNED_LONG
+  #elif SIZE_MAX == ULLONG_MAX
+    #define my_MPI_SIZE_T MPI_UNSIGNED_LONG_LONG
+  #else
+    #error "Error: Type of size_t not supported by Reduce_size_t_Max"
+  #endif
+
+  // Perform the reduction
+  size_t out;
+  MPI_Allreduce(&in, &out, 1, my_MPI_SIZE_T, MPI_MAX, world);
+  return out;
+}
+
   #ifdef PARTICLES
 /* MPI reduction wrapper for sum(part_int)*/
 Real ReducePartIntSum(part_int_t x)

src/mpi/mpi_routines.h

@@ -166,6 +166,14 @@ Real ReduceRealMin(Real x);
 /* MPI reduction wrapper for avg(Real)*/
 Real ReduceRealAvg(Real x);
 
+/*!
+ * \brief MPI reduction wrapper to find the maximum of a size_t variable
+ *
+ * \param in The rank-local value to be reduced
+ * \return size_t The global reduced value
+ */
+size_t Reduce_size_t_Max(size_t in);
+
     #ifdef PARTICLES
 /* MPI reduction wrapper for sum(part_int)*/
 Real ReducePartIntSum(part_int_t x);

src/utils/cuda_utilities.cpp

@@ -1,6 +1,37 @@
+/*!
+ * \file cuda_utilities.cpp
+ * \brief Implementation file for cuda_utilities.h
+ *
+ */
 #include "../utils/cuda_utilities.h"
 
+#include <iomanip>
+#include <sstream>
+
+#include "../io/io.h"
+#include "../mpi/mpi_routines.h"
+
 namespace cuda_utilities
 {
+void Print_GPU_Memory_Usage(std::string const &additional_text)
+{
+  // Get the memory usage
+  size_t gpu_free_memory, gpu_total_memory;
+  CudaSafeCall(cudaMemGetInfo(&gpu_free_memory, &gpu_total_memory));
+
+  // Assuming that all GPUs in the system have the same amount of memory
+  size_t const gpu_used_memory = Reduce_size_t_Max(gpu_total_memory - gpu_free_memory);
+
+  Real const percent_used = 100.0 * (static_cast<Real>(gpu_used_memory) / static_cast<Real>(gpu_total_memory));
+
+  // Prep the message to print
+  std::stringstream output_message_stream;
+  output_message_stream << std::fixed << std::setprecision(2);
+  output_message_stream << "Percentage of GPU memory used: " << percent_used << "%. GPU memory used "
+                        << std::to_string(gpu_used_memory) << ", GPU total memory " << std::to_string(gpu_total_memory)
+                        << additional_text << std::endl;
+  std::string output_message = output_message_stream.str();
 
+  chprintf(output_message.c_str());
+}
 }  // end namespace cuda_utilities

src/utils/cuda_utilities.h

@@ -7,6 +7,8 @@
 
 #pragma once
 
+#include <string>
+
 // Local Includes
 #include "../global/global.h"
 #include "../global/global_cuda.h"
@@ -121,4 +123,13 @@ struct AutomaticLaunchParams {
   int numBlocks;
 };
 // =====================================================================
-}  // end namespace cuda_utilities
+
+// =====================================================================
+/*!
+ * \brief Print the current GPU memory usage to standard out
+ *
+ * \param additional_text Any additional text to be appended to the end of the message
+ */
+void Print_GPU_Memory_Usage(std::string const &additional_text = "");
+// =====================================================================
+}  // end namespace cuda_utilities

src/utils/gpu.hpp

@@ -66,6 +66,7 @@ static constexpr int maxWarpsPerBlock = 1024 / WARPSIZE;
   #define cudaPointerAttributes              hipPointerAttribute_t
   #define cudaPointerGetAttributes           hipPointerGetAttributes
   #define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize
+  #define cudaMemGetInfo                     hipMemGetInfo
 
   // Texture definitions
   #define cudaArray           hipArray