slight refactoring to let us remove the SET_MPI_GRID macro

cholla-hydro · Jul 26, 2024 · b8aaa7f · b8aaa7f
1 parent f6e4913
commit b8aaa7f
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 28 deletions.
diff --git a/builds/make.inc.template b/builds/make.inc.template
@@ -4,9 +4,6 @@
 #To use MPI, DFLAGS must include -DMPI_CHOLLA
 DFLAGS += -DMPI_CHOLLA
 
-#Set the MPI Processes grid [nproc_x, nproc_y, nproc_z]
-#DFLAGS += -DSET_MPI_GRID
-
 # Single or double precision
 #DFLAGS += -DPRECISION=1
 DFLAGS += -DPRECISION=2

diff --git a/src/global/global.cpp b/src/global/global.cpp
@@ -103,10 +103,10 @@ char *Trim(char *s)
 
 // NOLINTNEXTLINE(cert-err58-cpp)
 // NOLINTNEXTLINE(*)
-const std::set<std::string> optionalParams = {
-    "flag_delta", "ddelta_dt",     "n_delta",      "Lz",          "Lx",       "phi",      "theta",   "delta",
-    "nzr",        "nxr",           "H0",           "Omega_M",     "Omega_L",  "Omega_R",  "Omega_K", "w0",
-    "wa",         "Init_redshift", "End_redshift", "tile_length", "n_proc_x", "n_proc_y", "n_proc_z"};  // NOLINT
+const std::set<std::string> optionalParams = {"flag_delta",   "ddelta_dt",  "n_delta", "Lz",  "Lx", "phi",
+                                              "theta",        "delta",      "nzr",     "nxr", "H0", "Omega_M",
+                                              "Omega_L",      "Omega_R",    "Omega_K", "w0",  "wa", "Init_redshift",
+                                              "End_redshift", "tile_length"};  // NOLINT
 
 bool Old_Style_Parse_Param(const char *name, const char *value, struct Parameters *parms);
 
@@ -381,15 +381,6 @@ bool Old_Style_Parse_Param(const char *name, const char *value, struct Parameter
     parms->tile_length = atof(value);
 #endif  // TILED_INITIAL_CONDITIONS
 
-#ifdef SET_MPI_GRID
-    // Set the MPI Processes grid [n_proc_x, n_proc_y, n_proc_z]
-  } else if (strcmp(name, "n_proc_x") == 0) {
-    parms->n_proc_x = atoi(value);
-  } else if (strcmp(name, "n_proc_y") == 0) {
-    parms->n_proc_y = atoi(value);
-  } else if (strcmp(name, "n_proc_z") == 0) {
-    parms->n_proc_z = atoi(value);
-#endif
   } else if (strcmp(name, "bc_potential_type") == 0) {
     parms->bc_potential_type = atoi(value);
 #ifdef CHEMISTRY_GPU
@@ -452,6 +443,25 @@ void Init_Param_Struct_Members(ParameterMap &pmap, struct Parameters *parms)
   parms->gamma = Real(pmap.value<double>("gamma"));
   CHOLLA_ASSERT(parms->gamma > 1.0, "gamma parameter must be greater than one.");
 
+  // Set the MPI Processes grid [n_proc_x, n_proc_y, n_proc_z]
+  if (pmap.has_param("n_proc_x") or pmap.has_param("n_proc_y") or pmap.has_param("n_proc_z")) {
+    parms->n_proc_x = pmap.value<int>("n_proc_x");
+    parms->n_proc_y = pmap.value<int>("n_proc_y");
+    parms->n_proc_z = pmap.value<int>("n_proc_z");
+    CHOLLA_ASSERT((parms->n_proc_x > 0) and (parms->n_proc_y > 0) and (parms->n_proc_z > 0),
+                  "When specified, n_proc_x, n_proc_y, and n_proc_z must be positive");
+    // the following check also implicitly ensures that n_proc_[xyz] are all 1 without MPI
+    int product = parms->n_proc_x * parms->n_proc_y * parms->n_proc_z;
+    CHOLLA_ASSERT(product == nproc,
+                  "The product of n_proc_x, n_proc_y, and n_proc_z is %d. It doesn't match the "
+                  "number of processes, %d",
+                  product, nproc);
+  } else {
+    parms->n_proc_x = 0;
+    parms->n_proc_y = 0;
+    parms->n_proc_z = 0;
+  }
+
 #ifdef TEMPERATURE_FLOOR
   if (not pmap.has_param("temperature_floor")) {
     chprintf("WARNING: parameter file doesn't include temperature_floor parameter. Defaulting to value of 0!\n");

diff --git a/src/global/global.h b/src/global/global.h
@@ -335,12 +335,11 @@ struct Parameters {
   Real tile_length;
 #endif  // TILED_INITIAL_CONDITIONS
 
-#ifdef SET_MPI_GRID
-  // Set the MPI Processes grid [n_proc_x, n_proc_y, n_proc_z]
+  // Set the MPI Processes grid [n_proc_x, n_proc_y, n_proc_z] (if they aren't provided, they are set to 0)
   int n_proc_x;
   int n_proc_y;
   int n_proc_z;
-#endif
+
   int bc_potential_type;
 #if defined(COOLING_GRACKLE) || defined(CHEMISTRY_GPU)
   char UVB_rates_file[MAXLEN];  // File for the UVB photoheating and

diff --git a/src/mpi/mpi_routines.cpp b/src/mpi/mpi_routines.cpp
@@ -303,15 +303,14 @@ void DomainDecompositionBLOCK(struct Parameters *P, struct Header *H, int nx_gin
     nproc_z = tmp;
   }
 
-  #ifdef SET_MPI_GRID
-  // Set the MPI Processes grid [n_proc_x, n_proc_y, n_proc_z]
-  nproc_x = P->n_proc_x;
-  nproc_y = P->n_proc_y;
-  nproc_z = P->n_proc_z;
-  chprintf("Setting MPI grid: nx=%d  ny=%d  nz=%d\n", nproc_x, nproc_y, nproc_z);
-  // chprintf("Setting MPI grid: nx=%d  ny=%d  nz=%d\n", P->n_proc_x,
-  // P->n_proc_y, P->n_proc_z);
-  #endif
+  if ((P->n_proc_x > 0) or (P->n_proc_y > 0) or (P->n_proc_z > 0)) {
+    // (in reality all or none of the parameters will be 0)
+    // Set the MPI Processes grid [n_proc_x, n_proc_y, n_proc_z]
+    nproc_x = P->n_proc_x;
+    nproc_y = P->n_proc_y;
+    nproc_z = P->n_proc_z;
+    chprintf("Setting MPI grid: nx=%d  ny=%d  nz=%d\n", nproc_x, nproc_y, nproc_z);
+  }
 
   // chprintf("Allocating tiling.\n");
   MPI_Barrier(world);