make arrays and scalar accessible from GPU

CarpetX/src/driver.cxx

@@ -1684,9 +1684,10 @@ void SetupGlobals() {
     arraygroupdata.data.resize(group.numtimelevels);
     arraygroupdata.valid.resize(group.numtimelevels);
     for (int tl = 0; tl < int(arraygroupdata.data.size()); ++tl) {
-      // TODO: Allocate in managed memory
-      arraygroupdata.data.at(tl).resize(arraygroupdata.numvars *
-                                        arraygroupdata.array_size);
+      arraygroupdata.data.at(tl).resize(
+          amrex::Box(amrex::IntVect(arraygroupdata.lbnd),
+                     amrex::IntVect(arraygroupdata.ubnd)),
+          arraygroupdata.numvars);
       why_valid_t why([]() { return "SetupGlobals"; });
       arraygroupdata.valid.at(tl).resize(arraygroupdata.numvars, why);
       for (int vi = 0; vi < arraygroupdata.numvars; ++vi) {
@@ -2167,7 +2168,14 @@ operator<<(YAML::Emitter &yaml,
   yaml << YAML::BeginMap;
   yaml << YAML::Key << "commongroupdata" << YAML::Value
        << (GHExt::CommonGroupData)arraygroupdata;
-  yaml << YAML::Key << "data" << YAML::Value << arraygroupdata.data;
+  yaml << YAML::Key << "data" << YAML::Value << YAML::Flow << YAML::BeginSeq;
+  for (int tl = 0; tl < int(arraygroupdata.data.size()); ++tl) {
+    yaml << YAML::BeginSeq;
+    for (int vi = 0; vi < arraygroupdata.numvars; ++vi)
+      yaml << *arraygroupdata.data.at(tl).dataPtr(vi);
+    yaml << YAML::EndSeq;
+  }
+  yaml << YAML::EndSeq;
   yaml << YAML::EndMap;
   return yaml;
 }

CarpetX/src/driver.hxx

@@ -14,6 +14,7 @@
 #include <AMReX_FluxRegister.H>
 #include <AMReX_Interpolater.H>
 #include <AMReX_MultiFab.H>
+#include <AMReX_FArrayBox.H>
 
 #include <yaml-cpp/yaml.h>
 
@@ -158,8 +159,7 @@ struct GHExt {
     // we assume that grid scalars only hold "analysis" data.
 
     struct ArrayGroupData : public CommonGroupData {
-      vector<vector<CCTK_REAL> >
-          data; // [time level][var index + grid point index]
+      vector<amrex::FArrayBox> data; // [time level]
       int array_size;
       int dimension;
       int activetimelevels;

CarpetX/src/io_openpmd.cxx

@@ -1131,8 +1131,7 @@ void carpetx_openpmd_t::InputOpenPMD(const cGH *const cctkGH,
         assert(cactus_dk > 0);
         assert(cactus_np > 0);
         for (int vi = 0; vi < numvars; ++vi) {
-          CCTK_REAL *const cactus_var_ptr =
-              groupdata.data.at(tl).data() + vi * cactus_np;
+          CCTK_REAL *const cactus_var_ptr = groupdata.data.at(tl).dataPtr(vi);
           if (input_ghosts || intbox == extbox) {
 #if OPENPMDAPI_VERSION_GE(0, 15, 0)
             record_components.at(vi).loadChunkRaw(cactus_var_ptr, start, count);
@@ -1747,8 +1746,7 @@ void carpetx_openpmd_t::OutputOpenPMD(const cGH *const cctkGH,
         assert(cactus_dk > 0);
         assert(cactus_np > 0);
         for (int vi = 0; vi < numvars; ++vi) {
-          const CCTK_REAL *const var_ptr =
-              groupdata.data.at(tl).data() + vi * cactus_np;
+          const CCTK_REAL *const var_ptr = groupdata.data.at(tl).dataPtr(vi);
           if (output_ghosts || intbox == extbox) {
             const CCTK_REAL *const ptr = var_ptr;
 #if OPENPMDAPI_VERSION_GE(0, 15, 0)

CarpetX/src/io_tsv.cxx

@@ -158,7 +158,7 @@ void WriteTSVScalars(const cGH *restrict cctkGH, const string &filename,
   file << cctkGH->cctk_iteration << sep << cctkGH->cctk_time;
   const int tl = 0;
   for (int vi = 0; vi < arraygroupdata.numvars; ++vi)
-    file << sep << arraygroupdata.data.at(tl).at(vi);
+    file << sep << *arraygroupdata.data.at(tl).dataPtr(vi);
   file << "\n";
 }
 
@@ -207,7 +207,7 @@ void WriteTSVArrays(const cGH *restrict cctkGH, const string &filename,
       file << sep << (dir == out_dir ? i : 0);
     const int tl = 0;
     for (int vi = 0; vi < arraygroupdata.numvars; ++vi)
-      file << sep << arraygroupdata.data.at(tl).at(np * vi + DI[out_dir] * i);
+      file << sep << arraygroupdata.data.at(tl).dataPtr(vi)[DI[out_dir] * i];
     file << "\n";
   }
 }

CarpetX/src/schedule.cxx

@@ -529,8 +529,7 @@ void enter_global_mode(cGH *restrict cctkGH) {
           const auto &restrict vars = arraygroupdata.data.at(tl);
           for (int vi = 0; vi < arraygroupdata.numvars; ++vi)
             cctkGH->data[arraygroupdata.firstvarindex + vi][tl] =
-                const_cast<CCTK_REAL *>(
-                    &vars.at(vi * arraygroupdata.array_size));
+                const_cast<CCTK_REAL *>(vars.dataPtr(vi));
         }
       }
     }

CarpetX/src/valid.cxx

@@ -237,7 +237,7 @@ void poison_invalid_ga(const int gi, const int vi, const int tl) {
   if (!valid.valid_int) {
     int dimension = arraygroupdata.dimension;
     CCTK_REAL *restrict const ptr =
-        const_cast<CCTK_REAL *>(&arraygroupdata.data.at(tl).at(vi));
+        const_cast<CCTK_REAL *>(arraygroupdata.data.at(tl).dataPtr(vi));
     const int *gsh = arraygroupdata.gsh;
     int n_elems = 1;
     for (int i = 0; i < dimension; i++)
@@ -486,7 +486,8 @@ void check_valid_ga(const int gi, const int vi, const int tl,
   assert(valid.valid_outer && valid.valid_ghosts);
 
   if (valid.valid_int) {
-    const CCTK_REAL *restrict const ptr = &arraygroupdata.data.at(tl).at(vi);
+    const CCTK_REAL *restrict const ptr =
+        arraygroupdata.data.at(tl).dataPtr(vi);
     int dimension = arraygroupdata.dimension;
     const int *gsh = arraygroupdata.gsh;
     int n_elems = 1;