From 60dc6bbab4a5c6a6863a9494a79e3cd10df03f31 Mon Sep 17 00:00:00 2001 From: Roland Haas Date: Wed, 26 Jul 2023 13:25:16 -0500 Subject: [PATCH 1/2] CarpetX: use AMReX FArrayBox for grid scalars and arrays --- CarpetX/src/driver.cxx | 15 ++++++++++++--- CarpetX/src/driver.hxx | 4 ++-- CarpetX/src/io_openpmd.cxx | 6 ++---- CarpetX/src/io_tsv.cxx | 4 ++-- CarpetX/src/schedule.cxx | 3 +-- CarpetX/src/valid.cxx | 5 +++-- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/CarpetX/src/driver.cxx b/CarpetX/src/driver.cxx index ccd9f8fb8..b1a6bd6a4 100644 --- a/CarpetX/src/driver.cxx +++ b/CarpetX/src/driver.cxx @@ -1685,8 +1685,10 @@ void SetupGlobals() { arraygroupdata.valid.resize(group.numtimelevels); for (int tl = 0; tl < int(arraygroupdata.data.size()); ++tl) { // TODO: Allocate in managed memory - arraygroupdata.data.at(tl).resize(arraygroupdata.numvars * - arraygroupdata.array_size); + arraygroupdata.data.at(tl).resize( + amrex::Box(amrex::IntVect(arraygroupdata.lbnd), + amrex::IntVect(arraygroupdata.ubnd)), + arraygroupdata.numvars); why_valid_t why([]() { return "SetupGlobals"; }); arraygroupdata.valid.at(tl).resize(arraygroupdata.numvars, why); for (int vi = 0; vi < arraygroupdata.numvars; ++vi) { @@ -2167,7 +2169,14 @@ operator<<(YAML::Emitter &yaml, yaml << YAML::BeginMap; yaml << YAML::Key << "commongroupdata" << YAML::Value << (GHExt::CommonGroupData)arraygroupdata; - yaml << YAML::Key << "data" << YAML::Value << arraygroupdata.data; + yaml << YAML::Key << "data" << YAML::Value << YAML::Flow << YAML::BeginSeq; + for (int tl = 0; tl < int(arraygroupdata.data.size()); ++tl) { + yaml << YAML::BeginSeq; + for (int vi = 0; vi < arraygroupdata.numvars; ++vi) + yaml << *arraygroupdata.data.at(tl).dataPtr(vi); + yaml << YAML::EndSeq; + } + yaml << YAML::EndSeq; yaml << YAML::EndMap; return yaml; } diff --git a/CarpetX/src/driver.hxx b/CarpetX/src/driver.hxx index 4f94a7d5a..ebf773151 100644 --- a/CarpetX/src/driver.hxx +++ b/CarpetX/src/driver.hxx @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -158,8 +159,7 @@ struct GHExt { // we assume that grid scalars only hold "analysis" data. struct ArrayGroupData : public CommonGroupData { - vector > - data; // [time level][var index + grid point index] + vector data; // [time level] int array_size; int dimension; int activetimelevels; diff --git a/CarpetX/src/io_openpmd.cxx b/CarpetX/src/io_openpmd.cxx index 33ae9d5e5..abfebd7ac 100644 --- a/CarpetX/src/io_openpmd.cxx +++ b/CarpetX/src/io_openpmd.cxx @@ -1131,8 +1131,7 @@ void carpetx_openpmd_t::InputOpenPMD(const cGH *const cctkGH, assert(cactus_dk > 0); assert(cactus_np > 0); for (int vi = 0; vi < numvars; ++vi) { - CCTK_REAL *const cactus_var_ptr = - groupdata.data.at(tl).data() + vi * cactus_np; + CCTK_REAL *const cactus_var_ptr = groupdata.data.at(tl).dataPtr(vi); if (input_ghosts || intbox == extbox) { #if OPENPMDAPI_VERSION_GE(0, 15, 0) record_components.at(vi).loadChunkRaw(cactus_var_ptr, start, count); @@ -1747,8 +1746,7 @@ void carpetx_openpmd_t::OutputOpenPMD(const cGH *const cctkGH, assert(cactus_dk > 0); assert(cactus_np > 0); for (int vi = 0; vi < numvars; ++vi) { - const CCTK_REAL *const var_ptr = - groupdata.data.at(tl).data() + vi * cactus_np; + const CCTK_REAL *const var_ptr = groupdata.data.at(tl).dataPtr(vi); if (output_ghosts || intbox == extbox) { const CCTK_REAL *const ptr = var_ptr; #if OPENPMDAPI_VERSION_GE(0, 15, 0) diff --git a/CarpetX/src/io_tsv.cxx b/CarpetX/src/io_tsv.cxx index 9576ae53d..85b86a9ea 100644 --- a/CarpetX/src/io_tsv.cxx +++ b/CarpetX/src/io_tsv.cxx @@ -158,7 +158,7 @@ void WriteTSVScalars(const cGH *restrict cctkGH, const string &filename, file << cctkGH->cctk_iteration << sep << cctkGH->cctk_time; const int tl = 0; for (int vi = 0; vi < arraygroupdata.numvars; ++vi) - file << sep << arraygroupdata.data.at(tl).at(vi); + file << sep << *arraygroupdata.data.at(tl).dataPtr(vi); file << "\n"; } @@ -207,7 +207,7 @@ void WriteTSVArrays(const cGH *restrict cctkGH, const string &filename, file << sep << (dir == out_dir ? i : 0); const int tl = 0; for (int vi = 0; vi < arraygroupdata.numvars; ++vi) - file << sep << arraygroupdata.data.at(tl).at(np * vi + DI[out_dir] * i); + file << sep << arraygroupdata.data.at(tl).dataPtr(vi)[DI[out_dir] * i]; file << "\n"; } } diff --git a/CarpetX/src/schedule.cxx b/CarpetX/src/schedule.cxx index 0c05a77fc..b7851554e 100644 --- a/CarpetX/src/schedule.cxx +++ b/CarpetX/src/schedule.cxx @@ -529,8 +529,7 @@ void enter_global_mode(cGH *restrict cctkGH) { const auto &restrict vars = arraygroupdata.data.at(tl); for (int vi = 0; vi < arraygroupdata.numvars; ++vi) cctkGH->data[arraygroupdata.firstvarindex + vi][tl] = - const_cast( - &vars.at(vi * arraygroupdata.array_size)); + const_cast(vars.dataPtr(vi)); } } } diff --git a/CarpetX/src/valid.cxx b/CarpetX/src/valid.cxx index e84061d16..48e4bcbbd 100644 --- a/CarpetX/src/valid.cxx +++ b/CarpetX/src/valid.cxx @@ -237,7 +237,7 @@ void poison_invalid_ga(const int gi, const int vi, const int tl) { if (!valid.valid_int) { int dimension = arraygroupdata.dimension; CCTK_REAL *restrict const ptr = - const_cast(&arraygroupdata.data.at(tl).at(vi)); + const_cast(arraygroupdata.data.at(tl).dataPtr(vi)); const int *gsh = arraygroupdata.gsh; int n_elems = 1; for (int i = 0; i < dimension; i++) @@ -486,7 +486,8 @@ void check_valid_ga(const int gi, const int vi, const int tl, assert(valid.valid_outer && valid.valid_ghosts); if (valid.valid_int) { - const CCTK_REAL *restrict const ptr = &arraygroupdata.data.at(tl).at(vi); + const CCTK_REAL *restrict const ptr = + arraygroupdata.data.at(tl).dataPtr(vi); int dimension = arraygroupdata.dimension; const int *gsh = arraygroupdata.gsh; int n_elems = 1; From 0a587b897b56b5d7d7b9891118961d9e4eb5f25c Mon Sep 17 00:00:00 2001 From: Roland Haas Date: Fri, 25 Aug 2023 15:12:02 -0500 Subject: [PATCH 2/2] CarpetX: remove TODO about managed data for arrays --- CarpetX/src/driver.cxx | 1 - 1 file changed, 1 deletion(-) diff --git a/CarpetX/src/driver.cxx b/CarpetX/src/driver.cxx index b1a6bd6a4..8c61fdb25 100644 --- a/CarpetX/src/driver.cxx +++ b/CarpetX/src/driver.cxx @@ -1684,7 +1684,6 @@ void SetupGlobals() { arraygroupdata.data.resize(group.numtimelevels); arraygroupdata.valid.resize(group.numtimelevels); for (int tl = 0; tl < int(arraygroupdata.data.size()); ++tl) { - // TODO: Allocate in managed memory arraygroupdata.data.at(tl).resize( amrex::Box(amrex::IntVect(arraygroupdata.lbnd), amrex::IntVect(arraygroupdata.ubnd)),