From d75c04b422dae567c9d3c74482b5be84d002ee57 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Mon, 20 Nov 2023 13:57:50 -0800 Subject: [PATCH 01/16] solve_bicgstab: use fewer MFs (#3635) ## Summary This PR cuts the number of MFs used in `solve_bicgstab`, saving on memory and LocalCopy operations. In particular, the MFs `ph` and `sh` are removed. ## Additional background This is a follow up to avoid-use-of-s and other PRs to improve `solve_bicgstab`. My own testing has shown that this PR gives the same results as before, but regression testing should be done to verify this in all cases. --- Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H | 25 ++++++++++------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H index da50aae7b89..4768f06fc41 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H @@ -90,14 +90,12 @@ MLCGSolverT::solve_bicgstab (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) const int ncomp = sol.nComp(); - MF ph = Lp.make(amrlev, mglev, sol.nGrowVect()); - MF sh = Lp.make(amrlev, mglev, sol.nGrowVect()); - ph.setVal(RT(0.0)); - sh.setVal(RT(0.0)); + MF p = Lp.make(amrlev, mglev, sol.nGrowVect()); + MF r = Lp.make(amrlev, mglev, sol.nGrowVect()); + p.setVal(RT(0.0)); // Make sure all entries are initialized to avoid errors + r.setVal(RT(0.0)); MF sorig = Lp.make(amrlev, mglev, nghost); - MF p = Lp.make(amrlev, mglev, nghost); - MF r = Lp.make(amrlev, mglev, nghost); MF rh = Lp.make(amrlev, mglev, nghost); MF v = Lp.make(amrlev, mglev, nghost); MF t = Lp.make(amrlev, mglev, nghost); @@ -151,8 +149,7 @@ MLCGSolverT::solve_bicgstab (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) MF::Saxpy(p, -omega, v, 0, 0, ncomp, nghost); // p += -omega*v MF::Xpay(p, beta, r, 0, 0, ncomp, nghost); // p = r + beta*p } - ph.LocalCopy(p,0,0,ncomp,nghost); - Lp.apply(amrlev, mglev, v, ph, MLLinOpT::BCMode::Homogeneous, MLLinOpT::StateMode::Correction); + Lp.apply(amrlev, mglev, v, p, MLLinOpT::BCMode::Homogeneous, MLLinOpT::StateMode::Correction); Lp.normalize(amrlev, mglev, v); RT rhTv = dotxy(rh,v); @@ -164,9 +161,10 @@ MLCGSolverT::solve_bicgstab (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) { ret = 2; break; } - MF::Saxpy(sol, alpha, ph, 0, 0, ncomp, nghost); // sol += alpha * ph - MF::Saxpy(r, -alpha, v, 0, 0, ncomp, nghost); // r += -alpha * v + MF::Saxpy(sol, alpha, p, 0, 0, ncomp, nghost); // sol += alpha * p + MF::Saxpy(r, -alpha, v, 0, 0, ncomp, nghost); // r += -alpha * v + rnorm = norm_inf(r); rnorm = norm_inf(r); if ( verbose > 2 && ParallelDescriptor::IOProcessor() ) @@ -179,8 +177,7 @@ MLCGSolverT::solve_bicgstab (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) if ( rnorm < eps_rel*rnorm0 || rnorm < eps_abs ) { break; } - sh.LocalCopy(r,0,0,ncomp,nghost); - Lp.apply(amrlev, mglev, t, sh, MLLinOpT::BCMode::Homogeneous, MLLinOpT::StateMode::Correction); + Lp.apply(amrlev, mglev, t, r, MLLinOpT::BCMode::Homogeneous, MLLinOpT::StateMode::Correction); Lp.normalize(amrlev, mglev, t); // // This is a little funky. I want to elide one of the reductions @@ -201,8 +198,8 @@ MLCGSolverT::solve_bicgstab (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) { ret = 3; break; } - MF::Saxpy(sol, omega, sh, 0, 0, ncomp, nghost); // sol += omega * sh - MF::Saxpy(r, -omega, t, 0, 0, ncomp, nghost); // r += -omega * t + MF::Saxpy(sol, omega, r, 0, 0, ncomp, nghost); // sol += omega * r + MF::Saxpy(r, -omega, t, 0, 0, ncomp, nghost); // r += -omega * t rnorm = norm_inf(r); From c51e0a3071f3f1d2a11b6c339bff58d4ba5e30e8 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 20 Nov 2023 13:59:06 -0800 Subject: [PATCH 02/16] CI: Ascent + Conduit (#3639) ## Summary For some reason, `AMReX_ASCENT` does not trigger `AMReX_CONDUIT` to be set, too. cc @cyrush ## Additional background X-ref: #3350 --- .github/workflows/ascent.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ascent.yml b/.github/workflows/ascent.yml index 83d2f7ebac3..d8217621a1b 100644 --- a/.github/workflows/ascent.yml +++ b/.github/workflows/ascent.yml @@ -26,7 +26,8 @@ jobs: -DCMAKE_BUILD_TYPE=Debug \ -DAMReX_ENABLE_TESTS=ON \ -DAMReX_FORTRAN=OFF \ - -DAMReX_ASCENT=ON + -DAMReX_ASCENT=ON \ + -DAMReX_CONDUIT=ON - name: Build run: | . /ascent_docker_setup_env.sh From 9e35dc19489dc5d312e92781cb0471d282cf8370 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 20 Nov 2023 21:04:03 -0800 Subject: [PATCH 03/16] Ascent: SoA Particle Support (#3350) ## Summary Add support for pure SoA layouted particle containers for Ascent. ## Additional background Follow-up to #2878. ## Checklist The proposed changes: - [ ] fix a bug or incorrect behavior in AMReX - [x] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --------- Co-authored-by: Andrew Myers --- Src/Extern/Conduit/AMReX_Conduit_Blueprint.H | 5 +- .../AMReX_Conduit_Blueprint_ParticlesI.H | 207 ++++++++---- .../Ascent_Insitu_SOA/CMakeLists.txt | 13 + Tests/Particles/Ascent_Insitu_SOA/GNUmakefile | 24 ++ Tests/Particles/Ascent_Insitu_SOA/inputs.rt | 10 + Tests/Particles/Ascent_Insitu_SOA/main.cpp | 306 ++++++++++++++++++ 6 files changed, 491 insertions(+), 74 deletions(-) create mode 100644 Tests/Particles/Ascent_Insitu_SOA/CMakeLists.txt create mode 100644 Tests/Particles/Ascent_Insitu_SOA/GNUmakefile create mode 100644 Tests/Particles/Ascent_Insitu_SOA/inputs.rt create mode 100644 Tests/Particles/Ascent_Insitu_SOA/main.cpp diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H index 6d23bcf07ed..9ac8eb31fd0 100644 --- a/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H +++ b/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H @@ -96,9 +96,8 @@ namespace amrex // coordset and fields used to represent the passed particle container. // This allows you to use unique names to wrap multiple particle containers // into a single blueprint tree. - template - void ParticleContainerToBlueprint (const ParticleContainer + void ParticleContainerToBlueprint (const ParticleContainer_impl &pc, const Vector &real_comp_names, diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H index f2d7d1ed2da..e4186ba247b 100644 --- a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H +++ b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H @@ -20,10 +20,9 @@ namespace amrex // Note: // This is a helper function, it's not part of the AMReX Blueprint Interface. //---------------------------------------------------------------------------// -template +template void -ParticleTileToBlueprint(const ParticleTile, +ParticleTileToBlueprint(const ParticleTile &ptile, const Vector &real_comp_names, @@ -31,15 +30,11 @@ ParticleTileToBlueprint(const ParticleTile); + int num_particles = ptile.size(); // knowing the above, we can zero copy the x,y,z positions + id, cpu // and any user fields in the AOS - // get the first particle's struct - const auto &pstruct = ptile.GetArrayOfStructs(); - // setup a blueprint description for the particle mesh // create a coordinate set std::string coordset_name = topology_name + "_coords"; @@ -63,29 +58,56 @@ ParticleTileToBlueprint(const ParticleTile(pstruct.data()); - char* pbuf = const_cast(pbuf_const); + if constexpr(ParticleType::is_soa_particle) + { + amrex::ignore_unused(pbuf); - ParticleReal* xp = reinterpret_cast(pbuf); pbuf += sizeof(ParticleReal); - n_coords["values/x"].set_external(xp, - num_particles, - 0, - struct_size); + const auto &soa = ptile.GetStructOfArrays(); + + // for soa entries, we can use standard strides, + // since these are contiguous arrays + + n_coords["values/x"].set_external(const_cast(&soa.GetRealData(0)[0]), + num_particles); #if AMREX_SPACEDIM > 1 - ParticleReal* yp = reinterpret_cast(pbuf); pbuf += sizeof(ParticleReal); - n_coords["values/y"].set_external(yp, - num_particles, - 0, - struct_size); + n_coords["values/y"].set_external(const_cast(&soa.GetRealData(1)[0]), + num_particles); #endif #if AMREX_SPACEDIM > 2 - ParticleReal* zp = reinterpret_cast(pbuf); pbuf += sizeof(ParticleReal); - n_coords["values/z"].set_external(zp, - num_particles, - 0, - struct_size); + n_coords["values/z"].set_external(const_cast(&soa.GetRealData(2)[0]), + num_particles); +#endif + } else + { + // get the first particle's struct + const auto &pstruct = ptile.GetArrayOfStructs(); + const int struct_size = sizeof(ParticleType); + + const char* pbuf_const = reinterpret_cast(pstruct.data()); + pbuf = const_cast(pbuf_const); + + ParticleReal* xp = reinterpret_cast(pbuf); pbuf += sizeof(ParticleReal); + n_coords["values/x"].set_external(xp, + num_particles, + 0, + struct_size); +#if AMREX_SPACEDIM > 1 + ParticleReal* yp = reinterpret_cast(pbuf); pbuf += sizeof(ParticleReal); + n_coords["values/y"].set_external(yp, + num_particles, + 0, + struct_size); +#endif +#if AMREX_SPACEDIM > 2 + ParticleReal* zp = reinterpret_cast(pbuf); pbuf += sizeof(ParticleReal); + n_coords["values/z"].set_external(zp, + num_particles, + 0, + struct_size); #endif + } // fields conduit::Node &n_fields = res["fields"]; @@ -95,20 +117,26 @@ ParticleTileToBlueprint(const ParticleTile(pbuf); pbuf += sizeof(ParticleReal); - conduit::Node &n_f = n_fields[real_comp_names.at(vname_real_idx)]; - n_f["topology"] = topology_name; - n_f["association"] = "element"; - n_f["values"].set_external(val, - num_particles, - 0, - struct_size); + constexpr int struct_size = sizeof(ParticleType); + constexpr int NStructReal = ParticleType::NReal; - vname_real_idx++; + // struct real fields, the first set are always the particle positions + // which we wrap above + for (int i = 0; i < NStructReal; i++) + { + ParticleReal* val = reinterpret_cast(pbuf); pbuf += sizeof(ParticleReal); + conduit::Node &n_f = n_fields[real_comp_names.at(vname_real_idx)]; + n_f["topology"] = topology_name; + n_f["association"] = "element"; + n_f["values"].set_external(val, + num_particles, + 0, + struct_size); + + vname_real_idx++; + } } //----------------------------------// @@ -116,44 +144,77 @@ ParticleTileToBlueprint(const ParticleTile(pbuf); pbuf += sizeof(int); - conduit::Node &n_f_id = n_fields[topology_name + "_id"]; + if constexpr(!ParticleType::is_soa_particle) + { + const int struct_size = sizeof(ParticleType); + + // id is the first int entry + int* id = reinterpret_cast(pbuf); pbuf += sizeof(int); + conduit::Node &n_f_id = n_fields[topology_name + "_id"]; + + n_f_id["topology"] = topology_name; + n_f_id["association"] = "element"; + n_f_id["values"].set_external(id, + num_particles, + 0, + struct_size); + + // cpu is the second int entry + int* cpu = reinterpret_cast(pbuf); pbuf += sizeof(int); + conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"]; + + n_f_cpu["topology"] = topology_name; + n_f_cpu["association"] = "element"; + n_f_cpu["values"].set_external(cpu, + num_particles, + 0, + struct_size); + } else { + const auto &soa = ptile.GetStructOfArrays(); + + // for soa entries, we can use standard strides, + // since these are contiguous arrays - n_f_id["topology"] = topology_name; - n_f_id["association"] = "element"; - n_f_id["values"].set_external(id, - num_particles, - 0, - struct_size); + // id is the first int entry + conduit::Node &n_f_id = n_fields[topology_name + "_id"]; - // cpu is the second int entry - int* cpu = reinterpret_cast(pbuf); pbuf += sizeof(int); - conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"]; + n_f_id["topology"] = topology_name; + n_f_id["association"] = "element"; + n_f_id["values"].set_external(const_cast(&soa.GetIntData(0)[0]), + num_particles); - n_f_cpu["topology"] = topology_name; - n_f_cpu["association"] = "element"; - n_f_cpu["values"].set_external(cpu, - num_particles, - 0, - struct_size); + // cpu is the second int entry + conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"]; + + n_f_cpu["topology"] = topology_name; + n_f_cpu["association"] = "element"; + n_f_cpu["values"].set_external(const_cast(&soa.GetIntData(0)[0]), + num_particles); + + } // -------------------------------- // user defined, integer aos fields // -------------------------------- int vname_int_idx = 0; - for (int i = 0; i < NStructInt; i++) + if constexpr(!ParticleType::is_soa_particle) { - int* val = reinterpret_cast(pbuf); pbuf += sizeof(int); - conduit::Node &n_f = n_fields[int_comp_names.at(vname_int_idx)]; - n_f["topology"] = topology_name; - n_f["association"] = "element"; - n_f["values"].set_external(val, - num_particles, - 0, - struct_size); - vname_int_idx++; + constexpr int struct_size = sizeof(ParticleType); + constexpr int NStructInt = ParticleType::NInt; + + for (int i = 0; i < NStructInt; i++) + { + int* val = reinterpret_cast(pbuf); pbuf += sizeof(int); + conduit::Node &n_f = n_fields[int_comp_names.at(vname_int_idx)]; + n_f["topology"] = topology_name; + n_f["association"] = "element"; + n_f["values"].set_external(val, + num_particles, + 0, + struct_size); + vname_int_idx++; + } } // ------------------------- @@ -193,10 +254,9 @@ ParticleTileToBlueprint(const ParticleTile +template void -ParticleContainerToBlueprint(const ParticleContainer &pc, const Vector &real_comp_names, @@ -209,8 +269,13 @@ ParticleContainerToBlueprint(const ParticleContainer; + using MyParConstIter = ParConstIter_impl; // // blueprint expects unique ids for each domain published diff --git a/Tests/Particles/Ascent_Insitu_SOA/CMakeLists.txt b/Tests/Particles/Ascent_Insitu_SOA/CMakeLists.txt new file mode 100644 index 00000000000..82216a02af8 --- /dev/null +++ b/Tests/Particles/Ascent_Insitu_SOA/CMakeLists.txt @@ -0,0 +1,13 @@ +if ( NOT AMReX_ASCENT ) + return () +endif () + +foreach(D IN LISTS AMReX_SPACEDIM) + set(_sources main.cpp) + set(_input_files inputs.rt ) + + setup_test(${D} _sources _input_files NTASKS 2) + + unset(_sources) + unset(_input_files) +endforeach() diff --git a/Tests/Particles/Ascent_Insitu_SOA/GNUmakefile b/Tests/Particles/Ascent_Insitu_SOA/GNUmakefile new file mode 100644 index 00000000000..660e4a13f22 --- /dev/null +++ b/Tests/Particles/Ascent_Insitu_SOA/GNUmakefile @@ -0,0 +1,24 @@ +AMREX_HOME = ../../../ + +DEBUG = FALSE + +DIM = 3 + +COMP = gcc + +USE_MPI = TRUE +USE_OMP = FALSE +USE_CUDA = FALSE + +TINY_PROFILE = TRUE +USE_PARTICLES = TRUE +USE_ASCENT = TRUE + +include $(AMREX_HOME)/Tools/GNUMake/Make.defs + +include ./Make.package +include $(AMREX_HOME)/Src/Base/Make.package +include $(AMREX_HOME)/Src/Particle/Make.package +include $(AMREX_HOME)/Src/Extern/Conduit/Make.package + +include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/Ascent_Insitu_SOA/inputs.rt b/Tests/Particles/Ascent_Insitu_SOA/inputs.rt new file mode 100644 index 00000000000..e34fda14923 --- /dev/null +++ b/Tests/Particles/Ascent_Insitu_SOA/inputs.rt @@ -0,0 +1,10 @@ +ascent.size = (32, 64, 64) +ascent.max_grid_size = 32 +ascent.is_periodic = 1 +ascent.num_ppc = 1 +ascent.nlevs = 1 + +ascent.num_runtime_real = 0 +ascent.num_runtime_int = 0 + +particles.do_tiling = 1 diff --git a/Tests/Particles/Ascent_Insitu_SOA/main.cpp b/Tests/Particles/Ascent_Insitu_SOA/main.cpp new file mode 100644 index 00000000000..46e2af98422 --- /dev/null +++ b/Tests/Particles/Ascent_Insitu_SOA/main.cpp @@ -0,0 +1,306 @@ +#include +#include +#include + +#if !defined(AMREX_PARTICLES) || !defined(AMREX_USE_CONDUIT) +#error Incompatible AMReX library configuration! This tutorial requires AMREX_PARTICLES and AMREX_USE_CONDUIT +#endif + +#include + +#include + +#include + + +using namespace amrex; + +static constexpr int NR = 7; +static constexpr int NI = 4; + +int num_runtime_real = 0; +int num_runtime_int = 0; + +bool remove_negative = true; + +void get_position_unit_cell (Real* r, const IntVect& nppc, int i_part) +{ + int nx = nppc[0]; +#if AMREX_SPACEDIM > 1 + int ny = nppc[1]; +#else + int ny = 1; +#endif +#if AMREX_SPACEDIM > 2 + int nz = nppc[2]; +#else + int nz = 1; +#endif + + int ix_part = i_part/(ny * nz); + int iy_part = (i_part % (ny * nz)) % ny; + int iz_part = (i_part % (ny * nz)) / ny; + + r[0] = (0.5+ix_part)/nx; + r[1] = (0.5+iy_part)/ny; + r[2] = (0.5+iz_part)/nz; +} + +class TestParticleContainer + : public amrex::ParticleContainerPureSoA +{ + +public: + + TestParticleContainer (const Vector & a_geom, + const Vector & a_dmap, + const Vector & a_ba, + const Vector & a_rr) + : amrex::ParticleContainerPureSoA(a_geom, a_dmap, a_ba, a_rr) + { + for (int i = 0; i < num_runtime_real; ++i) + { + AddRealComp(true); + } + for (int i = 0; i < num_runtime_int; ++i) + { + AddIntComp(true); + } + } + + void InitParticles (const amrex::IntVect& a_num_particles_per_cell) + { + BL_PROFILE("InitParticles"); + + const int lev = 0; // only add particles on level 0 + const Real* dx = Geom(lev).CellSize(); + const Real* plo = Geom(lev).ProbLo(); + + const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], + *a_num_particles_per_cell[1], + *a_num_particles_per_cell[2]); + + for(MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) + { + const Box& tile_box = mfi.tilebox(); + + std::array, NR> host_real; + std::array, NI> host_int; + + std::vector > host_runtime_real(NumRuntimeRealComps()); + std::vector > host_runtime_int(NumRuntimeIntComps()); + + for (IntVect iv = tile_box.smallEnd(); iv <= tile_box.bigEnd(); tile_box.next(iv)) + { + for (int i_part=0; i_part(id)); + host_int[1].push_back(ParallelDescriptor::MyProc()); + host_real[0].push_back(static_cast (plo[0] + (iv[0] + r[0])*dx[0])); +#if AMREX_SPACEDIM > 1 + host_real[1].push_back(static_cast (plo[1] + (iv[1] + r[1])*dx[1])); +#endif +#if AMREX_SPACEDIM > 2 + host_real[2].push_back(static_cast (plo[2] + (iv[2] + r[2])*dx[2])); +#endif + + for (int i = AMREX_SPACEDIM; i < NR; ++i) + host_real[i].push_back(static_cast(id)); + for (int i = 2; i < NI; ++i) + host_int[i].push_back(static_cast(id)); + for (int i = 0; i < NumRuntimeRealComps(); ++i) + host_runtime_real[i].push_back(static_cast(id)); + for (int i = 0; i < NumRuntimeIntComps(); ++i) + host_runtime_int[i].push_back(static_cast(id)); + } + } + + auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex()); + auto old_size = particle_tile.size(); + auto new_size = old_size + host_real[0].size(); + particle_tile.resize(new_size); + + auto& soa = particle_tile.GetStructOfArrays(); + for (int i = 0; i < NR; ++i) + { + Gpu::copyAsync(Gpu::hostToDevice, + host_real[i].begin(), + host_real[i].end(), + soa.GetRealData(i).begin() + old_size); + } + + for (int i = 0; i < NI; ++i) + { + Gpu::copyAsync(Gpu::hostToDevice, + host_int[i].begin(), + host_int[i].end(), + soa.GetIntData(i).begin() + old_size); + } + for (int i = 0; i < NumRuntimeRealComps(); ++i) + { + Gpu::copyAsync(Gpu::hostToDevice, + host_runtime_real[i].begin(), + host_runtime_real[i].end(), + soa.GetRealData(NR+i).begin() + old_size); + } + + for (int i = 0; i < NumRuntimeIntComps(); ++i) + { + Gpu::copyAsync(Gpu::hostToDevice, + host_runtime_int[i].begin(), + host_runtime_int[i].end(), + soa.GetIntData(NI+i).begin() + old_size); + } + + Gpu::streamSynchronize(); + } + + Redistribute(); + } +}; + +struct TestParams +{ + IntVect size; + int max_grid_size; + int num_ppc; + int is_periodic; + int nlevs; +}; + +void testAscent (); + +int main (int argc, char* argv[]) +{ + amrex::Initialize(argc,argv); + + amrex::Print() << "Running redistribute test \n"; + testAscent(); + + amrex::Finalize(); +} + +void get_test_params (TestParams& params, const std::string& prefix) +{ + ParmParse pp(prefix); + pp.get("size", params.size); + pp.get("max_grid_size", params.max_grid_size); + pp.get("num_ppc", params.num_ppc); + pp.get("is_periodic", params.is_periodic); + pp.get("nlevs", params.nlevs); + pp.query("num_runtime_real", num_runtime_real); + pp.query("num_runtime_int", num_runtime_int); +} + +void testAscent () +{ + BL_PROFILE("testAscent"); + TestParams params; + get_test_params(params, "ascent"); + + int is_per[BL_SPACEDIM]; + for (int & d : is_per) + d = params.is_periodic; + + Vector rr(params.nlevs-1); + for (int lev = 1; lev < params.nlevs; lev++) + rr[lev-1] = IntVect(AMREX_D_DECL(2,2,2)); + + RealBox real_box; + for (int n = 0; n < BL_SPACEDIM; n++) + { + real_box.setLo(n, 0.0); + real_box.setHi(n, params.size[n]); + } + + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); + IntVect domain_hi(AMREX_D_DECL(params.size[0]-1,params.size[1]-1,params.size[2]-1)); + const Box base_domain(domain_lo, domain_hi); + + Vector geom(params.nlevs); + geom[0].define(base_domain, &real_box, CoordSys::cartesian, is_per); + for (int lev = 1; lev < params.nlevs; lev++) { + geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), + &real_box, CoordSys::cartesian, is_per); + } + + Vector ba(params.nlevs); + Vector dm(params.nlevs); + auto lo = IntVect(AMREX_D_DECL(0, 0, 0)); + IntVect size = params.size; + for (int lev = 0; lev < params.nlevs; ++lev) + { + ba[lev].define(Box(lo, lo+params.size-1)); + ba[lev].maxSize(params.max_grid_size); + dm[lev].define(ba[lev]); + lo += size/2; + size *= 2; + } + + TestParticleContainer pc(geom, dm, ba, rr); + + int npc = params.num_ppc; + auto nppc = IntVect(AMREX_D_DECL(npc, npc, npc)); + + amrex::Print() << "About to initialize particles \n"; + + pc.InitParticles(nppc); + + { + conduit::Node bp_mesh; + /* TODO + amrex::MultiLevelToBlueprint( + nlev, + amrex::GetVecOfConstPtrs(mf), + varnames, + geom, + time, + iteration, + warpx.refRatio(), + bp_mesh + ); + */ + + // wrap pc for current species into a blueprint topology + std::string const prefix = "particle"; + Vector particle_varnames; + for (int i = 0; i < pc.NumRealComps(); ++i) { + particle_varnames.push_back(prefix + "_real_" + std::to_string(i)); + } + Vector particle_int_varnames; + for (int i = 0; i < pc.NumIntComps(); ++i) { + particle_int_varnames.push_back(prefix + "_int_" + std::to_string(i)); + } + + amrex::ParticleContainerToBlueprint(pc, + particle_varnames, + particle_int_varnames, + bp_mesh, + prefix); + // publish + ascent::Ascent ascent; + conduit::Node opts; + opts["exceptions"] = "catch"; + opts["mpi_comm"] = MPI_Comm_c2f(ParallelDescriptor::Communicator()); + ascent.open(opts); + ascent.publish(bp_mesh); + + // If you want to save blueprint HDF5 files w/o using an Ascent + // extract, you can call the following AMReX helper: + // const auto step = istep[0]; + // amrex::WriteBlueprintFiles(bp_mesh, "bp_export", step, "hdf5"); + + // render + conduit::Node actions; + ascent.execute(actions); + ascent.close(); + } + + // the way this test is set up, if we make it here we pass + amrex::Print() << "pass \n"; +} + From 60b45bc19b691c1187f4cfd9cd39b2744bf7e35a Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 27 Nov 2023 13:57:39 -0800 Subject: [PATCH 04/16] MLEBABecLap: Support Robin BC at Domain Boundaries (#3617) --- .../MLMG/AMReX_MLABecLaplacian.H | 58 +++++++++++-------- Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H | 4 +- Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H | 13 +++-- Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp | 10 ++++ Src/LinearSolvers/MLMG/AMReX_MLLinOp.H | 32 +++++----- 5 files changed, 71 insertions(+), 46 deletions(-) diff --git a/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H b/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H index 9498af0e620..5c90a4e21ff 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H @@ -190,15 +190,15 @@ public: Array const& flux, FAB const& sol, int face_only, int ncomp); -protected: - - bool m_needs_update = true; - RT m_a_scalar = std::numeric_limits::quiet_NaN(); RT m_b_scalar = std::numeric_limits::quiet_NaN(); Vector > m_a_coeffs; Vector > > m_b_coeffs; +protected: + + bool m_needs_update = true; + Vector m_is_singular; [[nodiscard]] bool supportRobinBC () const noexcept override { return true; } @@ -474,29 +474,29 @@ MLABecLaplacianT::applyMetricTermsCoeffs () // \tilde{alpha}_i = alpha_i + (1-B) beta_{i+1/2} / h^2 // \tilde{rhs}_i = rhs_i + A beta_{i+1/2} / h^2 // -template -void -MLABecLaplacianT::applyRobinBCTermsCoeffs () +namespace detail { +template +void applyRobinBCTermsCoeffs (LP& linop) { - if (!(this->hasRobinBC())) { return; } + using RT = typename LP::RT; - const int ncomp = this->getNComp(); + const int ncomp = linop.getNComp(); bool reset_alpha = false; - if (m_a_scalar == RT(0.0)) { - m_a_scalar = RT(1.0); + if (linop.m_a_scalar == RT(0.0)) { + linop.m_a_scalar = RT(1.0); reset_alpha = true; } - const RT bovera = m_b_scalar/m_a_scalar; + const RT bovera = linop.m_b_scalar/linop.m_a_scalar; - for (int amrlev = 0; amrlev < this->m_num_amr_levels; ++amrlev) { + for (int amrlev = 0; amrlev < linop.NAMRLevels(); ++amrlev) { const int mglev = 0; - const Box& domain = this->m_geom[amrlev][mglev].Domain(); - const RT dxi = static_cast(this->m_geom[amrlev][mglev].InvCellSize(0)); - const RT dyi = static_cast((AMREX_SPACEDIM >= 2) ? this->m_geom[amrlev][mglev].InvCellSize(1) : Real(1.0)); - const RT dzi = static_cast((AMREX_SPACEDIM == 3) ? this->m_geom[amrlev][mglev].InvCellSize(2) : Real(1.0)); + const Box& domain = linop.Geom(amrlev,mglev).Domain(); + const RT dxi = static_cast(linop.Geom(amrlev,mglev).InvCellSize(0)); + const RT dyi = static_cast((AMREX_SPACEDIM >= 2) ? linop.Geom(amrlev,mglev).InvCellSize(1) : Real(1.0)); + const RT dzi = static_cast((AMREX_SPACEDIM == 3) ? linop.Geom(amrlev,mglev).InvCellSize(2) : Real(1.0)); if (reset_alpha) { - m_a_coeffs[amrlev][mglev].setVal(RT(0.0)); + linop.m_a_coeffs[amrlev][mglev].setVal(RT(0.0)); } MFItInfo mfi_info; @@ -505,20 +505,20 @@ MLABecLaplacianT::applyRobinBCTermsCoeffs () #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(m_a_coeffs[amrlev][mglev], mfi_info); mfi.isValid(); ++mfi) + for (MFIter mfi(linop.m_a_coeffs[amrlev][mglev], mfi_info); mfi.isValid(); ++mfi) { const Box& vbx = mfi.validbox(); - auto const& afab = m_a_coeffs[amrlev][mglev].array(mfi); + auto const& afab = linop.m_a_coeffs[amrlev][mglev].array(mfi); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto const& bfab = m_b_coeffs[amrlev][mglev][idim].const_array(mfi); + auto const& bfab = linop.m_b_coeffs[amrlev][mglev][idim].const_array(mfi); const Box& blo = amrex::adjCellLo(vbx,idim); const Box& bhi = amrex::adjCellHi(vbx,idim); bool outside_domain_lo = !(domain.contains(blo)); bool outside_domain_hi = !(domain.contains(bhi)); if ((!outside_domain_lo) && (!outside_domain_hi)) { continue; } for (int icomp = 0; icomp < ncomp; ++icomp) { - auto const& rbc = (*(this->m_robin_bcval[amrlev]))[mfi].const_array(icomp*3); - if (this->m_lobc_orig[icomp][idim] == LinOpBCType::Robin && outside_domain_lo) + auto const& rbc = (*(linop.m_robin_bcval[amrlev]))[mfi].const_array(icomp*3); + if (linop.m_lobc_orig[icomp][idim] == LinOpBCType::Robin && outside_domain_lo) { if (idim == 0) { RT fac = bovera*dxi*dxi; @@ -546,7 +546,7 @@ MLABecLaplacianT::applyRobinBCTermsCoeffs () }); } } - if (this->m_hibc_orig[icomp][idim] == LinOpBCType::Robin && outside_domain_hi) + if (linop.m_hibc_orig[icomp][idim] == LinOpBCType::Robin && outside_domain_hi) { if (idim == 0) { RT fac = bovera*dxi*dxi; @@ -579,6 +579,16 @@ MLABecLaplacianT::applyRobinBCTermsCoeffs () } } } +} // namespace detail + +template +void +MLABecLaplacianT::applyRobinBCTermsCoeffs () +{ + if (this->hasRobinBC()) { + detail::applyRobinBCTermsCoeffs(*this); + } +} template void diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H index 175f34ae54b..d4b3718212a 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H @@ -131,6 +131,8 @@ public: RT location; }; + Vector > m_robin_bcval; + protected: bool m_has_metric_term = false; @@ -182,8 +184,6 @@ protected: }; Vector > > m_bcondloc; - Vector > m_robin_bcval; - // used to save interpolation coefficients of the first interior cells mutable Vector> > m_undrrelxr; diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H index 6187c479e4c..448a5aaedf4 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H @@ -114,6 +114,8 @@ public: void getEBFluxes (const Vector& a_flux, const Vector& a_sol) const override; + void applyRobinBCTermsCoeffs (); + #if defined(AMREX_USE_HYPRE) && (AMREX_SPACEDIM > 1) [[nodiscard]] std::unique_ptr makeHypre (Hypre::Interface hypre_interface) const override; #endif @@ -122,6 +124,11 @@ public: [[nodiscard]] std::unique_ptr makePETSc () const override; #endif + Real m_a_scalar = std::numeric_limits::quiet_NaN(); + Real m_b_scalar = std::numeric_limits::quiet_NaN(); + Vector > m_a_coeffs; + Vector > > m_b_coeffs; + protected: int m_ncomp = 1; @@ -131,10 +138,6 @@ protected: Location m_beta_loc; // Location of coefficients: face centers or face centroids Location m_phi_loc; // Location of solution variable: cell centers or cell centroids - Real m_a_scalar = std::numeric_limits::quiet_NaN(); - Real m_b_scalar = std::numeric_limits::quiet_NaN(); - Vector > m_a_coeffs; - Vector > > m_b_coeffs; Vector > m_cc_mask; Vector > m_eb_phi; @@ -154,6 +157,8 @@ protected: const Vector& b_eb); void averageDownCoeffs (); void averageDownCoeffsToCoarseAmrLevel (int flev); + + [[nodiscard]] bool supportRobinBC () const noexcept override { return true; } }; } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp index b37537645e6..247eeeea25c 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp @@ -685,6 +685,8 @@ MLEBABecLap::prepareForSolve () MLCellABecLap::prepareForSolve(); + applyRobinBCTermsCoeffs(); + averageDownCoeffs(); if (m_eb_phi[0]) { @@ -1285,6 +1287,14 @@ MLEBABecLap::getEBFluxes (const Vector& a_flux, const VectorhasRobinBC()) { + detail::applyRobinBCTermsCoeffs(*this); + } +} + #if defined(AMREX_USE_HYPRE) && (AMREX_SPACEDIM > 1) std::unique_ptr MLEBABecLap::makeHypre (Hypre::Interface hypre_interface) const diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H index 706fe679d7e..a889f7514af 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H @@ -488,6 +488,22 @@ public: [[nodiscard]] bool isMFIterSafe (int amrlev, int mglev1, int mglev2) const; + //! Return the number of AMR levels + [[nodiscard]] int NAMRLevels () const noexcept { return m_num_amr_levels; } + + //! Return the number of MG levels at given AMR level + [[nodiscard]] int NMGLevels (int amrlev) const noexcept { return m_num_mg_levels[amrlev]; } + + [[nodiscard]] const Geometry& Geom (int amr_lev, int mglev=0) const noexcept { return m_geom[amr_lev][mglev]; } + + // BC + Vector > m_lobc; + Vector > m_hibc; + // Need to save the original copy because we change the BC type to + // Neumann for inhomogeneous Neumann and Robin. + Vector > m_lobc_orig; + Vector > m_hibc_orig; + protected: static constexpr int mg_coarsen_ratio = 2; @@ -544,14 +560,6 @@ protected: }; std::unique_ptr m_raii_comm; - // BC - Vector > m_lobc; - Vector > m_hibc; - // Need to save the original copy because we change the BC type to - // Neumann for inhomogeneous Neumann and Robin. - Vector > m_lobc_orig; - Vector > m_hibc_orig; - Array m_domain_bloc_lo {{AMREX_D_DECL(0._rt,0._rt,0._rt)}}; Array m_domain_bloc_hi {{AMREX_D_DECL(0._rt,0._rt,0._rt)}}; @@ -561,20 +569,12 @@ protected: const MF* m_coarse_data_for_bc = nullptr; MF m_coarse_data_for_bc_raii; - //! Return the number of AMR levels - [[nodiscard]] int NAMRLevels () const noexcept { return m_num_amr_levels; } - - //! Return the number of MG levels at given AMR level - [[nodiscard]] int NMGLevels (int amrlev) const noexcept { return m_num_mg_levels[amrlev]; } - //! Return AMR refinement ratios [[nodiscard]] const Vector& AMRRefRatio () const noexcept { return m_amr_ref_ratio; } //! Return AMR refinement ratio at given AMR level [[nodiscard]] int AMRRefRatio (int amr_lev) const noexcept { return m_amr_ref_ratio[amr_lev]; } - [[nodiscard]] const Geometry& Geom (int amr_lev, int mglev=0) const noexcept { return m_geom[amr_lev][mglev]; } - [[nodiscard]] FabFactory const* Factory (int amr_lev, int mglev=0) const noexcept { return m_factory[amr_lev][mglev].get(); } From 4b64003c80099627e9da856801517844f2bb21b8 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Mon, 27 Nov 2023 15:58:46 -0600 Subject: [PATCH 05/16] Comments: correct some typos (#3641) ## Summary This PR fixes some typos and spelling errors in the comments that codespell seems to have missed. --- CHANGES | 6 +++--- Src/AmrCore/AMReX_FillPatcher.H | 8 ++++---- Src/AmrCore/AMReX_InterpFaceRegister.H | 4 ++-- Src/Base/AMReX_BCRec.H | 2 +- Src/Base/AMReX_FArrayBox.H | 2 +- Src/Base/AMReX_FabArray.H | 4 ++-- Src/Base/AMReX_MultiFab.H | 4 ++-- Src/Base/AMReX_MultiFabUtil.H | 2 +- Src/Base/AMReX_MultiFabUtil.cpp | 2 +- Src/Base/AMReX_iMultiFab.H | 4 ++-- Src/EB/AMReX_EB_utils.cpp | 2 +- Src/F_Interfaces/Base/AMReX_parallel_mod.F90 | 2 +- Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H | 10 +++++----- Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.H | 2 +- Src/LinearSolvers/MLMG/AMReX_MLLinOp.H | 2 +- Src/LinearSolvers/MLMG/AMReX_MLMG.H | 2 +- Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H | 8 ++++---- Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H | 2 +- Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp | 2 +- Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp | 2 +- 20 files changed, 36 insertions(+), 36 deletions(-) diff --git a/CHANGES b/CHANGES index a7a22fd97bd..f9f2d4febe8 100644 --- a/CHANGES +++ b/CHANGES @@ -72,7 +72,7 @@ -- We weren't defining cent_hat out far enough (#3548) - -- Add Fortran inteface for FillCoarsePatch for face variables (#3542) + -- Add Fortran interface for FillCoarsePatch for face variables (#3542) -- print_state/printCell: Make it work without managed memory (#3543) @@ -114,7 +114,7 @@ -- Simplify filterParticles Kernel (#3510) - -- Generatize particle-to-cell assignment function (#3499) + -- Generalize particle-to-cell assignment function (#3499) Follow-on to 3499 (#3514) ParticleLocator: Make Assignor optional template parameter (#3515) @@ -302,7 +302,7 @@ # 23.07 - -- Allow users to change the default vector growth stategy (#3389) + -- Allow users to change the default vector growth strategy (#3389) -- Communications arena implementation (#3388) diff --git a/Src/AmrCore/AMReX_FillPatcher.H b/Src/AmrCore/AMReX_FillPatcher.H index 46d1107dea3..d36b3529efd 100644 --- a/Src/AmrCore/AMReX_FillPatcher.H +++ b/Src/AmrCore/AMReX_FillPatcher.H @@ -15,13 +15,13 @@ namespace amrex { * with interpolation of the coarse data. Then it fills the fine ghost * cells overlapping fine level valid cells with the fine level data. If * the valid cells of the destination need to be filled, it will be done as - * well. Finally, it will fill the physical bounbary using the user + * well. Finally, it will fill the physical boundary using the user * provided functor. The `fill` member function can be used to do the * operations just described. Alternatively, one can also use the * `fillCoarseFineBounary` to fill the ghost cells at the coarse/fine * boundary only. Then one can manually call FillBoundary to fill the other * ghost cells, and use the physical BC functor to handle the physical - * boundeary. + * boundary. * * The communication of the coarse data needed for spatial interpolation is * optimized at the cost of being error-prone. One must follow the @@ -42,7 +42,7 @@ namespace amrex { * * (3) When to destroy? Usually, we do time steppig on a coarse level * first. Then we recursively do time stepping on fine levels. After the - * finer level finishes, we do reflux and averge the fine data down to the + * finer level finishes, we do reflux and average the fine data down to the * coarse level. After that we should destroy the FillPatcher object * associated with these two levels, because the coarse data stored in the * object has become outdated. For AmrCore based codes, you could use @@ -118,7 +118,7 @@ public: * \param fbc for filling fine level physical BC * \param fbccomp starting component of the fine level BC functor * \param bcs BCRec specifying physical boundary types - * \parame bcscomp starting component of the BCRec Vector. + * \param bcscomp starting component of the BCRec Vector. * \param pre_interp optional pre-interpolation hook for modifying the coarse data * \param post_interp optional post-interpolation hook for modifying the fine data */ diff --git a/Src/AmrCore/AMReX_InterpFaceRegister.H b/Src/AmrCore/AMReX_InterpFaceRegister.H index 5e9f92784e7..a63c2c23e4c 100644 --- a/Src/AmrCore/AMReX_InterpFaceRegister.H +++ b/Src/AmrCore/AMReX_InterpFaceRegister.H @@ -10,7 +10,7 @@ namespace amrex { /** * \brief InterpFaceRegister is a coarse/fine boundary register for - * interpolation of face data at the coarse/fine boundadry. + * interpolation of face data at the coarse/fine boundary. */ class InterpFaceRegister @@ -31,7 +31,7 @@ public: Geometry const& fgeom, IntVect const& ref_ratio); /** - * \brief Defines an InterpFaceRegister objecct. + * \brief Defines an InterpFaceRegister object. * * \param fba The fine level BoxArray * \param fdm The fine level DistributionMapping diff --git a/Src/Base/AMReX_BCRec.H b/Src/Base/AMReX_BCRec.H index c39634cfb09..268147a3a04 100644 --- a/Src/Base/AMReX_BCRec.H +++ b/Src/Base/AMReX_BCRec.H @@ -43,7 +43,7 @@ public: {} /* * \brief Yet another constructor. Inherits bndry types from bc_domain - * when bx lies on edge of domain otherwise gets interior Dirchlet. + * when bx lies on edge of domain otherwise gets interior Dirichlet. */ AMREX_GPU_HOST_DEVICE BCRec (const Box& bx, diff --git a/Src/Base/AMReX_FArrayBox.H b/Src/Base/AMReX_FArrayBox.H index 084b38ce46b..b6dc4e887c8 100644 --- a/Src/Base/AMReX_FArrayBox.H +++ b/Src/Base/AMReX_FArrayBox.H @@ -116,7 +116,7 @@ public: * \brief Pure virtual function. Derived classes MUST override this * function to skip over the next FAB f in the istream, under the * assumption that the header for the FAB f has already been - * skpped over. + * skipped over. */ virtual void skip (std::istream& is, FArrayBox& f) const = 0; diff --git a/Src/Base/AMReX_FabArray.H b/Src/Base/AMReX_FabArray.H index e507dab153b..96efc1f18fa 100644 --- a/Src/Base/AMReX_FabArray.H +++ b/Src/Base/AMReX_FabArray.H @@ -339,7 +339,7 @@ public: /** * \brief Construct an empty FabArray that has a default Arena. If - * `define` is called later with a nulltpr as MFInfo's arena, the + * `define` is called later with a nullptr as MFInfo's arena, the * default Arena `a` will be used. If the arena in MFInfo is not a * nullptr, the MFInfo's arena will be used. */ @@ -2827,7 +2827,7 @@ template void FabArray::shift (const IntVect& v) { - clearThisBD(); // The new boxarry will have a different ID. + clearThisBD(); // The new boxarray will have a different ID. boxarray.shift(v); addThisBD(); #ifdef AMREX_USE_OMP diff --git a/Src/Base/AMReX_MultiFab.H b/Src/Base/AMReX_MultiFab.H index 416c4540da5..1a6c1d7f15f 100644 --- a/Src/Base/AMReX_MultiFab.H +++ b/Src/Base/AMReX_MultiFab.H @@ -50,7 +50,7 @@ public: /** * \brief Constructs an empty MultiFab. Data can be defined at a later * time using the define member functions inherited from FabArray. If - * `define` is called later with a nulltpr as MFInfo's arena, the default + * `define` is called later with a nullptr as MFInfo's arena, the default * Arena `a` will be used. If the arena in MFInfo is not a nullptr, the * MFInfo's arena will be used. */ @@ -60,7 +60,7 @@ public: * \brief * Constructs a MultiFab * \param bs a valid region - * \param dm a DistribuionMapping + * \param dm a DistributionMapping * \param ncomp number of components * \param ngrow number of cells the region grows * \param info MFInfo diff --git a/Src/Base/AMReX_MultiFabUtil.H b/Src/Base/AMReX_MultiFabUtil.H index ad1fa669f32..064b8a1b19f 100644 --- a/Src/Base/AMReX_MultiFabUtil.H +++ b/Src/Base/AMReX_MultiFabUtil.H @@ -375,7 +375,7 @@ namespace amrex void FillRandom (MultiFab& mf, int scomp, int ncomp); /** - * \brief Fill MultiFab with random numbers from nornmal distribution + * \brief Fill MultiFab with random numbers from normal distribution * * All cells including ghost cells are filled. * diff --git a/Src/Base/AMReX_MultiFabUtil.cpp b/Src/Base/AMReX_MultiFabUtil.cpp index 93ba453cc07..5520c6164d7 100644 --- a/Src/Base/AMReX_MultiFabUtil.cpp +++ b/Src/Base/AMReX_MultiFabUtil.cpp @@ -477,7 +477,7 @@ namespace amrex auto tmptype = type; tmptype.set(dir); if (dir >= AMREX_SPACEDIM || !tmptype.nodeCentered()) { - amrex::Abort("average_down_edges: not face index type"); + amrex::Abort("average_down_edges: not edge index type"); } const int ncomp = crse.nComp(); if (isMFIterSafe(fine, crse)) diff --git a/Src/Base/AMReX_iMultiFab.H b/Src/Base/AMReX_iMultiFab.H index 519ab9c82a5..eb1e350433a 100644 --- a/Src/Base/AMReX_iMultiFab.H +++ b/Src/Base/AMReX_iMultiFab.H @@ -24,7 +24,7 @@ namespace amrex { * member functions are defined for I/O and simple arithmetic operations on * these aggregate objects. * -8 This class does NOT provide a copy constructor or assignment operator. +* This class does NOT provide a copy constructor or assignment operator. */ class iMultiFab : @@ -42,7 +42,7 @@ public: /** * \brief Constructs an empty iMultiFab. Data can be defined at a later * time using the define member functions inherited from FabArray. If - * `define` is called later with a nulltpr as MFInfo's arena, the default + * `define` is called later with a nullptr as MFInfo's arena, the default * Arena `a` will be used. If the arena in MFInfo is not a nullptr, the * MFInfo's arena will be used. */ diff --git a/Src/EB/AMReX_EB_utils.cpp b/Src/EB/AMReX_EB_utils.cpp index 948a3d5db20..857a8eb08a8 100644 --- a/Src/EB/AMReX_EB_utils.cpp +++ b/Src/EB/AMReX_EB_utils.cpp @@ -31,7 +31,7 @@ facets_nearest_pt (IntVect const& ind_pt, IntVect const& ind_loop, RealVect cons RealVect const& eb_normal, RealVect const& eb_p0, GpuArray const& dx) { - // Enumerate the possible EB facet edges invovlved. + // Enumerate the possible EB facet edges involved. int n_facets = 0; IntVect ind_facets {AMREX_D_DECL(0, 0, 0)}; for (int d = 0; d < AMREX_SPACEDIM; ++d) { diff --git a/Src/F_Interfaces/Base/AMReX_parallel_mod.F90 b/Src/F_Interfaces/Base/AMReX_parallel_mod.F90 index 00fcc275c89..68cf647ff7f 100644 --- a/Src/F_Interfaces/Base/AMReX_parallel_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_parallel_mod.F90 @@ -71,7 +71,7 @@ subroutine amrex_parallel_init (comm) if (present(comm) .and. .not.flag) then if (comm .ne. MPI_COMM_WORLD) then - stop "MPI has not been initialized. How come we are given a communciator?" + stop "MPI has not been initialized. How come we are given a communicator?" endif end if diff --git a/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H b/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H index 5c90a4e21ff..cc9c361e916 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H @@ -72,7 +72,7 @@ public: /** * Sets alpha as a scalar field to values from a single component - * mutlifab. + * multifab. * * \param [in] amrlev The level of the multifab for the solver, with * \p amrlev = 0 always being the lowest level in the @@ -88,12 +88,12 @@ public: /** * Sets alpha as a single scalar constant value across - * the mutlifab. + * the multifab. * * \param [in] amrlev The level of the multifab for the solver, with * \p amrlev = 0 always being the lowest level in the * AMR hierarchy represented in the solve. - * \param [in] alpha Single scalar value to populate across mutlifab. + * \param [in] alpha Single scalar value to populate across multifab. */ template , @@ -118,12 +118,12 @@ public: /** * Sets beta as a single scalar constant value across - * the mutlifabs (one for each dimension). + * the multifabs (one for each dimension). * * \param [in] amrlev The level of the multifab for the solver, with * \p amrlev = 0 always being the lowest level in the * AMR hierarchy represented in the solve. - * \param [in] beta Single scalar value to populate across mutlifabs. + * \param [in] beta Single scalar value to populate across multifabs. */ template , diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.H b/Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.H index 0c4937d2de5..05b3250caf8 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.H @@ -18,7 +18,7 @@ namespace amrex { // where phi and rhs are nodal multifab, and sigma is a tensor constant // with only diagonal components. The EB is assumed to be Dirichlet. // -// del dot (simga grad phi) - alpha/r^2 phi = rhs, for RZ where alpha is a +// del dot (sigma grad phi) - alpha/r^2 phi = rhs, for RZ where alpha is a // scalar constant that is zero by default. class MLEBNodeFDLaplacian diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H index a889f7514af..b8aa71eebd0 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H @@ -287,7 +287,7 @@ public: * \param famrlev fine AMR level * \param fine fine level data * \param crse coarse level data - * \parame nghost number of ghost cells + * \param nghost number of ghost cells */ virtual void interpolationAmr (int famrlev, MF& fine, const MF& crse, IntVect const& nghost) const = 0; diff --git a/Src/LinearSolvers/MLMG/AMReX_MLMG.H b/Src/LinearSolvers/MLMG/AMReX_MLMG.H index f1fed2d3db3..70e7e121486 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLMG.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLMG.H @@ -1370,7 +1370,7 @@ MLMGT::mgFcycle () } } -// At the true bottom of the coarset AMR level. +// At the true bottom of the coarsest AMR level. // in : Residual (res) as b // out : Correction (cor) as x template diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H index d1ae9e0b7ed..32c75224e78 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H @@ -2406,7 +2406,7 @@ void mlndlap_fillijmat_aa_cpu (Box const& ndbx, Real f2xmy = Real(2.0)*facx - facy; Real fmx2y = Real(2.0)*facy - facx; - // Note that ccdom has been grown at peridoci boundaries. + // Note that ccdom has been grown at periodic boundaries. const Box& nddom = amrex::surroundingNodes(ccdom); constexpr auto gidmax = std::numeric_limits::max(); @@ -2552,7 +2552,7 @@ void mlndlap_fillijmat_ha_cpu (Box const& ndbx, Real facx = Real(1.0/6.0)*dxinv[0]*dxinv[0]; Real facy = Real(1.0/6.0)*dxinv[1]*dxinv[1]; - // Note that ccdom has been grown at peridoci boundaries. + // Note that ccdom has been grown at periodic boundaries. const Box& nddom = amrex::surroundingNodes(ccdom); constexpr auto gidmax = std::numeric_limits::max(); @@ -2708,7 +2708,7 @@ void mlndlap_fillijmat_cs_cpu (Box const& ndbx, Real f2xmy = Real(2.0)*facx - facy; Real fmx2y = Real(2.0)*facy - facx; - // Note that ccdom has been grown at peridoci boundaries. + // Note that ccdom has been grown at periodic boundaries. const Box& nddom = amrex::surroundingNodes(ccdom); constexpr auto gidmax = std::numeric_limits::max(); @@ -3355,7 +3355,7 @@ void mlndlap_fillijmat_cs_gpu (const int ps, const int i, const int j, const int fp = fm = Real(0.0); } - // Note that nddom has been grown at peridoci boundaries. + // Note that nddom has been grown at periodic boundaries. const Box& nddom = amrex::surroundingNodes(ccdom); constexpr auto gidmax = std::numeric_limits::max(); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H index adbf00da231..bb2ec8dd4f5 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H @@ -6,7 +6,7 @@ namespace amrex { -// del dot (sigma grah phi) = rhs +// del dot (sigma grad phi) = rhs // where phi and rhs are nodal, and sigma is cell-centered. class MLNodeLaplacian diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp index 4a749a1ed08..d96e183af4d 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp @@ -947,7 +947,7 @@ MLNodeLaplacian::checkPoint (std::string const& file_name) const HeaderFile.precision(17); - // MLLinop stuff + // MLLinOp stuff HeaderFile << "verbose = " << verbose << "\n" << "nlevs = " << NAMRLevels() << "\n" << "do_agglomeration = " << info.do_agglomeration << "\n" diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp index 0fb9e2ba33b..41b52833e2c 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp @@ -234,7 +234,7 @@ void MLNodeLinOp_set_dot_mask (MultiFab& dot_mask, iMultiFab const& omask, Geome Box nddomain = amrex::surroundingNodes(geom.Domain()); if (strategy != MLNodeLinOp::CoarseningStrategy::Sigma) { - nddomain.grow(1000); // hack to avoid masks being modified at Neuman boundary + nddomain.grow(1000); // hack to avoid masks being modified at Neumann boundary } #ifdef AMREX_USE_OMP From 4b8dd0373188ca6505e3a9dfaaafbaf4b7fe5ffe Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Tue, 28 Nov 2023 19:07:39 -0600 Subject: [PATCH 06/16] Comments on FAB-related classes and methods (#3636) New or updated comments for various FAB-related classes and methods. Descriptions are tweaked to match standard comment conventions and make IntelliSense happy. For example, some descriptions get moved to be just before their respective class declarations. --- Src/Base/AMReX_BaseFab.H | 96 ++++++++++++------------- Src/Base/AMReX_BoxArray.H | 12 ++-- Src/Base/AMReX_FArrayBox.H | 2 - Src/Base/AMReX_FabArray.H | 120 ++++++++++++++++---------------- Src/Base/AMReX_FabArrayBase.H | 6 ++ Src/Base/AMReX_FabConv.H | 86 +++++++++++------------ Src/Base/AMReX_MultiFab.H | 98 ++++++++++++++------------ Src/Base/AMReX_MultiFabUtil.H | 42 +++++++---- Src/Base/AMReX_MultiFabUtil.cpp | 3 - 9 files changed, 243 insertions(+), 222 deletions(-) diff --git a/Src/Base/AMReX_BaseFab.H b/Src/Base/AMReX_BaseFab.H index 99132038397..006d7639ad6 100644 --- a/Src/Base/AMReX_BaseFab.H +++ b/Src/Base/AMReX_BaseFab.H @@ -90,54 +90,6 @@ makeArray4 (T* p, Box const& bx, int ncomp) noexcept return Array4{p, amrex::begin(bx), amrex::end(bx), ncomp}; } -/** -* \brief A Fortran Array-like Object -* BaseFab emulates the Fortran array concept. -* Useful operations can be performed upon -* BaseFabs in C++, and they provide a convenient interface to -* Fortran when it is necessary to retreat into that language. - -* BaseFab is a template class. Through use of the -* template, a BaseFab may be based upon any class. So far at least, -* most applications have been based upon simple types like integers, -* real*4s, or real*8s. Most applications do not use BaseFabs -* directly, but utilize specialized classes derived from BaseFab. - -* Classes derived from BaseFab include FArrayBox, IArrayBox, TagBox, -* Mask, EBFArrayBox, EBCellFlag and CutFab. - -* BaseFab objects depend on the dimensionality of space -* (indirectly through the DOMAIN Box member). It is -* typical to define the macro SPACEDIM to be 1, 2, or 3 to indicate -* the dimension of space. See the discussion of class Box for more -* information. A BaseFab contains a Box DOMAIN, which indicates the -* integer indexing space over which the array is defined. A BaseFab -* also has NVAR components. By components, we mean that for each -* point in the rectangular indexing space, there are NVAR values -* associated with that point. A Fortran array corresponding to a -* BaseFab would have (SPACEDIM+1) dimensions. - -* By design, the array layout in a BaseFab mirrors that of a -* Fortran array. The first index (x direction for example) varies -* most rapidly, the next index (y direction), if any, varies next -* fastest. The component index varies last, after all the spatial -* indices. - -* It is sometimes convenient to be able to treat a sub-array within an -* existing BaseFab as a BaseFab in its own right. This is often -* referred to as aliasing the BaseFab. Note that when aliasing is -* used, the BaseFabs domain will not, in general, be the same as the -* parent BaseFabs domain, nor will the number of components. -* BaseFab is a dimension dependent class, so SPACEDIM must be -* defined as either 1, 2, or 3 when compiling. - -* This is NOT a polymorphic class. - -* It does NOT provide a copy constructor or assignment operator. - -* T MUST have a default constructor and an assignment operator. -*/ - template typename std::enable_if::value>::type placementNew (T* const /*ptr*/, Long /*n*/) @@ -178,6 +130,54 @@ placementDelete (T* const ptr, Long n) }); } +/** + * \brief A FortranArrayBox(FAB)-like object + * + * BaseFab emulates the Fortran array concept. + * Useful operations can be performed upon + * BaseFabs in C++, and they provide a convenient interface to + * Fortran when it is necessary to retreat into that language. + * + * BaseFab is a template class. Through use of the + * template, a BaseFab may be based upon any class. So far at least, + * most applications have been based upon simple types like integers, + * real*4s, or real*8s. Most applications do not use BaseFabs + * directly, but utilize specialized classes derived from BaseFab. + * + * Classes derived from BaseFab include FArrayBox, IArrayBox, TagBox, + * Mask, EBFArrayBox, EBCellFlag and CutFab. + * + * BaseFab objects depend on the dimensionality of space + * (indirectly through the DOMAIN Box member). It is + * typical to define the macro SPACEDIM to be 1, 2, or 3 to indicate + * the dimension of space. See the discussion of class Box for more + * information. A BaseFab contains a Box DOMAIN, which indicates the + * integer indexing space over which the array is defined. A BaseFab + * also has NVAR components. By components, we mean that for each + * point in the rectangular indexing space, there are NVAR values + * associated with that point. A Fortran array corresponding to a + * BaseFab would have (SPACEDIM+1) dimensions. + * + * By design, the array layout in a BaseFab mirrors that of a + * Fortran array. The first index (x direction for example) varies + * most rapidly, the next index (y direction), if any, varies next + * fastest. The component index varies last, after all the spatial + * indices. + * + * It is sometimes convenient to be able to treat a sub-array within an + * existing BaseFab as a BaseFab in its own right. This is often + * referred to as aliasing the BaseFab. Note that when aliasing is + * used, the BaseFabs domain will not, in general, be the same as the + * parent BaseFabs domain, nor will the number of components. + * BaseFab is a dimension dependent class, so SPACEDIM must be + * defined as either 1, 2, or 3 when compiling. + * + * This is NOT a polymorphic class. + * + * It does NOT provide a copy constructor or assignment operator. + * + * \tparam T MUST have a default constructor and an assignment operator. + */ template class BaseFab : public DataAllocator diff --git a/Src/Base/AMReX_BoxArray.H b/Src/Base/AMReX_BoxArray.H index 807cd9d8516..94358f580a4 100644 --- a/Src/Base/AMReX_BoxArray.H +++ b/Src/Base/AMReX_BoxArray.H @@ -515,16 +515,16 @@ struct BATransformer // for backward compatibility using BndryBATransformer = BATransformer; -/** -* \brief A collection of Boxes stored in an Array. It is a -* reference-counted concrete class, not a polymorphic one; i.e. you -* cannot use any of the List member functions with a BoxList. -*/ - class MFIter; class AmrMesh; class FabArrayBase; +/** + * \brief A collection of Boxes stored in an Array. + * + * It is a reference-counted concrete class, not a polymorphic one; i.e. you + * cannot use any of the List member functions with a BoxList. + */ class BoxArray { public: diff --git a/Src/Base/AMReX_FArrayBox.H b/Src/Base/AMReX_FArrayBox.H index b6dc4e887c8..2dda4e6b0a0 100644 --- a/Src/Base/AMReX_FArrayBox.H +++ b/Src/Base/AMReX_FArrayBox.H @@ -26,7 +26,6 @@ class FArrayBox; * primarily for FArrayBox implementers; i.e. user's shouldn't * call any of the member functions in this class directly. */ - class FABio // NOLINT(cppcoreguidelines-special-member-functions) { public: @@ -224,7 +223,6 @@ private: * This class does NOT provide a copy constructor or assignment operator, * but it has a move constructor. */ - class FArrayBox : public BaseFab diff --git a/Src/Base/AMReX_FabArray.H b/Src/Base/AMReX_FabArray.H index 96efc1f18fa..56300d4cfef 100644 --- a/Src/Base/AMReX_FabArray.H +++ b/Src/Base/AMReX_FabArray.H @@ -56,64 +56,11 @@ Long nBytesOwned (T const&) noexcept { return 0; } template Long nBytesOwned (BaseFab const& fab) noexcept { return fab.nBytesOwned(); } -/* - A Collection of Fortran Array-like Objects - - - The FabArray class implements a collection (stored as an array) of - Fortran array-like objects. The parameterized type FAB is intended to be - any class derived from BaseFab. For example, FAB may be a BaseFab of - integers, so we could write: - - FabArray > int_fabs; - - Then int_fabs is a FabArray that can hold a collection of BaseFab - objects. - - FabArray is not just a general container class for Fortran arrays. It is - intended to hold "grid" data for use in finite difference calculations in - which the data is defined on a union of (usually disjoint) rectangular - regions embedded in a uniform index space. This region, called the valid - region, is represented by a BoxArray. For the purposes of this discussion, - the Kth Box in the BoxArray represents the interior region of the Kth grid. - - Since the intent is to be used with finite difference calculations a - FabArray also includes the notion of a boundary region for each grid. The - boundary region is specified by the ngrow parameter which tells the FabArray - to allocate each FAB to be ngrow cells larger in all directions than the - underlying Box. The larger region covered by the union of all the FABs is - called the region of definition. The underlying notion is that the valid - region contains the grid interior data and the region of definition includes - the interior region plus the boundary areas. - - Operations are available to copy data from the valid regions into these - boundary areas where the two overlap. The number of components, that is, - the number of values that can be stored in each cell of a FAB, is either - given as an argument to the constructor or is inherent in the definition of - the underlying FAB. Each FAB in the FabArray will have the same number of - components. - - In summary, a FabArray is an array of FABs. The Kth element contains a FAB - that holds the data for the Kth grid, a Box that defines the valid region - of the Kth grid. - - A typical use for a FabArray would be to hold the solution vector or - right-hand-side when solving a linear system of equations on a union of - rectangular grids. The copy operations would be used to copy data from the - valid regions of neighboring grids into the boundary regions after each - relaxation step of the iterative method. If a multigrid method is used, a - FabArray could be used to hold the data at each level in the multigrid - hierarchy. - - This class is a concrete class not a polymorphic one. - - This class does NOT provide a copy constructor or assignment operator. -*/ - -// -// alloc: allocate memory or not -// +/** + * \brief FabArray memory allocation information + */ struct MFInfo { + // alloc: allocate memory or not bool alloc = true; Arena* arena = nullptr; Vector tags; @@ -314,6 +261,60 @@ Add (FabArray& dst, FabArray const& src, int srccomp, int dstcomp, int } } +/** + * \brief An Array of FortranArrayBox(FAB)-like Objects + * + * The FabArray class implements a collection (stored as an array) of + * Fortran array box-like ( \p FAB ) objects. The parameterized type \p FAB is intended to be + * any class derived from BaseFab. For example, \p FAB may be a BaseFab of + * integers, so we could write: + * + * FabArray > int_fabs; + * + * Then int_fabs is a FabArray that can hold a collection of BaseFab + * objects. + * + * FabArray is not just a general container class for Fortran arrays. It is + * intended to hold "grid" data for use in finite difference calculations in + * which the data is defined on a union of (usually disjoint) rectangular + * regions embedded in a uniform index space. This region, called the valid + * region, is represented by a BoxArray. For the purposes of this discussion, + * the Kth Box in the BoxArray represents the interior region of the Kth grid. + * + * Since the intent is to be used with finite difference calculations a + * FabArray also includes the notion of a boundary region for each grid. The + * boundary region is specified by the ngrow parameter which tells the FabArray + * to allocate each \p FAB to be ngrow cells larger in all directions than the + * underlying Box. The larger region covered by the union of all the \p FABs is + * called the region of definition. The underlying notion is that the valid + * region contains the grid interior data and the region of definition includes + * the interior region plus the boundary areas. + * + * Operations are available to copy data from the valid regions into these + * boundary areas where the two overlap. The number of components, that is, + * the number of values that can be stored in each cell of a \p FAB, is either + * given as an argument to the constructor or is inherent in the definition of + * the underlying \p FAB. Each \p FAB in the FabArray will have the same number of + * components. + * + * In summary, a FabArray is an array of \p FABs. The Kth element contains a \p FAB + * that holds the data for the Kth grid, a Box that defines the valid region + * of the Kth grid. + * + * A typical use for a FabArray would be to hold the solution vector or + * right-hand-side when solving a linear system of equations on a union of + * rectangular grids. The copy operations would be used to copy data from the + * valid regions of neighboring grids into the boundary regions after each + * relaxation step of the iterative method. If a multigrid method is used, a + * FabArray could be used to hold the data at each level in the multigrid + * hierarchy. + * + * This class is a concrete class not a polymorphic one. + * + * This class does NOT provide a copy constructor or assignment operator. + * + * \tparam FAB FortranArrayBox-like object. Typically a derived class of BaseFab. Not to be confused with FabArrayBase. + */ template class FabArray : @@ -338,8 +339,9 @@ public: FabArray () noexcept; /** - * \brief Construct an empty FabArray that has a default Arena. If - * `define` is called later with a nullptr as MFInfo's arena, the + * \brief Construct an empty FabArray that has a default Arena. + * + * If `define` is called later with a nullptr as MFInfo's arena, the * default Arena `a` will be used. If the arena in MFInfo is not a * nullptr, the MFInfo's arena will be used. */ diff --git a/Src/Base/AMReX_FabArrayBase.H b/Src/Base/AMReX_FabArrayBase.H index 29d3d63b29e..d8bc4411874 100644 --- a/Src/Base/AMReX_FabArrayBase.H +++ b/Src/Base/AMReX_FabArrayBase.H @@ -28,6 +28,12 @@ template class FabArray; namespace EB2 { class IndexSpace; } +/** + * \brief Base class for FabArray. + * + * Not to be confused with FArrayBox or `FAB` shorthands. + * Can be read as FArrayBox-like Array Base. + */ class FabArrayBase { friend class MFIter; diff --git a/Src/Base/AMReX_FabConv.H b/Src/Base/AMReX_FabConv.H index 78554000f25..25dae063de7 100644 --- a/Src/Base/AMReX_FabConv.H +++ b/Src/Base/AMReX_FabConv.H @@ -13,20 +13,18 @@ namespace amrex { -// -// A Descriptor of the Long Integer type - /** -* This class is meant to hold all information needed to completely -* describe the "int" or "Long" type on a machine. To describe an integer both -* the number of bytes and their ordering, relative to canonical -* ordering 1 .. sizeof(Long), needs to be specified. -* This allows us to write out integers in the native format on a machine, -* and then by also saving the IntDescriptor, we can read them back in on -* another machine and have enough information to construct the exact same -* values. -*/ - + * \brief A Descriptor of the Long Integer type + * + * This class is meant to hold all information needed to completely + * describe the "int" or "Long" type on a machine. To describe an integer both + * the number of bytes and their ordering, relative to canonical + * ordering 1 .. sizeof(Long), needs to be specified. + * This allows us to write out integers in the native format on a machine, + * and then by also saving the IntDescriptor, we can read them back in on + * another machine and have enough information to construct the exact same + * values. + */ class IntDescriptor { @@ -72,39 +70,37 @@ std::ostream& operator<< (std::ostream& os, const IntDescriptor& id); //! std::istream& operator>> (std::istream& is, IntDescriptor& id); - - //A Descriptor of the Real Type - /** -* \brief This class is meant to hold all information needed to completely -* describe the "Real" floating-point type on a machine. By "Real" here we -* mean either the "float" or "double" type that this version of AMReX -* was built with, which corresponds to whether BL_USE_FLOAT or -* BL_USE_DOUBLE was used to build the version of the library. -* -* To describe a "Real" type two arrays are needed: one detailing the ordering -* of the bytes in the Real, relative to the canonical ordering -* 1 .. sizeof(Real) and the other detailing the format of the floating-point -* number. -* -* The array detailing the format of a floating-point number is an eight-element -* array of longs containing the following information: -* -* format[0] = number of bits per number -* format[1] = number of bits in exponent -* format[2] = number of bits in mantissa -* format[3] = start bit of sign -* format[4] = start bit of exponent -* format[5] = start bit of mantissa -* format[6] = high order mantissa bit (CRAY needs this) -* format[7] = bias of exponent -* -* This allows us to write out "Real"s in the native format on a machine, -* and then by also saving the IntDescriptor, we can read them back in on -* another machine and have enough information to construct the exact same -* "Real" values, provided the Reals have the same size on the two machines. -*/ - + * \brief A Descriptor of the Real Type + * + * This class is meant to hold all information needed to completely + * describe the "Real" floating-point type on a machine. By "Real" here we + * mean either the "float" or "double" type that this version of AMReX + * was built with, which corresponds to whether BL_USE_FLOAT or + * BL_USE_DOUBLE was used to build the version of the library. + * + * To describe a "Real" type two arrays are needed: one detailing the ordering + * of the bytes in the Real, relative to the canonical ordering + * 1 .. sizeof(Real) and the other detailing the format of the floating-point + * number. + * + * The array detailing the format of a floating-point number is an eight-element + * array of longs containing the following information: + * + * format[0] = number of bits per number + * format[1] = number of bits in exponent + * format[2] = number of bits in mantissa + * format[3] = start bit of sign + * format[4] = start bit of exponent + * format[5] = start bit of mantissa + * format[6] = high order mantissa bit (CRAY needs this) + * format[7] = bias of exponent + * + * This allows us to write out "Real"s in the native format on a machine, + * and then by also saving the IntDescriptor, we can read them back in on + * another machine and have enough information to construct the exact same + * "Real" values, provided the Reals have the same size on the two machines. + */ class RealDescriptor { public: diff --git a/Src/Base/AMReX_MultiFab.H b/Src/Base/AMReX_MultiFab.H index 1a6c1d7f15f..fdf2e67cbd8 100644 --- a/Src/Base/AMReX_MultiFab.H +++ b/Src/Base/AMReX_MultiFab.H @@ -24,16 +24,17 @@ using fMultiFab = FabArray >; class iMultiFab; /** - * \brief - * A collection (stored as an array) of FArrayBox objects. + * \brief A collection (stored as an array) of FArrayBox objects. + * * This class is useful for storing floating point data on a domain defined by * a union of rectangular regions embedded in a uniform index space. * MultiFab class extends the function of the underlying FabArray class just * as the FArrayBox class extends the function of BaseFab. - * Additional member functions are defined for I/O and simple arithmetic operations on these aggregate objects. + * Additional member functions are defined for I/O and simple arithmetic + * operations on these aggregate objects. + * * This class does NOT provide a copy constructor or assignment operator. */ - class MultiFab : public FabArray @@ -41,34 +42,36 @@ class MultiFab public: /** - * \brief Constructs an empty MultiFab. Data can be defined at a later - * time using the define member functions inherited - * from FabArray. + * \brief Constructs an empty MultiFab. + * + * Data can be defined at a later time using the define member functions + * inherited from FabArray. */ MultiFab () noexcept; /** - * \brief Constructs an empty MultiFab. Data can be defined at a later - * time using the define member functions inherited from FabArray. If - * `define` is called later with a nullptr as MFInfo's arena, the default - * Arena `a` will be used. If the arena in MFInfo is not a nullptr, the - * MFInfo's arena will be used. + * \brief Constructs an empty MultiFab. + * + * Data can be defined at a later time using the define member functions. + * If `define` is called later with a nullptr as MFInfo's arena, the + * default Arena `a` will be used. If the arena in MFInfo is not a + * nullptr, the MFInfo's arena will be used. */ explicit MultiFab (Arena* a) noexcept; /** - * \brief - * Constructs a MultiFab - * \param bs a valid region - * \param dm a DistributionMapping + * \brief Constructs a MultiFab + * + * The size of the FArrayBox is given by the Box grown by \p ngrow, and + * the number of components is given by \p ncomp. If \p info is set to + * not allocating memory, then no FArrayBoxes are allocated at + * this time but can be defined later. + * + * \param bxs a valid region + * \param dm a DistribuionMapping * \param ncomp number of components * \param ngrow number of cells the region grows * \param info MFInfo - - * The size of the FArrayBox is given by the Box grown by ngrow, and - * the number of components is given by ncomp. If info is set to - * not allocating memory, then no FArrayBoxes are allocated at - * this time but can be defined later. */ MultiFab (const BoxArray& bxs, const DistributionMapping& dm, @@ -95,10 +98,11 @@ public: #endif /** - * \brief Make an alias MultiFab. maketype must be - * amrex::make_alias. scomp is the starting component of the - * alias and ncomp is the number of components in the new aliasing - * MultiFab. + * \brief Make an alias MultiFab. + * + * Note that \p maketype must be `amrex::make_alias`, + * \p scomp is the starting component of the alias, and + * \p ncomp is the number of components in the new aliasing MultiFab. */ MultiFab (const MultiFab& rhs, MakeType maketype, int scomp, int ncomp); @@ -135,11 +139,13 @@ public: #endif MultiFab& operator= (Real r); - // + /** - * \brief Returns the minimum value contained in component comp of the - * MultiFab. The parameter nghost determines the number of - * boundary cells to search for the minimum. The default is to + * \brief Returns the minimum value contained in component \p comp of the + * MultiFab. + * + * The parameter \p nghost determines the number of + * boundary cells to search for the minimum. The default is to * search only the valid regions of the FArrayBoxes. */ [[nodiscard]] Real min (int comp, @@ -154,16 +160,18 @@ public: int nghost = 0, bool local = false) const; /** - * \brief Returns the maximum value contained in component comp of the - * MultiFab. The parameter nghost determines the number of - * boundary cells to search for the maximum. The default is to + * \brief Returns the maximum value contained in component \p comp of the + * MultiFab. + * + * The parameter \p nghost determines the number of + * boundary cells to search for the maximum. The default is to * search only the valid regions of the FArrayBoxes. */ [[nodiscard]] Real max (int comp, int nghost = 0, bool local = false) const; /** - * \brief Identical to the previous max() function, but confines its + * \brief Identical to the previous `max()` function, but confines its * search to intersection of Box b and the MultiFab. */ [[nodiscard]] Real max (const Box& region, @@ -191,7 +199,7 @@ public: /** * \brief Returns the maximum *absolute* values contained in - * each component of "comps" of the MultiFab. "nghost" ghost cells are used. + * each component of \p comps of the MultiFab. \p nghost ghost cells are used. */ [[nodiscard]] Vector norm0 (const Vector& comps, int nghost = 0, bool local = false, bool ignore_covered = false ) const; [[nodiscard]] Vector norminf (const Vector& comps, int nghost = 0, bool local = false, bool ignore_covered = false) const { @@ -199,13 +207,14 @@ public: } /** - * \brief Returns the L1 norm of component "comp" over the MultiFab. + * \brief Returns the L1 norm of component \p comp over the MultiFab. + * * No ghost cells are used. This version has no double counting for nodal data. */ [[nodiscard]] Real norm1 (int comp, const Periodicity& period, bool ignore_covered = false) const; /** - * \brief Returns the L1 norm of component "comp" over the MultiFab. - * ngrow ghost cells are used. + * \brief Returns the L1 norm of component \p comp over the MultiFab. + * \p ngrow ghost cells are used. */ [[nodiscard]] Real norm1 (int comp = 0, int ngrow = 0, bool local = false) const; /** @@ -214,12 +223,12 @@ public: */ [[nodiscard]] Vector norm1 (const Vector& comps, int ngrow = 0, bool local = false) const; /** - * \brief Returns the L2 norm of component "comp" over the MultiFab. + * \brief Returns the L2 norm of component \p comp over the MultiFab. * No ghost cells are used. */ [[nodiscard]] Real norm2 (int comp = 0) const; /** - * \brief Returns the L2 norm of component "comp" over the MultiFab. + * \brief Returns the L2 norm of component \p comp over the MultiFab. * No ghost cells are used. This version has no double counting for nodal data. */ [[nodiscard]] Real norm2 (int comp, const Periodicity& period) const; @@ -236,16 +245,17 @@ public: using FabArray::sum; /** - * \brief Same as sum with local=false, but for non-cell-centered data, this - * skips non-unique points that are owned by multiple boxes. + * \brief Same as sum with \p local =false, but for non-cell-centered data, this + * skips non-unique points that are owned by multiple boxes. */ [[nodiscard]] Real sum_unique (int comp = 0, bool local = false, const Periodicity& period = Periodicity::NonPeriodic()) const; /** - * \brief Adds the scalar value val to the value of each cell in the - * specified subregion of the MultiFab. The subregion consists - * of the num_comp components starting at component comp. + * \brief Adds the scalar value \p val to the value of each cell in the + * specified subregion of the MultiFab. + * + * The subregion consists of the \p num_comp components starting at component \p comp. * The value of nghost specifies the number of cells in the * boundary region of each FArrayBox in the subregion that should * be modified. diff --git a/Src/Base/AMReX_MultiFabUtil.H b/Src/Base/AMReX_MultiFabUtil.H index 064b8a1b19f..29af89ba88e 100644 --- a/Src/Base/AMReX_MultiFabUtil.H +++ b/Src/Base/AMReX_MultiFabUtil.H @@ -19,67 +19,76 @@ namespace amrex const MultiFab& nd, int scomp, int ncomp, int ngrow = 0); - //! Average edge-based MultiFab onto cell-centered MultiFab. This fills in - //! ngrow ghost cells in the cell-centered MultiFab. Both cell centered and - //! edge centered MultiFabs need to have ngrow ghost values + /** + * \brief Average edge-based MultiFab onto cell-centered MultiFab. + * + * This fills in \p ngrow ghost cells in the cell-centered MultiFab. Both cell centered and + * edge centered MultiFabs need to have \p ngrow ghost values. + */ void average_edge_to_cellcenter (MultiFab& cc, int dcomp, const Vector& edge, int ngrow = 0); - //! Average face-based MultiFab onto cell-centered MultiFab. void average_face_to_cellcenter (MultiFab& cc, int dcomp, const Vector& fc, int ngrow = 0); - + //! Average face-based FabArray onto cell-centered FabArray. template && IsFabArray_v, int> = 0> void average_face_to_cellcenter (CMF& cc, int dcomp, const Array& fc, int ngrow = 0); - + //! Average face-based MultiFab onto cell-centered MultiFab with geometric weighting. void average_face_to_cellcenter (MultiFab& cc, const Vector& fc, const Geometry& geom); + //! Average face-based MultiFab onto cell-centered MultiFab with geometric weighting. void average_face_to_cellcenter (MultiFab& cc, const Array& fc, const Geometry& geom); - - //! Average cell-centered MultiFab onto face-based MultiFab. + //! Average cell-centered MultiFab onto face-based MultiFab with geometric weighting. void average_cellcenter_to_face (const Vector& fc, const MultiFab& cc, const Geometry& geom, int ncomp = 1, bool use_harmonic_averaging = false); + //! Average cell-centered MultiFab onto face-based MultiFab with geometric weighting. void average_cellcenter_to_face (const Array& fc, const MultiFab& cc, const Geometry& geom, int ncomp = 1, bool use_harmonic_averaging = false); - //! Average fine face-based MultiFab onto crse face-based MultiFab. + //! Average fine face-based FabArray onto crse face-based FabArray. template ::value,int> = 0> void average_down_faces (const Vector& fine, const Vector& crse, const IntVect& ratio, int ngcrse = 0); + //! Average fine face-based FabArray onto crse face-based FabArray. template ::value,int> = 0> void average_down_faces (const Vector& fine, const Vector& crse, int ratio, int ngcrse = 0); + //! Average fine face-based FabArray onto crse face-based FabArray. template ::value,int> = 0> void average_down_faces (const Array& fine, const Array& crse, const IntVect& ratio, int ngcrse = 0); + //! Average fine face-based FabArray onto crse face-based FabArray. template ::value,int> = 0> void average_down_faces (const Array& fine, const Array& crse, int ratio, int ngcrse = 0); - //! This version does average down for one direction. - //! It uses the IndexType of MultiFabs to determine the direction. - //! It is expected that one direction is nodal and the rest are cell-centered. + /** + * \brief This version does average down for one face direction. + * + * It uses the IndexType of MultiFabs to determine the direction. + * It is expected that one direction is nodal and the rest are cell-centered. + */ template void average_down_faces (const FabArray& fine, FabArray& crse, const IntVect& ratio, int ngcrse=0); @@ -117,9 +126,12 @@ namespace amrex int ngcrse = 0, bool mfiter_is_definitely_safe=false); - //! Average fine cell-based MultiFab onto crse cell-centered MultiFab using - //! volume-weighting. This routine DOES NOT assume that the crse BoxArray is - //! a coarsened version of the fine BoxArray. + /** + * \brief Volume weighed average of fine MultiFab onto coarse MultiFab. + * + * Both MultiFabs are assumed to be cell-centered. This routine DOES NOT assume that + * the crse BoxArray is a coarsened version of the fine BoxArray. + */ void average_down (const MultiFab& S_fine, MultiFab& S_crse, const Geometry& fgeom, const Geometry& cgeom, int scomp, int ncomp, const IntVect& ratio); diff --git a/Src/Base/AMReX_MultiFabUtil.cpp b/Src/Base/AMReX_MultiFabUtil.cpp index 5520c6164d7..27efbffca0f 100644 --- a/Src/Base/AMReX_MultiFabUtil.cpp +++ b/Src/Base/AMReX_MultiFabUtil.cpp @@ -308,9 +308,6 @@ namespace amrex // ************************************************************************************************************* - // Average fine cell-based MultiFab onto crse cell-centered MultiFab. - // We do NOT assume that the coarse layout is a coarsened version of the fine layout. - // This version DOES use volume-weighting. void average_down (const MultiFab& S_fine, MultiFab& S_crse, const Geometry& fgeom, const Geometry& cgeom, int scomp, int ncomp, int rr) From 44c5c7a5da23c1a5e7e899612b1df696cea7e4fe Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 29 Nov 2023 15:07:44 -0800 Subject: [PATCH 07/16] When checking for periodic outs on GPU, copy full particle data (#3646) Fixes a bug in mapped coordinates when GPUs are enabled. The proposed changes: - [x] fix a bug or incorrect behavior in AMReX - [ ] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --- Src/Particle/AMReX_ParticleUtil.H | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Src/Particle/AMReX_ParticleUtil.H b/Src/Particle/AMReX_ParticleUtil.H index d4b13030c04..5430cd34037 100644 --- a/Src/Particle/AMReX_ParticleUtil.H +++ b/Src/Particle/AMReX_ParticleUtil.H @@ -487,11 +487,7 @@ partitionParticlesByDest (PTile& ptile, const PLocator& ploc, CellAssignor&& ass } else { - amrex::Particle<0> p_prime; - AMREX_D_TERM(p_prime.pos(0) = src_data.pos(0, i+this_offset);, - p_prime.pos(1) = src_data.pos(1, i+this_offset);, - p_prime.pos(2) = src_data.pos(2, i+this_offset);); - + auto p_prime = src_data.getSuperParticle(i+this_offset); enforcePeriodic(p_prime, plo, phi, rlo, rhi, is_per); auto tup_prime = ploc(p_prime, lev_min, lev_max, nGrow, assignor); assigned_grid = amrex::get<0>(tup_prime); From 0cd1385bde604099e5822d636f85d724d74d756e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 29 Nov 2023 15:41:35 -0800 Subject: [PATCH 08/16] Fix: nosmt OMP Threads Default (#3647) ## Summary Fix that `OMP_NUM_THREADS` was ignored in non-verbose runs. ## Additional background Follow-up to #3607 ## Checklist The proposed changes: - [x] fix a bug or incorrect behavior in AMReX - [ ] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --- Src/Base/AMReX_OpenMP.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Src/Base/AMReX_OpenMP.cpp b/Src/Base/AMReX_OpenMP.cpp index 5ddd9944411..c0c33ce962f 100644 --- a/Src/Base/AMReX_OpenMP.cpp +++ b/Src/Base/AMReX_OpenMP.cpp @@ -152,12 +152,13 @@ namespace amrex::OpenMP // default or OMP_NUM_THREADS environment variable } else if (omp_threads == "nosmt") { char const *env_omp_num_threads = std::getenv("OMP_NUM_THREADS"); - if (env_omp_num_threads != nullptr && amrex::system::verbose > 1) { + if (env_omp_num_threads == nullptr) { + omp_set_num_threads(numUniquePhysicalCores()); + } + else if (amrex::system::verbose > 1) { amrex::Print() << "amrex.omp_threads was set to nosmt," << "but OMP_NUM_THREADS was set. Will keep " << "OMP_NUM_THREADS=" << env_omp_num_threads << ".\n"; - } else { - omp_set_num_threads(numUniquePhysicalCores()); } } else { std::optional num_omp_threads = to_int(omp_threads); From 9b733ec45cd93a80234a7c98248b6eb4816589d5 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Wed, 29 Nov 2023 17:43:44 -0600 Subject: [PATCH 09/16] solve_cg: avoid use of MF `z` (#3637) ## Summary The method `solve_cg` is modified so that the MF `z` is no longer used, thus saving on memory usage. ## Additional background This is similar to the PR that changed `solve_bicgstab` that avoided use of `s`. --- Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H index 4768f06fc41..3afa56ee245 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H @@ -262,7 +262,6 @@ MLCGSolverT::solve_cg (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) MF sorig = Lp.make(amrlev, mglev, nghost); MF r = Lp.make(amrlev, mglev, nghost); - MF z = Lp.make(amrlev, mglev, nghost); MF q = Lp.make(amrlev, mglev, nghost); sorig.LocalCopy(sol,0,0,ncomp,nghost); @@ -295,9 +294,7 @@ MLCGSolverT::solve_cg (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) for (; iter <= maxiter; ++iter) { - z.LocalCopy(r,0,0,ncomp,nghost); - - RT rho = dotxy(z,r); + RT rho = dotxy(r,r); if ( rho == 0 ) { @@ -305,12 +302,12 @@ MLCGSolverT::solve_cg (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) } if (iter == 1) { - p.LocalCopy(z,0,0,ncomp,nghost); + p.LocalCopy(r,0,0,ncomp,nghost); } else { RT beta = rho/rho_1; - MF::Xpay(p, beta, z, 0, 0, ncomp, nghost); // p = z + beta * p + MF::Xpay(p, beta, r, 0, 0, ncomp, nghost); // p = r + beta * p } Lp.apply(amrlev, mglev, q, p, MLLinOpT::BCMode::Homogeneous, MLLinOpT::StateMode::Correction); From 2f47fa7361bbf5793503cfb31b717bece889bde0 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Fri, 1 Dec 2023 09:54:52 -0800 Subject: [PATCH 10/16] Update CHANGES for 23.12 (#3651) --- CHANGES | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/CHANGES b/CHANGES index f9f2d4febe8..82b28a03e87 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,44 @@ +# 23.12 + + -- solve_cg: avoid use of MF `z` (#3637) + + -- Fix: nosmt OMP Threads Default (#3647) + `amrex.omp_threads`: Can Avoid SMT (#3607) + + -- When checking for periodic outs on GPU, copy full particle data (#3646) + + -- MLEBABecLap: Support Robin BC at Domain Boundaries (#3617) + + -- Ascent: SoA Particle Support (#3350) + + -- solve_bicgstab: use fewer MFs (#3635) + + -- solve_bicgstab: cut use of `s` (#3629) + + -- Bug fix for amrex::Subtract when called with interger nghost (#3634) + + -- Fix typo in `MLMGT::getGradSolution` when `MF` is different from `AMF` (#3631) + + -- SUNDIALS: Use sunrealtype instead of realtype (#3632) + + -- SYCL: Use get_multi_ptr instead of get_pointer (#3630) + + -- Plotfile Tools: GPU support (#3626) + + -- solve_cg: use linop.make instead of MF constructor (#3627) + + -- CArena: shrink_in_place and operator<< (#3621) + + -- solve_bicgstab: use linop.make instead of MF constructor (#3619) + + -- replace AMREX_DEVICE_COMPILE with AMREX_IF_ON_DEVICE and AMREX_IF_ON_HOST (#3591) + + -- [Breaking] Prefix `amrex_` to each plotfile Tool (#3600) + + -- FillRandom: Use MKL host API (#3536) + + -- use hipPointerAttribute_t.type as HIP is removing hipPointerAttribute_t.memoryType (#3610) + # 23.11 -- Give FlashFluxRegisters ways to accumulate data in registers (#3597) From eb388ae90a7534b32c67f0410cd595a39bc362f9 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Sat, 2 Dec 2023 14:45:29 -0800 Subject: [PATCH 11/16] Work around compiler bug in nvcc 12.2 by using functor instead of lambda (#3653) This fixes an issue seen in ERF on garuda. The proposed changes: - [ ] fix a bug or incorrect behavior in AMReX - [ ] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --- Src/Particle/AMReX_ParticleIO.H | 38 ++++++++++++--------------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index ea24b1e5f81..e6969c9b1bd 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -32,6 +32,14 @@ ParticleContainer_impl + AMREX_GPU_HOST_DEVICE + int operator() (const P& p) const { + return p.id() > 0; + } +}; + template class Allocator, class CellAssignor> void @@ -77,10 +85,7 @@ ParticleContainer_impl int - { - return p.id() > 0; - }, true); + FilterPositiveID{}, true); } template 0; - }); + FilterPositiveID{}); } template 0; - }); + FilterPositiveID{}); } template 0; - }); + FilterPositiveID{}); } template 0; - }); + FilterPositiveID{}); } template 0; - }); + FilterPositiveID{}); } template Date: Sat, 2 Dec 2023 14:45:59 -0800 Subject: [PATCH 12/16] Disable CodeQL scheduled jobs on forks (#3649) --- .github/workflows/codeql.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0e03e384633..f240930bd8b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -14,6 +14,7 @@ concurrency: jobs: analyze: + if: ${{ github.repository == 'AMReX-Codes/amrex' || github.event_name != 'schedule' }} name: Analyze runs-on: ubuntu-latest permissions: From 72c333d39ce9a7ba2e2bec85b2d9ef4f5230ca9b Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Sat, 2 Dec 2023 14:46:30 -0800 Subject: [PATCH 13/16] Robustify the Cache Cleanup Scripts (#3650) Make changes to handle the cases where the workflow names contain spaces. Note that none of the workflow names in amrex has spaces. --- .github/workflows/cleanup-cache-postpr.yml | 5 ++++- .github/workflows/cleanup-cache.yml | 13 ++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index 73d6eaf0903..978e9c28f04 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -31,7 +31,10 @@ jobs: set +e keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) + # $keys might contain spaces. Thus we set IFS to \n. + IFS=$'\n' for k in $keys do - gh actions-cache delete $k -R $REPO -B $BRANCH --confirm + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm done + unset IFS diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index f224ace0ef3..cafae38bf1c 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -27,7 +27,7 @@ jobs: EVENT=${{ github.event.workflow_run.event }} # Triggering workflow run name (e.g., LinuxClang) - WORKFLOW_NAME=${{ github.event.workflow_run.name }} + WORKFLOW_NAME="${{ github.event.workflow_run.name }}" if [[ $EVENT == "pull_request" ]]; then gh run download ${{ github.event.workflow_run.id }} -n pr_number @@ -45,16 +45,19 @@ jobs: # The goal is to keep the last used key of each job and delete all others. # something like ccache-LinuxClang- - keyprefix=ccache-${WORKFLOW_NAME}- + keyprefix="ccache-${WORKFLOW_NAME}-" - cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key $keyprefix | awk -F '-git-' '{print $1}' | sort | uniq) + cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq) # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". + # It might also contain spaces. Thus we set IFS to \n. + IFS=$'\n' for j in $cached_jobs do - old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key ${j}-git- --sort last-used | cut -f 1 | tail -n +2) + old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) for k in $old_keys do - gh actions-cache delete $k -R $REPO -B $BRANCH --confirm + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm done done + unset IFS From 2f1b1d1206eec72dbbc364c5c95c5e515f4fe479 Mon Sep 17 00:00:00 2001 From: Candace Gilet Date: Sat, 2 Dec 2023 17:47:38 -0500 Subject: [PATCH 14/16] Clarify documentation on setEBDirchlet() and fix link to AMReX-Hydro (#3652) ## Summary ## Additional background ## Checklist The proposed changes: - [ ] fix a bug or incorrect behavior in AMReX - [ ] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --- Docs/sphinx_documentation/source/LinearSolvers.rst | 9 ++++++--- Docs/sphinx_documentation/source/conf.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Docs/sphinx_documentation/source/LinearSolvers.rst b/Docs/sphinx_documentation/source/LinearSolvers.rst index e7266f02243..fb7138e1f4d 100644 --- a/Docs/sphinx_documentation/source/LinearSolvers.rst +++ b/Docs/sphinx_documentation/source/LinearSolvers.rst @@ -483,7 +483,9 @@ To set homogeneous Dirichlet boundary conditions, call ml_ebabeclap->setEBHomogDirichlet(lev, coeff); where coeff can be a real number (i.e. the value is the same at every cell) -or is the MultiFab holding the coefficient of the gradient at each cell with an EB face. +or a MultiFab holding the coefficient of the gradient at each cell with an EB face. +In other words, coeff is :math:`\beta` in the canonical form given in equation :eq:`eqn::abeclap` +located at the EB surface centroid. To set inhomogeneous Dirichlet boundary conditions, call @@ -494,8 +496,9 @@ To set inhomogeneous Dirichlet boundary conditions, call ml_ebabeclap->setEBDirichlet(lev, phi_on_eb, coeff); where phi_on_eb is the MultiFab holding the Dirichlet values in every cut cell, -and coeff again is a real number (i.e. the value is the same at every cell) -or a MultiFab holding the coefficient of the gradient at each cell with an EB face. +and coeff again is a real number +or a MultiFab holding the coefficient of the gradient at each cell with an EB face, +i.e. :math:`\beta` in equation :eq:`eqn::abeclap` located at the EB surface centroid. Currently there are options to define the face-based coefficients on face centers vs face centroids, and to interpret the solution variable diff --git a/Docs/sphinx_documentation/source/conf.py b/Docs/sphinx_documentation/source/conf.py index dc29ab6e041..8cb17c78e1f 100644 --- a/Docs/sphinx_documentation/source/conf.py +++ b/Docs/sphinx_documentation/source/conf.py @@ -42,7 +42,7 @@ def get_amrex_version(): intersphinx_mapping = { 'amrex_tutorials': ('https://amrex-codes.github.io/amrex/tutorials_html/', None), - 'amrex_hydro':('https://amrex-codes.github.io/amrex/hydro_html/', None) + 'amrex_hydro':('https://amrex-fluids.github.io/amrex-hydro/docs_html/', None) } # Add any paths that contain templates here, relative to this directory. From efd77ffab39c4e32e2b9e4a4fd446a49e22f0c9a Mon Sep 17 00:00:00 2001 From: Ann Almgren Date: Sat, 2 Dec 2023 16:51:48 -0800 Subject: [PATCH 15/16] two separate fixes -- particle_compare and ref_ratio=1 (#3655) ## Summary ## Additional background ## Checklist The proposed changes: - [ ] fix a bug or incorrect behavior in AMReX - [X] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --- Src/AmrCore/AMReX_AmrMesh.cpp | 2 +- .../Postprocessing/C_Src/particle_compare.cpp | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index efccf318315..c0e191d79e4 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -1082,7 +1082,7 @@ AmrMesh::checkInput () for (int i = 0; i < max_level; i++) { if (MaxRefRatio(i) < 2) { - amrex::Error("Amr::checkInput: bad ref_ratios"); + amrex::Warning("Amr::checkInput: ref_ratios all equal to one!"); } } diff --git a/Tools/Postprocessing/C_Src/particle_compare.cpp b/Tools/Postprocessing/C_Src/particle_compare.cpp index 9967625cca6..e9efc69cf62 100644 --- a/Tools/Postprocessing/C_Src/particle_compare.cpp +++ b/Tools/Postprocessing/C_Src/particle_compare.cpp @@ -438,12 +438,15 @@ int main_main() std::string fn2; std::string pt; Real rtol = 0.0; + Real atol = 0.0; int farg=1; while (farg <= narg) { const std::string fname = amrex::get_command_argument(farg); if (fname == "-r" || fname == "--rel_tol") { rtol = std::stod(amrex::get_command_argument(++farg)); + } else if (fname == "--abs_tol") { + atol = std::stod(amrex::get_command_argument(++farg)); } else { break; } @@ -472,6 +475,7 @@ int main_main() << "\n" << " optional arguments:\n" << " -r|--rel_tol rtol : relative tolerance (default is 0)\n" + << " --abs_tol atol : absolute tolerance (default is 0)\n" << std::endl; return EXIT_SUCCESS; } @@ -555,16 +559,25 @@ int main_main() int exit_code = 0; for (int i = 0; i < header1.num_comp; ++i) { - if (global_norms[i+header1.num_comp] > rtol) exit_code = 1; + if (global_norms[i ] > atol && + global_norms[i+header1.num_comp] > rtol) exit_code = 1; } if (exit_code == 0) { - amrex::Print() << " PARTICLES AGREE to relative tolerance " << rtol << "\n"; + if (atol > 0.) { + amrex::Print() << " PARTICLES AGREE to relative tolerance " << rtol << " and/or absolute tolerance " << atol << "\n"; + } else { + amrex::Print() << " PARTICLES AGREE to relative tolerance " << rtol << "\n"; + } } else { - amrex::Print() << " PARTICLES DISAGREE to relative tolerance " << rtol << "\n"; + if (atol > 0.) { + amrex::Print() << " PARTICLES DISAGREE to relative tolerance " << rtol << " and/or absolute tolerance " << atol << "\n"; + } else { + amrex::Print() << " PARTICLES DISAGREE to relative tolerance " << rtol << "\n"; + } } return exit_code; From edb4c25027efbbc465c88d453441dcd7115d8651 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 6 Dec 2023 09:44:58 -0800 Subject: [PATCH 16/16] GNU Make: Fix name collision for aurora (#3656) --- Tools/GNUMake/Make.machines | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/GNUMake/Make.machines b/Tools/GNUMake/Make.machines index 1e6293dd001..db91bb99947 100644 --- a/Tools/GNUMake/Make.machines +++ b/Tools/GNUMake/Make.machines @@ -123,9 +123,11 @@ ifeq ($(findstring asterix, $(host_name)), asterix) endif ifeq ($(findstring aurora, $(host_name)), aurora) +ifneq ($(findstring alcf.anl.gov, $(host_name)),alcf.anl.gov) which_site := hs which_computer := aurora endif +endif ifeq ($(findstring kestrel, $(NREL_CLUSTER)), kestrel) which_site := nrel