From 18b8dbddfefae1091dc647a9700589f09e3bab9c Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 2 Feb 2023 16:41:47 -0500 Subject: [PATCH 01/44] output time and step number to a file rather than screen. Will continue with more meaningful outputs --- Source/IO.H | 9 +++++++++ Source/IO.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/Source/IO.H b/Source/IO.H index e851f0a2..c159b065 100644 --- a/Source/IO.H +++ b/Source/IO.H @@ -6,6 +6,15 @@ #include #include +void +InitializeReducedData0D(); + +void +WriteReducedData0D(const amrex::MultiFab& state, + const FlavoredNeutrinoContainer& neutrinos, + const amrex::Geometry& geom, amrex::Real time, + int step); + void WritePlotFile (const amrex::MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, diff --git a/Source/IO.cpp b/Source/IO.cpp index 8d5bd473..7123c4ff 100644 --- a/Source/IO.cpp +++ b/Source/IO.cpp @@ -12,6 +12,33 @@ using namespace amrex; +const char* filename0D = "reduced0D.dat"; + +void +InitializeReducedData0D() +{ + std::ofstream outfile; + outfile.open(filename0D, std::ofstream::out); + outfile << "1:step\t"; + outfile << "2:time(s)\t"; + outfile << std::endl; + outfile.close(); +} + +void +WriteReducedData0D(const amrex::MultiFab& state, + const FlavoredNeutrinoContainer& neutrinos, + const amrex::Geometry& geom, amrex::Real time, + int step) +{ + std::ofstream outfile; + outfile.open(filename0D, std::ofstream::app); + outfile << step << "\t"; + outfile << time << "\t"; + outfile << std::endl; + outfile.close(); +} + void WritePlotFile (const amrex::MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, From 3b372b1ee2bacd20bce48dd90e3f1617b26f9eb5 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 3 Feb 2023 10:58:28 -0500 Subject: [PATCH 02/44] move all data reduction including file IO to a separate data reduction class --- Source/IO.cpp | 27 --------------------------- Source/Make.package | 2 ++ Source/main.cpp | 5 ++++- 3 files changed, 6 insertions(+), 28 deletions(-) diff --git a/Source/IO.cpp b/Source/IO.cpp index 7123c4ff..8d5bd473 100644 --- a/Source/IO.cpp +++ b/Source/IO.cpp @@ -12,33 +12,6 @@ using namespace amrex; -const char* filename0D = "reduced0D.dat"; - -void -InitializeReducedData0D() -{ - std::ofstream outfile; - outfile.open(filename0D, std::ofstream::out); - outfile << "1:step\t"; - outfile << "2:time(s)\t"; - outfile << std::endl; - outfile.close(); -} - -void -WriteReducedData0D(const amrex::MultiFab& state, - const FlavoredNeutrinoContainer& neutrinos, - const amrex::Geometry& geom, amrex::Real time, - int step) -{ - std::ofstream outfile; - outfile.open(filename0D, std::ofstream::app); - outfile << step << "\t"; - outfile << time << "\t"; - outfile << std::endl; - outfile.close(); -} - void WritePlotFile (const amrex::MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, diff --git a/Source/Make.package b/Source/Make.package index 191e7a7f..6d4f1cd6 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -1,4 +1,5 @@ CEXE_sources += main.cpp +CEXE_sources += DataReducer.cpp CEXE_sources += IO.cpp CEXE_sources += FlavoredNeutrinoContainerInit.cpp CEXE_sources += Evolve.cpp @@ -11,3 +12,4 @@ CEXE_headers += Particles.H CEXE_headers += IO.H CEXE_headers += Parameters.H CEXE_headers += ParticleInterpolator.H +CEXE_headers += DataReducer.H diff --git a/Source/main.cpp b/Source/main.cpp index b17266da..5d9443d3 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -31,6 +31,7 @@ #include "Evolve.H" #include "Constants.H" #include "IO.H" +#include "DataReducer.H" using namespace amrex; @@ -118,10 +119,12 @@ void evolve_flavor(const TestParams* parms) deposit_to_mesh(neutrinos_old, state, geom); // Write plotfile after initialization + DataReducer rd; if (not parms->do_restart) { // If we have just initialized, then always save the particle data for reference const int write_particles_after_init = 1; WritePlotFile(state, neutrinos_old, geom, initial_time, initial_step, write_particles_after_init); + rd.InitializeFiles(); } amrex::Print() << "Done. " << std::endl; @@ -176,7 +179,7 @@ void evolve_flavor(const TestParams* parms) const int step = integrator.get_step_number(); const Real time = integrator.get_time(); - amrex::Print() << "Completed time step: " << step << " t = " << time << " s. ct = " << PhysConst::c * time << " cm" << std::endl; + rd.WriteReducedData0D(geom, state, neutrinos, time, step+1); run_fom += neutrinos.TotalNumberOfParticles(); From ad2a207f02793b59b51e60ef8de1100d336c7963 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 3 Feb 2023 11:00:59 -0500 Subject: [PATCH 03/44] added missing files for the DataReducer class --- Source/DataReducer.H | 26 +++++++++++++++++ Source/DataReducer.cpp | 66 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 Source/DataReducer.H create mode 100644 Source/DataReducer.cpp diff --git a/Source/DataReducer.H b/Source/DataReducer.H new file mode 100644 index 00000000..09886e88 --- /dev/null +++ b/Source/DataReducer.H @@ -0,0 +1,26 @@ +#ifndef DATAREDUCER_H_ +#define DATAREDUCER_H_ + +#include +#include +#include +#include +#include +#include + +class DataReducer{ +private: + const char* filename0D = "reduced0D.dat"; + +public: + void + InitializeFiles(); + + void + WriteReducedData0D(const amrex::Geometry& geom, + const MultiFab& state, + const FlavoredNeutrinoContainer& neutrinos, + amrex::Real time, int step); +}; + +#endif diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp new file mode 100644 index 00000000..3a052868 --- /dev/null +++ b/Source/DataReducer.cpp @@ -0,0 +1,66 @@ +#include "Evolve.H" +#include "Constants.H" +#include "DataReducer.H" +#include + +void +DataReducer::InitializeFiles(){ + std::ofstream outfile; + outfile.open(filename0D, std::ofstream::out); + outfile << "1:step\t"; + outfile << "2:time(s)\t"; + outfile << "3:N00(cm^-3)\t"; + outfile << "3:N11(cm^-3)\t"; + outfile << std::endl; + outfile.close(); +} + +void +DataReducer::WriteReducedData0D(const amrex::Geometry& geom, + const MultiFab& state, + const FlavoredNeutrinoContainer& neutrinos, + const amrex::Real time, const int step) +{ + // define the reduction operator to get the summed number of neutrinos in each state + /*ReduceOps reduce_op; + ReduceData reduce_data(reduce_op); + using ReduceTuple = typename decltype(reduce_data)::Type; + for (MFIter mfi(state); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.fabbox(); + auto const& fab = state.array(mfi); + reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple + { + Real N00 = fab(i,j,k,GIdx::N00_Re); + Real N11 = fab(i,j,k,GIdx::N11_Re); + return {N00,N11}; + }); + } + + // extract the reduced values from the combined reduced data structure + auto rv = reduce_data.value(); + rd.N00 = amrex::get<0>(rv); + rd.N11 = amrex::get<1>(rv);*/ + + // reduce across MPI ranks + //ParallelDescriptor::ReduceRealMax(rd.N00); + //ParallelDescriptor::ReduceRealMax(rd.N11); + + int ncells = geom.Domain().volume(); + + Real N00 = state.sum(GIdx::N00_Re) / ncells; + Real N11 = state.sum(GIdx::N11_Re) / ncells; +#if NF > 2 + Real N22 = state.sum(GIdx::N22_Re) / ncells; +#endif + + // write to file + std::ofstream outfile; + outfile.open(filename0D, std::ofstream::app); + outfile << step << "\t"; + outfile << time << "\t"; + outfile << N00 << "\t"; + outfile << N11 << "\t"; + outfile << std::endl; + outfile.close(); +} From 38b408b522545dd21854674ce15a1a60f08675d6 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 3 Feb 2023 11:06:50 -0500 Subject: [PATCH 04/44] bring amrex up to 23.02 --- amrex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amrex b/amrex index ff9e8324..eefe2463 160000 --- a/amrex +++ b/amrex @@ -1 +1 @@ -Subproject commit ff9e832483f142a34da72f9bea5fed72e49fea33 +Subproject commit eefe2463884081a27025973d4a99dc909657b4f0 From f855cb931a9f4e13fbc4949f8f0be1da46997de3 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 3 Feb 2023 14:06:20 -0500 Subject: [PATCH 05/44] added parallel reduction operation over grid. ParReduce is a bit more straightforward than doing the array reduction followed by a parallel reduction --- Source/DataReducer.cpp | 55 ++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 3a052868..b953dd6b 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -21,38 +21,35 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, const FlavoredNeutrinoContainer& neutrinos, const amrex::Real time, const int step) { - // define the reduction operator to get the summed number of neutrinos in each state - /*ReduceOps reduce_op; - ReduceData reduce_data(reduce_op); - using ReduceTuple = typename decltype(reduce_data)::Type; - for (MFIter mfi(state); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.fabbox(); - auto const& fab = state.array(mfi); - reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple - { - Real N00 = fab(i,j,k,GIdx::N00_Re); - Real N11 = fab(i,j,k,GIdx::N11_Re); - return {N00,N11}; - }); - } - - // extract the reduced values from the combined reduced data structure - auto rv = reduce_data.value(); - rd.N00 = amrex::get<0>(rv); - rd.N11 = amrex::get<1>(rv);*/ + // get index volume of the domain + int ncells = geom.Domain().volume(); - // reduce across MPI ranks - //ParallelDescriptor::ReduceRealMax(rd.N00); - //ParallelDescriptor::ReduceRealMax(rd.N11); + // sample per-particle simple reduction + //Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_HOST_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); + //ParallelDescriptor::ReduceRealMin(pupt_min); - int ncells = geom.Domain().volume(); + // sample grid simple reduction + //Real N00 = state.sum(GIdx::N00_Re) / ncells; + + //=============================// + // Do reductions over the grid // + //=============================// + // first, get a reference to the data arrays + auto const& ma = state.const_arrays(); + IntVect nghost(AMREX_D_DECL(0, 0, 0)); - Real N00 = state.sum(GIdx::N00_Re) / ncells; - Real N11 = state.sum(GIdx::N11_Re) / ncells; -#if NF > 2 - Real N22 = state.sum(GIdx::N22_Re) / ncells; -#endif + // use the ParReduce function to define reduction operator + GpuTuple result = ParReduce(TypeList{}, + TypeList{}, + state, nghost, + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple { + Array4 const& a = ma[box_no]; + Real N00 = a(i,j,k,GIdx::N00_Re); + Real N11 = a(i,j,k,GIdx::N11_Re); + return {N00, N11}; + }); + Real N00 = amrex::get<0>(result)/ncells; + Real N11 = amrex::get<1>(result)/ncells; // write to file std::ofstream outfile; From 40b56857858ca252be132724639517393ac7e554 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 3 Feb 2023 15:29:32 -0500 Subject: [PATCH 06/44] modify reduction operator to deal with an ArithmeticArray, which is an array that includes basic arithmetic operations --- Source/ArithmeticArray.H | 93 ++++++++++++++++++++++++++++++++++++++++ Source/DataReducer.cpp | 27 ++++++------ Source/Make.package | 1 + 3 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 Source/ArithmeticArray.H diff --git a/Source/ArithmeticArray.H b/Source/ArithmeticArray.H new file mode 100644 index 00000000..36d32403 --- /dev/null +++ b/Source/ArithmeticArray.H @@ -0,0 +1,93 @@ +template +class ArithmeticArray{ +public: + std::array data; + + AMREX_GPU_HOST_DEVICE + ArithmeticArray(){} + + AMREX_GPU_HOST_DEVICE + ArithmeticArray(T in){ + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + ArithmeticArray(ArithmeticArray in){ + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + ArithmeticArray& operator=(const ArithmeticArray& input){ + for(size_t i=0; i& operator=(const double input){ + for(size_t i=0; i operator*(const double scale) const{ + ArithmeticArray result; + for(size_t i=0; i operator/(const double scale) const{ + double inv_scale = 1./scale; + return operator*(inv_scale); + } + template + AMREX_GPU_HOST_DEVICE + const ArithmeticArray operator+(const ArithmeticArray& input) const{ + ArithmeticArray result; + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + const ArithmeticArray operator-(const ArithmeticArray& input) const{ + ArithmeticArray result; + for(size_t i=0; i operator-() const{ + ArithmeticArray result; + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + void operator+=(const ArithmeticArray& input){ + for(size_t i=0; i + AMREX_GPU_HOST_DEVICE + bool operator==(const ArithmeticArray& input){ + bool isequal = true; + for(size_t i=0; i void @@ -39,25 +40,25 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, IntVect nghost(AMREX_D_DECL(0, 0, 0)); // use the ParReduce function to define reduction operator - GpuTuple result = ParReduce(TypeList{}, - TypeList{}, - state, nghost, - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& a = ma[box_no]; - Real N00 = a(i,j,k,GIdx::N00_Re); - Real N11 = a(i,j,k,GIdx::N11_Re); - return {N00, N11}; - }); - Real N00 = amrex::get<0>(result)/ncells; - Real N11 = amrex::get<1>(result)/ncells; + GpuTuple< ArithmeticArray > result = ParReduce(TypeList{}, + TypeList >{}, + state, nghost, + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple > { + Array4 const& a = ma[box_no]; + ArithmeticArray Ndiag; + Ndiag[0] = a(i,j,k,GIdx::N00_Re); + Ndiag[1] = a(i,j,k,GIdx::N11_Re); + return {Ndiag}; + }); + ArithmeticArray N = amrex::get<0>(result) / ncells; // write to file std::ofstream outfile; outfile.open(filename0D, std::ofstream::app); outfile << step << "\t"; outfile << time << "\t"; - outfile << N00 << "\t"; - outfile << N11 << "\t"; + outfile << N[0] << "\t"; + outfile << N[1] << "\t"; outfile << std::endl; outfile.close(); } diff --git a/Source/Make.package b/Source/Make.package index 6d4f1cd6..b038507d 100644 --- a/Source/Make.package +++ b/Source/Make.package @@ -13,3 +13,4 @@ CEXE_headers += IO.H CEXE_headers += Parameters.H CEXE_headers += ParticleInterpolator.H CEXE_headers += DataReducer.H +CEXE_headers += ArithmeticArray.H \ No newline at end of file From 6cfd7edbdbfaea77db6367c913b4413f1dfcfb39 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Fri, 3 Feb 2023 16:33:56 -0500 Subject: [PATCH 07/44] use code generator to make data reduction work for any number of flavors --- Scripts/symbolic_hermitians/generate_code.py | 22 +++++++++ Source/DataReducer.cpp | 49 ++++++++++++++------ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index b0f00a57..6011ca9a 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -168,6 +168,28 @@ def delete_generated_files(): code.append(string1+deplist[icomp]+string2+flist[icomp]+string3+string4[ivar]) write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_deposit_to_mesh_fill")) + #======================# + # DataReducer.cpp_fill # + #======================# + tails = ["","bar"] + code = [] + for t in tails: + # diagonal averages + N = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::N{}{}_{}"+t+")") + Nlist = N.header_diagonals(); + for i in range(len(Nlist)): + code.append("Ndiag"+t+"["+str(i)+"] = "+Nlist[i]+";") + + # off-diagonal magnitude + mag2 = 0 + for i in range(N.size): + for j in range(i+1,N.size): + re,im = N.H[i,j].as_real_imag() + mag2 += re**2 + im**2 + code.append("offdiag_mag2 += "+sympy.cxxcode(sympy.simplify(mag2))+";") + + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill")) + #==================# # Evolve.H_M2_fill # #==================# diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index f862d58e..1d963328 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -8,10 +8,16 @@ void DataReducer::InitializeFiles(){ std::ofstream outfile; outfile.open(filename0D, std::ofstream::out); - outfile << "1:step\t"; - outfile << "2:time(s)\t"; - outfile << "3:N00(cm^-3)\t"; - outfile << "3:N11(cm^-3)\t"; + int j=0; + j++; outfile << j<<":step\t"; + j++; outfile << j<<":time(s)\t"; + j++; outfile << j<<":tot(cm^-3)\t"; + j++; outfile << j<<":Ndiff(cm^-3)\t"; + for(int i=0; i > result = ParReduce(TypeList{}, - TypeList >{}, - state, nghost, - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple > { + GpuTuple< ArithmeticArray, ArithmeticArray, Real > result = + ParReduce(TypeList{}, + TypeList, ArithmeticArray, Real >{}, + state, nghost, + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple, ArithmeticArray, Real > { Array4 const& a = ma[box_no]; - ArithmeticArray Ndiag; - Ndiag[0] = a(i,j,k,GIdx::N00_Re); - Ndiag[1] = a(i,j,k,GIdx::N11_Re); - return {Ndiag}; + ArithmeticArray Ndiag, Ndiagbar; + Real offdiag_mag2 = 0; + #include "generated_files/DataReducer.cpp_fill" + return {Ndiag, Ndiagbar, offdiag_mag2}; }); - ArithmeticArray N = amrex::get<0>(result) / ncells; + ArithmeticArray N = amrex::get<0>(result) / ncells; + ArithmeticArray Nbar = amrex::get<1>(result) / ncells; + Real offdiag_mag = sqrt(amrex::get<2>(result)) / ncells; // write to file std::ofstream outfile; + Real Ntot=0, Ndiff=0; outfile.open(filename0D, std::ofstream::app); outfile << step << "\t"; outfile << time << "\t"; - outfile << N[0] << "\t"; - outfile << N[1] << "\t"; + for(int i=0; i Date: Fri, 3 Feb 2023 17:12:16 -0500 Subject: [PATCH 08/44] add Tr(rho) to the scalar output. Also only output to the reduction file if the processor id is 0 --- Source/DataReducer.cpp | 65 ++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 1d963328..963e7685 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -18,6 +18,7 @@ DataReducer::InitializeFiles(){ j++; outfile << j << ":N"< reduce_ops; + auto particleResult = amrex::ParticleReduce< ReduceData< amrex::Real> >(neutrinos, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { + Real tracerho = 0; + tracerho += p.rdata(PIdx::f00_Re); + tracerho += p.rdata(PIdx::f11_Re); + tracerho += p.rdata(PIdx::f00_Rebar); + tracerho += p.rdata(PIdx::f11_Rebar); + return GpuTuple{tracerho}; + }, reduce_ops); + Real TrRho = amrex::get<0>(particleResult); + ParallelDescriptor::ReduceRealSum(TrRho); + + // sample per-particle simple reduction //Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_HOST_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); //ParallelDescriptor::ReduceRealMin(pupt_min); @@ -52,32 +71,42 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, state, nghost, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple, ArithmeticArray, Real > { Array4 const& a = ma[box_no]; + + // Doing the actual work ArithmeticArray Ndiag, Ndiagbar; Real offdiag_mag2 = 0; #include "generated_files/DataReducer.cpp_fill" return {Ndiag, Ndiagbar, offdiag_mag2}; + }); + + // retrieve the reduced data values ArithmeticArray N = amrex::get<0>(result) / ncells; ArithmeticArray Nbar = amrex::get<1>(result) / ncells; Real offdiag_mag = sqrt(amrex::get<2>(result)) / ncells; - // write to file - std::ofstream outfile; - Real Ntot=0, Ndiff=0; - outfile.open(filename0D, std::ofstream::app); - outfile << step << "\t"; - outfile << time << "\t"; - for(int i=0; i Date: Fri, 3 Feb 2023 17:29:54 -0500 Subject: [PATCH 09/44] generate particle reduction code to work with any number of particles --- Scripts/symbolic_hermitians/generate_code.py | 14 ++++++++++++++ Source/DataReducer.cpp | 5 +---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 6011ca9a..22bb1c56 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -168,6 +168,20 @@ def delete_generated_files(): code.append(string1+deplist[icomp]+string2+flist[icomp]+string3+string4[ivar]) write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_deposit_to_mesh_fill")) + #================================# + # DataReducer.cpp_fill_particles # + #================================# + tails = ["","bar"] + code = [] + for t in tails: + # diagonal averages + N = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") + Nlist = N.header_diagonals(); + for i in range(len(Nlist)): + code.append("tracerho += "+Nlist[i]+";") + + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill_particles")) + #======================# # DataReducer.cpp_fill # #======================# diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 963e7685..b26520e2 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -40,10 +40,7 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, auto particleResult = amrex::ParticleReduce< ReduceData< amrex::Real> >(neutrinos, [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { Real tracerho = 0; - tracerho += p.rdata(PIdx::f00_Re); - tracerho += p.rdata(PIdx::f11_Re); - tracerho += p.rdata(PIdx::f00_Rebar); - tracerho += p.rdata(PIdx::f11_Rebar); + #include "generated_files/DataReducer.cpp_fill_particles" return GpuTuple{tracerho}; }, reduce_ops); Real TrRho = amrex::get<0>(particleResult); From fba2efc03920f5550c60bb5468d3e65fab54af64 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 16 Feb 2023 10:44:02 -0500 Subject: [PATCH 10/44] added HighFive submodule --- .gitmodules | 3 +++ submodules/HighFive | 1 + 2 files changed, 4 insertions(+) create mode 160000 submodules/HighFive diff --git a/.gitmodules b/.gitmodules index c5213734..c709196d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "amrex"] path = amrex url = https://github.com/dwillcox/amrex.git +[submodule "submodules/HighFive"] + path = submodules/HighFive + url = https://github.com/BlueBrain/HighFive.git diff --git a/submodules/HighFive b/submodules/HighFive new file mode 160000 index 00000000..4d29deeb --- /dev/null +++ b/submodules/HighFive @@ -0,0 +1 @@ +Subproject commit 4d29deebf939ba47f8f28f0594b813fd9289c0fd From 24f5537add758a5419fdb9f41165e61f59ed6608 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 1 Mar 2023 11:15:42 -0500 Subject: [PATCH 11/44] first bit of support for hdf5 realtime io --- Source/DataReducer.H | 11 ++++++++++- Source/DataReducer.cpp | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/Source/DataReducer.H b/Source/DataReducer.H index 09886e88..b698b5b4 100644 --- a/Source/DataReducer.H +++ b/Source/DataReducer.H @@ -7,12 +7,21 @@ #include #include #include +#include <../submodules/HighFive/include/highfive/H5File.hpp> +#include <../submodules/HighFive/include/highfive/H5DataSpace.hpp> +#include <../submodules/HighFive/include/highfive/H5DataSet.hpp> class DataReducer{ private: + +#ifdef AMREX_USE_HDF5 + const char* filename0D = "reduced0D.h5"; + const HighFive::DataSpace dataspace = HighFive::DataSpace({0}, {HighFive::DataSpace::UNLIMITED}); +#else const char* filename0D = "reduced0D.dat"; +#endif -public: +public: void InitializeFiles(); diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index b26520e2..8ea63462 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -3,9 +3,38 @@ #include "DataReducer.H" #include "ArithmeticArray.H" #include +#ifdef AMREX_USE_HDF5 +#include <../submodules/HighFive/include/highfive/H5File.hpp> +#include <../submodules/HighFive/include/highfive/H5DataSpace.hpp> +#include <../submodules/HighFive/include/highfive/H5DataSet.hpp> +#endif + +#ifdef AMREX_USE_HDF5 +// append a single scalar to the specified file and dataset +template +void append_0D(HighFive::File& file0D, const char* datasetname, const T value){ + HighFive::DataSet dataset_step = file0D.getDataSet(datasetname); + std::vector dims = dataset_step.getDimensions(); + dims[0] ++; + dataset_step.resize(dims); + dataset_step.select({dims[0]-1},{1}).write((int[1]){value}); +} +#endif void DataReducer::InitializeFiles(){ + +#ifdef AMREX_USE_HDF5 + + using namespace HighFive; + File file0D(filename0D, File::Truncate | File::Create); + + DataSetCreateProps props; + props.add(Chunking(std::vector{1})); + file0D.createDataSet("step", dataspace, create_datatype(), props); + +#else + std::ofstream outfile; outfile.open(filename0D, std::ofstream::out); int j=0; @@ -21,6 +50,8 @@ DataReducer::InitializeFiles(){ j++; outfile << j<<":sumTrRho\t"; outfile << std::endl; outfile.close(); + +#endif } void @@ -86,6 +117,10 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, // write to file // //===============// if(ParallelDescriptor::MyProc()==0){ +#ifdef AMREX_USE_HDF5 + HighFive::File file0D(filename0D, HighFive::File::ReadWrite); + append_0D(file0D, "step", step); +#else std::ofstream outfile; Real Ntot=0, Ndiff=0; outfile.open(filename0D, std::ofstream::app); @@ -105,5 +140,6 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, outfile << TrRho << "\t"; outfile << std::endl; outfile.close(); +#endif } } From 90fe5ecae9e0338b9c34358290267cd95bd6b4ee Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Wed, 1 Mar 2023 13:21:12 -0500 Subject: [PATCH 12/44] added hdf5 support for all reduced data --- Source/DataReducer.cpp | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 8ea63462..78aa7d23 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -3,6 +3,7 @@ #include "DataReducer.H" #include "ArithmeticArray.H" #include +#include #ifdef AMREX_USE_HDF5 #include <../submodules/HighFive/include/highfive/H5File.hpp> #include <../submodules/HighFive/include/highfive/H5DataSpace.hpp> @@ -12,12 +13,12 @@ #ifdef AMREX_USE_HDF5 // append a single scalar to the specified file and dataset template -void append_0D(HighFive::File& file0D, const char* datasetname, const T value){ +void append_0D(HighFive::File& file0D, const std::string& datasetname, const T value){ HighFive::DataSet dataset_step = file0D.getDataSet(datasetname); std::vector dims = dataset_step.getDimensions(); dims[0] ++; dataset_step.resize(dims); - dataset_step.select({dims[0]-1},{1}).write((int[1]){value}); + dataset_step.select({dims[0]-1},{1}).write((T[1]){value}); } #endif @@ -32,7 +33,15 @@ DataReducer::InitializeFiles(){ DataSetCreateProps props; props.add(Chunking(std::vector{1})); file0D.createDataSet("step", dataspace, create_datatype(), props); - + file0D.createDataSet("time(s)", dataspace, create_datatype(), props); + file0D.createDataSet("Ntot(cm^-3)", dataspace, create_datatype(), props); + file0D.createDataSet("Ndiff(cm^-3)", dataspace, create_datatype(), props); + for(int i=0; i(), props); + file0D.createDataSet(std::string("N")+std::to_string(i)+std::to_string(i)+std::string("bar(cm^-3)"), dataspace, create_datatype(), props); + } + file0D.createDataSet("N_offdiag(cm^-3)", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrRho", dataspace, create_datatype(), props); #else std::ofstream outfile; @@ -40,7 +49,7 @@ DataReducer::InitializeFiles(){ int j=0; j++; outfile << j<<":step\t"; j++; outfile << j<<":time(s)\t"; - j++; outfile << j<<":tot(cm^-3)\t"; + j++; outfile << j<<":Ntot(cm^-3)\t"; j++; outfile << j<<":Ndiff(cm^-3)\t"; for(int i=0; i Nbar = amrex::get<1>(result) / ncells; Real offdiag_mag = sqrt(amrex::get<2>(result)) / ncells; + // calculate net number of neutrinos and antineutrinos + Real Ntot=0, Ndiff=0; + for(int i=0; i Date: Wed, 1 Mar 2023 14:48:18 -0500 Subject: [PATCH 13/44] added support for outputting the net energy of the system. Required adding one more scalar variable to each particle. This is a good metric to track numerical errors - the number should remain constant --- Scripts/symbolic_hermitians/generate_code.py | 9 +++++- Source/DataReducer.H | 2 ++ Source/DataReducer.cpp | 34 ++++++++++++-------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 22bb1c56..967cb887 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -99,6 +99,7 @@ def delete_generated_files(): for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() + code += ["TrHf"] code = [code[i]+"," for i in range(len(code))] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "FlavoredNeutrinoContainer.H_fill")) @@ -115,6 +116,7 @@ def delete_generated_files(): for v in vars: A = HermitianMatrix(args.N, v+"{}{}_{}"+t) code += A.header() + code += ["TrHf"] code_string = 'attribute_names = {"time", "x", "y", "z", "pupx", "pupy", "pupz", "pupt", ' code = ['"{}"'.format(c) for c in code] code_string = code_string + ", ".join(code) + "};" @@ -178,7 +180,7 @@ def delete_generated_files(): N = HermitianMatrix(args.N, "p.rdata(PIdx::f{}{}_{}"+t+")") Nlist = N.header_diagonals(); for i in range(len(Nlist)): - code.append("tracerho += "+Nlist[i]+";") + code.append("Trf += "+Nlist[i]+";") write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill_particles")) @@ -420,6 +422,11 @@ def sgn(t,var): # Write out dFdt->F code.append(dFdt.code()) + + # store Tr(H*F) for estimating numerical errors + TrHf = (H*F).trace(); + code.append(["p.rdata(PIdx::TrHf) += "+sympy.cxxcode(sympy.simplify(TrHf))+";"]) + code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) diff --git a/Source/DataReducer.H b/Source/DataReducer.H index b698b5b4..008ad833 100644 --- a/Source/DataReducer.H +++ b/Source/DataReducer.H @@ -7,9 +7,11 @@ #include #include #include +#ifdef AMREX_USE_HDF5 #include <../submodules/HighFive/include/highfive/H5File.hpp> #include <../submodules/HighFive/include/highfive/H5DataSpace.hpp> #include <../submodules/HighFive/include/highfive/H5DataSet.hpp> +#endif class DataReducer{ private: diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 78aa7d23..e5698c28 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -41,7 +41,8 @@ DataReducer::InitializeFiles(){ file0D.createDataSet(std::string("N")+std::to_string(i)+std::to_string(i)+std::string("bar(cm^-3)"), dataspace, create_datatype(), props); } file0D.createDataSet("N_offdiag(cm^-3)", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrRho", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); #else std::ofstream outfile; @@ -56,7 +57,8 @@ DataReducer::InitializeFiles(){ j++; outfile << j << ":N"< reduce_ops; - auto particleResult = amrex::ParticleReduce< ReduceData< amrex::Real> >(neutrinos, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - Real tracerho = 0; - #include "generated_files/DataReducer.cpp_fill_particles" - return GpuTuple{tracerho}; - }, reduce_ops); - Real TrRho = amrex::get<0>(particleResult); - ParallelDescriptor::ReduceRealSum(TrRho); - + amrex::ReduceOps reduce_ops; + auto particleResult = amrex::ParticleReduce< ReduceData >(neutrinos, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { + Real TrHf = p.rdata(PIdx::TrHf); + Real Trf = 0; +#include "generated_files/DataReducer.cpp_fill_particles" + return GpuTuple{Trf,TrHf}; + }, reduce_ops); + Real Trf = amrex::get<0>(particleResult); + Real TrHf = amrex::get<1>(particleResult); + ParallelDescriptor::ReduceRealSum(Trf); + ParallelDescriptor::ReduceRealSum(TrHf); // sample per-particle simple reduction //Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_HOST_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); @@ -144,7 +148,8 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, append_0D(file0D, std::string("N")+std::to_string(i)+std::to_string(i)+std::string("bar(cm^-3)"), Nbar[i]); } append_0D(file0D, "N_offdiag(cm^-3)", offdiag_mag); - append_0D(file0D, "sumTrRho", TrRho); + append_0D(file0D, "sumTrf", Trf); + append_0D(file0D, "sumTrHf", TrHf); #else std::ofstream outfile; outfile.open(filename0D, std::ofstream::app); @@ -157,7 +162,8 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, outfile << Nbar[i] << "\t"; } outfile << offdiag_mag << "\t"; - outfile << TrRho << "\t"; + outfile << Trf << "\t"; + outfile << TrHf << "\t"; outfile << std::endl; outfile.close(); #endif From ea9d796eed724eccf5af047c5d237e4db0de8bdd Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 2 Mar 2023 13:58:26 -0500 Subject: [PATCH 14/44] remove commented reduction examples --- Source/DataReducer.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index e5698c28..6f1fed38 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -91,13 +91,6 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, ParallelDescriptor::ReduceRealSum(Trf); ParallelDescriptor::ReduceRealSum(TrHf); - // sample per-particle simple reduction - //Real pupt_min = amrex::ReduceMin(*this, [=] AMREX_GPU_HOST_DEVICE (const FlavoredNeutrinoContainer::ParticleType& p) -> Real { return p.rdata(PIdx::pupt); }); - //ParallelDescriptor::ReduceRealMin(pupt_min); - - // sample grid simple reduction - //Real N00 = state.sum(GIdx::N00_Re) / ncells; - //=============================// // Do reductions over the grid // //=============================// From 74cf458354b267a16e44e8d9833e569d229effa8 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Thu, 2 Mar 2023 14:04:11 -0500 Subject: [PATCH 15/44] total energy should account for the relative number of real neutrinos present in a particular neutrino particle. --- Scripts/symbolic_hermitians/generate_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 967cb887..331d5c93 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -425,7 +425,7 @@ def sgn(t,var): # store Tr(H*F) for estimating numerical errors TrHf = (H*F).trace(); - code.append(["p.rdata(PIdx::TrHf) += "+sympy.cxxcode(sympy.simplify(TrHf))+";"]) + code.append(["p.rdata(PIdx::TrHf) += p.rdata(PIdx::N"+t+") * ("+sympy.cxxcode(sympy.simplify(TrHf))+");"]) code = [line for sublist in code for line in sublist] write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill")) From 72dd52e359da7146c08c60ef60882fb945a280d1 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 28 Mar 2023 18:00:39 -0400 Subject: [PATCH 16/44] only output particle info at first timestep if it is output at any time --- Source/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/main.cpp b/Source/main.cpp index 5d9443d3..0b6eac4b 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -122,7 +122,7 @@ void evolve_flavor(const TestParams* parms) DataReducer rd; if (not parms->do_restart) { // If we have just initialized, then always save the particle data for reference - const int write_particles_after_init = 1; + const int write_particles_after_init = (parms->write_plot_particles_every>0); WritePlotFile(state, neutrinos_old, geom, initial_time, initial_step, write_particles_after_init); rd.InitializeFiles(); } From 781961763741886884c7445f5422c02fe8608cb9 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Tue, 28 Mar 2023 18:12:22 -0400 Subject: [PATCH 17/44] add fluxes to realtime data output --- Scripts/symbolic_hermitians/generate_code.py | 11 ++- Source/DataReducer.cpp | 82 ++++++++++++++------ 2 files changed, 68 insertions(+), 25 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 331d5c93..98d536a6 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -193,8 +193,17 @@ def delete_generated_files(): # diagonal averages N = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::N{}{}_{}"+t+")") Nlist = N.header_diagonals(); + Fx = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Fx{}{}_{}"+t+")") + Fxlist = Fx.header_diagonals(); + Fy = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Fy{}{}_{}"+t+")") + Fylist = Fy.header_diagonals(); + Fz = HermitianMatrix(args.N, "a(i\,j\,k\,GIdx::Fz{}{}_{}"+t+")") + Fzlist = Fz.header_diagonals(); for i in range(len(Nlist)): code.append("Ndiag"+t+"["+str(i)+"] = "+Nlist[i]+";") + code.append("Fxdiag"+t+"["+str(i)+"] = "+Fxlist[i]+";") + code.append("Fydiag"+t+"["+str(i)+"] = "+Fylist[i]+";") + code.append("Fzdiag"+t+"["+str(i)+"] = "+Fzlist[i]+";") # off-diagonal magnitude mag2 = 0 @@ -202,7 +211,7 @@ def delete_generated_files(): for j in range(i+1,N.size): re,im = N.H[i,j].as_real_imag() mag2 += re**2 + im**2 - code.append("offdiag_mag2 += "+sympy.cxxcode(sympy.simplify(mag2))+";") + code.append("N_offdiag_mag2 += "+sympy.cxxcode(sympy.simplify(mag2))+";") write_code(code, os.path.join(args.emu_home, "Source/generated_files", "DataReducer.cpp_fill")) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 6f1fed38..9085b06a 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -34,13 +34,19 @@ DataReducer::InitializeFiles(){ props.add(Chunking(std::vector{1})); file0D.createDataSet("step", dataspace, create_datatype(), props); file0D.createDataSet("time(s)", dataspace, create_datatype(), props); - file0D.createDataSet("Ntot(cm^-3)", dataspace, create_datatype(), props); - file0D.createDataSet("Ndiff(cm^-3)", dataspace, create_datatype(), props); + file0D.createDataSet("Ntot(1|ccm)", dataspace, create_datatype(), props); + file0D.createDataSet("Ndiff(1|ccm)", dataspace, create_datatype(), props); for(int i=0; i(), props); - file0D.createDataSet(std::string("N")+std::to_string(i)+std::to_string(i)+std::string("bar(cm^-3)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("N")+std::to_string(i)+std::to_string(i)+std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("N")+std::to_string(i)+std::to_string(i)+std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fx")+std::to_string(i)+std::to_string(i)+std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fy")+std::to_string(i)+std::to_string(i)+std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fz")+std::to_string(i)+std::to_string(i)+std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fx")+std::to_string(i)+std::to_string(i)+std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fy")+std::to_string(i)+std::to_string(i)+std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fz")+std::to_string(i)+std::to_string(i)+std::string("bar(1|ccm)"), dataspace, create_datatype(), props); } - file0D.createDataSet("N_offdiag(cm^-3)", dataspace, create_datatype(), props); + file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); #else @@ -50,13 +56,19 @@ DataReducer::InitializeFiles(){ int j=0; j++; outfile << j<<":step\t"; j++; outfile << j<<":time(s)\t"; - j++; outfile << j<<":Ntot(cm^-3)\t"; - j++; outfile << j<<":Ndiff(cm^-3)\t"; + j++; outfile << j<<":Ntot(1|ccm)\t"; + j++; outfile << j<<":Ndiff(1|ccm)\t"; for(int i=0; i, ArithmeticArray, Real > result = - ParReduce(TypeList{}, - TypeList, ArithmeticArray, Real >{}, + GpuTuple< ArithmeticArray, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > result = + ParReduce(TypeList{}, + TypeList, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray >{}, state, nghost, - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> GpuTuple, ArithmeticArray, Real > { + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> + GpuTuple, ArithmeticArray, Real , ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray, ArithmeticArray > { Array4 const& a = ma[box_no]; // Doing the actual work - ArithmeticArray Ndiag, Ndiagbar; - Real offdiag_mag2 = 0; + ArithmeticArray Ndiag, Ndiagbar; + ArithmeticArray Fxdiag, Fxdiagbar; + ArithmeticArray Fydiag, Fydiagbar; + ArithmeticArray Fzdiag, Fzdiagbar; + Real N_offdiag_mag2 = 0; #include "generated_files/DataReducer.cpp_fill" - return {Ndiag, Ndiagbar, offdiag_mag2}; + return {Ndiag, Ndiagbar, N_offdiag_mag2, Fxdiag, Fydiag, Fzdiag, Fxdiagbar,Fydiagbar,Fzdiagbar}; }); // retrieve the reduced data values ArithmeticArray N = amrex::get<0>(result) / ncells; ArithmeticArray Nbar = amrex::get<1>(result) / ncells; - Real offdiag_mag = sqrt(amrex::get<2>(result)) / ncells; + Real N_offdiag_mag = sqrt(amrex::get<2>(result)) / ncells; + ArithmeticArray Fx = amrex::get<3>(result) / ncells; + ArithmeticArray Fy = amrex::get<4>(result) / ncells; + ArithmeticArray Fz = amrex::get<5>(result) / ncells; + ArithmeticArray Fxbar = amrex::get<6>(result) / ncells; + ArithmeticArray Fybar = amrex::get<7>(result) / ncells; + ArithmeticArray Fzbar = amrex::get<8>(result) / ncells; // calculate net number of neutrinos and antineutrinos Real Ntot=0, Ndiff=0; @@ -134,13 +156,19 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom, HighFive::File file0D(filename0D, HighFive::File::ReadWrite); append_0D(file0D, "step", step); append_0D(file0D, "time(s)", time); - append_0D(file0D, "Ntot(cm^-3)", Ntot); - append_0D(file0D, "Ndiff(cm^-3)", Ndiff); + append_0D(file0D, "Ntot(1|ccm)", Ntot); + append_0D(file0D, "Ndiff(1|ccm)", Ndiff); for(int i=0; i Date: Tue, 28 Mar 2023 18:15:59 -0400 Subject: [PATCH 18/44] comment out print lines so initial conditions functions can be used more cleanly in scripts --- .../initial_conditions/initial_condition_tools.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Scripts/initial_conditions/initial_condition_tools.py b/Scripts/initial_conditions/initial_condition_tools.py index 16c0f625..efa8ef6c 100644 --- a/Scripts/initial_conditions/initial_condition_tools.py +++ b/Scripts/initial_conditions/initial_condition_tools.py @@ -116,7 +116,7 @@ def minerbo_Z(fluxfac): print("Failed to converge on a solution.") assert(False) - print("fluxfac=",fluxfac," Z=",Z) + #print("fluxfac=",fluxfac," Z=",Z) return Z # angular structure as determined by the Levermore closure @@ -161,7 +161,7 @@ def levermore_v(fluxfac): print("Failed to converge on a solution.") assert(False) - print("fluxfac=",fluxfac," v=",v) + #print("fluxfac=",fluxfac," v=",v) return v # interpolate the levermore closure @@ -251,10 +251,10 @@ def moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, direction_g # double check that the number densities are correct particle_n = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]]) particle_fmag = np.sum(particles[:,rkey[nvarname]] * particles[:,rkey[fvarname]] * mu[:,nu_nubar, flavor]) - print("nu/nubar,flavor =", nu_nubar, flavor) - print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) - print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) - print() + #print("nu/nubar,flavor =", nu_nubar, flavor) + #print("output/input ndens =",particle_n, nnu[nu_nubar,flavor]) + #print("output/input fluxfac =",particle_fmag / particle_n, fluxfac[nu_nubar,flavor]) + #print() return particles From 1e45e098460c92a2e5541fb8b55f9008c8ca72a3 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 13:04:18 -0400 Subject: [PATCH 19/44] Reduced max grid size on FFI test to force MPI parallelism --- sample_inputs/inputs_fast_flavor_nonzerok | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 1996e71a..dc858a8e 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -21,7 +21,7 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 1000 +max_grid_size = 10 # Number of steps to run nsteps = 5000 From ce52078e4068ac3d639a5ed02f0f5377b3e34a10 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 13:21:58 -0400 Subject: [PATCH 20/44] Run fiducial simulation in test suite. Will use it to test data reduction scripts as well. --- Jenkinsfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 4d401eb9..279df650 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,6 +53,14 @@ pipeline { } }} + stage('Fiducial 2F GPU Binary'){ steps{ + dir('Exec'){ + sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' + sh 'python ../Scripts/data_reduction/reduce_data.py' + } + }} + } // stages{ post { From 864f05a4b2fc02f80e66ed4037bcad1196ccfa93 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 13:34:13 -0400 Subject: [PATCH 21/44] Get rid of mpi errors in stdout --- Jenkinsfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 279df650..56b5acb8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,11 @@ pipeline { triggers { pollSCM('') } // Run tests whenever a new commit is detected. agent { dockerfile {args '--gpus all'}} // Use the Dockerfile defined in the root Flash-X directory + environment { + // Get rid of Read -1, expected , errno =1 error + // See https://github.com/open-mpi/ompi/issues/4948 + OMPI_MCA_btl_vader_single_copy_mechanism = 'none' + } stages { //=============================// From f0c5d5a1d41816d2d6c384122f309b0c090eca03 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 13:35:17 -0400 Subject: [PATCH 22/44] Reduce the number of steps the fiducial test takes to reduce testing time --- sample_inputs/inputs_1d_fiducial | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 833421e8..8ccf8e7d 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -23,7 +23,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 5000 +nsteps = 2000 # Simulation end time end_time = 5.0e-9 From d02b6db64067265942421cd8261c36fc10a0c262 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 13:36:27 -0400 Subject: [PATCH 23/44] install yt to be able to test data reduction scripts --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bc578e3c..a581d770 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,6 @@ FROM nvidia/cuda:11.4.0-devel-ubuntu20.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 -RUN pip3 install numpy matplotlib h5py scipy sympy +RUN pip3 install numpy matplotlib h5py scipy sympy yt ENV USER=jenkins ENV LOGNAME=jenkins From 2da4e430cd9f637f185d536e358260c86c8a47bb Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 13:53:48 -0400 Subject: [PATCH 24/44] Add more data reduction scripts to fiducial test --- Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 56b5acb8..f7ddcbdf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,6 +63,13 @@ pipeline { sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' sh 'python ../Scripts/data_reduction/reduce_data.py' + sh 'python ../Scripts/data_reduction/reduce_data_fft.py' + sh 'python ../Scripts/data_reduction/combine_files plt _reduced_data.h5' + sh 'python ../Scripts/data_reduction/combine_files plt _reduced_data_fft_power.h5' + sh 'python ../Scripts/babysitting/avgfee.py' + sh 'python ../Scripts/babysitting/power_spectrum.py' + sh 'python ../Scripts/data_reduction/convertToHDF5.py' + archiveArtifacts artifacts: '*.pdf' } }} From f57f90fdad955d467e4a9e3e54b466fce3867004 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 14:14:05 -0400 Subject: [PATCH 25/44] First attempt at including HFD5 tests in jenkins --- Jenkinsfile | 9 +++++++++ makefiles/GNUmakefile_jenkins_HDF5 | 31 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 makefiles/GNUmakefile_jenkins_HDF5 diff --git a/Jenkinsfile b/Jenkinsfile index f7ddcbdf..a83bf1c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,6 +73,15 @@ pipeline { } }} + stage('Fiducial 3F CPU HDF5'){ steps{ + dir('Exec'){ + sh 'cp makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'make realclean; make generate; make -j' + sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' + sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' + } + }} + } // stages{ post { diff --git a/makefiles/GNUmakefile_jenkins_HDF5 b/makefiles/GNUmakefile_jenkins_HDF5 new file mode 100644 index 00000000..f5a8af6f --- /dev/null +++ b/makefiles/GNUmakefile_jenkins_HDF5 @@ -0,0 +1,31 @@ +EMU_HOME ?= ../ +AMREX_HOME ?= ../amrex + +DIM = 3 + +NUM_FLAVORS = 3 + +SHAPE_FACTOR_ORDER = 2 + +COMP = gnu + +DEBUG = FALSE + +USE_MPI = TRUE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = FALSE +AMREX_CUDA_ARCH=75 + +USE_HDF5=TRUE +HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/serial + +TINY_PROFILE = TRUE +USE_PARTICLES = TRUE + +PRECISION = DOUBLE + +Bpack := +Blocs := . + +include ../Make.Emu From 8f4b45a3102575706202b8fbb2fa629531d7581d Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 14:32:43 -0400 Subject: [PATCH 26/44] Remove extraneous plot files to avoid accumuation of 'old' files during testing --- Jenkinsfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a83bf1c3..5f6cbc72 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,13 +32,15 @@ pipeline { sh 'python ../Scripts/initial_conditions/st0_msw_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_msw_test' sh 'python ../Scripts/tests/msw_test.py' + sh 'rm -rf plt*' } }} stage('Bipolar'){ steps{ dir('Exec'){ - sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' + sh 'python ../Scripts/initial_conditions/st1_bipolar_test.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_bipolar_test' + sh 'rm -rf plt*' } }} @@ -47,6 +49,7 @@ pipeline { sh 'python ../Scripts/initial_conditions/st2_2beam_fast_flavor.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor' sh 'python ../Scripts/tests/fast_flavor_test.py' + sh 'rm -rf plt*' } }} @@ -55,6 +58,7 @@ pipeline { sh 'python ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_fast_flavor_nonzerok' sh 'python ../Scripts/tests/fast_flavor_k_test.py' + sh 'rm -rf plt*' } }} @@ -70,6 +74,7 @@ pipeline { sh 'python ../Scripts/babysitting/power_spectrum.py' sh 'python ../Scripts/data_reduction/convertToHDF5.py' archiveArtifacts artifacts: '*.pdf' + sh 'rm -rf plt*' } }} @@ -79,6 +84,7 @@ pipeline { sh 'make realclean; make generate; make -j' sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' + sh 'rm -rf plt*' } }} From ec0213ffa6b3e5daea366a0d2e65e79aa18c9d39 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 14:57:29 -0400 Subject: [PATCH 27/44] move amrex to submodules directory --- .gitmodules | 2 +- amrex => submodules/amrex | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename amrex => submodules/amrex (100%) diff --git a/.gitmodules b/.gitmodules index c709196d..f2322e41 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1,5 @@ [submodule "amrex"] - path = amrex + path = submodules/amrex url = https://github.com/dwillcox/amrex.git [submodule "submodules/HighFive"] path = submodules/HighFive diff --git a/amrex b/submodules/amrex similarity index 100% rename from amrex rename to submodules/amrex From cfcadaab237267d84f841695a0a8fdc9f8fdf9e4 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 15:04:11 -0400 Subject: [PATCH 28/44] Clean up makefiles to make them a little bit more user friendly and transparent --- Exec/.gitignore | 1 + Exec/GNUmakefile | 28 ---------------------------- Make.Emu | 9 +++++++-- makefiles/GNUmakefile_cassowary | 19 +++++++++++++++++++ makefiles/GNUmakefile_ganon | 23 +++-------------------- makefiles/GNUmakefile_jenkins | 18 +++--------------- makefiles/GNUmakefile_jenkins_HDF5 | 18 +++--------------- 7 files changed, 36 insertions(+), 80 deletions(-) delete mode 100644 Exec/GNUmakefile create mode 100644 makefiles/GNUmakefile_cassowary diff --git a/Exec/.gitignore b/Exec/.gitignore index e02b640b..44236a37 100644 --- a/Exec/.gitignore +++ b/Exec/.gitignore @@ -5,3 +5,4 @@ plt* tmp_build_dir Backtrace.* *.png +*.dat diff --git a/Exec/GNUmakefile b/Exec/GNUmakefile deleted file mode 100644 index 3577d5e8..00000000 --- a/Exec/GNUmakefile +++ /dev/null @@ -1,28 +0,0 @@ -EMU_HOME ?= ../ -AMREX_HOME ?= ../amrex - -SHAPE_FACTOR_ORDER ?= 2 -NUM_FLAVORS ?= 3 - -COMP = gnu - -DEBUG = FALSE - -USE_MPI = TRUE -USE_OMP = FALSE -USE_ACC = FALSE -USE_CUDA = FALSE -AMREX_CUDA_ARCH=60 - -USE_HDF5=FALSE -HDF5_HOME=/usr/local/hdf5-1.12.2_gnu9.4.0 - -TINY_PROFILE = TRUE -USE_PARTICLES = TRUE - -PRECISION = DOUBLE - -Bpack := -Blocs := . - -include $(EMU_HOME)/Make.Emu diff --git a/Make.Emu b/Make.Emu index 1b756472..d1081c91 100644 --- a/Make.Emu +++ b/Make.Emu @@ -1,6 +1,11 @@ -NUM_FLAVORS ?= 2 -SHAPE_FACTOR_ORDER ?= 2 +# things that used to be defined in the makefile in Exec DIM = 3 +TINY_PROFILE = TRUE +USE_PARTICLES = TRUE +PRECISION = DOUBLE +Bpack := +Blocs := . + TOP := $(EMU_HOME) diff --git a/makefiles/GNUmakefile_cassowary b/makefiles/GNUmakefile_cassowary new file mode 100644 index 00000000..4ed76b6f --- /dev/null +++ b/makefiles/GNUmakefile_cassowary @@ -0,0 +1,19 @@ +NUM_FLAVORS = 2 +SHAPE_FACTOR_ORDER = 2 + +COMP = gnu + +DEBUG = FALSE + +USE_MPI = TRUE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = FALSE +AMREX_CUDA_ARCH=60 + +USE_HDF5=FALSE +HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/serial + +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex +include ../Make.Emu diff --git a/makefiles/GNUmakefile_ganon b/makefiles/GNUmakefile_ganon index e6c6fa45..209b1344 100644 --- a/makefiles/GNUmakefile_ganon +++ b/makefiles/GNUmakefile_ganon @@ -1,9 +1,5 @@ -EMU_HOME ?= ../ -AMREX_HOME ?= ../amrex - -DIM = 3 - NUM_FLAVORS = 2 +SHAPE_FACTOR_ORDER = 2 COMP = gnu @@ -15,19 +11,6 @@ USE_ACC = FALSE USE_CUDA = FALSE USE_HDF5 = FALSE -TINY_PROFILE = TRUE -USE_PARTICLES = TRUE - -PRECISION = DOUBLE - -Bpack := -Blocs := . - -ifeq ($(USE_HDF5), TRUE) -HDF5_HOME = /usr/local/hdf5-1.12.0_gnu7.5.0 -DEFINES += -DAMREX_USE_HDF5 -INCLUDE_LOCATIONS += $(HDF5_HOME)/include -LIBRARIES += -L$(HDF5_HOME)/lib -lhdf5 -lz -ldl -endif - +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex include ../Make.Emu diff --git a/makefiles/GNUmakefile_jenkins b/makefiles/GNUmakefile_jenkins index da1d700d..431901c8 100644 --- a/makefiles/GNUmakefile_jenkins +++ b/makefiles/GNUmakefile_jenkins @@ -1,15 +1,9 @@ -EMU_HOME ?= ../ -AMREX_HOME ?= ../amrex - -DIM = 3 - NUM_FLAVORS = 2 - SHAPE_FACTOR_ORDER = 2 COMP = gnu -DEBUG = FALSE +DEBUG = FALSE USE_MPI = TRUE USE_OMP = FALSE @@ -17,12 +11,6 @@ USE_ACC = FALSE USE_CUDA = TRUE AMREX_CUDA_ARCH=75 -TINY_PROFILE = TRUE -USE_PARTICLES = TRUE - -PRECISION = DOUBLE - -Bpack := -Blocs := . - +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex include ../Make.Emu diff --git a/makefiles/GNUmakefile_jenkins_HDF5 b/makefiles/GNUmakefile_jenkins_HDF5 index f5a8af6f..6526aac2 100644 --- a/makefiles/GNUmakefile_jenkins_HDF5 +++ b/makefiles/GNUmakefile_jenkins_HDF5 @@ -1,15 +1,9 @@ -EMU_HOME ?= ../ -AMREX_HOME ?= ../amrex - -DIM = 3 - NUM_FLAVORS = 3 - SHAPE_FACTOR_ORDER = 2 COMP = gnu -DEBUG = FALSE +DEBUG = FALSE USE_MPI = TRUE USE_OMP = FALSE @@ -20,12 +14,6 @@ AMREX_CUDA_ARCH=75 USE_HDF5=TRUE HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/serial -TINY_PROFILE = TRUE -USE_PARTICLES = TRUE - -PRECISION = DOUBLE - -Bpack := -Blocs := . - +EMU_HOME = .. +AMREX_HOME = ../submodules/amrex include ../Make.Emu From 8dc682b7d86a761c546828d99a0f3d7d3468fb35 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 15:10:41 -0400 Subject: [PATCH 29/44] fix glob search string so reduction script doesnt match to reduced data files --- Scripts/data_reduction/reduce_data_fft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/data_reduction/reduce_data_fft.py b/Scripts/data_reduction/reduce_data_fft.py index 9a5527b8..7abbe3fe 100755 --- a/Scripts/data_reduction/reduce_data_fft.py +++ b/Scripts/data_reduction/reduce_data_fft.py @@ -9,7 +9,7 @@ parser.add_argument("-o", "--output", type=str, default="reduced_data_fft.h5", help="Name of the output file (default: reduced_data_fft.h5)") args = parser.parse_args() -directories = sorted(glob.glob("plt*")) +directories = sorted(glob.glob("plt?????")) t = [] From ba0ae7a9ad1d0bb85a026e47489da8966ff07b38 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 15:21:57 -0400 Subject: [PATCH 30/44] fix python script filename --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5f6cbc72..1a8a48d6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,8 +68,8 @@ pipeline { sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial' sh 'python ../Scripts/data_reduction/reduce_data.py' sh 'python ../Scripts/data_reduction/reduce_data_fft.py' - sh 'python ../Scripts/data_reduction/combine_files plt _reduced_data.h5' - sh 'python ../Scripts/data_reduction/combine_files plt _reduced_data_fft_power.h5' + sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5' + sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5' sh 'python ../Scripts/babysitting/avgfee.py' sh 'python ../Scripts/babysitting/power_spectrum.py' sh 'python ../Scripts/data_reduction/convertToHDF5.py' From 260ccb7162c2303aa6af9b763c4284d277c6b269 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 15:33:33 -0400 Subject: [PATCH 31/44] further reduce number of steps for test speed --- sample_inputs/inputs_1d_fiducial | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 8ccf8e7d..c3e89d37 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -23,7 +23,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 2000 +nsteps = 1000 # Simulation end time end_time = 5.0e-9 From 9cc1f4e390460db98465105ad222ae4a86fd020c Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 15:44:47 -0400 Subject: [PATCH 32/44] fix some paths to get hdf5 working in tests (hopefully) --- Jenkinsfile | 2 +- makefiles/GNUmakefile_jenkins_HDF5 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1a8a48d6..bb4b90bb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,7 +80,7 @@ pipeline { stage('Fiducial 3F CPU HDF5'){ steps{ dir('Exec'){ - sh 'cp makefiles/GNUmakefile_jenkins GNUmakefile' + sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5 GNUmakefile' sh 'make realclean; make generate; make -j' sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' diff --git a/makefiles/GNUmakefile_jenkins_HDF5 b/makefiles/GNUmakefile_jenkins_HDF5 index 6526aac2..1a5b3993 100644 --- a/makefiles/GNUmakefile_jenkins_HDF5 +++ b/makefiles/GNUmakefile_jenkins_HDF5 @@ -12,7 +12,7 @@ USE_CUDA = FALSE AMREX_CUDA_ARCH=75 USE_HDF5=TRUE -HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/serial +HDF5_HOME=/usr/lib/x86_64-linux-gnu/hdf5/openmpi EMU_HOME = .. AMREX_HOME = ../submodules/amrex From 9baf856a3f213da50de8cad50fdfa3ef5704661d Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 16:06:18 -0400 Subject: [PATCH 33/44] Protect reduced data file initialization to only occur for IO processor. Mixed in with some formatting changes from VS Code. --- Source/DataReducer.cpp | 119 ++++++++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 49 deletions(-) diff --git a/Source/DataReducer.cpp b/Source/DataReducer.cpp index 9085b06a..cb283d11 100644 --- a/Source/DataReducer.cpp +++ b/Source/DataReducer.cpp @@ -22,59 +22,80 @@ void append_0D(HighFive::File& file0D, const std::string& datasetname, const T v } #endif -void -DataReducer::InitializeFiles(){ +void DataReducer::InitializeFiles() +{ + if (ParallelDescriptor::IOProcessor()) + { #ifdef AMREX_USE_HDF5 - using namespace HighFive; - File file0D(filename0D, File::Truncate | File::Create); - - DataSetCreateProps props; - props.add(Chunking(std::vector{1})); - file0D.createDataSet("step", dataspace, create_datatype(), props); - file0D.createDataSet("time(s)", dataspace, create_datatype(), props); - file0D.createDataSet("Ntot(1|ccm)", dataspace, create_datatype(), props); - file0D.createDataSet("Ndiff(1|ccm)", dataspace, create_datatype(), props); - for(int i=0; i(), props); - file0D.createDataSet(std::string("N")+std::to_string(i)+std::to_string(i)+std::string("bar(1|ccm)"), dataspace, create_datatype(), props); - file0D.createDataSet(std::string("Fx")+std::to_string(i)+std::to_string(i)+std::string("(1|ccm)"), dataspace, create_datatype(), props); - file0D.createDataSet(std::string("Fy")+std::to_string(i)+std::to_string(i)+std::string("(1|ccm)"), dataspace, create_datatype(), props); - file0D.createDataSet(std::string("Fz")+std::to_string(i)+std::to_string(i)+std::string("(1|ccm)"), dataspace, create_datatype(), props); - file0D.createDataSet(std::string("Fx")+std::to_string(i)+std::to_string(i)+std::string("bar(1|ccm)"), dataspace, create_datatype(), props); - file0D.createDataSet(std::string("Fy")+std::to_string(i)+std::to_string(i)+std::string("bar(1|ccm)"), dataspace, create_datatype(), props); - file0D.createDataSet(std::string("Fz")+std::to_string(i)+std::to_string(i)+std::string("bar(1|ccm)"), dataspace, create_datatype(), props); - } - file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); - file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); + using namespace HighFive; + File file0D(filename0D, File::Truncate | File::Create); + + DataSetCreateProps props; + props.add(Chunking(std::vector{1})); + file0D.createDataSet("step", dataspace, create_datatype(), props); + file0D.createDataSet("time(s)", dataspace, create_datatype(), props); + file0D.createDataSet("Ntot(1|ccm)", dataspace, create_datatype(), props); + file0D.createDataSet("Ndiff(1|ccm)", dataspace, create_datatype(), props); + for (int i = 0; i < NUM_FLAVORS; i++) + { + file0D.createDataSet(std::string("N") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("N") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fx") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fy") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fz") + std::to_string(i) + std::to_string(i) + std::string("(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fx") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fy") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + file0D.createDataSet(std::string("Fz") + std::to_string(i) + std::to_string(i) + std::string("bar(1|ccm)"), dataspace, create_datatype(), props); + } + file0D.createDataSet("N_offdiag_mag(1|ccm)", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrf", dataspace, create_datatype(), props); + file0D.createDataSet("sumTrHf", dataspace, create_datatype(), props); + #else - - std::ofstream outfile; - outfile.open(filename0D, std::ofstream::out); - int j=0; - j++; outfile << j<<":step\t"; - j++; outfile << j<<":time(s)\t"; - j++; outfile << j<<":Ntot(1|ccm)\t"; - j++; outfile << j<<":Ndiff(1|ccm)\t"; - for(int i=0; i Date: Sun, 2 Apr 2023 16:42:13 -0400 Subject: [PATCH 34/44] cache submodules for faster testing --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bb4b90bb..b473dc29 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -97,7 +97,7 @@ pipeline { deleteDirs: true, disableDeferredWipeout: false, notFailBuild: true, - patterns: [[pattern: 'amrex', type: 'EXCLUDE']] ) // allow amrex to be cached + patterns: [[pattern: 'submodules', type: 'EXCLUDE']] ) // allow submodules to be cached } } From 4388d0cbfd37f5763116a963174d69a668f464f0 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 16:42:47 -0400 Subject: [PATCH 35/44] ignore hdf5 files --- Exec/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/Exec/.gitignore b/Exec/.gitignore index 44236a37..6d1de408 100644 --- a/Exec/.gitignore +++ b/Exec/.gitignore @@ -6,3 +6,4 @@ tmp_build_dir Backtrace.* *.png *.dat +*.h5 From c34908d3d78aca4c579663ad1ad4a4657d464b65 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 16:44:17 -0400 Subject: [PATCH 36/44] Create a three-flavor version of the initial condition script. --- Jenkinsfile | 2 +- .../st4_linear_moment_ffi_3F.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 Scripts/initial_conditions/st4_linear_moment_ffi_3F.py diff --git a/Jenkinsfile b/Jenkinsfile index b473dc29..d7d2a7a3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -82,7 +82,7 @@ pipeline { dir('Exec'){ sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5 GNUmakefile' sh 'make realclean; make generate; make -j' - sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py' + sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' sh 'rm -rf plt*' } diff --git a/Scripts/initial_conditions/st4_linear_moment_ffi_3F.py b/Scripts/initial_conditions/st4_linear_moment_ffi_3F.py new file mode 100644 index 00000000..93bdc96c --- /dev/null +++ b/Scripts/initial_conditions/st4_linear_moment_ffi_3F.py @@ -0,0 +1,45 @@ +import h5py +import numpy as np +import sys +import os +importpath = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(importpath) +sys.path.append(importpath+"/../data_analysis") +from initial_condition_tools import uniform_sphere, write_particles, moment_interpolate_particles, linear_interpolate +import amrex_plot_tools as amrex + +# generation parameters +# MUST MATCH THE INPUTS IN THE EMU INPUT FILE! +NF = 3 +nphi_equator = 16 +theta = 0 +thetabar = 3.14159265359 +phi = 0 +phibar=0 +ndens = 4.891290819e+32 +ndensbar = 4.891290819e+32 +fluxfac = .333333333333333 +fluxfacbar = .333333333333333 +energy_erg = 50 * 1e6*amrex.eV + +# flux factor vectors +fhat = np.array([np.cos(phi) *np.sin(theta ), + np.sin(phi) *np.sin(theta ), + np.cos(theta )]) +fhatbar = np.array([np.cos(phibar)*np.sin(thetabar), + np.sin(phibar)*np.sin(thetabar), + np.cos(thetabar)]) + +nnu = np.zeros((2,NF)) +nnu[0,0] = ndens +nnu[1,0] = ndensbar +nnu[:,1:] = 0 + +fnu = np.zeros((2,NF,3)) +fnu[0,0,:] = ndens * fluxfac * fhat +fnu[1,0,:] = ndensbar * fluxfacbar * fhatbar +fnu[:,1:,:] = 0 + +particles = moment_interpolate_particles(nphi_equator, nnu, fnu, energy_erg, uniform_sphere, linear_interpolate) # [particle, variable] + +write_particles(np.array(particles), NF, "particle_input.dat") From de0f7492b1e239f9ec793e8228ccf166b5e170ad Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 16:50:50 -0400 Subject: [PATCH 37/44] Add understandable error message that works even when the code is not compiled in debug mode to save future headache --- Source/FlavoredNeutrinoContainerInit.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/FlavoredNeutrinoContainerInit.cpp b/Source/FlavoredNeutrinoContainerInit.cpp index 47089138..e47d042a 100644 --- a/Source/FlavoredNeutrinoContainerInit.cpp +++ b/Source/FlavoredNeutrinoContainerInit.cpp @@ -29,6 +29,7 @@ Gpu::ManagedVector > read_particle_data(std::strin ss = std::stringstream(line); int NF_in; ss >> NF_in; + if(NF_in != NUM_FLAVORS) amrex::Print() << "Error: number of flavors in particle data file does not match the number of flavors Emu was compiled for." << std::endl; AMREX_ASSERT(NF_in == NUM_FLAVORS); while(std::getline(file, line)){ From af20c1b1258f115cdee33a48a248740115836003 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 17:25:10 -0400 Subject: [PATCH 38/44] Add babysitting plot scripts to plot reduced data output. Add to Jenkins to make sure they work and the reduced data look good. --- Exec/.gitignore | 1 + Jenkinsfile | 2 + Scripts/babysitting/avgfee.py | 3 +- Scripts/babysitting/avgfee_HDF5.py | 76 ++++++++++++++++++++++++++ Scripts/babysitting/avgfee_gnuplot.plt | 7 +++ 5 files changed, 88 insertions(+), 1 deletion(-) create mode 100755 Scripts/babysitting/avgfee_HDF5.py create mode 100644 Scripts/babysitting/avgfee_gnuplot.plt diff --git a/Exec/.gitignore b/Exec/.gitignore index 6d1de408..7797b25a 100644 --- a/Exec/.gitignore +++ b/Exec/.gitignore @@ -7,3 +7,4 @@ Backtrace.* *.png *.dat *.h5 +*.pdf diff --git a/Jenkinsfile b/Jenkinsfile index d7d2a7a3..84c48283 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,6 +73,7 @@ pipeline { sh 'python ../Scripts/babysitting/avgfee.py' sh 'python ../Scripts/babysitting/power_spectrum.py' sh 'python ../Scripts/data_reduction/convertToHDF5.py' + sh 'gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt' archiveArtifacts artifacts: '*.pdf' sh 'rm -rf plt*' } @@ -84,6 +85,7 @@ pipeline { sh 'make realclean; make generate; make -j' sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py' sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial' + sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py' sh 'rm -rf plt*' } }} diff --git a/Scripts/babysitting/avgfee.py b/Scripts/babysitting/avgfee.py index 4b17d038..58f6c09e 100755 --- a/Scripts/babysitting/avgfee.py +++ b/Scripts/babysitting/avgfee.py @@ -1,4 +1,5 @@ -# Run from /ocean/projects/phy200048p/shared to generate plot showing time evolution of at different dimensionalities +# plots and as a function of time +# without reference to the reduced data file outputs import os os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' diff --git a/Scripts/babysitting/avgfee_HDF5.py b/Scripts/babysitting/avgfee_HDF5.py new file mode 100755 index 00000000..57c2f8eb --- /dev/null +++ b/Scripts/babysitting/avgfee_HDF5.py @@ -0,0 +1,76 @@ +# plots and as a function of time +# assuming the code was compiled with HDF5 and wrote the file reduced0D.h5 + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import numpy as np +import matplotlib.pyplot as plt +import glob +import h5py +import matplotlib as mpl +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator,LogLocator) + + +base=["N","Fx","Fy","Fz"] +diag_flavor=["00","11"]#,"22"] +offdiag_flavor=["01"]#,"02","12"] +re=["Re","Im"] +# real/imag +R=0 +I=1 + + +###################### +# read averaged data # +###################### +avgData = h5py.File("reduced0D.h5","r") +t=np.array(avgData["time(s)"])*1e9 +N00=np.array(avgData["N00(1|ccm)"]) +Noffdiag = np.array(avgData["N_offdiag_mag(1|ccm)"]) +avgData.close() + +################ +# plot options # +################ +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +#mpl.rc('text', usetex=True) +mpl.rcParams['xtick.major.size'] = 7 +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 +mpl.rcParams['axes.linewidth'] = 2 + + +fig, ax = plt.subplots(1,1, figsize=(6,5)) + +############## +# formatting # +############## +ax.axhline(1./3., color="green") +ax.set_ylabel(r"$\langle N\rangle$ (cm$^{-3}$)") +ax.set_xlabel(r"$t\,(10^{-9}\,\mathrm{s})$") +ax.tick_params(axis='both', which='both', direction='in', right=True,top=True) +ax.xaxis.set_minor_locator(AutoMinorLocator()) +ax.yaxis.set_minor_locator(AutoMinorLocator()) +ax.minorticks_on() +ax.grid(which='both') + +############# +# plot data # +############# +ax.plot(t, N00) + +############ +# save pdf # +############ +plt.savefig("avgfee.pdf", bbox_inches="tight") + +# same for f_e\mu +ax.semilogy(t, Noffdiag) +plt.savefig("avgfemu.pdf", bbox_inches="tight") diff --git a/Scripts/babysitting/avgfee_gnuplot.plt b/Scripts/babysitting/avgfee_gnuplot.plt new file mode 100644 index 00000000..234139eb --- /dev/null +++ b/Scripts/babysitting/avgfee_gnuplot.plt @@ -0,0 +1,7 @@ +set term pdf +set output 'avgfee.pdf' +set key autotitle columnhead +set xlabel 'time (s)' +set ylabel 'N00 (cm^-3)' +set yrange [0:*] +plot 'reduced0D.dat' u 2:5 w l \ No newline at end of file From d70197447ed9be1498d6e0de6becd675a2958bb4 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 17:55:04 -0400 Subject: [PATCH 39/44] add gnuplot library --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a581d770..1c2d49fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM nvidia/cuda:11.4.0-devel-ubuntu20.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update -RUN apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 +RUN apt-get install -y python3 python3-pip gfortran build-essential libhdf5-openmpi-dev openmpi-bin pkg-config libopenmpi-dev openmpi-bin libblas-dev liblapack-dev libpnetcdf-dev git python-is-python3 gnuplot RUN pip3 install numpy matplotlib h5py scipy sympy yt ENV USER=jenkins ENV LOGNAME=jenkins From 70c4a4ab8431b0d24d5feed0c8f2dede8eec6960 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 17:55:35 -0400 Subject: [PATCH 40/44] Rename one of the makefiles to be a clear default --- makefiles/{GNUmakefile_ganon => GNUmakefile_default} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename makefiles/{GNUmakefile_ganon => GNUmakefile_default} (57%) diff --git a/makefiles/GNUmakefile_ganon b/makefiles/GNUmakefile_default similarity index 57% rename from makefiles/GNUmakefile_ganon rename to makefiles/GNUmakefile_default index 209b1344..74d83b04 100644 --- a/makefiles/GNUmakefile_ganon +++ b/makefiles/GNUmakefile_default @@ -3,12 +3,12 @@ SHAPE_FACTOR_ORDER = 2 COMP = gnu -DEBUG = TRUE +DEBUG = FALSE -USE_MPI = TRUE -USE_OMP = FALSE -USE_ACC = FALSE -USE_CUDA = FALSE +USE_MPI = FALSE +USE_OMP = FALSE +USE_ACC = FALSE +USE_CUDA = FALSE USE_HDF5 = FALSE EMU_HOME = .. From 5fd7b3b6c86262d64ed180a3b67a2c15130d61db Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 17:55:56 -0400 Subject: [PATCH 41/44] draft update of the readme --- README.md | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d32d487c..103482f9 100644 --- a/README.md +++ b/README.md @@ -31,30 +31,35 @@ If you would like to run Emu, first clone Emu with the AMReX submodule: git clone --recurse-submodules https://github.com/AMReX-Astro/Emu.git ``` -Then change directories to `Emu/Exec`. +Then change directories to `Emu/Exec`. Before each compilation, you must symbolically generate Emu source code for +the number of neutrino flavors you wish to use and specify a few other compile-time settings in a file called `GNUmakefile`. -Before each compilation, you must symbolically generate Emu source code for -the number of neutrino flavors you wish to use. Do this like: +Copy in a default makefile. In this file you can specify the number of neutrino flavors, whether to compile for GPUs, etc. We have set the defaults to 2 neutrino flavors, order 2 PIC shape factors, and compiling for a single CPU. +```cp ../GNUmakefile_default GNUmakefile``` +Compiling occurs in two stages. We first have to generate code according to the number of neutrino flavors. ``` -make generate NUM_FLAVORS=2 +make generate ``` - -Then compile Emu with `make`, e.g.: - +Then we have to compile Emu. ``` -make NUM_FLAVORS=2 +make -j ``` -Emu parameters are set in an input file, and we provide a series of sample -input files for various simulation setups in `Emu/sample_inputs`. - -You can run the MSW setup in Emu by doing: +The initial particle distribution is set by an ASCII particle data file. You can generate the data file with our initial condition scripts. For instance, if we want to simulate a two-beam fast flavor instability, generate the initial conditions using +```python3 ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py``` +You should now see a new file called `particle_input.dat`. +The parameters for the simulation are set in input files. These include information about things like the size of the domain, the number of grid cells, and fundamental neutrino properties. Run the fast flavor test simulation using the particle distribution generated previously using one of the test input files stored in `sample_inputs` ``` ./main3d.gnu.TPROF.MPI.ex inputs_msw_test ``` +We have a number of data reduction, analysis, and visualization scripts in the `Scripts` directory. Generate a PDF file titled `avgfee.pdf` showing the time evolution of the average number density of electron neutrinos using +``` +gnuplot ../Scripts/babysitting/avgfee_gnuplot.plt +``` + # Open Source Development Emu is an open-source code under active development, and we welcome any From 2ea3ae5f74b689e8a42d369d89eaadbd5511c743 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 17:58:05 -0400 Subject: [PATCH 42/44] line spacing --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 103482f9..9708b159 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,9 @@ Then change directories to `Emu/Exec`. Before each compilation, you must symboli the number of neutrino flavors you wish to use and specify a few other compile-time settings in a file called `GNUmakefile`. Copy in a default makefile. In this file you can specify the number of neutrino flavors, whether to compile for GPUs, etc. We have set the defaults to 2 neutrino flavors, order 2 PIC shape factors, and compiling for a single CPU. -```cp ../GNUmakefile_default GNUmakefile``` +``` +cp ../GNUmakefile_default GNUmakefile +``` Compiling occurs in two stages. We first have to generate code according to the number of neutrino flavors. ``` @@ -47,7 +49,9 @@ make -j ``` The initial particle distribution is set by an ASCII particle data file. You can generate the data file with our initial condition scripts. For instance, if we want to simulate a two-beam fast flavor instability, generate the initial conditions using -```python3 ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py``` +``` +python3 ../Scripts/initial_conditions/st3_2beam_fast_flavor_nonzerok.py +``` You should now see a new file called `particle_input.dat`. The parameters for the simulation are set in input files. These include information about things like the size of the domain, the number of grid cells, and fundamental neutrino properties. Run the fast flavor test simulation using the particle distribution generated previously using one of the test input files stored in `sample_inputs` From 5414929812a1bffa697c701f677290d82de80023 Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 18:05:26 -0400 Subject: [PATCH 43/44] A few more tweaks going through the readme and testing --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9708b159..bcabfc9c 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ If you would like to run Emu, first clone Emu with the AMReX submodule: ``` git clone --recurse-submodules https://github.com/AMReX-Astro/Emu.git +git submodule update ``` Then change directories to `Emu/Exec`. Before each compilation, you must symbolically generate Emu source code for @@ -36,7 +37,7 @@ the number of neutrino flavors you wish to use and specify a few other compile-t Copy in a default makefile. In this file you can specify the number of neutrino flavors, whether to compile for GPUs, etc. We have set the defaults to 2 neutrino flavors, order 2 PIC shape factors, and compiling for a single CPU. ``` -cp ../GNUmakefile_default GNUmakefile +cp ../makefiles/GNUmakefile_default GNUmakefile ``` Compiling occurs in two stages. We first have to generate code according to the number of neutrino flavors. @@ -56,7 +57,7 @@ You should now see a new file called `particle_input.dat`. The parameters for the simulation are set in input files. These include information about things like the size of the domain, the number of grid cells, and fundamental neutrino properties. Run the fast flavor test simulation using the particle distribution generated previously using one of the test input files stored in `sample_inputs` ``` -./main3d.gnu.TPROF.MPI.ex inputs_msw_test +./main3d.gnu.TPROF.ex ../sample_inputs/inputs_fast_flavor_nonzerok ``` We have a number of data reduction, analysis, and visualization scripts in the `Scripts` directory. Generate a PDF file titled `avgfee.pdf` showing the time evolution of the average number density of electron neutrinos using From 7b97ed60e418f393a7aa391e70f00de93b278f7a Mon Sep 17 00:00:00 2001 From: Sherwood Richers Date: Sun, 2 Apr 2023 18:19:39 -0400 Subject: [PATCH 44/44] updated change log for next version --- CHANGES.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 6017bdfb..bf3777e0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,16 @@ +# 1.4 + + * Restructured initial conditions to have python scripts generate particle distributions instead of doing so inside the C++ code. This improves the code's flexibility. + + * Add option to output data in HDF5 format. Post-processing scripts only work with the original binary format, since yt only reads the binary format. + + * Add realtime output of scalar quantities to make basic analysis many times faster than with the post-processing scripts. + + * Include all of the basic post-processing scripts with Emu itself to avoid keeping multiple incompatible copies of them. + +# 1.3 + + * Incorporated various feature additions used for _Code Comparison for Fast Flavor Instability Simulations_ (https://doi.org/10.1103/PhysRevD.106.043011) # 1.2