Skip to content

Commit

Permalink
Add more timers and prints
Browse files Browse the repository at this point in the history
  • Loading branch information
@eebasso
eebasso committed Nov 13, 2023
1 parent fb03069 commit 6b9afc9
Show file tree
Hide file tree
Showing 3 changed files with 321 additions and 8 deletions.
2 changes: 2 additions & 0 deletions Src/LinearSolvers/MLMG/AMReX_MLLinOp.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1380,6 +1380,7 @@ MLLinOpT<MF>::make (Vector<Vector<MF> >& mf, IntVect const& ng) const
for (int alev = 0; alev < m_num_amr_levels; ++alev) {
mf[alev].resize(m_num_mg_levels[alev]);
for (int mlev = 0; mlev < m_num_mg_levels[alev]; ++mlev) {
amrex::Print() << "MLLinOpT<MF>::make(mf,ng): call on MLLinOpT<MF>::make(amrlev,mglev,ng)\n";
mf[alev][mlev] = make(alev, mlev, ng);
}
}
Expand All @@ -1389,6 +1390,7 @@ template <typename MF>
MF
MLLinOpT<MF>::make (int amrlev, int mglev, IntVect const& ng) const
{
amrex::Print() << "MLLinOpT<MF>::make(amrlev,mglev,ng) called\n";
return MF(amrex::convert(m_grids[amrlev][mglev], m_ixtype),
m_dmap[amrlev][mglev], getNComp(), ng, MFInfo(),
*m_factory[amrlev][mglev]);
Expand Down
50 changes: 46 additions & 4 deletions Src/LinearSolvers/MLMG/AMReX_MLMG.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ private:
Vector<Vector<MF> > rescor; //!< = res - L(cor)
//! Residual of the correction form

enum timer_types { solve_time=0, iter_time, bottom_time, ntimers };
enum timer_types { solve_time=0, iter_time, bottom_time, oneIter_time, smooth_time, mgVcycle_time, mgFcycle_time, mgVcycle_nonsmooth_time, ntimers };
Vector<double> timer;

RT m_rhsnorm0 = RT(-1.0);
Expand Down Expand Up @@ -346,6 +346,8 @@ MLMGT<MF>::solve (const Vector<AMF*>& a_sol, const Vector<AMF const*>& a_rhs,
{
BL_PROFILE("MLMG::solve()");

verbose = 5;

if constexpr (std::is_same<AMF,MultiFab>()) {
if (checkpoint_file != nullptr) {
checkPoint(a_sol, a_rhs, a_tol_rel, a_tol_abs, checkpoint_file);
Expand Down Expand Up @@ -517,9 +519,16 @@ MLMGT<MF>::solve (const Vector<AMF*>& a_sol, const Vector<AMF const*>& a_rhs,
ParallelContext::CommunicatorSub());
if (ParallelContext::MyProcSub() == 0)
{
amrex::AllPrint() << "MLMG: Timers: Solve = " << timer[solve_time]
amrex::AllPrint() << "MLMG: Timers:"
<< " Solve = " << timer[solve_time]
<< " Iter = " << timer[iter_time]
<< " Bottom = " << timer[bottom_time] << "\n";
<< " Bottom = " << timer[bottom_time]
<< " OneIter = " << timer[oneIter_time]
<< " Smooth = " << timer[smooth_time]
<< " mgVcycle = " << timer[mgVcycle_time]
<< " mgVcycle non-smooth = " << timer[mgVcycle_nonsmooth_time]
<< " mgFcycle = " << timer[mgFcycle_time]
<< "\n";
}
}

Expand Down Expand Up @@ -788,6 +797,7 @@ MLMGT<MF>::compResidual (const Vector<MF*>& a_res, const Vector<MF*>& a_sol,
{
if (sol_is_alias[alev])
{
amrex::Print() << "MLMG::compResidual: linop.make(...) called\n";
sol[alev] = linop.make(alev, 0, ng_sol);
}
sol[alev].LocalCopy(*a_sol[alev], 0, 0, ncomp, IntVect(0));
Expand Down Expand Up @@ -979,6 +989,7 @@ MLMGT<MF>::prepareForSolve (Vector<AMF*> const& a_sol, Vector<AMF const*> const&
}
if (!sol_is_alias[alev]) {
if (!solve_called) {
amrex::Print() << "MLMG::prepareForSolve: 1st linop.make(...) called\n";
sol[alev] = linop.make(alev, 0, ng_sol);
}
sol[alev].LocalCopy(*a_sol[alev], 0, 0, ncomp, IntVect(0));
Expand All @@ -992,6 +1003,7 @@ MLMGT<MF>::prepareForSolve (Vector<AMF*> const& a_sol, Vector<AMF const*> const&
{
if (cf_strategy == CFStrategy::ghostnodes) { ng_rhs = IntVect(linop.getNGrow(alev)); }
if (!solve_called) {
amrex::Print() << "MLMG::prepareForSolve: 2nd linop.make(...) called\n";
rhs[alev] = linop.make(alev, 0, ng_rhs);
}
rhs[alev].LocalCopy(*a_rhs[alev], 0, 0, ncomp, ng_rhs);
Expand Down Expand Up @@ -1028,6 +1040,7 @@ MLMGT<MF>::prepareForSolve (Vector<AMF*> const& a_sol, Vector<AMF const*> const&
IntVect ng = linop.isCellCentered() ? IntVect(0) : IntVect(1);
if (cf_strategy == CFStrategy::ghostnodes) { ng = ng_rhs; }
if (!solve_called) {
amrex::Print() << "MLMG::prepareForSolve: 3rd+4th linop.make(Vector<Vector<MF>,ng) called\n";
linop.make(res, ng);
linop.make(rescor, ng);
}
Expand All @@ -1052,6 +1065,7 @@ MLMGT<MF>::prepareForSolve (Vector<AMF*> const& a_sol, Vector<AMF const*> const&
if (!solve_called) {
IntVect _ng = ng;
if (cf_strategy == CFStrategy::ghostnodes) { _ng=IntVect(linop.getNGrow(alev,mglev)); }
amrex::Print() << "MLMG::prepareForSolve: 5th linop.make(...) called\n";
cor[alev][mglev] = linop.make(alev, mglev, _ng);
}
cor[alev][mglev].setVal(RT(0.0));
Expand All @@ -1068,6 +1082,7 @@ MLMGT<MF>::prepareForSolve (Vector<AMF*> const& a_sol, Vector<AMF const*> const&
if (!solve_called) {
IntVect _ng = ng;
if (cf_strategy == CFStrategy::ghostnodes) { _ng=IntVect(linop.getNGrow(alev,mglev)); }
amrex::Print() << "MLMG::prepareForSolve: 6th linop.make(...) called\n";
cor_hold[alev][mglev] = linop.make(alev, mglev, _ng);
}
cor_hold[alev][mglev].setVal(RT(0.0));
Expand All @@ -1079,6 +1094,7 @@ MLMGT<MF>::prepareForSolve (Vector<AMF*> const& a_sol, Vector<AMF const*> const&
if (!solve_called) {
IntVect _ng = ng;
if (cf_strategy == CFStrategy::ghostnodes) { _ng=IntVect(linop.getNGrow(alev)); }
amrex::Print() << "MLMG::prepareForSolve: 7th linop.make(...) called\n";
cor_hold[alev][0] = linop.make(alev, 0, _ng);
}
cor_hold[alev][0].setVal(RT(0.0));
Expand Down Expand Up @@ -1142,6 +1158,7 @@ template <typename MF>
void MLMGT<MF>::oneIter (int iter)
{
BL_PROFILE("MLMG::oneIter()");
auto oneIter_start_time = amrex::second();

for (int alev = finest_amr_lev; alev > 0; --alev)
{
Expand Down Expand Up @@ -1204,6 +1221,8 @@ void MLMGT<MF>::oneIter (int iter)
}

linop.averageDownAndSync(sol);

timer[oneIter_time] += amrex::second()- oneIter_start_time;
}

template <typename MF>
Expand All @@ -1222,6 +1241,8 @@ void
MLMGT<MF>::mgVcycle (int amrlev, int mglev_top)
{
BL_PROFILE("MLMG::mgVcycle()");
auto mgVcycle_start_time = amrex::second();
auto mgVcycle_nonsmooth_start_time = amrex::second();

const int mglev_bottom = linop.NMGLevels(amrlev) - 1;

Expand All @@ -1239,7 +1260,11 @@ MLMGT<MF>::mgVcycle (int amrlev, int mglev_top)
cor[amrlev][mglev].setVal(RT(0.0));
bool skip_fillboundary = true;
for (int i = 0; i < nu1; ++i) {
timer[mgVcycle_nonsmooth_time] += amrex::second() - mgVcycle_nonsmooth_start_time;
auto smooth_start_time = amrex::second();
linop.smooth(amrlev, mglev, cor[amrlev][mglev], res[amrlev][mglev], skip_fillboundary);
timer[smooth_time] += amrex::second() - smooth_start_time;
mgVcycle_nonsmooth_start_time = amrex::second();
skip_fillboundary = false;
}

Expand Down Expand Up @@ -1286,8 +1311,12 @@ MLMGT<MF>::mgVcycle (int amrlev, int mglev_top)
cor[amrlev][mglev_bottom].setVal(RT(0.0));
bool skip_fillboundary = true;
for (int i = 0; i < nu1; ++i) {
timer[mgVcycle_nonsmooth_time] += amrex::second() - mgVcycle_nonsmooth_start_time;
auto smooth_start_time = amrex::second();
linop.smooth(amrlev, mglev_bottom, cor[amrlev][mglev_bottom],
res[amrlev][mglev_bottom], skip_fillboundary);
timer[smooth_time] += amrex::second() - smooth_start_time;
mgVcycle_nonsmooth_start_time = amrex::second();
skip_fillboundary = false;
}
if (verbose >= 4)
Expand All @@ -1313,7 +1342,11 @@ MLMGT<MF>::mgVcycle (int amrlev, int mglev_top)
<< " UP: Norm before smooth " << norm << "\n";
}
for (int i = 0; i < nu2; ++i) {
timer[mgVcycle_nonsmooth_time] += amrex::second() - mgVcycle_nonsmooth_start_time;
auto smooth_start_time = amrex::second();
linop.smooth(amrlev, mglev, cor[amrlev][mglev], res[amrlev][mglev]);
timer[smooth_time] += amrex::second() - smooth_start_time;
mgVcycle_nonsmooth_start_time = amrex::second();
}

if (cf_strategy == CFStrategy::ghostnodes) { computeResOfCorrection(amrlev, mglev); }
Expand All @@ -1326,6 +1359,8 @@ MLMGT<MF>::mgVcycle (int amrlev, int mglev_top)
<< " UP: Norm after smooth " << norm << "\n";
}
}
timer[mgVcycle_time] += amrex::second() - mgVcycle_start_time;
timer[mgVcycle_nonsmooth_time] += amrex::second() - mgVcycle_nonsmooth_start_time;
}

// FMG cycle on the coarsest AMR level.
Expand All @@ -1336,6 +1371,7 @@ void
MLMGT<MF>::mgFcycle ()
{
BL_PROFILE("MLMG::mgFcycle()");
auto mgFcycle_start_time = amrex::second();

#ifdef AMREX_USE_EB
AMREX_ASSERT(linop.isCellCentered());
Expand Down Expand Up @@ -1368,9 +1404,10 @@ MLMGT<MF>::mgFcycle ()
mgVcycle(amrlev, mglev);
cor[amrlev][mglev].LocalAdd(cor_hold[amrlev][mglev], 0, 0, ncomp, nghost);
}
timer[mgFcycle_time] += amrex::second() - mgFcycle_start_time;
}

// At the true bottom of the coarset AMR level.
// At the true bottom of the coarsest AMR level.
// in : Residual (res) as b
// out : Correction (cor) as x
template <typename MF>
Expand Down Expand Up @@ -1434,7 +1471,9 @@ MLMGT<MF>::actualBottomSolve ()
{
bool skip_fillboundary = true;
for (int i = 0; i < nuf; ++i) {
auto smooth_start_time = amrex::second();
linop.smooth(amrlev, mglev, x, b, skip_fillboundary);
timer[smooth_time] += amrex::second() - smooth_start_time;
skip_fillboundary = false;
}
}
Expand All @@ -1445,6 +1484,7 @@ MLMGT<MF>::actualBottomSolve ()
if (linop.isBottomSingular() && linop.getEnforceSingularSolvable())
{
const IntVect ng = b.nGrowVect();
amrex::Print() << "MLMG::actualBottomSolve: linop.make(...) called\n";
raii_b = linop.make(amrlev, mglev, ng);
raii_b.LocalCopy(b, 0, 0, ncomp, ng);
bottom_b = &raii_b;
Expand Down Expand Up @@ -1508,7 +1548,9 @@ MLMGT<MF>::actualBottomSolve ()
}
const int n = (ret==0) ? nub : nuf;
for (int i = 0; i < n; ++i) {
auto smooth_start_time = amrex::second();
linop.smooth(amrlev, mglev, x, b);
timer[smooth_time] += amrex::second() - smooth_start_time;
}
}
}
Expand Down
Loading

0 comments on commit 6b9afc9

Please sign in to comment.