From 121f9e83d4d8f95e4849fa601fada4b374ef318c Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Mon, 21 Aug 2023 14:40:01 -0400 Subject: [PATCH 01/39] Cleanup of some style issues --- palace/drivers/basesolver.hpp | 2 +- palace/drivers/drivensolver.cpp | 18 ++---- palace/drivers/drivensolver.hpp | 2 +- palace/drivers/eigensolver.cpp | 7 +- palace/drivers/eigensolver.hpp | 2 +- palace/drivers/electrostaticsolver.cpp | 16 ++--- palace/drivers/electrostaticsolver.hpp | 2 +- palace/drivers/magnetostaticsolver.cpp | 16 ++--- palace/drivers/magnetostaticsolver.hpp | 2 +- palace/drivers/transientsolver.cpp | 1 - palace/drivers/transientsolver.hpp | 2 +- palace/fem/coefficient.hpp | 1 - palace/fem/multigrid.hpp | 18 +++--- palace/linalg/errorestimator.cpp | 88 +++++++++++++------------- palace/linalg/fluxprojector.cpp | 11 ++-- palace/linalg/fluxprojector.hpp | 6 +- palace/models/postoperator.cpp | 1 - palace/models/postoperator.hpp | 2 +- palace/utils/errorindicators.cpp | 5 +- palace/utils/errorindicators.hpp | 5 +- 20 files changed, 95 insertions(+), 112 deletions(-) diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index 7a3c69614..7fd554fd5 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -18,7 +18,7 @@ class ParMesh; namespace palace { -struct ErrorIndicators; +class ErrorIndicators; class IoData; class PostOperator; class Timer; diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index dde09da90..a610921ac 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -118,7 +118,6 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator // simply by setting diagonal entries of the system matrix for the corresponding dofs. // Because the Dirichlet BC is always homogenous, no special elimination is required on // the RHS. Assemble the linear system for the initial frequency (so we can call - // KspSolver::SetOperators). Compute everything at the first frequency step. auto K = spaceop.GetStiffnessMatrix(Operator::DIAG_ONE); auto C = spaceop.GetDampingMatrix(Operator::DIAG_ZERO); @@ -147,7 +146,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators indicators(spaceop.GlobalTrueVSize()); const auto error_reducer = ErrorReductionOperator(); - auto update_error_indicators = + auto UpdateErrorIndicators = [&estimator, &indicators, &error_reducer, &postop](const auto &E) { BlockTimer bt(Timer::ESTSOLVE); @@ -182,13 +181,11 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator BlockTimer bt1(Timer::SOLVE); Mpi::Print("\n"); ksp.Mult(RHS, E); - - update_error_indicators(E); + UpdateErrorIndicators(E); // Compute B = -1/(iω) ∇ x E on the true dofs, and set the internal GridFunctions in // PostOperator for all postprocessing operations. BlockTimer bt2(Timer::POSTPRO); - double E_elec = 0.0, E_mag = 0.0; Curl->Mult(E, B); B *= -1.0 / (1i * omega); @@ -214,7 +211,6 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator // Increment frequency. omega += delta_omega; } - SaveMetadata(ksp); return indicators; } @@ -276,7 +272,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // the online stage. ErrorIndicators indicators(spaceop.GlobalTrueVSize()); ErrorReductionOperator error_reducer; - auto update_error_indicators = [&estimator, &indicators, &error_reducer](const auto &E) + auto UpdateErrorIndicators = [&estimator, &indicators, &error_reducer](const auto &E) { BlockTimer bt(Timer::ESTSOLVE); error_reducer(indicators, estimator(E)); @@ -288,11 +284,10 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // of the RomOperator. BlockTimer bt1(Timer::CONSTRUCTPROM); prom.SolveHDM(omega0, E); // Print matrix stats at first HDM solve - update_error_indicators(E); - + UpdateErrorIndicators(E); prom.AddHDMSample(omega0, E); prom.SolveHDM(omega0 + (nstep - step0 - 1) * delta_omega, E); - update_error_indicators(E); + UpdateErrorIndicators(E); prom.AddHDMSample(omega0 + (nstep - step0 - 1) * delta_omega, E); // Greedy procedure for basis construction (offline phase). Basis is initialized with @@ -315,7 +310,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator iter - iter0 + 1, prom.GetReducedDimension(), omega_star * f0, omega_star, max_error); prom.SolveHDM(omega_star, E); - update_error_indicators(E); + UpdateErrorIndicators(E); prom.AddHDMSample(omega_star, E); iter++; } @@ -376,7 +371,6 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator step++; omega += delta_omega; } - return indicators; } diff --git a/palace/drivers/drivensolver.hpp b/palace/drivers/drivensolver.hpp index 97a3d5990..bf54d5007 100644 --- a/palace/drivers/drivensolver.hpp +++ b/palace/drivers/drivensolver.hpp @@ -19,7 +19,7 @@ namespace palace { class CurlFluxErrorEstimator; -struct ErrorIndicators; +class ErrorIndicators; class IoData; class LumpedPortOperator; class PostOperator; diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index d848d8332..727753da9 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -263,7 +263,7 @@ ErrorIndicators EigenSolver::Solve(const std::vectorGetEigenvector(i, E); - Curl->Mult(E, B); B *= -1.0 / (1i * omega); - if (i < iodata.solver.eigenmode.n) { // Only update the error indicator for targetted modes. - update_error_indicators(E); + UpdateErrorIndicators(E); } postop.SetEGridFunction(E); postop.SetBGridFunction(B); - postop.UpdatePorts(spaceop.GetLumpedPortOp(), omega.real()); // Postprocess the mode. diff --git a/palace/drivers/eigensolver.hpp b/palace/drivers/eigensolver.hpp index c53b6a980..fed7c4d40 100644 --- a/palace/drivers/eigensolver.hpp +++ b/palace/drivers/eigensolver.hpp @@ -19,7 +19,7 @@ class ParMesh; namespace palace { -struct ErrorIndicators; +class ErrorIndicators; class IoData; class LumpedPortOperator; class PostOperator; diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index c4290c253..e5287d1f6 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -68,16 +68,15 @@ ElectrostaticSolver::Solve(const std::vector> &me linalg::Norml2(laplaceop.GetComm(), RHS)); // Next terminal. - ++step; + step++; } + // Construct estimator and reducer for error indicators. GradFluxErrorEstimator estimator(iodata, laplaceop.GetMaterialOp(), mesh, laplaceop.GetH1Space()); - - // Evaluate error estimator and reduce over all auto indicators = ErrorIndicators(laplaceop.GlobalTrueVSize()); ErrorReductionOperator error_reducer; - auto update_error_indicators = + auto UpdateErrorIndicators = [&estimator, &indicators, &error_reducer](const auto &V) { BlockTimer bt1(Timer::ESTSOLVE); @@ -85,7 +84,7 @@ ElectrostaticSolver::Solve(const std::vector> &me }; // Compute and reduce the error indicators for each solution. - std::for_each(V.begin(), V.end(), update_error_indicators); + std::for_each(V.begin(), V.end(), UpdateErrorIndicators); // Register the indicator field used to drive the overall adaptation. postop.SetIndicatorGridFunction(indicators.local_error_indicators); @@ -122,7 +121,6 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & // for all postprocessing operations. E = 0.0; Grad->AddMult(V[i], E, -1.0); - postop.SetEGridFunction(E); postop.SetVGridFunction(V[i]); double Ue = postop.GetEFieldEnergy(); @@ -137,13 +135,13 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & // Diagonal: C_ii = 2 U_e(V_i) / V_i². C(i, i) = Cm(i, i) = 2.0 * Ue; - ++i; + i++; } // Off-diagonals: C_ij = U_e(V_i + V_j) / (V_i V_j) - 1/2 (V_i/V_j C_ii + V_j/V_i C_jj). - for (i = 0; i < C.Height(); ++i) + for (i = 0; i < C.Height(); i++) { - for (int j = 0; j < C.Width(); ++j) + for (int j = 0; j < C.Width(); j++) { if (j < i) { diff --git a/palace/drivers/electrostaticsolver.hpp b/palace/drivers/electrostaticsolver.hpp index 550be706d..61cdb3ba2 100644 --- a/palace/drivers/electrostaticsolver.hpp +++ b/palace/drivers/electrostaticsolver.hpp @@ -23,7 +23,7 @@ class ParMesh; namespace palace { -struct ErrorIndicators; +class ErrorIndicators; class IoData; class LaplaceOperator; class PostOperator; diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index aba77563f..b10102e83 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -68,7 +68,7 @@ MagnetostaticSolver::Solve(const std::vector> &me linalg::Norml2(curlcurlop.GetComm(), RHS)); // Next source. - ++step; + step++; } CurlFluxErrorEstimator estimator(iodata, curlcurlop.GetMaterialOp(), mesh, @@ -77,14 +77,14 @@ MagnetostaticSolver::Solve(const std::vector> &me // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators indicators(curlcurlop.GlobalTrueVSize()); ErrorReductionOperator error_reducer; - auto update_error_indicators =[&estimator, &indicators, &error_reducer](const auto &A) + auto UpdateErrorIndicators =[&estimator, &indicators, &error_reducer](const auto &A) { BlockTimer bt(Timer::ESTSOLVE); error_reducer(indicators, estimator(A)); }; // Compute and reduce the error indicators for each solution. - std::for_each(A.begin(), A.end(), update_error_indicators); + std::for_each(A.begin(), A.end(), UpdateErrorIndicators); // Register the indicator field used to drive the overall adaptation. postop.SetIndicatorGridFunction(indicators.local_error_indicators); @@ -141,13 +141,13 @@ void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator // Diagonal: M_ii = 2 U_m(A_i) / I_i². M(i, i) = Mm(i, i) = 2.0 * Um / (Iinc(i) * Iinc(i)); - ++i; + i++; } // Off-diagonals: M_ij = U_m(A_i + A_j) / (I_i I_j) - 1/2 (I_i/I_j M_ii + I_j/I_i M_jj). - for (i = 0; i < M.Height(); ++i) + for (i = 0; i < M.Height(); i++) { - for (int j = 0; j < M.Width(); ++j) + for (int j = 0; j < M.Width(); j++) { if (j < i) { @@ -214,10 +214,10 @@ void MagnetostaticSolver::PostprocessTerminals(const SurfaceCurrentOperator &sur mat(i, j) * scale, table.w, table.p, (idx2 == surf_j_op.rbegin()->first) ? "" : ","); // clang-format on - ++j; + j++; } output.print("\n"); - ++i; + i++; } }; const double H = iodata.DimensionalizeValue(IoData::ValueType::INDUCTANCE, 1.0); diff --git a/palace/drivers/magnetostaticsolver.hpp b/palace/drivers/magnetostaticsolver.hpp index 42c737c64..7c262ad08 100644 --- a/palace/drivers/magnetostaticsolver.hpp +++ b/palace/drivers/magnetostaticsolver.hpp @@ -21,7 +21,7 @@ namespace palace { class CurlCurlOperator; -struct ErrorIndicators; +class ErrorIndicators; class IoData; class PostOperator; class SurfaceCurrentOperator; diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 36184ad86..f92560293 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -125,7 +125,6 @@ TransientSolver::Solve(const std::vector> &mesh) // Increment time step. step++; } - SaveMetadata(timeop.GetLinearSolver()); return ErrorIndicators(spaceop.GlobalTrueVSize()); } diff --git a/palace/drivers/transientsolver.hpp b/palace/drivers/transientsolver.hpp index f770d8969..b155695a4 100644 --- a/palace/drivers/transientsolver.hpp +++ b/palace/drivers/transientsolver.hpp @@ -19,7 +19,7 @@ class ParMesh; namespace palace { -struct ErrorIndicators; +class ErrorIndicators; class IoData; class LumpedPortOperator; class PostOperator; diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index 9b3cd68e6..464c9e536 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -499,7 +499,6 @@ class CurlFluxCoefficient : public mfem::VectorCoefficient const mfem::IntegrationPoint &ip) override { V.SetSize(3); - coef.Eval(muinv, T, ip); X.GetCurl(T, curl); muinv.Mult(curl, V); diff --git a/palace/fem/multigrid.hpp b/palace/fem/multigrid.hpp index 0f7b4c29b..71726f97d 100644 --- a/palace/fem/multigrid.hpp +++ b/palace/fem/multigrid.hpp @@ -136,22 +136,21 @@ template inline mfem::ParFiniteElementSpaceHierarchy ConstructFiniteElementSpaceHierarchy( int mg_max_levels, bool mg_legacy_transfer, int pa_order_threshold, const std::vector> &mesh, - const std::vector> &fecs, int dim = 1, - int ordering = mfem::Ordering::byNODES) + const std::vector> &fecs, int dim = 1) { MFEM_VERIFY(!mesh.empty() && !fecs.empty(), "Empty mesh or FE collection for FE space construction!"); int coarse_mesh_l = std::max(0, static_cast(mesh.size() + fecs.size()) - 1 - mg_max_levels); - auto *fespace = new mfem::ParFiniteElementSpace(mesh[coarse_mesh_l].get(), fecs[0].get(), - dim, ordering); + auto *fespace = + new mfem::ParFiniteElementSpace(mesh[coarse_mesh_l].get(), fecs[0].get(), dim); mfem::ParFiniteElementSpaceHierarchy fespaces(mesh[coarse_mesh_l].get(), fespace, false, true); // h-refinement for (std::size_t l = coarse_mesh_l + 1; l < mesh.size(); l++) { - fespace = new mfem::ParFiniteElementSpace(mesh[l].get(), fecs[0].get(), dim, ordering); + fespace = new mfem::ParFiniteElementSpace(mesh[l].get(), fecs[0].get(), dim); auto *P = new ParOperator( std::make_unique(fespaces.GetFinestFESpace(), *fespace), @@ -162,8 +161,7 @@ inline mfem::ParFiniteElementSpaceHierarchy ConstructFiniteElementSpaceHierarchy // p-refinement for (std::size_t l = 1; l < fecs.size(); l++) { - fespace = - new mfem::ParFiniteElementSpace(mesh.back().get(), fecs[l].get(), dim, ordering); + fespace = new mfem::ParFiniteElementSpace(mesh.back().get(), fecs[l].get(), dim); ParOperator *P; if (!mg_legacy_transfer && mfem::DeviceCanUseCeed()) @@ -194,11 +192,11 @@ mfem::ParFiniteElementSpaceHierarchy ConstructFiniteElementSpaceHierarchy( int mg_max_levels, bool mg_legacy_transfer, int pa_order_threshold, const Container &mesh, const std::vector> &fecs, const mfem::Array &dbc_marker, std::vector> &dbc_tdof_lists, - int dim = 1, int ordering = mfem::Ordering::byNODES) + int dim = 1) { dbc_tdof_lists.clear(); - auto fespaces = ConstructFiniteElementSpaceHierarchy( - mg_max_levels, mg_legacy_transfer, pa_order_threshold, mesh, fecs, dim, ordering); + auto fespaces = ConstructFiniteElementSpaceHierarchy(mg_max_levels, mg_legacy_transfer, + pa_order_threshold, mesh, fecs, dim); for (int l = 0; l < fespaces.GetNumLevels(); l++) { fespaces.GetFESpaceAtLevel(l).GetEssentialTrueDofs(dbc_marker, diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 813a5b122..f91cf7f41 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -22,18 +22,19 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( : mat_op(mat_op), fes(fes), smooth_flux_fecs(ConstructFECollections( iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, - iodata.solver.linear.mg_coarsen_type, false)), // TODO: pc_lor? + iodata.solver.linear.mg_coarsen_type, iodata.solver.linear.pc_mat_lor)), smooth_flux_fes(ConstructFiniteElementSpaceHierarchy( iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), - smooth_projector(smooth_flux_fes, iodata.solver.linear.tol, 200, 0, iodata.solver.pa_order_threshold), + smooth_projector(smooth_flux_fes, iodata.solver.linear.tol, 200, 0, + iodata.solver.pa_order_threshold), coarse_flux_fec(iodata.solver.order, mesh.back()->Dimension(), mfem::BasisType::GaussLobatto), coarse_flux_fes(mesh.back().get(), &coarse_flux_fec, mesh.back()->Dimension()), scalar_mass_matrices(fes.GetNE()), smooth_to_coarse_embed(fes.GetNE()) { mfem::MassIntegrator mass_integrator; - for (int e = 0; e < fes.GetNE(); ++e) + for (int e = 0; e < fes.GetNE(); e++) { // Loop over each element, and save an elemental mass matrix. // Will exploit the fact that vector L2 mass matrix components are independent. @@ -73,8 +74,8 @@ Vector CurlFluxErrorEstimator::operator()(const ComplexVector &v) const ComplexVector(rhs_from_coef(smooth_flux_fes.GetFinestFESpace(), real_coef), rhs_from_coef(smooth_flux_fes.GetFinestFESpace(), imag_coef)); - // Given the RHS vector of non-smooth flux, construct a flux projector and - // perform mass matrix inversion in the appropriate space, giving f = M⁻¹ f̂. + // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass + // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. auto build_flux = [](const FluxProjector &proj, const ComplexVector &flux_coef) { // Use a copy construction to match appropriate size. @@ -84,8 +85,8 @@ Vector CurlFluxErrorEstimator::operator()(const ComplexVector &v) const }; auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); - // Given a complex solution represented with a PetscParVector, build a - // ComplexGridFunction for evaluation. + // Given a complex solution represented with a PetscParVector, build a ComplexGridFunction + // for evaluation. auto build_func = [](const ComplexVector &f, mfem::ParFiniteElementSpace &fes) { mfem::ParComplexGridFunction flux(&fes); @@ -103,13 +104,12 @@ Vector CurlFluxErrorEstimator::operator()(const ComplexVector &v) const coarse_flux_func.real().ProjectCoefficient(real_coef); coarse_flux_func.imag().ProjectCoefficient(imag_coef); - // Loop over elements, embed the smooth flux into the coarse flux space, then - // compute squared integral using a component-wise mass matrix. + // Loop over elements, embed the smooth flux into the coarse flux space, then compute + // squared integral using a component-wise mass matrix. Vector smooth_vec, coarse_vec, sub_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - - for (int e = 0; e < fes.GetNE(); ++e) + for (int e = 0; e < fes.GetNE(); e++) { // real smooth_flux_func.real().GetElementDofValues(e, smooth_vec); @@ -117,14 +117,14 @@ Vector CurlFluxErrorEstimator::operator()(const ComplexVector &v) const const int ndof = coarse_vec.Size() / 3; sub_vec.SetSize(ndof); - for (int c = 0; c < 3; ++c) + for (int c = 0; c < 3; c++) { sub_vec.MakeRef(coarse_vec, c * ndof); normalization += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); } smooth_to_coarse_embed[e].AddMult(smooth_vec, coarse_vec, -1.0); - for (int c = 0; c < 3; ++c) + for (int c = 0; c < 3; c++) { sub_vec.MakeRef(coarse_vec, c * ndof); estimates[e] += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); @@ -134,14 +134,14 @@ Vector CurlFluxErrorEstimator::operator()(const ComplexVector &v) const smooth_flux_func.imag().GetElementDofValues(e, smooth_vec); coarse_flux_func.imag().GetElementDofValues(e, coarse_vec); - for (int c = 0; c < 3; ++c) + for (int c = 0; c < 3; c++) { sub_vec.MakeRef(coarse_vec, c * ndof); normalization += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); } smooth_to_coarse_embed[e].AddMult(smooth_vec, coarse_vec, -1.0); - for (int c = 0; c < 3; ++c) + for (int c = 0; c < 3; c++) { sub_vec.MakeRef(coarse_vec, c * ndof); estimates[e] += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); @@ -152,10 +152,8 @@ Vector CurlFluxErrorEstimator::operator()(const ComplexVector &v) const Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); normalization = std::sqrt(normalization); - std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); - return estimates; } @@ -183,8 +181,8 @@ Vector CurlFluxErrorEstimator::operator()(const Vector &v) const const auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fes.GetFinestFESpace(), coef); - // Given the RHS vector of non-smooth flux, construct a flux projector and - // perform mass matrix inversion in the appropriate space, giving f = M⁻¹ f̂. + // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass + // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. auto build_flux = [](const FluxProjector &proj, const Vector &flux_coef) { // Use a copy construction to match appropriate size. @@ -194,8 +192,8 @@ Vector CurlFluxErrorEstimator::operator()(const Vector &v) const }; auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); - // Given a complex solution represented with a PetscParVector, build a - // ComplexGridFunction for evaluation. + // Given a complex solution represented with a PetscParVector, build a ComplexGridFunction + // for evaluation. auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fes) { mfem::ParGridFunction flux(&fes); @@ -208,26 +206,26 @@ Vector CurlFluxErrorEstimator::operator()(const Vector &v) const mfem::ParGridFunction coarse_flux_func(&coarse_flux_fes); coarse_flux_func.ProjectCoefficient(coef); - // Loop over elements, embed the smooth flux into the coarse flux space, then - // compute squared integral using a component-wise mass matrix. + // Loop over elements, embed the smooth flux into the coarse flux space, then compute + // squared integral using a component-wise mass matrix. Vector smooth_vec, coarse_vec, sub_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - for (int e = 0; e < fes.GetNE(); ++e) + for (int e = 0; e < fes.GetNE(); e++) { smooth_flux_func.GetElementDofValues(e, smooth_vec); coarse_flux_func.GetElementDofValues(e, coarse_vec); const int ndof = coarse_vec.Size() / 3; sub_vec.SetSize(ndof); - for (int c = 0; c < 3; ++c) + for (int c = 0; c < 3; c++) { sub_vec.MakeRef(coarse_vec, c * ndof); normalization += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); } smooth_to_coarse_embed[e].AddMult(smooth_vec, coarse_vec, -1.0); - for (int c = 0; c < 3; ++c) + for (int c = 0; c < 3; c++) { sub_vec.MakeRef(coarse_vec, c * ndof); estimates[e] += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); @@ -240,6 +238,7 @@ Vector CurlFluxErrorEstimator::operator()(const Vector &v) const normalization = std::sqrt(normalization); std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); + return estimates; } @@ -249,13 +248,16 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( mfem::ParFiniteElementSpace &fes) : mat_op(mat_op), fes(fes), smooth_flux_fecs(ConstructFECollections( - iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, + iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_coarsen_type, false)), - smooth_flux_component_fes(ConstructFiniteElementSpaceHierarchy( - iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), - smooth_flux_fes(mesh.back().get(), smooth_flux_fecs.back().get(), mesh.back()->Dimension()), - smooth_projector(smooth_flux_component_fes, iodata.solver.linear.tol, 200, 0, iodata.solver.pa_order_threshold), + smooth_flux_component_fes( + ConstructFiniteElementSpaceHierarchy( + iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, + iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), + smooth_flux_fes(mesh.back().get(), smooth_flux_fecs.back().get(), + mesh.back()->Dimension()), + smooth_projector(smooth_flux_component_fes, iodata.solver.linear.tol, 200, 0, + iodata.solver.pa_order_threshold), coarse_flux_fec(iodata.solver.order, mesh.back()->Dimension(), mfem::BasisType::GaussLobatto), coarse_flux_fes(mesh.back().get(), &coarse_flux_fec, mesh.back()->Dimension()), @@ -263,7 +265,7 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( { mfem::MassIntegrator mass_integrator; - for (int e = 0; e < fes.GetNE(); ++e) + for (int e = 0; e < fes.GetNE(); e++) { // Loop over each element, and save an elemental mass matrix. // Will exploit the fact that vector L2 mass matrix components are independent. @@ -299,9 +301,8 @@ Vector GradFluxErrorEstimator::operator()(const Vector &v) const }; auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fes, coef); - // Given the RHS vector of non-smooth flux, construct a flux projector and - // perform component wise mass matrix inversion in the appropriate space, - // giving fᵢ = M⁻¹ f̂ᵢ. + // Given the RHS vector of non-smooth flux, construct a flux projector and perform + // component wise mass matrix inversion in the appropriate space, giving fᵢ = M⁻¹ f̂ᵢ. auto build_flux = [](const FluxProjector &proj, Vector &rhs) { // Use a copy construction to match appropriate size. @@ -314,7 +315,7 @@ Vector GradFluxErrorEstimator::operator()(const Vector &v) const MFEM_ASSERT(ndof % 3 == 0, "!"); Vector flux_comp, rhs_comp; - for (std::size_t i = 0; i < 3; ++i) + for (int i = 0; i < 3; i++) { flux_comp.MakeRef(flux, i * stride, stride); rhs_comp.MakeRef(rhs, i * stride, stride); @@ -341,7 +342,7 @@ Vector GradFluxErrorEstimator::operator()(const Vector &v) const Vector coarse_vec, smooth_vec, coarse_sub_vec, smooth_sub_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - for (int e = 0; e < fes.GetNE(); ++e) + for (int e = 0; e < fes.GetNE(); e++) { coarse_flux.GetElementDofValues(e, coarse_vec); smooth_flux_func.GetElementDofValues(e, smooth_vec); @@ -355,18 +356,15 @@ Vector GradFluxErrorEstimator::operator()(const Vector &v) const coarse_sub_vec.SetSize(ndof); smooth_sub_vec.SetSize(ndof); - for (int c = 0; c < 3; ++c) + for (int c = 0; c < 3; c++) { coarse_sub_vec.MakeRef(coarse_vec, c * ndof); smooth_sub_vec.MakeRef(smooth_vec, c * ndof); normalization += scalar_mass_matrices[e].InnerProduct(coarse_sub_vec, coarse_sub_vec); - - // Embed - smooth_to_coarse_embed[e].AddMult(smooth_sub_vec, coarse_sub_vec, -1.0); - - // Integrate - estimates[e] += scalar_mass_matrices[e].InnerProduct(coarse_sub_vec, coarse_sub_vec); + smooth_to_coarse_embed[e].AddMult(smooth_sub_vec, coarse_sub_vec, -1.0); // Embed + estimates[e] += scalar_mass_matrices[e].InnerProduct(coarse_sub_vec, + coarse_sub_vec); // Integrate } estimates[e] = std::sqrt(estimates[e]); diff --git a/palace/linalg/fluxprojector.cpp b/palace/linalg/fluxprojector.cpp index 2111fc144..92be72e65 100644 --- a/palace/linalg/fluxprojector.cpp +++ b/palace/linalg/fluxprojector.cpp @@ -17,7 +17,8 @@ namespace palace // Given a finite element space hierarchy, construct a vector of mass matrix // operators corresponding to each level. -std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHierarchy &h, int pa_order_threshold) +std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHierarchy &h, + int pa_order_threshold) { constexpr int skip_zeros = 0; const bool is_scalar_FE_space = @@ -25,7 +26,7 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie // Assemble the bilinear form operator auto M = std::make_unique(h.GetNumLevels()); - for (int l = 0; l < h.GetNumLevels(); ++l) + for (int l = 0; l < h.GetNumLevels(); l++) { auto &h_l = h.GetFESpaceAtLevel(l); auto m = std::make_unique(&h_l); @@ -51,12 +52,14 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie } FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, - double tol, int max_it, int print_level, int pa_order_threshold) + double tol, int max_it, int print_level, + int pa_order_threshold) : M(BuildMassMatrixOperator(smooth_flux_fes, pa_order_threshold)) { // The system matrix for the projection is real and SPD. For the coarse-level AMG solve, // we don't use an exact solve on the coarsest level. - auto amg = std::make_unique>(std::make_unique(1, 1, 0)); + auto amg = + std::make_unique>(std::make_unique(1, 1, 0)); auto gmg = std::make_unique>( std::move(amg), smooth_flux_fes, nullptr, 1, 1, 2, 1.0, 0.0, true, pa_order_threshold); diff --git a/palace/linalg/fluxprojector.hpp b/palace/linalg/fluxprojector.hpp index d3fb43ecc..eccda9fc0 100644 --- a/palace/linalg/fluxprojector.hpp +++ b/palace/linalg/fluxprojector.hpp @@ -33,8 +33,8 @@ class FluxProjector mutable Vector tmp; public: - FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, - double tol = 1e-12, int max_it = 200, int print_level = 1, int pa_order_threshold = 1); + FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, double tol = 1e-12, + int max_it = 200, int print_level = 1, int pa_order_threshold = 1); // Given a vector of dof defining the flux, compute the smooth flux IN PLACE. void Mult(Vector &x) const @@ -53,8 +53,6 @@ class FluxProjector y = x; Mult(y); } - - // TODO: Add in an ArrayMult optimization }; } // namespace palace diff --git a/palace/models/postoperator.cpp b/palace/models/postoperator.cpp index 578805b69..a49fc6978 100644 --- a/palace/models/postoperator.cpp +++ b/palace/models/postoperator.cpp @@ -657,7 +657,6 @@ void PostOperator::WriteFields(int step, double time) const paraview.SetTime(time); paraview_bdr.SetCycle(step); paraview_bdr.SetTime(time); - if (first_save) { mfem::ParMesh &mesh = diff --git a/palace/models/postoperator.hpp b/palace/models/postoperator.hpp index 8a10ed064..943a1dfa2 100644 --- a/palace/models/postoperator.hpp +++ b/palace/models/postoperator.hpp @@ -21,7 +21,7 @@ namespace palace { class CurlCurlOperator; -struct ErrorIndicators; +class ErrorIndicators; class IoData; class LaplaceOperator; class LumpedPortOperator; diff --git a/palace/utils/errorindicators.cpp b/palace/utils/errorindicators.cpp index 67ba34631..ca7eec892 100644 --- a/palace/utils/errorindicators.cpp +++ b/palace/utils/errorindicators.cpp @@ -11,8 +11,7 @@ namespace palace void ErrorReductionOperator::operator()(ErrorIndicators &ebar, const Vector &ind, double p) const { - - // Compute the global indicator across all processors + // Compute the global indicator across all processors. auto comm = Mpi::World(); double candidate_global_error_indicator = std::transform_reduce(ind.begin(), ind.end(), 0.0, std::plus(), @@ -45,7 +44,7 @@ void ErrorReductionOperator::operator()(ErrorIndicators &ebar, const Vector &ind } // Another sample has been added, increment for the running average lambda. - ++n; + n++; } } // namespace palace diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index ca8782172..b6d551f25 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -10,9 +10,10 @@ namespace palace { // Storage for error estimation results from the solve. Required in the AMR loop. An error -// indicator is non-negative, whilst an error estimate is signed. -struct ErrorIndicators +// indicator is non-negative, while an error estimate is signed. +class ErrorIndicators { +public: ErrorIndicators(int ndof) : ndof(ndof) {} ErrorIndicators() = delete; ErrorIndicators(const ErrorIndicators &) = default; From bf7349404f1051b13118eef1224f23682b2451c3 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Mon, 2 Oct 2023 14:35:01 -0400 Subject: [PATCH 02/39] Fixes to all the driver classes to better call the estimators - Change the estimator class to report the array and the normalization - Change the driver calls to better record timings, and to better output the computed fields --- palace/drivers/basesolver.cpp | 63 +++++++++++++++++++++ palace/drivers/basesolver.hpp | 10 +++- palace/drivers/drivensolver.cpp | 75 +++++++++++++++---------- palace/drivers/drivensolver.hpp | 3 +- palace/drivers/eigensolver.cpp | 33 +++++++---- palace/drivers/eigensolver.hpp | 3 +- palace/drivers/electrostaticsolver.cpp | 49 +++++++++------- palace/drivers/electrostaticsolver.hpp | 9 ++- palace/drivers/magnetostaticsolver.cpp | 50 ++++++++++------- palace/drivers/magnetostaticsolver.hpp | 9 ++- palace/drivers/transientsolver.cpp | 31 +++++++++- palace/drivers/transientsolver.hpp | 3 +- palace/linalg/errorestimator.cpp | 26 ++++----- palace/linalg/errorestimator.hpp | 15 ++--- palace/linalg/fluxprojector.cpp | 4 +- palace/main.cpp | 2 +- palace/utils/errorindicators.cpp | 78 ++++++++++++++++++-------- palace/utils/errorindicators.hpp | 64 +++++++++++++++++---- palace/utils/timer.hpp | 4 +- 19 files changed, 383 insertions(+), 148 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index 71b4ab7aa..e9b0c37d0 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -566,6 +566,69 @@ void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double Mpi::Barrier(); } +void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, double time, + const ErrorIndicators &indicators) const +{ + if (post_dir.length() == 0) + { + return; + } + + // Write the indicator statistics + if (root) + { + std::string path = post_dir + "error-indicators.csv"; + auto output = OutputFile(path, (step > 0)); + if (step == 0) + { + // clang-format off + output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", + name, table.w1, + "Global Rel.", table.w, + "Element Min.", table.w, + "Element Max.", table.w, + "Element Mean", table.w, + "Normalization", table.w); + // clang-format on + } + + // clang-format off + output.print("{:{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", + time, table.w1, table.p1, + indicators.GetGlobalErrorIndicator(), table.w, table.p, + indicators.GetMinErrorIndicator(), table.w, table.p, + indicators.GetMaxErrorIndicator(), table.w, table.p, + indicators.GetMeanErrorIndicator(), table.w, table.p, + indicators.GetNormalization(), table.w, table.p); + // clang-format on + } +} + +void BaseSolver::PostprocessErrorIndicators(const std::string &name, + const ErrorIndicators &indicators) const +{ + if (post_dir.length() == 0) + { + return; + } + + // Write the indicator statistics + if (root) + { + std::string path = post_dir + "error-indicators.csv"; + auto output = OutputFile(path, true); + // clang-format off + output.print("{:>{}s},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", + name, table.w1, + indicators.GetGlobalErrorIndicator(), table.w, table.p, + indicators.GetMinErrorIndicator(), table.w, table.p, + indicators.GetMaxErrorIndicator(), table.w, table.p, + indicators.GetMeanErrorIndicator(), table.w, table.p, + indicators.GetNormalization(), table.w, table.p); + // clang-format on + } +} + template void BaseSolver::SaveMetadata(const KspSolver &) const; template void BaseSolver::SaveMetadata(const ComplexKspSolver &) const; diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index 7fd554fd5..ed415ca89 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -67,13 +67,21 @@ class BaseSolver double time) const; void PostprocessFields(const PostOperator &postop, int step, double time) const; + // Postprocess granular error indicator to file. + void PostprocessErrorIndicators(const std::string &name, int step, double time, + const ErrorIndicators &indicators) const; + // Append combined error indicator to file. + void PostprocessErrorIndicators(const std::string &name, + const ErrorIndicators &indicators) const; + public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, const char *git_tag = nullptr); virtual ~BaseSolver() = default; // Performs a solve using the mesh sequence, and report error indicators. - virtual ErrorIndicators Solve(const std::vector> &mesh) const = 0; + virtual ErrorIndicators + Solve(const std::vector> &mesh) const = 0; // These methods write different simulation metadata to a JSON file in post_dir. void SaveMetadata(const mfem::ParFiniteElementSpaceHierarchy &fespaces) const; diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index a610921ac..b5bed2ae9 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -26,7 +26,8 @@ namespace palace using namespace std::complex_literals; -ErrorIndicators DrivenSolver::Solve(const std::vector> &mesh) const +ErrorIndicators +DrivenSolver::Solve(const std::vector> &mesh) const { // Set up the spatial discretization and frequency sweep. BlockTimer bt0(Timer::CONSTRUCT); @@ -95,18 +96,18 @@ ErrorIndicators DrivenSolver::Solve(const std::vector 0, "No excitation specified for driven simulation!"); } Mpi::Print("\n"); - - auto estimator = [&](){ - BlockTimer bt_estconstruct(Timer::ESTCONSTRUCT); + // Initalize the error estimation operator. + auto estimator = [&]() + { + BlockTimer bt(Timer::ESTCONSTRUCT); return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), mesh, - spaceop.GetNDSpace()); + spaceop.GetNDSpace()); }(); // Main frequency sweep loop. - return adaptive ? SweepAdaptive(spaceop, postop, estimator, nstep, step0, omega0, - delta_omega) - : SweepUniform(spaceop, postop, estimator, nstep, step0, omega0, - delta_omega); + return adaptive + ? SweepAdaptive(spaceop, postop, estimator, nstep, step0, omega0, delta_omega) + : SweepUniform(spaceop, postop, estimator, nstep, step0, omega0, delta_omega); } ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &postop, @@ -144,15 +145,20 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator B = 0.0; // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators indicators(spaceop.GlobalTrueVSize()); - const auto error_reducer = ErrorReductionOperator(); - auto UpdateErrorIndicators = - [&estimator, &indicators, &error_reducer, &postop](const auto &E) + ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); + const ErrorReductionOperator ErrorReducer; + auto UpdateErrorIndicators = [this, &estimator, &indicators, &ErrorReducer, + &postop](const auto &E, int step, double f) { - BlockTimer bt(Timer::ESTSOLVE); - auto ind = estimator(E); - postop.SetIndicatorGridFunction(ind); - error_reducer(indicators, std::move(ind)); + BlockTimer bt0(Timer::ESTSOLVE); + auto estimate = estimator(E); + BlockTimer bt1(Timer::POSTPRO); + // Write the indicator for this mode. + postop.SetIndicatorGridFunction(estimate.indicators); + PostprocessErrorIndicators( + "f (GHz)", step, f, + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorReducer(indicators, std::move(estimate)); }; // Main frequency sweep loop. @@ -181,7 +187,10 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator BlockTimer bt1(Timer::SOLVE); Mpi::Print("\n"); ksp.Mult(RHS, E); - UpdateErrorIndicators(E); + + // Compute the error indicators, and post process the indicator field. + UpdateErrorIndicators(E, step, + iodata.DimensionalizeValue(IoData::ValueType::FREQUENCY, omega)); // Compute B = -1/(iω) ∇ x E on the true dofs, and set the internal GridFunctions in // PostOperator for all postprocessing operations. @@ -212,6 +221,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator omega += delta_omega; } SaveMetadata(ksp); + PostprocessErrorIndicators("Mean", indicators); return indicators; } @@ -221,7 +231,6 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator double delta_omega) const { // Configure default parameters if not specified. - BlockTimer bt0(Timer::CONSTRUCT); double offline_tol = iodata.solver.driven.adaptive_tol; int nmax = iodata.solver.driven.adaptive_nmax; int ncand = iodata.solver.driven.adaptive_ncand; @@ -268,14 +277,21 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator prom.Initialize(omega0, delta_omega, nstep - step0, nmax); spaceop.GetWavePortOp().SetSuppressOutput(true); // Suppress wave port output for offline - // The error indicators will be calculated for each HDM sample rather than for + // The error indicators will be calculated for each HDM sample rather than for // the online stage. - ErrorIndicators indicators(spaceop.GlobalTrueVSize()); - ErrorReductionOperator error_reducer; - auto UpdateErrorIndicators = [&estimator, &indicators, &error_reducer](const auto &E) + ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); + ErrorReductionOperator ErrorReducer; + auto UpdateErrorIndicators = [this, &estimator, &indicators, + &ErrorReducer](const auto &E, int step, double frequency) { - BlockTimer bt(Timer::ESTSOLVE); - error_reducer(indicators, estimator(E)); + BlockTimer bt0(Timer::ESTSOLVE); + auto estimate = estimator(E); + BlockTimer bt1(Timer::POSTPRO); + // Write the indicator for this mode. + PostprocessErrorIndicators( + "f (GHz)", step, frequency, + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorReducer(indicators, std::move(estimate)); }; // Initialize the basis with samples from the top and bottom of the frequency @@ -284,10 +300,10 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // of the RomOperator. BlockTimer bt1(Timer::CONSTRUCTPROM); prom.SolveHDM(omega0, E); // Print matrix stats at first HDM solve - UpdateErrorIndicators(E); + UpdateErrorIndicators(E, 0, omega0 * f0); prom.AddHDMSample(omega0, E); prom.SolveHDM(omega0 + (nstep - step0 - 1) * delta_omega, E); - UpdateErrorIndicators(E); + UpdateErrorIndicators(E, 1, (omega0 + (nstep - step0 - 1) * delta_omega) * f0); prom.AddHDMSample(omega0 + (nstep - step0 - 1) * delta_omega, E); // Greedy procedure for basis construction (offline phase). Basis is initialized with @@ -310,7 +326,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator iter - iter0 + 1, prom.GetReducedDimension(), omega_star * f0, omega_star, max_error); prom.SolveHDM(omega_star, E); - UpdateErrorIndicators(E); + UpdateErrorIndicators(E, iter - iter0 + 1, omega_star * f0); prom.AddHDMSample(omega_star, E); iter++; } @@ -322,7 +338,8 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator SaveMetadata(prom.GetLinearSolver()); // Set the indicator field to the combined field for postprocessing. - postop.SetIndicatorGridFunction(indicators.local_error_indicators); + PostprocessErrorIndicators("Mean", indicators); // Report the mean value + postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); // Main fast frequency sweep loop (online phase). BlockTimer bt2(Timer::CONSTRUCT); diff --git a/palace/drivers/drivensolver.hpp b/palace/drivers/drivensolver.hpp index bf54d5007..7e5a92a5c 100644 --- a/palace/drivers/drivensolver.hpp +++ b/palace/drivers/drivensolver.hpp @@ -62,7 +62,8 @@ class DrivenSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators Solve(const std::vector> &mesh) const final; + ErrorIndicators + Solve(const std::vector> &mesh) const final; }; } // namespace palace diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 727753da9..d1923d4bc 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -24,7 +24,8 @@ namespace palace using namespace std::complex_literals; -ErrorIndicators EigenSolver::Solve(const std::vector> &mesh) const +ErrorIndicators +EigenSolver::Solve(const std::vector> &mesh) const { // Construct and extract the system matrices defining the eigenvalue problem. The diagonal // values for the mass matrix PEC dof shift the Dirichlet eigenvalues out of the @@ -251,8 +252,12 @@ ErrorIndicators EigenSolver::Solve(const std::vectorSetLinearSolver(*ksp); // TODO: Add a block timer for estimate construction - CurlFluxErrorEstimator estimator(iodata, spaceop.GetMaterialOp(), mesh, - spaceop.GetNDSpace()); + auto estimator = [&]() + { + BlockTimer bt(Timer::ESTCONSTRUCT); + return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), mesh, + spaceop.GetNDSpace()); + }(); // Eigenvalue problem solve. BlockTimer bt1(Timer::SOLVE); @@ -261,15 +266,20 @@ ErrorIndicators EigenSolver::Solve(const std::vector> &mesh) const final; + ErrorIndicators + Solve(const std::vector> &mesh) const final; }; } // namespace palace diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index e5287d1f6..8a834b490 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -72,32 +72,23 @@ ElectrostaticSolver::Solve(const std::vector> &me } // Construct estimator and reducer for error indicators. - GradFluxErrorEstimator estimator(iodata, laplaceop.GetMaterialOp(), mesh, - laplaceop.GetH1Space()); - auto indicators = ErrorIndicators(laplaceop.GlobalTrueVSize()); - ErrorReductionOperator error_reducer; - auto UpdateErrorIndicators = - [&estimator, &indicators, &error_reducer](const auto &V) + auto estimator = [&]() { - BlockTimer bt1(Timer::ESTSOLVE); - error_reducer(indicators, estimator(V)); - }; - - // Compute and reduce the error indicators for each solution. - std::for_each(V.begin(), V.end(), UpdateErrorIndicators); - - // Register the indicator field used to drive the overall adaptation. - postop.SetIndicatorGridFunction(indicators.local_error_indicators); + BlockTimer bt(Timer::ESTCONSTRUCT); + return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), mesh, + laplaceop.GetH1Space()); + }(); // Postprocess the capacitance matrix from the computed field solutions. BlockTimer bt1(Timer::POSTPRO); SaveMetadata(ksp); - Postprocess(laplaceop, postop, V); - return indicators; + return Postprocess(laplaceop, postop, estimator, V); } -void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, - const std::vector &V) const +ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, + PostOperator &postop, + const GradFluxErrorEstimator &estimator, + const std::vector &V) const { // Postprocess the Maxwell capacitance matrix. See p. 97 of the COMSOL AC/DC Module manual // for the associated formulas based on the electric field energy based on a unit voltage @@ -105,11 +96,28 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & // charges from the prescribed voltage to get C directly as: // Q_i = ∫ ρ dV = ∫ ∇ ⋅ (ε E) dV = ∫ (ε E) ⋅ n dS // and C_ij = Q_i/V_j. The energy formulation avoids having to locally integrate E = -∇V. + // Additionally compute error estimates for each terminal. auto Grad = laplaceop.GetGradMatrix(); const std::map> &terminal_sources = laplaceop.GetSources(); int nstep = static_cast(terminal_sources.size()); mfem::DenseMatrix C(nstep), Cm(nstep); Vector E(Grad->Height()), Vij(Grad->Width()); + ErrorIndicators indicators(laplaceop.GlobalTrueVSize(), laplaceop.GetComm()); + const ErrorReductionOperator ErrorReducer; + auto UpdateErrorIndicators = [this, &estimator, &indicators, &ErrorReducer, + &postop](const auto &V, int i, double idx) + { + BlockTimer bt0(Timer::ESTSOLVE); + auto estimate = estimator(V); + ErrorReducer(indicators, estimate); + BlockTimer bt1(Timer::POSTPRO); + // Write the indicator for this mode. + postop.SetIndicatorGridFunction(estimate.indicators); + PostprocessErrorIndicators( + "i", i, idx, + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorReducer(indicators, std::move(estimate)); + }; if (iodata.solver.electrostatic.n_post > 0) { Mpi::Print("\n"); @@ -127,6 +135,7 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & PostprocessDomains(postop, "i", i, idx, Ue, 0.0, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, Ue, 0.0, 1.0, 0.0); PostprocessProbes(postop, "i", i, idx); + UpdateErrorIndicators(V[i], i, idx); if (i < iodata.solver.electrostatic.n_post) { PostprocessFields(postop, i, idx); @@ -166,6 +175,8 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & mfem::DenseMatrix Cinv(C); Cinv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(terminal_sources, C, Cinv, Cm); + PostprocessErrorIndicators("Mean", indicators); + return indicators; } void ElectrostaticSolver::PostprocessTerminals( diff --git a/palace/drivers/electrostaticsolver.hpp b/palace/drivers/electrostaticsolver.hpp index 61cdb3ba2..5891aa293 100644 --- a/palace/drivers/electrostaticsolver.hpp +++ b/palace/drivers/electrostaticsolver.hpp @@ -24,6 +24,7 @@ namespace palace { class ErrorIndicators; +class GradFluxErrorEstimator; class IoData; class LaplaceOperator; class PostOperator; @@ -35,8 +36,9 @@ class Timer; class ElectrostaticSolver : public BaseSolver { private: - void Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, - const std::vector &V) const; + ErrorIndicators Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, + const GradFluxErrorEstimator &estimator, + const std::vector &V) const; void PostprocessTerminals(const std::map> &terminal_sources, const mfem::DenseMatrix &C, const mfem::DenseMatrix &Cinv, @@ -45,7 +47,8 @@ class ElectrostaticSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators Solve(const std::vector> &mesh) const final; + ErrorIndicators + Solve(const std::vector> &mesh) const final; }; } // namespace palace diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index b10102e83..1d65f920a 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -71,33 +71,23 @@ MagnetostaticSolver::Solve(const std::vector> &me step++; } - CurlFluxErrorEstimator estimator(iodata, curlcurlop.GetMaterialOp(), mesh, - curlcurlop.GetNDSpace()); - - // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators indicators(curlcurlop.GlobalTrueVSize()); - ErrorReductionOperator error_reducer; - auto UpdateErrorIndicators =[&estimator, &indicators, &error_reducer](const auto &A) + auto estimator = [&]() { - BlockTimer bt(Timer::ESTSOLVE); - error_reducer(indicators, estimator(A)); - }; - - // Compute and reduce the error indicators for each solution. - std::for_each(A.begin(), A.end(), UpdateErrorIndicators); - - // Register the indicator field used to drive the overall adaptation. - postop.SetIndicatorGridFunction(indicators.local_error_indicators); + BlockTimer bt(Timer::ESTCONSTRUCT); + return CurlFluxErrorEstimator(iodata, curlcurlop.GetMaterialOp(), mesh, + curlcurlop.GetNDSpace()); + }(); // Postprocess the capacitance matrix from the computed field solutions. BlockTimer bt1(Timer::POSTPRO); SaveMetadata(ksp); - Postprocess(curlcurlop, postop, A); - return indicators; + return Postprocess(curlcurlop, postop, estimator, A); } -void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, - const std::vector &A) const +ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, + PostOperator &postop, + const CurlFluxErrorEstimator &estimator, + const std::vector &A) const { // Postprocess the Maxwell inductance matrix. See p. 97 of the COMSOL AC/DC Module manual // for the associated formulas based on the magnetic field energy based on a current @@ -112,6 +102,23 @@ void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator mfem::DenseMatrix M(nstep), Mm(nstep); Vector B(Curl->Height()), Aij(Curl->Width()); Vector Iinc(nstep); + + // Initialize structures for storing and reducing the results of error estimation. + ErrorIndicators indicators(curlcurlop.GlobalTrueVSize(), curlcurlop.GetComm()); + ErrorReductionOperator ErrorReducer; + auto UpdateErrorIndicators = [this, &estimator, &indicators, &ErrorReducer, + &postop](const auto &A, int i, double idx) + { + BlockTimer bt0(Timer::ESTSOLVE); + auto estimate = estimator(A); + BlockTimer bt1(Timer::POSTPRO); + // Write the indicator for this mode. + postop.SetIndicatorGridFunction(estimate.indicators); + PostprocessErrorIndicators( + "i", i, idx, + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorReducer(indicators, std::move(estimate)); + }; if (iodata.solver.magnetostatic.n_post > 0) { Mpi::Print("\n"); @@ -133,6 +140,7 @@ void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator PostprocessDomains(postop, "i", i, idx, 0.0, Um, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, 0.0, Um, 0.0, Iinc(i)); PostprocessProbes(postop, "i", i, idx); + UpdateErrorIndicators(A[i], i, idx); if (i < iodata.solver.magnetostatic.n_post) { PostprocessFields(postop, i, idx); @@ -172,6 +180,8 @@ void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator mfem::DenseMatrix Minv(M); Minv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(surf_j_op, M, Minv, Mm); + PostprocessErrorIndicators("Mean", indicators); + return indicators; } void MagnetostaticSolver::PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, diff --git a/palace/drivers/magnetostaticsolver.hpp b/palace/drivers/magnetostaticsolver.hpp index 7c262ad08..b224c2538 100644 --- a/palace/drivers/magnetostaticsolver.hpp +++ b/palace/drivers/magnetostaticsolver.hpp @@ -21,6 +21,7 @@ namespace palace { class CurlCurlOperator; +class CurlFluxErrorEstimator; class ErrorIndicators; class IoData; class PostOperator; @@ -33,8 +34,9 @@ class Timer; class MagnetostaticSolver : public BaseSolver { private: - void Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, - const std::vector &A) const; + ErrorIndicators Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, + const CurlFluxErrorEstimator &estimator, + const std::vector &A) const; void PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, const mfem::DenseMatrix &M, const mfem::DenseMatrix &Minv, @@ -43,7 +45,8 @@ class MagnetostaticSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators Solve(const std::vector> &mesh) const final; + ErrorIndicators + Solve(const std::vector> &mesh) const final; }; } // namespace palace diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index f92560293..ccbb92005 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -77,6 +77,28 @@ TransientSolver::Solve(const std::vector> &mesh) } Mpi::Print("\n"); + // Initialize structures for storing and reducing the results of error estimation. + auto estimator = [&]() + { + BlockTimer bt(Timer::ESTCONSTRUCT); + return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), mesh, + spaceop.GetNDSpace()); + }(); + ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); + ErrorReductionOperator ErrorReducer; + auto UpdateErrorIndicators = [this, &estimator, &indicators, &ErrorReducer, + &postop](const auto &E, int step, double time) + { + BlockTimer bt0(Timer::ESTSOLVE); + auto estimate = estimator(E); + BlockTimer bt1(Timer::POSTPRO); + postop.SetIndicatorGridFunction(estimate.indicators); + PostprocessErrorIndicators( + "t (ns)", step, time, + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorReducer(indicators, std::move(estimate)); + }; + // Main time integration loop. int step = 0; double t = -delta_t; @@ -118,6 +140,12 @@ TransientSolver::Solve(const std::vector> &mesh) E_elec + E_mag); } + if (step > 0) + { + // Calculate and record the error indicators. + UpdateErrorIndicators(E, step, t); + } + // Postprocess port voltages/currents and optionally write solution to disk. Postprocess(postop, spaceop.GetLumpedPortOp(), spaceop.GetSurfaceCurrentOp(), step, t, J_coef(t), E_elec, E_mag, !iodata.solver.transient.only_port_post); @@ -126,7 +154,8 @@ TransientSolver::Solve(const std::vector> &mesh) step++; } SaveMetadata(timeop.GetLinearSolver()); - return ErrorIndicators(spaceop.GlobalTrueVSize()); + PostprocessErrorIndicators("Mean", step, t, indicators); + return indicators; } std::function TransientSolver::GetTimeExcitation(bool dot) const { diff --git a/palace/drivers/transientsolver.hpp b/palace/drivers/transientsolver.hpp index b155695a4..85dba778d 100644 --- a/palace/drivers/transientsolver.hpp +++ b/palace/drivers/transientsolver.hpp @@ -50,7 +50,8 @@ class TransientSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators Solve(const std::vector> &mesh) const override; + ErrorIndicators + Solve(const std::vector> &mesh) const override; }; } // namespace palace diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index f91cf7f41..c98f80a44 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -47,7 +47,7 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( } } -Vector CurlFluxErrorEstimator::operator()(const ComplexVector &v) const +IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVector &v) const { mfem::ParComplexGridFunction field(&fes); field.real().SetFromTrueDofs(v.Real()); @@ -154,10 +154,10 @@ Vector CurlFluxErrorEstimator::operator()(const ComplexVector &v) const normalization = std::sqrt(normalization); std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); - return estimates; + return {estimates, normalization}; } -Vector CurlFluxErrorEstimator::operator()(const Vector &v) const +IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v) const { mfem::ParGridFunction field(&fes); field.SetFromTrueDofs(v); @@ -238,8 +238,7 @@ Vector CurlFluxErrorEstimator::operator()(const Vector &v) const normalization = std::sqrt(normalization); std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); - - return estimates; + return {estimates, normalization}; } GradFluxErrorEstimator::GradFluxErrorEstimator( @@ -250,10 +249,9 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( smooth_flux_fecs(ConstructFECollections( iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_coarsen_type, false)), - smooth_flux_component_fes( - ConstructFiniteElementSpaceHierarchy( - iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), + smooth_flux_component_fes(ConstructFiniteElementSpaceHierarchy( + iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, + iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), smooth_flux_fes(mesh.back().get(), smooth_flux_fecs.back().get(), mesh.back()->Dimension()), smooth_projector(smooth_flux_component_fes, iodata.solver.linear.tol, 200, 0, @@ -278,7 +276,7 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( } } -Vector GradFluxErrorEstimator::operator()(const Vector &v) const +IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v) const { mfem::ParGridFunction field(&fes); field.SetFromTrueDofs(v); @@ -369,13 +367,9 @@ Vector GradFluxErrorEstimator::operator()(const Vector &v) const estimates[e] = std::sqrt(estimates[e]); } - Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); normalization = std::sqrt(normalization); - std::for_each(estimates.begin(), estimates.end(), - [&normalization](auto &x) { x /= normalization; }); - if constexpr (false) { // Debugging branch generates some intermediate fields for paraview. @@ -394,6 +388,8 @@ Vector GradFluxErrorEstimator::operator()(const Vector &v) const paraview.Save(); } - return estimates; + std::for_each(estimates.begin(), estimates.end(), + [&normalization](auto &x) { x /= normalization; }); + return {estimates, normalization}; } } // namespace palace diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 4c373280a..0f7d6286b 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -7,6 +7,7 @@ #include #include "linalg/fluxprojector.hpp" +#include "utils/errorindicators.hpp" namespace palace { @@ -21,11 +22,11 @@ class PetscParVector; // Class used for computing curl flux error estimate, // i.e. || μ⁻¹∇ × V - F ||_K -// where F denotes a smooth reconstruction of μ⁻¹∇ × V +// where F denotes a smooth reconstruction of μ⁻¹∇ × V. class CurlFluxErrorEstimator { const MaterialOperator &mat_op; - // The finite element space used to represent V + // The finite element space used to represent V. mfem::ParFiniteElementSpace &fes; std::vector> smooth_flux_fecs; @@ -45,19 +46,19 @@ class CurlFluxErrorEstimator mfem::ParFiniteElementSpace &fes); // Compute elemental error indicators given a complex vector of true DOF. - Vector operator()(const ComplexVector &v) const; + IndicatorsAndNormalization operator()(const ComplexVector &v) const; // Compute elemental error indicators given a vector of true DOF. - Vector operator()(const Vector &v) const; + IndicatorsAndNormalization operator()(const Vector &v) const; }; // Class used for computing grad flux error estimate, // i.e. || ϵ ∇ ϕ - F ||_K -// where F denotes a smooth reconstruction of ϵ ∇ ϕ +// where F denotes a smooth reconstruction of ϵ ∇ ϕ. class GradFluxErrorEstimator { const MaterialOperator &mat_op; - // The finite element space used to represent ϕ + // The finite element space used to represent ϕ. mfem::ParFiniteElementSpace &fes; // Collections and spaces for the smooth flux. Note the hierarchy uses the @@ -81,7 +82,7 @@ class GradFluxErrorEstimator mfem::ParFiniteElementSpace &fes); // Compute elemental error indicators given a vector of true DOF. - Vector operator()(const Vector &v) const; + IndicatorsAndNormalization operator()(const Vector &v) const; }; } // namespace palace diff --git a/palace/linalg/fluxprojector.cpp b/palace/linalg/fluxprojector.cpp index 92be72e65..e688cb529 100644 --- a/palace/linalg/fluxprojector.cpp +++ b/palace/linalg/fluxprojector.cpp @@ -43,7 +43,9 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie } auto M_l = std::make_unique( - fem::AssembleOperator(std::move(m), true, (l > 0) ? pa_order_threshold : 100, skip_zeros), h_l); + fem::AssembleOperator(std::move(m), true, (l > 0) ? pa_order_threshold : 100, + skip_zeros), + h_l); // Set the essential dofs (none). M->AddOperator(std::move(M_l)); diff --git a/palace/main.cpp b/palace/main.cpp index 24d274233..f62689334 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -189,7 +189,7 @@ int main(int argc, char *argv[]) auto solver_output = solver->Solve(mesh); Mpi::Print(world_comm, "Normalized Error Indicator: {:.3e}", - solver_output.global_error_indicator); + solver_output.GetGlobalErrorIndicator()); // Print timing summary. BlockTimer::Print(world_comm); diff --git a/palace/utils/errorindicators.cpp b/palace/utils/errorindicators.cpp index ca7eec892..409fdb5cd 100644 --- a/palace/utils/errorindicators.cpp +++ b/palace/utils/errorindicators.cpp @@ -8,40 +8,74 @@ namespace palace { -void ErrorReductionOperator::operator()(ErrorIndicators &ebar, const Vector &ind, +ErrorIndicators::ErrorIndicators(const IndicatorsAndNormalization &indicators, + int global_true_v_size, MPI_Comm comm) + : local_error_indicators(indicators.indicators), global_true_v_size(global_true_v_size), + comm(comm), normalization(indicators.normalization) +{ + // Compute the global indicator across all processors. + constexpr int p = 2; + global_error_indicator = + std::transform_reduce(local_error_indicators.begin(), local_error_indicators.end(), + 0.0, std::plus(), [&p](auto val) { return std::pow(val, p); }); + Mpi::GlobalSum(1, &global_error_indicator, comm); + global_error_indicator = std::pow(global_error_indicator, 1.0 / p); + min = local_error_indicators.Min(); + max = local_error_indicators.Max(); + int size = local_error_indicators.Size(); + Mpi::GlobalMin(1, &min, comm); + Mpi::GlobalMax(1, &max, comm); + Mpi::GlobalSum(1, &size, comm); + mean = global_error_indicator / size; +} + +void ErrorReductionOperator::operator()(ErrorIndicators &ebar, + const IndicatorsAndNormalization &ind, double p) const { + if (n == 0) + { + // No direct reduction necessary. + ebar = ErrorIndicators(ind, ebar.global_true_v_size, ebar.comm); + n++; + return; + } + + const auto comm = ebar.GetComm(); + // Compute the global indicator across all processors. - auto comm = Mpi::World(); double candidate_global_error_indicator = - std::transform_reduce(ind.begin(), ind.end(), 0.0, std::plus(), + std::transform_reduce(ind.indicators.begin(), ind.indicators.end(), 0.0, std::plus(), [&p](auto val) { return std::pow(val, p); }); - Mpi::GlobalSum(1, &candidate_global_error_indicator, comm); - candidate_global_error_indicator = std::pow(candidate_global_error_indicator, 1.0 / p); - ebar.global_error_indicator = - std::max(ebar.global_error_indicator, candidate_global_error_indicator); + // Update the global error indicator and local error indicators to the new mean normalized + // values. The average local indicator is used rather than the indicator for the maximum + // error to drive the adaptation, to account for a local error that might be marginally + // important to many solves, rather than only large in one solve. - // update the average local indicator. Using running average update rather - // than sum and final division to maintain validity at all times. + // TODO: Could alternatively consider the maximum. + ebar.global_error_indicator = + (n * ebar.global_error_indicator + candidate_global_error_indicator) / (n + 1); auto running_average = [this](const auto &xbar, const auto &x) { return (xbar * n + x) / (n + 1); }; - if (n > 0) - { - MFEM_VERIFY(ebar.local_error_indicators.Size() == ind.Size(), - "Local error indicator vectors mismatch."); - // Combine these error indicators into the current average. - std::transform(ebar.local_error_indicators.begin(), ebar.local_error_indicators.end(), - ind.begin(), ebar.local_error_indicators.begin(), running_average); - } - else - { - // This is the first sample, just steal the data. - ebar.local_error_indicators = ind; - } + MFEM_VERIFY(ebar.local_error_indicators.Size() == ind.indicators.Size(), + "Local error indicator vectors mismatch."); + // Combine these error indicators into the current average. + std::transform(ebar.local_error_indicators.begin(), ebar.local_error_indicators.end(), + ind.indicators.begin(), ebar.local_error_indicators.begin(), + running_average); + + // Assumes that the global error indicator is already reduced across processors. + ebar.min = ebar.local_error_indicators.Min(); + ebar.max = ebar.local_error_indicators.Max(); + int size = ebar.local_error_indicators.Size(); + Mpi::GlobalMin(1, &ebar.min, comm); + Mpi::GlobalMax(1, &ebar.max, comm); + Mpi::GlobalSum(1, &size, comm); + ebar.mean = ebar.global_error_indicator / size; // Another sample has been added, increment for the running average lambda. n++; diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index b6d551f25..646ea9da8 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -5,16 +5,33 @@ #define PALACE_UTILS_ERROR_INDICATORS_HPP #include "linalg/vector.hpp" +#include "utils/communication.hpp" namespace palace { +class ErrorReductionOperator; -// Storage for error estimation results from the solve. Required in the AMR loop. An error -// indicator is non-negative, while an error estimate is signed. +// Unnormalized error indicators and the normalization factor. +struct IndicatorsAndNormalization +{ + Vector indicators; + double normalization; +}; + +// Storage for error estimation results from the solve. Required in the AMR loop. This is +// richer than the IndicatorsAndNormalization because it stores derived quantities, and a +// communicator for use in adaptation. class ErrorIndicators { public: - ErrorIndicators(int ndof) : ndof(ndof) {} + // Construct an Error indicator from an initial set of indicators + explicit ErrorIndicators(const IndicatorsAndNormalization &indicators, + int global_true_v_size, MPI_Comm comm); + // Construct an empty ErrorIndicators. + explicit ErrorIndicators(int global_true_v_size, MPI_Comm comm) + : global_true_v_size(global_true_v_size), comm(comm) + { + } ErrorIndicators() = delete; ErrorIndicators(const ErrorIndicators &) = default; ErrorIndicators(ErrorIndicators &&) = default; @@ -22,28 +39,55 @@ class ErrorIndicators ErrorIndicators &operator=(ErrorIndicators &&) = default; ~ErrorIndicators() = default; + // Return the average normalized local error indicators. + const auto &GetLocalErrorIndicators() const { return local_error_indicators; } + // Return the global error indicator. + auto GetGlobalErrorIndicator() const { return global_error_indicator; } + // Return the largest normalized local error indicator. + auto GetMaxErrorIndicator() const { return max; } + // Return the smallest normalized local error indicator. + auto GetMinErrorIndicator() const { return min; } + // Return the mean normalized local error indicator. + auto GetMeanErrorIndicator() const { return mean; } + // Return the normalization constant for the absolute error. + auto GetNormalization() const { return normalization; } + // The communicator used in any reductions over processors. + const MPI_Comm &GetComm() { return comm; } + // Return the global number of true dofs associated with this set of error indicators. + auto GlobalTrueVSize() const { return global_true_v_size; } + +protected: + friend class ErrorReductionOperator; // Elemental localized error indicators. Used for marking elements for // refinement and coarsening. Vector local_error_indicators; - // Global error indicator. Used for driving AMR and diagnostics. This - // combines the local error indicators across all ranks. This number is the - // same on all ranks. + // Global error indicator. Used for driving AMR and diagnostics. double global_error_indicator = 0; - // Number of global dof in the mesh. - int ndof; + // Global number of true dof in the finite element solution for which this indicator is + // calculated. + int global_true_v_size; + // Communicator used in calculation of the global error indicator. + MPI_Comm comm; + // Statistics, updated simultaneously with the global error indicator. + double min, max, mean; + // Mean normalization constant. + double normalization; }; // Operator for performing reduction of a vector of local indicators into a // global running total for use in adaptation. class ErrorReductionOperator { - // number of samples. mutability required to ensure operation. + // Number of samples. Mutability required to guarantee operation. mutable int n = 0; public: // Reduce a vector indicators, i, computed with norm p, into the combined // error indicator e. - void operator()(ErrorIndicators &e, const Vector &i, double p = 2) const; + void operator()(ErrorIndicators &e, const IndicatorsAndNormalization &i, + double p = 2) const; + // Resets the internal counter for number of samples. + void Reset() { n = 0; } }; } // namespace palace diff --git a/palace/utils/timer.hpp b/palace/utils/timer.hpp index 9a5ab2fa3..15a305164 100644 --- a/palace/utils/timer.hpp +++ b/palace/utils/timer.hpp @@ -28,8 +28,8 @@ class Timer { INIT = 0, CONSTRUCT, - WAVEPORT, // Wave port solver - ESTCONSTRUCT, // Construction of estimate operator + WAVEPORT, // Wave port solver + ESTCONSTRUCT, // Construction of estimate operator SOLVE, PRECONDITIONER, // Linear solver COARSESOLVE, // Linear solver From a97349cadff23b236c8ad63ceb1d4a23f15e8971 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Tue, 3 Oct 2023 10:16:28 -0400 Subject: [PATCH 03/39] Fixes to better handle normalization --- palace/drivers/basesolver.cpp | 11 +++++----- palace/drivers/basesolver.hpp | 7 +++--- palace/drivers/drivensolver.cpp | 12 +++++++---- palace/drivers/eigensolver.cpp | 6 ++++-- palace/drivers/electrostaticsolver.cpp | 6 ++++-- palace/drivers/magnetostaticsolver.cpp | 6 ++++-- palace/drivers/transientsolver.cpp | 16 +++++++------- palace/linalg/errorestimator.cpp | 30 ++++++++++++++++++-------- palace/linalg/errorestimator.hpp | 7 +++--- palace/main.cpp | 2 +- 10 files changed, 64 insertions(+), 39 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index e9b0c37d0..be62bced6 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -567,7 +567,8 @@ void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double } void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, double time, - const ErrorIndicators &indicators) const + const ErrorIndicators &indicators, + bool normalized) const { if (post_dir.length() == 0) { @@ -584,10 +585,10 @@ void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, d // clang-format off output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", name, table.w1, - "Global Rel.", table.w, - "Element Min.", table.w, - "Element Max.", table.w, - "Element Mean", table.w, + normalized ? "Sum Rel." : "Sum Abs.", table.w, + normalized ? "Min Elem Rel." : "Min Elem Abs.", table.w, + normalized ? "Max Elem Rel." : "Max Elem Abs.", table.w, + normalized ? "Mean Elem Rel." : "Mean Elem Abs.", table.w, "Normalization", table.w); // clang-format on } diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index ed415ca89..e411c0f24 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -67,10 +67,11 @@ class BaseSolver double time) const; void PostprocessFields(const PostOperator &postop, int step, double time) const; - // Postprocess granular error indicator to file. + // Postprocess granular error indicator to file. The argument normalized indicates if + // supplied indicators have already been normalized. void PostprocessErrorIndicators(const std::string &name, int step, double time, - const ErrorIndicators &indicators) const; - // Append combined error indicator to file. + const ErrorIndicators &indicators, bool normalized) const; + // Write a string labeled error indicator. Used for writing statistics. void PostprocessErrorIndicators(const std::string &name, const ErrorIndicators &indicators) const; diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index b5bed2ae9..0c944413c 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -151,13 +151,15 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &postop](const auto &E, int step, double f) { BlockTimer bt0(Timer::ESTSOLVE); - auto estimate = estimator(E); + constexpr bool normalized = true; + auto estimate = estimator(E, normalized); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. postop.SetIndicatorGridFunction(estimate.indicators); PostprocessErrorIndicators( "f (GHz)", step, f, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + normalized); ErrorReducer(indicators, std::move(estimate)); }; @@ -285,12 +287,14 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &ErrorReducer](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTSOLVE); - auto estimate = estimator(E); + constexpr bool normalized = true; + auto estimate = estimator(E, normalized); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. PostprocessErrorIndicators( "f (GHz)", step, frequency, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + normalized); ErrorReducer(indicators, std::move(estimate)); }; diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index d1923d4bc..6e34bcfac 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -272,13 +272,15 @@ EigenSolver::Solve(const std::vector> &mesh) cons [this, &estimator, &indicators, &ErrorReducer, &postop](const auto &E, int i) { BlockTimer bt0(Timer::ESTSOLVE); - auto ind = estimator(E); + constexpr bool normalized = true; + auto ind = estimator(E, normalized); BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(ind.indicators); // Write the indicator for this mode. PostprocessErrorIndicators( "m", i, i + 1, - ErrorIndicators{ind, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorIndicators{ind, indicators.GlobalTrueVSize(), indicators.GetComm()}, + normalized); ErrorReducer(indicators, std::move(ind)); }; diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 8a834b490..3f9fcb4c6 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -108,14 +108,16 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, &postop](const auto &V, int i, double idx) { BlockTimer bt0(Timer::ESTSOLVE); - auto estimate = estimator(V); + constexpr bool normalized = true; + auto estimate = estimator(V, normalized); ErrorReducer(indicators, estimate); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. postop.SetIndicatorGridFunction(estimate.indicators); PostprocessErrorIndicators( "i", i, idx, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + normalized); ErrorReducer(indicators, std::move(estimate)); }; if (iodata.solver.electrostatic.n_post > 0) diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index 1d65f920a..b19389fe1 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -110,13 +110,15 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, &postop](const auto &A, int i, double idx) { BlockTimer bt0(Timer::ESTSOLVE); - auto estimate = estimator(A); + constexpr bool normalized = true; + auto estimate = estimator(A, normalized); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. postop.SetIndicatorGridFunction(estimate.indicators); PostprocessErrorIndicators( "i", i, idx, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + normalized); ErrorReducer(indicators, std::move(estimate)); }; if (iodata.solver.magnetostatic.n_post > 0) diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index ccbb92005..568720de5 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -90,12 +90,15 @@ TransientSolver::Solve(const std::vector> &mesh) &postop](const auto &E, int step, double time) { BlockTimer bt0(Timer::ESTSOLVE); - auto estimate = estimator(E); + // Initial flux of zero would return nan. + bool constexpr normalized = false; + auto estimate = estimator(E, normalized); BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(estimate.indicators); PostprocessErrorIndicators( "t (ns)", step, time, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}); + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + normalized); ErrorReducer(indicators, std::move(estimate)); }; @@ -140,11 +143,8 @@ TransientSolver::Solve(const std::vector> &mesh) E_elec + E_mag); } - if (step > 0) - { - // Calculate and record the error indicators. - UpdateErrorIndicators(E, step, t); - } + // Calculate and record the error indicators. + UpdateErrorIndicators(E, step, t); // Postprocess port voltages/currents and optionally write solution to disk. Postprocess(postop, spaceop.GetLumpedPortOp(), spaceop.GetSurfaceCurrentOp(), step, t, @@ -154,7 +154,7 @@ TransientSolver::Solve(const std::vector> &mesh) step++; } SaveMetadata(timeop.GetLinearSolver()); - PostprocessErrorIndicators("Mean", step, t, indicators); + PostprocessErrorIndicators("Mean", indicators); return indicators; } std::function TransientSolver::GetTimeExcitation(bool dot) const diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index c98f80a44..f65808136 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -47,7 +47,8 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( } } -IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVector &v) const +IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVector &v, + bool normalize) const { mfem::ParComplexGridFunction field(&fes); field.real().SetFromTrueDofs(v.Real()); @@ -152,12 +153,16 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVecto Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); normalization = std::sqrt(normalization); - std::for_each(estimates.begin(), estimates.end(), - [&normalization](auto &x) { x /= normalization; }); + if (normalize) + { + std::for_each(estimates.begin(), estimates.end(), + [&normalization](auto &x) { x /= normalization; }); + } return {estimates, normalization}; } -IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v) const +IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v, + bool normalize) const { mfem::ParGridFunction field(&fes); field.SetFromTrueDofs(v); @@ -236,8 +241,11 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v) c Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); normalization = std::sqrt(normalization); - std::for_each(estimates.begin(), estimates.end(), - [&normalization](auto &x) { x /= normalization; }); + if (normalize) + { + std::for_each(estimates.begin(), estimates.end(), + [&normalization](auto &x) { x /= normalization; }); + } return {estimates, normalization}; } @@ -276,7 +284,8 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( } } -IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v) const +IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v, + bool normalize) const { mfem::ParGridFunction field(&fes); field.SetFromTrueDofs(v); @@ -388,8 +397,11 @@ IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v) c paraview.Save(); } - std::for_each(estimates.begin(), estimates.end(), - [&normalization](auto &x) { x /= normalization; }); + if (normalize) + { + std::for_each(estimates.begin(), estimates.end(), + [&normalization](auto &x) { x /= normalization; }); + } return {estimates, normalization}; } } // namespace palace diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 0f7d6286b..8d9b9a672 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -46,10 +46,11 @@ class CurlFluxErrorEstimator mfem::ParFiniteElementSpace &fes); // Compute elemental error indicators given a complex vector of true DOF. - IndicatorsAndNormalization operator()(const ComplexVector &v) const; + IndicatorsAndNormalization operator()(const ComplexVector &v, + bool normalize = true) const; // Compute elemental error indicators given a vector of true DOF. - IndicatorsAndNormalization operator()(const Vector &v) const; + IndicatorsAndNormalization operator()(const Vector &v, bool normalize = true) const; }; // Class used for computing grad flux error estimate, @@ -82,7 +83,7 @@ class GradFluxErrorEstimator mfem::ParFiniteElementSpace &fes); // Compute elemental error indicators given a vector of true DOF. - IndicatorsAndNormalization operator()(const Vector &v) const; + IndicatorsAndNormalization operator()(const Vector &v, bool normalize = true) const; }; } // namespace palace diff --git a/palace/main.cpp b/palace/main.cpp index f62689334..a19902b58 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -188,7 +188,7 @@ int main(int argc, char *argv[]) // Run the problem driver. auto solver_output = solver->Solve(mesh); - Mpi::Print(world_comm, "Normalized Error Indicator: {:.3e}", + Mpi::Print(world_comm, "Error Estimate: {:.3e}\n", solver_output.GetGlobalErrorIndicator()); // Print timing summary. From 21c80224c29b4138f8e78871e524b8a43e0e881b Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Tue, 3 Oct 2023 10:20:19 -0400 Subject: [PATCH 04/39] Adding reference data for the error indicator outputs --- .../ref/cavity/impedance/error-indicators.csv | 17 ++ test/ref/cavity/pec/error-indicators.csv | 17 ++ test/ref/coaxial/matched/error-indicators.csv | 203 ++++++++++++++++++ test/ref/coaxial/open/error-indicators.csv | 203 ++++++++++++++++++ .../cpw/lumped_adaptive/error-indicators.csv | 11 + .../cpw/lumped_uniform/error-indicators.csv | 17 ++ .../cpw/wave_adaptive/error-indicators.csv | 11 + .../ref/cpw/wave_uniform/error-indicators.csv | 17 ++ test/ref/rings/error-indicators.csv | 4 + test/ref/spheres/error-indicators.csv | 4 + test/testcase.jl | 33 ++- 11 files changed, 536 insertions(+), 1 deletion(-) create mode 100644 test/ref/cavity/impedance/error-indicators.csv create mode 100644 test/ref/cavity/pec/error-indicators.csv create mode 100644 test/ref/coaxial/matched/error-indicators.csv create mode 100644 test/ref/coaxial/open/error-indicators.csv create mode 100644 test/ref/cpw/lumped_adaptive/error-indicators.csv create mode 100644 test/ref/cpw/lumped_uniform/error-indicators.csv create mode 100644 test/ref/cpw/wave_adaptive/error-indicators.csv create mode 100644 test/ref/cpw/wave_uniform/error-indicators.csv create mode 100644 test/ref/rings/error-indicators.csv create mode 100644 test/ref/spheres/error-indicators.csv diff --git a/test/ref/cavity/impedance/error-indicators.csv b/test/ref/cavity/impedance/error-indicators.csv new file mode 100644 index 000000000..c6bf6809c --- /dev/null +++ b/test/ref/cavity/impedance/error-indicators.csv @@ -0,0 +1,17 @@ + m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization + 1.000000e+00, +8.211772836e-03, +9.214906601e-05, +6.517143357e-04, +1.425655006e-05, +6.964432246e-01 + 2.000000e+00, +7.376205407e-03, +1.120202798e-04, +5.122686573e-04, +1.280591217e-05, +4.545414625e-01 + 3.000000e+00, +7.965439262e-03, +1.093565445e-04, +6.153862588e-04, +1.382888761e-05, +5.168938560e-01 + 4.000000e+00, +8.289455502e-03, +6.591010796e-05, +8.975703487e-04, +1.439141580e-05, +7.254717341e-01 + 5.000000e+00, +1.680165239e-02, +2.687010660e-04, +1.173134621e-03, +2.916953541e-05, +7.021945474e-01 + 6.000000e+00, +1.724222225e-02, +2.679598776e-04, +1.285586152e-03, +2.993441363e-05, +6.917663295e-01 + 7.000000e+00, +1.541389559e-02, +2.982919759e-04, +1.088771718e-03, +2.676023539e-05, +6.836966993e-01 + 8.000000e+00, +1.589306594e-02, +2.709869621e-04, +1.182747694e-03, +2.759212837e-05, +7.774852285e-01 + 9.000000e+00, +2.161744447e-02, +3.939248466e-04, +1.590949614e-03, +3.753028555e-05, +1.082707044e+00 + 1.000000e+01, +2.568013719e-02, +3.749226902e-04, +2.345062994e-03, +4.458357151e-05, +1.143427856e+00 + 1.100000e+01, +1.171936131e-02, +2.548287031e-04, +8.771786578e-04, +2.034611339e-05, +8.880851774e-01 + 1.200000e+01, +2.424099505e-02, +3.311784727e-04, +2.230405100e-03, +4.208506085e-05, +8.563666051e-01 + 1.300000e+01, +2.164706035e-02, +1.439791555e-04, +2.192101984e-03, +3.758170199e-05, +1.123784326e+00 + 1.400000e+01, +2.570506911e-02, +1.191879068e-04, +3.229781992e-03, +4.462685610e-05, +1.125048653e+00 + 1.500000e+01, +2.690501452e-02, +4.630239998e-04, +1.771500567e-03, +4.671009465e-05, +8.954818243e-01 + Mean, +1.698058608e-02, +3.754014884e-04, +1.180747207e-03, +2.948018416e-05, +6.964432246e-01 diff --git a/test/ref/cavity/pec/error-indicators.csv b/test/ref/cavity/pec/error-indicators.csv new file mode 100644 index 000000000..43f7be263 --- /dev/null +++ b/test/ref/cavity/pec/error-indicators.csv @@ -0,0 +1,17 @@ + m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization + 1.000000e+00, +8.211772151e-03, +9.214886120e-05, +6.517144290e-04, +1.425654887e-05, +6.964432123e-01 + 2.000000e+00, +7.376205453e-03, +1.120202072e-04, +5.122678696e-04, +1.280591225e-05, +4.545414720e-01 + 3.000000e+00, +7.965439326e-03, +1.093559864e-04, +6.153852901e-04, +1.382888772e-05, +5.168938061e-01 + 4.000000e+00, +8.289455200e-03, +6.591004762e-05, +8.975704258e-04, +1.439141528e-05, +7.254717195e-01 + 5.000000e+00, +1.680164760e-02, +2.686712679e-04, +1.173091559e-03, +2.916952709e-05, +7.021936550e-01 + 6.000000e+00, +1.724222427e-02, +2.679523802e-04, +1.285583367e-03, +2.993441713e-05, +6.917666416e-01 + 7.000000e+00, +1.541388964e-02, +2.981725401e-04, +1.088519843e-03, +2.676022508e-05, +6.836999314e-01 + 8.000000e+00, +1.589306788e-02, +2.709682463e-04, +1.182728017e-03, +2.759213174e-05, +7.774837036e-01 + 9.000000e+00, +2.161744444e-02, +3.939247063e-04, +1.590950711e-03, +3.753028548e-05, +1.082707023e+00 + 1.000000e+01, +2.568013806e-02, +3.749227088e-04, +2.345063718e-03, +4.458357302e-05, +1.143427857e+00 + 1.100000e+01, +1.171936187e-02, +2.548287393e-04, +8.771787445e-04, +2.034611435e-05, +8.880851653e-01 + 1.200000e+01, +2.424099394e-02, +3.311788874e-04, +2.230405084e-03, +4.208505892e-05, +8.563665736e-01 + 1.300000e+01, +2.164706073e-02, +1.439799500e-04, +2.192105601e-03, +3.758170266e-05, +1.123784343e+00 + 1.400000e+01, +2.570507021e-02, +1.191879086e-04, +3.229783591e-03, +4.462685800e-05, +1.125048641e+00 + 1.500000e+01, +2.690494118e-02, +4.630260842e-04, +1.771482822e-03, +4.670996732e-05, +8.954790492e-01 + Mean, +1.698058080e-02, +3.754082954e-04, +1.180749794e-03, +2.948017499e-05, +6.964432123e-01 diff --git a/test/ref/coaxial/matched/error-indicators.csv b/test/ref/coaxial/matched/error-indicators.csv new file mode 100644 index 000000000..ae715ec78 --- /dev/null +++ b/test/ref/coaxial/matched/error-indicators.csv @@ -0,0 +1,203 @@ + t (ns), Sum Abs., Min Elem Abs., Max Elem Abs., Mean Elem Abs., Normalization + 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 + 3.747406e-02, +5.656919329e-06, +4.843867778e-19, +1.429146542e-06, +4.419468225e-08, +1.015026012e-04 + 7.494811e-02, +1.331394786e-05, +1.254533371e-18, +3.354187386e-06, +1.040152177e-07, +1.286467416e-04 + 1.124222e-01, +4.533166152e-06, +8.678124436e-19, +1.144058873e-06, +3.541536056e-08, +1.363432445e-04 + 1.498962e-01, +5.316539312e-06, +5.368957501e-19, +1.352378049e-06, +4.153546338e-08, +2.237453773e-04 + 1.873703e-01, +2.287184397e-05, +3.014012761e-18, +5.789590613e-06, +1.786862810e-07, +5.988251413e-04 + 2.248443e-01, +1.598438319e-05, +2.227475254e-18, +4.071906511e-06, +1.248779937e-07, +1.231318974e-03 + 2.623184e-01, +1.150013517e-05, +5.207497192e-18, +3.089062447e-06, +8.984480602e-08, +2.183514588e-03 + 2.997925e-01, +2.092441719e-05, +1.273586617e-17, +5.172537100e-06, +1.634720093e-07, +3.443881315e-03 + 3.372665e-01, +3.023129661e-05, +1.998967366e-17, +8.595200612e-06, +2.361820048e-07, +4.896471117e-03 + 3.747406e-01, +2.966149891e-05, +5.653700090e-17, +8.160478469e-06, +2.317304602e-07, +6.288238192e-03 + 4.122146e-01, +3.934747640e-05, +1.413811959e-17, +1.043816140e-05, +3.074021594e-07, +7.236958436e-03 + 4.496887e-01, +5.496375443e-05, +1.412220851e-16, +1.376602993e-05, +4.294043315e-07, +7.438657288e-03 + 4.871627e-01, +5.317715171e-05, +1.765081252e-16, +1.345278693e-05, +4.154464977e-07, +7.655736081e-03 + 5.246368e-01, +6.345987516e-05, +3.583095408e-16, +1.616235728e-05, +4.957802747e-07, +1.144507368e-02 + 5.621109e-01, +1.000188545e-04, +6.686901513e-16, +2.890344809e-05, +7.813973009e-07, +2.129749051e-02 + 5.995849e-01, +1.569853883e-04, +6.894682576e-16, +5.054827882e-05, +1.226448346e-06, +3.669225071e-02 + 6.370590e-01, +2.371414637e-04, +2.987326546e-15, +7.811657098e-05, +1.852667685e-06, +5.641515864e-02 + 6.745330e-01, +3.436988518e-04, +2.164626179e-15, +1.084030298e-04, +2.685147279e-06, +7.828737440e-02 + 7.120071e-01, +4.503449422e-04, +8.429520619e-15, +1.334532774e-04, +3.518319861e-06, +9.885303849e-02 + 7.494811e-01, +5.387201987e-04, +2.207111440e-14, +1.460192421e-04, +4.208751553e-06, +1.137546488e-01 + 7.869552e-01, +5.854634658e-04, +3.677392161e-14, +1.426211788e-04, +4.573933327e-06, +1.195362049e-01 + 8.244293e-01, +5.945252634e-04, +1.530577540e-13, +1.553115760e-04, +4.644728621e-06, +1.188356113e-01 + 8.619033e-01, +6.277794936e-04, +8.375922406e-13, +1.446548827e-04, +4.904527294e-06, +1.309570623e-01 + 8.993774e-01, +8.196023636e-04, +3.513223930e-12, +2.064570783e-04, +6.403143466e-06, +1.857784636e-01 + 9.368514e-01, +1.245063027e-03, +1.222917751e-11, +3.796113154e-04, +9.727054900e-06, +2.867343617e-01 + 9.743255e-01, +1.824265443e-03, +3.836943631e-11, +5.734398443e-04, +1.425207377e-05, +4.167726757e-01 + 1.011800e+00, +2.450890627e-03, +1.109093309e-10, +7.521115017e-04, +1.914758302e-05, +5.549120055e-01 + 1.049274e+00, +3.008970317e-03, +2.905634904e-10, +8.708173546e-04, +2.350758060e-05, +6.782193157e-01 + 1.086748e+00, +3.391209593e-03, +6.708727454e-10, +8.861180161e-04, +2.649382494e-05, +7.649055092e-01 + 1.124222e+00, +3.537834071e-03, +1.316367029e-09, +8.504123136e-04, +2.763932868e-05, +8.022591780e-01 + 1.161696e+00, +3.511779316e-03, +2.035245965e-09, +8.840350012e-04, +2.743577591e-05, +8.009806200e-01 + 1.199170e+00, +3.584287871e-03, +1.923005472e-09, +8.159752392e-04, +2.800224899e-05, +8.143332817e-01 + 1.236644e+00, +4.160738357e-03, +3.636929129e-09, +8.694210879e-04, +3.250576841e-05, +9.294499013e-01 + 1.274118e+00, +5.352965496e-03, +1.872212829e-08, +1.361582719e-03, +4.182004294e-05, +1.185780064e+00 + 1.311592e+00, +6.884036966e-03, +5.579982808e-08, +1.934136740e-03, +5.378153880e-05, +1.534661136e+00 + 1.349066e+00, +8.393574758e-03, +1.270371513e-07, +2.361243463e-03, +6.557480279e-05, +1.896361173e+00 + 1.386540e+00, +9.587144869e-03, +2.368634260e-07, +2.543459607e-03, +7.489956929e-05, +2.199964362e+00 + 1.424014e+00, +1.030847844e-02, +3.669558013e-07, +2.421466310e-03, +8.053498780e-05, +2.397709121e+00 + 1.461488e+00, +1.058479827e-02, +4.699423782e-07, +2.459542459e-03, +8.269373645e-05, +2.477248853e+00 + 1.498962e+00, +1.065114462e-02, +5.188096886e-07, +2.478084260e-03, +8.321206733e-05, +2.474045346e+00 + 1.536436e+00, +1.088707747e-02, +7.069806642e-07, +2.347785517e-03, +8.505529270e-05, +2.473655996e+00 + 1.573910e+00, +1.161350503e-02, +1.366657071e-06, +2.453010503e-03, +9.073050803e-05, +2.579211451e+00 + 1.611384e+00, +1.284893913e-02, +2.472581913e-06, +2.368957242e-03, +1.003823369e-04, +2.837696929e+00 + 1.648859e+00, +1.431859015e-02, +3.857804555e-06, +3.068473841e-03, +1.118639856e-04, +3.200566296e+00 + 1.686333e+00, +1.568466897e-02, +5.309872328e-06, +3.469083241e-03, +1.225364763e-04, +3.569988955e+00 + 1.723807e+00, +1.672073192e-02, +6.527746680e-06, +3.499417907e-03, +1.306307181e-04, +3.860977237e+00 + 1.761281e+00, +1.735917503e-02, +7.046758600e-06, +3.289526113e-03, +1.356185549e-04, +4.031108681e+00 + 1.798755e+00, +1.767118125e-02, +6.301192058e-06, +3.511818706e-03, +1.380561035e-04, +4.086235159e+00 + 1.836229e+00, +1.781514385e-02, +5.066557197e-06, +3.414133307e-03, +1.391808113e-04, +4.072115006e+00 + 1.873703e+00, +1.795734365e-02, +9.785529399e-06, +3.394013663e-03, +1.402917473e-04, +4.053080663e+00 + 1.911177e+00, +1.820297964e-02, +2.153285398e-05, +3.420849872e-03, +1.422107784e-04, +4.080884955e+00 + 1.948651e+00, +1.856531991e-02, +3.797781532e-05, +3.234892491e-03, +1.450415618e-04, +4.169137462e+00 + 1.986125e+00, +1.899015404e-02, +5.721899093e-05, +3.396745163e-03, +1.483605785e-04, +4.292812394e+00 + 2.023599e+00, +1.939524081e-02, +6.688605498e-05, +3.250581882e-03, +1.515253188e-04, +4.411264745e+00 + 2.061073e+00, +1.970534620e-02, +8.499883944e-05, +3.312961586e-03, +1.539480172e-04, +4.493132813e+00 + 2.098547e+00, +1.988283045e-02, +6.241843571e-05, +3.301605857e-03, +1.553346129e-04, +4.527959658e+00 + 2.136021e+00, +1.993607356e-02, +7.807678532e-05, +3.172906476e-03, +1.557505747e-04, +4.524294614e+00 + 2.173495e+00, +1.989960314e-02, +4.892651018e-05, +3.293718894e-03, +1.554656495e-04, +4.500332857e+00 + 2.210969e+00, +1.981531470e-02, +6.512326289e-05, +3.115928939e-03, +1.548071461e-04, +4.473391974e+00 + 2.248443e+00, +1.972658289e-02, +1.542799260e-04, +3.228336522e-03, +1.541139288e-04, +4.453059505e+00 + 2.285917e+00, +1.965740166e-02, +2.721087012e-04, +3.180446561e-03, +1.535734504e-04, +4.440114615e+00 + 2.323392e+00, +1.959904710e-02, +3.980665603e-04, +3.107500278e-03, +1.531175555e-04, +4.429876188e+00 + 2.360866e+00, +1.953282422e-02, +4.003191673e-04, +3.189015057e-03, +1.526001892e-04, +4.416676540e+00 + 2.398340e+00, +1.944869644e-02, +5.464091449e-04, +2.981980322e-03, +1.519429410e-04, +4.396869067e+00 + 2.435814e+00, +1.933806042e-02, +4.155885733e-04, +3.140904648e-03, +1.510785971e-04, +4.369773349e+00 + 2.473288e+00, +1.920073395e-02, +4.446796335e-04, +3.059026781e-03, +1.500057340e-04, +4.337298959e+00 + 2.510762e+00, +1.905522216e-02, +3.406503094e-04, +3.038745856e-03, +1.488689231e-04, +4.303041277e+00 + 2.548236e+00, +1.892141280e-02, +1.821447779e-04, +3.082892837e-03, +1.478235375e-04, +4.271004959e+00 + 2.585710e+00, +1.879970377e-02, +8.506630768e-05, +2.887937022e-03, +1.468726857e-04, +4.243717289e+00 + 2.623184e+00, +1.867373516e-02, +1.004256193e-04, +3.051640523e-03, +1.458885559e-04, +4.219927993e+00 + 2.660658e+00, +1.852070404e-02, +1.196737664e-04, +2.938772398e-03, +1.446930003e-04, +4.193080930e+00 + 2.698132e+00, +1.830775799e-02, +1.071572521e-04, +2.968150322e-03, +1.430293593e-04, +4.152255197e+00 + 2.735606e+00, +1.800218944e-02, +1.320213032e-04, +2.975736659e-03, +1.406421050e-04, +4.086458524e+00 + 2.773080e+00, +1.760560892e-02, +9.506619049e-05, +2.829530695e-03, +1.375438197e-04, +3.991233526e+00 + 2.810554e+00, +1.716131434e-02, +7.317905711e-05, +2.954310998e-03, +1.340727683e-04, +3.874560228e+00 + 2.848028e+00, +1.672503706e-02, +4.353864457e-05, +2.810145825e-03, +1.306643520e-04, +3.757798136e+00 + 2.885502e+00, +1.634861093e-02, +2.010916948e-05, +2.884632875e-03, +1.277235229e-04, +3.667738464e+00 + 2.922976e+00, +1.607034327e-02, +7.273872449e-06, +2.862475829e-03, +1.255495568e-04, +3.620214926e+00 + 2.960451e+00, +1.586218226e-02, +9.724124591e-06, +2.809723296e-03, +1.239232989e-04, +3.604829965e+00 + 2.997925e+00, +1.562061453e-02, +1.277801775e-05, +2.899257047e-03, +1.220360510e-04, +3.584446446e+00 + 3.035399e+00, +1.522782614e-02, +1.233271900e-05, +2.721416546e-03, +1.189673917e-04, +3.511909116e+00 + 3.072873e+00, +1.458718436e-02, +1.012356779e-05, +2.782997074e-03, +1.139623778e-04, +3.352720251e+00 + 3.110347e+00, +1.366320273e-02, +7.294721134e-06, +2.762079550e-03, +1.067437713e-04, +3.102486911e+00 + 3.147821e+00, +1.251768483e-02, +4.420404497e-06, +2.455956528e-03, +9.779441270e-05, +2.794887923e+00 + 3.185295e+00, +1.133578878e-02, +2.426618937e-06, +2.045393417e-03, +8.856084987e-05, +2.497506493e+00 + 3.222769e+00, +1.036779337e-02, +1.089662298e-06, +2.156643879e-03, +8.099838569e-05, +2.287032135e+00 + 3.260243e+00, +9.775726200e-03, +7.775481997e-07, +2.063789951e-03, +7.637286094e-05, +2.199308734e+00 + 3.297717e+00, +9.532734097e-03, +6.108364044e-07, +2.153749292e-03, +7.447448513e-05, +2.194284318e+00 + 3.335191e+00, +9.398661259e-03, +8.216045578e-07, +2.135688547e-03, +7.342704109e-05, +2.187377027e+00 + 3.372665e+00, +9.084908004e-03, +4.990304560e-07, +2.040467598e-03, +7.097584378e-05, +2.107636508e+00 + 3.410139e+00, +8.406765257e-03, +2.134421440e-07, +2.145048813e-03, +6.567785357e-05, +1.925405968e+00 + 3.447613e+00, +7.345670737e-03, +2.218387340e-07, +1.993262694e-03, +5.738805263e-05, +1.652352466e+00 + 3.485087e+00, +6.047746915e-03, +3.358002123e-07, +1.633531207e-03, +4.724802278e-05, +1.332422196e+00 + 3.522561e+00, +4.746403684e-03, +5.213325624e-07, +1.144017220e-03, +3.708127878e-05, +1.031986493e+00 + 3.560035e+00, +3.769640682e-03, +7.310928528e-07, +8.020516437e-04, +2.945031782e-05, +8.248254263e-01 + 3.597509e+00, +3.333383968e-03, +7.229543071e-07, +7.732301888e-04, +2.604206225e-05, +7.467641311e-01 + 3.634984e+00, +3.288196120e-03, +5.789534976e-07, +8.209433323e-04, +2.568903219e-05, +7.476238539e-01 + 3.672458e+00, +3.293284151e-03, +8.017459876e-07, +7.778578450e-04, +2.572878243e-05, +7.473541982e-01 + 3.709932e+00, +3.117292905e-03, +4.714346501e-07, +8.140977754e-04, +2.435385082e-05, +7.038828045e-01 + 3.747406e+00, +2.729593477e-03, +5.298794400e-07, +7.857588551e-04, +2.132494904e-05, +6.126782673e-01 + 3.784880e+00, +2.180764435e-03, +1.216180507e-06, +6.625316701e-04, +1.703722215e-05, +4.890491612e-01 + 3.822354e+00, +1.586088428e-03, +2.384923310e-06, +4.875486123e-04, +1.239131585e-05, +3.558862685e-01 + 3.859828e+00, +1.066809700e-03, +3.511481065e-06, +3.033193261e-04, +8.334450780e-06, +2.372274680e-01 + 3.897302e+00, +7.233506887e-04, +4.708543875e-06, +1.463309998e-04, +5.651177256e-06, +1.561842423e-01 + 3.934776e+00, +6.203759622e-04, +3.573109435e-06, +1.545835338e-04, +4.846687204e-06, +1.265111525e-01 + 3.972250e+00, +6.226640636e-04, +4.851805508e-06, +1.561429812e-04, +4.864562997e-06, +1.273690025e-01 + 4.009724e+00, +6.087857616e-04, +4.093612728e-06, +1.485704094e-04, +4.756138763e-06, +1.276030578e-01 + 4.047198e+00, +5.400191444e-04, +3.742942640e-06, +1.478591037e-04, +4.218899565e-06, +1.171779781e-01 + 4.084672e+00, +4.401182610e-04, +4.379634926e-06, +1.273842804e-04, +3.438423914e-06, +9.815150341e-02 + 4.122146e+00, +3.324057867e-04, +3.390263835e-06, +9.627154190e-05, +2.596920209e-06, +7.584800868e-02 + 4.159620e+00, +2.370075783e-04, +4.831244850e-06, +6.338579248e-05, +1.851621706e-06, +5.553016806e-02 + 4.197094e+00, +1.811418739e-04, +5.641649752e-06, +3.631936955e-05, +1.415170890e-06, +4.116690846e-02 + 4.234568e+00, +1.557249175e-04, +2.668727064e-06, +2.221487415e-05, +1.216600918e-06, +3.415513459e-02 + 4.272043e+00, +1.465187122e-04, +1.674662472e-06, +2.170675982e-05, +1.144677439e-06, +3.215710958e-02 + 4.309517e+00, +1.405423708e-04, +1.924531635e-06, +2.149418194e-05, +1.097987272e-06, +3.160248470e-02 + 4.346991e+00, +1.357847027e-04, +1.605849913e-06, +2.192148955e-05, +1.060817990e-06, +3.065472344e-02 + 4.384465e+00, +1.258550769e-04, +1.284603768e-06, +2.046648332e-05, +9.832427880e-07, +2.904522959e-02 + 4.421939e+00, +1.169869496e-04, +1.278082689e-06, +2.159284942e-05, +9.139605438e-07, +2.713926010e-02 + 4.459413e+00, +1.110703423e-04, +1.509710125e-06, +2.079338969e-05, +8.677370496e-07, +2.542558185e-02 + 4.496887e+00, +1.057701136e-04, +5.790173697e-07, +2.100616047e-05, +8.263290126e-07, +2.427050930e-02 + 4.534361e+00, +1.026251685e-04, +7.810732718e-07, +2.116271293e-05, +8.017591292e-07, +2.376362919e-02 + 4.571835e+00, +1.011917516e-04, +5.963699975e-07, +2.046230884e-05, +7.905605596e-07, +2.366920631e-02 + 4.609309e+00, +1.001634828e-04, +8.947493041e-07, +2.134477255e-05, +7.825272090e-07, +2.354631490e-02 + 4.646783e+00, +9.707341248e-05, +3.485090573e-07, +2.031439706e-05, +7.583860350e-07, +2.295738668e-02 + 4.684257e+00, +9.167802949e-05, +3.892732641e-07, +2.066985033e-05, +7.162346054e-07, +2.162696749e-02 + 4.721731e+00, +8.383810226e-05, +2.180435689e-07, +2.059725781e-05, +6.549851739e-07, +1.951430734e-02 + 4.759205e+00, +7.380265099e-05, +2.824981367e-07, +1.856655811e-05, +5.765832109e-07, +1.682504287e-02 + 4.796679e+00, +6.231475191e-05, +3.941097394e-07, +1.489212560e-05, +4.868339993e-07, +1.398756018e-02 + 4.834153e+00, +5.205405315e-05, +9.286816014e-08, +1.031777329e-05, +4.066722902e-07, +1.158631725e-02 + 4.871627e+00, +4.563461801e-05, +2.947284352e-07, +9.944042992e-06, +3.565204532e-07, +1.014939239e-02 + 4.909101e+00, +4.281885346e-05, +2.428193082e-07, +9.489779373e-06, +3.345222927e-07, +9.727415693e-03 + 4.946576e+00, +4.221795835e-05, +3.153021840e-07, +1.010970714e-05, +3.298277996e-07, +9.783286724e-03 + 4.984050e+00, +4.193062278e-05, +1.303308212e-07, +9.664688337e-06, +3.275829905e-07, +9.697488727e-03 + 5.021524e+00, +3.983627118e-05, +2.205996708e-07, +9.929565063e-06, +3.112208686e-07, +9.137274998e-03 + 5.058998e+00, +3.557555069e-05, +4.469304141e-07, +9.812803111e-06, +2.779339898e-07, +8.056891307e-03 + 5.096472e+00, +2.974014514e-05, +3.707371569e-07, +8.638593197e-06, +2.323448839e-07, +6.593883084e-03 + 5.133946e+00, +2.283060266e-05, +1.698975080e-07, +6.683001397e-06, +1.783640832e-07, +4.979337600e-03 + 5.171420e+00, +1.666241488e-05, +3.152853125e-07, +4.477294455e-06, +1.301751163e-07, +3.482333541e-03 + 5.208894e+00, +1.250577556e-05, +1.567320721e-07, +2.385719075e-06, +9.770137157e-08, +2.384295551e-03 + 5.246368e+00, +1.120766321e-05, +2.127374541e-07, +2.042794930e-06, +8.755986884e-08, +1.893867546e-03 + 5.283842e+00, +1.074669371e-05, +1.995526713e-07, +2.278776543e-06, +8.395854458e-08, +1.857462365e-03 + 5.321316e+00, +1.005005277e-05, +3.020728485e-07, +2.048453621e-06, +7.851603724e-08, +1.884026962e-03 + 5.358790e+00, +1.027674820e-05, +1.968047729e-07, +2.146664014e-06, +8.028709530e-08, +1.781458878e-03 + 5.396264e+00, +9.992556903e-06, +2.897865156e-07, +2.019372889e-06, +7.806685080e-08, +1.539662271e-03 + 5.433738e+00, +8.130179588e-06, +9.113607665e-08, +1.644339079e-06, +6.351702803e-08, +1.215813603e-03 + 5.471212e+00, +6.073721937e-06, +1.478699844e-07, +1.146645281e-06, +4.745095264e-08, +8.794652134e-04 + 5.508686e+00, +6.698146250e-06, +9.692032714e-08, +9.740127564e-07, +5.232926758e-08, +5.925767029e-04 + 5.546160e+00, +7.402378271e-06, +3.073582384e-07, +1.094274341e-06, +5.783108024e-08, +4.025875389e-04 + 5.583635e+00, +6.171372548e-06, +1.350664345e-07, +9.761640377e-07, +4.821384803e-08, +3.250012544e-04 + 5.621109e+00, +5.253380349e-06, +9.249781991e-08, +7.532475805e-07, +4.104203398e-08, +3.144552651e-04 + 5.658583e+00, +6.034991990e-06, +1.524640444e-07, +8.379384916e-07, +4.714837492e-08, +3.143233654e-04 + 5.696057e+00, +6.954997371e-06, +1.738910829e-07, +1.032800032e-06, +5.433591696e-08, +3.053781280e-04 + 5.733531e+00, +6.618311823e-06, +1.393653595e-07, +9.706910964e-07, +5.170556112e-08, +2.905274734e-04 + 5.771005e+00, +4.983685205e-06, +7.755106256e-08, +7.959431861e-07, +3.893504067e-08, +2.767846518e-04 + 5.808479e+00, +5.498601999e-06, +5.420219015e-08, +8.069512442e-07, +4.295782812e-08, +2.672840112e-04 + 5.845953e+00, +6.952197828e-06, +1.517744765e-07, +1.004761387e-06, +5.431404553e-08, +2.606043537e-04 + 5.883427e+00, +6.329200774e-06, +9.133367320e-08, +8.427280265e-07, +4.944688105e-08, +2.537151852e-04 + 5.920901e+00, +5.215815525e-06, +2.025761161e-08, +8.189533922e-07, +4.074855879e-08, +2.439748930e-04 + 5.958375e+00, +5.305475054e-06, +1.698310034e-07, +6.730686943e-07, +4.144902386e-08, +2.288006244e-04 + 5.995849e+00, +6.176556969e-06, +1.308471229e-07, +9.587659988e-07, +4.825435132e-08, +2.091970502e-04 + 6.033323e+00, +6.645688022e-06, +8.608171152e-09, +8.996642786e-07, +5.191943767e-08, +1.868959008e-04 + 6.070797e+00, +5.398007444e-06, +1.508347448e-07, +8.539658993e-07, +4.217193316e-08, +1.639107998e-04 + 6.108271e+00, +4.639738945e-06, +1.461885578e-07, +5.846045529e-07, +3.624796051e-08, +1.453359455e-04 + 6.145745e+00, +6.145503917e-06, +8.241360256e-08, +9.145264944e-07, +4.801174935e-08, +1.340081739e-04 + 6.183219e+00, +6.485408052e-06, +6.256044891e-08, +9.129625606e-07, +5.066725041e-08, +1.300350501e-04 + 6.220694e+00, +5.542152865e-06, +8.574375172e-08, +7.821942712e-07, +4.329806926e-08, +1.300871660e-04 + 6.258168e+00, +4.985301527e-06, +1.082258939e-07, +7.100283107e-07, +3.894766818e-08, +1.297254971e-04 + 6.295642e+00, +5.334225761e-06, +4.704333371e-08, +7.980112259e-07, +4.167363876e-08, +1.253586089e-04 + 6.333116e+00, +6.421230064e-06, +2.899044502e-07, +9.204389046e-07, +5.016585987e-08, +1.158965803e-04 + 6.370590e+00, +6.073729696e-06, +8.837745352e-08, +7.781934516e-07, +4.745101325e-08, +1.015407485e-04 + 6.408064e+00, +4.266464125e-06, +7.836990080e-08, +6.328354800e-07, +3.333175098e-08, +8.295426644e-05 + 6.445538e+00, +5.024336673e-06, +9.562368737e-08, +6.546446017e-07, +3.925263026e-08, +6.413712406e-05 + 6.483012e+00, +6.379473131e-06, +2.230146922e-07, +9.421359087e-07, +4.983963384e-08, +4.719626400e-05 + 6.520486e+00, +5.693647315e-06, +1.453003864e-07, +7.169231018e-07, +4.448161965e-08, +3.428721749e-05 + 6.557960e+00, +4.451825872e-06, +6.643434571e-08, +7.111725148e-07, +3.477988963e-08, +2.871636923e-05 + 6.595434e+00, +4.899534092e-06, +1.455463377e-07, +7.337732989e-07, +3.827761010e-08, +2.830713259e-05 + 6.632908e+00, +5.715247683e-06, +1.362646790e-07, +8.893278507e-07, +4.465037252e-08, +2.868930397e-05 + 6.670382e+00, +5.627211459e-06, +1.506603539e-07, +7.797612740e-07, +4.396258952e-08, +2.757669223e-05 + 6.707856e+00, +4.948318503e-06, +1.411310733e-07, +7.615922799e-07, +3.865873830e-08, +2.478379048e-05 + 6.745330e+00, +4.504619390e-06, +8.895057287e-08, +7.798763972e-07, +3.519233898e-08, +2.028553556e-05 + 6.782804e+00, +4.993991381e-06, +8.728597097e-08, +7.967551049e-07, +3.901555766e-08, +1.596593978e-05 + 6.820278e+00, +5.747900143e-06, +1.409301742e-07, +8.337881468e-07, +4.490546987e-08, +1.262078790e-05 + 6.857752e+00, +5.247854702e-06, +1.813322114e-07, +7.726407318e-07, +4.099886486e-08, +9.582347785e-06 + 6.895227e+00, +3.985177625e-06, +3.643398367e-09, +6.312565334e-07, +3.113420020e-08, +7.451330914e-06 + 6.932701e+00, +4.586522913e-06, +8.433043328e-08, +6.862489896e-07, +3.583221026e-08, +7.714034018e-06 + 6.970175e+00, +5.649889468e-06, +9.980671974e-08, +8.434616317e-07, +4.413976147e-08, +8.153445503e-06 + 7.007649e+00, +5.253206883e-06, +6.315370615e-08, +6.957073006e-07, +4.104067878e-08, +8.013246854e-06 + 7.045123e+00, +3.957428597e-06, +3.892861524e-08, +6.417155614e-07, +3.091741091e-08, +7.103149370e-06 + 7.082597e+00, +4.067556090e-06, +1.030158119e-07, +5.483078992e-07, +3.177778195e-08, +6.251988925e-06 + 7.120071e+00, +5.339246881e-06, +3.932281422e-08, +8.433152884e-07, +4.171286626e-08, +7.632283077e-06 + 7.157545e+00, +5.327917643e-06, +1.177532816e-07, +6.938631104e-07, +4.162435658e-08, +7.329495899e-06 + 7.195019e+00, +3.969772189e-06, +7.950843774e-08, +6.036690615e-07, +3.101384523e-08, +6.045781511e-06 + 7.232493e+00, +3.715635311e-06, +1.404290934e-07, +5.594075907e-07, +2.902840086e-08, +5.792378480e-06 + 7.269967e+00, +5.014967369e-06, +1.612835104e-07, +7.753574291e-07, +3.917943257e-08, +6.622438552e-06 + 7.307441e+00, +5.352771106e-06, +1.184136830e-07, +7.375169940e-07, +4.181852426e-08, +7.298828834e-06 + 7.344915e+00, +4.153470584e-06, +4.480562392e-08, +6.123684096e-07, +3.244898894e-08, +5.538450050e-06 + 7.382389e+00, +3.599983803e-06, +3.650956171e-08, +5.446721100e-07, +2.812487346e-08, +5.496706240e-06 + 7.419863e+00, +4.830766906e-06, +5.884627608e-08, +7.174872509e-07, +3.774036645e-08, +6.712351610e-06 + 7.457337e+00, +5.146870332e-06, +1.163179384e-07, +7.555277815e-07, +4.020992447e-08, +6.428644479e-06 + 7.494811e+00, +4.397400790e-06, +1.080429328e-07, +5.798177323e-07, +3.435469367e-08, +6.238567259e-06 + Mean, +4.855456549e-03, +2.700723228e-04, +3.584548856e-04, +3.793325429e-05, +0.000000000e+00 diff --git a/test/ref/coaxial/open/error-indicators.csv b/test/ref/coaxial/open/error-indicators.csv new file mode 100644 index 000000000..692e67174 --- /dev/null +++ b/test/ref/coaxial/open/error-indicators.csv @@ -0,0 +1,203 @@ + t (ns), Sum Abs., Min Elem Abs., Max Elem Abs., Mean Elem Abs., Normalization + 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 + 3.747406e-02, +5.656919329e-06, +4.856284755e-19, +1.429146542e-06, +4.419468226e-08, +1.015026012e-04 + 7.494811e-02, +1.331394786e-05, +1.256204662e-18, +3.354187386e-06, +1.040152177e-07, +1.286467416e-04 + 1.124222e-01, +4.533166150e-06, +8.691346112e-19, +1.144058873e-06, +3.541536055e-08, +1.363432444e-04 + 1.498962e-01, +5.316539313e-06, +5.386495332e-19, +1.352378050e-06, +4.153546339e-08, +2.237453773e-04 + 1.873703e-01, +2.287184397e-05, +3.012395724e-18, +5.789590612e-06, +1.786862810e-07, +5.988251413e-04 + 2.248443e-01, +1.598438319e-05, +2.221781868e-18, +4.071906511e-06, +1.248779937e-07, +1.231318974e-03 + 2.623184e-01, +1.150013517e-05, +5.210625999e-18, +3.089062447e-06, +8.984480601e-08, +2.183514588e-03 + 2.997925e-01, +2.092441719e-05, +1.272709259e-17, +5.172537100e-06, +1.634720093e-07, +3.443881315e-03 + 3.372665e-01, +3.023129661e-05, +1.999099091e-17, +8.595200611e-06, +2.361820048e-07, +4.896471117e-03 + 3.747406e-01, +2.966149891e-05, +5.653422694e-17, +8.160478469e-06, +2.317304602e-07, +6.288238192e-03 + 4.122146e-01, +3.934747640e-05, +1.412915615e-17, +1.043816139e-05, +3.074021593e-07, +7.236958436e-03 + 4.496887e-01, +5.496375444e-05, +1.412118829e-16, +1.376602993e-05, +4.294043315e-07, +7.438657288e-03 + 4.871627e-01, +5.317715171e-05, +1.764915993e-16, +1.345278693e-05, +4.154464977e-07, +7.655736081e-03 + 5.246368e-01, +6.345987516e-05, +3.583062602e-16, +1.616235728e-05, +4.957802747e-07, +1.144507368e-02 + 5.621109e-01, +1.000188545e-04, +6.686919397e-16, +2.890344809e-05, +7.813973009e-07, +2.129749051e-02 + 5.995849e-01, +1.569853883e-04, +6.894487631e-16, +5.054827882e-05, +1.226448346e-06, +3.669225071e-02 + 6.370590e-01, +2.371414637e-04, +2.987336867e-15, +7.811657098e-05, +1.852667685e-06, +5.641515864e-02 + 6.745330e-01, +3.436988518e-04, +2.164168027e-15, +1.084030298e-04, +2.685147279e-06, +7.828737440e-02 + 7.120071e-01, +4.503449422e-04, +8.434257674e-15, +1.334532774e-04, +3.518319861e-06, +9.885303849e-02 + 7.494811e-01, +5.387201988e-04, +2.210553102e-14, +1.460192421e-04, +4.208751553e-06, +1.137546488e-01 + 7.869552e-01, +5.854634659e-04, +3.699123455e-14, +1.426211788e-04, +4.573933327e-06, +1.195362049e-01 + 8.244293e-01, +5.945252635e-04, +1.542889987e-13, +1.553115760e-04, +4.644728621e-06, +1.188356113e-01 + 8.619033e-01, +6.277794936e-04, +8.438991416e-13, +1.446548827e-04, +4.904527294e-06, +1.309570623e-01 + 8.993774e-01, +8.196023636e-04, +3.542625876e-12, +2.064570783e-04, +6.403143466e-06, +1.857784636e-01 + 9.368514e-01, +1.245063027e-03, +1.235452852e-11, +3.796113156e-04, +9.727054902e-06, +2.867343617e-01 + 9.743255e-01, +1.824265442e-03, +3.885931837e-11, +5.734398442e-04, +1.425207377e-05, +4.167726757e-01 + 1.011800e+00, +2.450890627e-03, +1.126668431e-10, +7.521115016e-04, +1.914758302e-05, +5.549120054e-01 + 1.049274e+00, +3.008970318e-03, +2.963478149e-10, +8.708173547e-04, +2.350758061e-05, +6.782193157e-01 + 1.086748e+00, +3.391209592e-03, +6.882830451e-10, +8.861180159e-04, +2.649382494e-05, +7.649055092e-01 + 1.124222e+00, +3.537834071e-03, +1.363904689e-09, +8.504123137e-04, +2.763932868e-05, +8.022591780e-01 + 1.161696e+00, +3.511779316e-03, +2.150129889e-09, +8.840350013e-04, +2.743577591e-05, +8.009806200e-01 + 1.199170e+00, +3.584287871e-03, +2.125157867e-09, +8.159752391e-04, +2.800224899e-05, +8.143332817e-01 + 1.236644e+00, +4.160738356e-03, +3.088716716e-09, +8.694210875e-04, +3.250576840e-05, +9.294499013e-01 + 1.274118e+00, +5.352965496e-03, +1.778269645e-08, +1.361582719e-03, +4.182004293e-05, +1.185780064e+00 + 1.311592e+00, +6.884036966e-03, +5.482403424e-08, +1.934136740e-03, +5.378153880e-05, +1.534661136e+00 + 1.349066e+00, +8.393574758e-03, +1.274739062e-07, +2.361243463e-03, +6.557480280e-05, +1.896361173e+00 + 1.386540e+00, +9.587144870e-03, +2.427922285e-07, +2.543459607e-03, +7.489956930e-05, +2.199964363e+00 + 1.424014e+00, +1.030847845e-02, +3.870793288e-07, +2.421466310e-03, +8.053498792e-05, +2.397709121e+00 + 1.461488e+00, +1.058479831e-02, +5.172682974e-07, +2.459542459e-03, +8.269373679e-05, +2.477248854e+00 + 1.498962e+00, +1.065114467e-02, +5.900129004e-07, +2.478084260e-03, +8.321206774e-05, +2.474045348e+00 + 1.536436e+00, +1.088707737e-02, +6.933528164e-07, +2.347785517e-03, +8.505529193e-05, +2.473655990e+00 + 1.573910e+00, +1.161350430e-02, +1.128992608e-06, +2.453010503e-03, +9.073050232e-05, +2.579211406e+00 + 1.611384e+00, +1.284893703e-02, +1.982498898e-06, +2.368957242e-03, +1.003823205e-04, +2.837696762e+00 + 1.648859e+00, +1.431858623e-02, +3.112369243e-06, +3.068473841e-03, +1.118639549e-04, +3.200565871e+00 + 1.686333e+00, +1.568466437e-02, +4.376324705e-06, +3.469083241e-03, +1.225364404e-04, +3.569988173e+00 + 1.723807e+00, +1.672073184e-02, +5.660135647e-06, +3.499417907e-03, +1.306307175e-04, +3.860976369e+00 + 1.761281e+00, +1.735919124e-02, +6.836616888e-06, +3.289526113e-03, +1.356186815e-04, +4.031109202e+00 + 1.798755e+00, +1.767123055e-02, +7.821966341e-06, +3.511818707e-03, +1.380564887e-04, +4.086241163e+00 + 1.836229e+00, +1.781523617e-02, +9.246723861e-06, +3.414133306e-03, +1.391815326e-04, +4.072134164e+00 + 1.873703e+00, +1.795745681e-02, +1.328997691e-05, +3.394013662e-03, +1.402926314e-04, +4.053121061e+00 + 1.911177e+00, +1.820302848e-02, +2.161284355e-05, +3.420849873e-03, +1.422111600e-04, +4.080942379e+00 + 1.948651e+00, +1.856514788e-02, +3.356781169e-05, +3.234892490e-03, +1.450402178e-04, +4.169173455e+00 + 1.986125e+00, +1.898957180e-02, +4.723137868e-05, +3.396745165e-03, +1.483560297e-04, +4.292737520e+00 + 2.023599e+00, +1.939414578e-02, +5.977348853e-05, +3.250581883e-03, +1.515167639e-04, +4.410955044e+00 + 2.061073e+00, +1.970394331e-02, +6.737231934e-05, +3.312961614e-03, +1.539370571e-04, +4.492517857e+00 + 2.098547e+00, +1.988190622e-02, +6.610344670e-05, +3.301605892e-03, +1.553273923e-04, +4.527215199e+00 + 2.136021e+00, +1.993731499e-02, +5.671443237e-05, +3.172906528e-03, +1.557602734e-04, +4.524104599e+00 + 2.173495e+00, +1.990562097e-02, +6.101946084e-05, +3.293718936e-03, +1.555126638e-04, +4.502015291e+00 + 2.210969e+00, +1.982893776e-02, +1.088878841e-04, +3.115929788e-03, +1.549135762e-04, +4.478491192e+00 + 2.248443e+00, +1.974876101e-02, +1.873801406e-04, +3.228307132e-03, +1.542871954e-04, +4.462065326e+00 + 2.285917e+00, +1.968329021e-02, +2.772981557e-04, +3.180410477e-03, +1.537757048e-04, +4.450543317e+00 + 2.323392e+00, +1.961439724e-02, +3.585051575e-04, +3.107072876e-03, +1.532374784e-04, +4.434922062e+00 + 2.360866e+00, +1.951363989e-02, +4.075955424e-04, +3.188334830e-03, +1.524503116e-04, +4.406444465e+00 + 2.398340e+00, +1.937089613e-02, +4.015208980e-04, +2.980831052e-03, +1.513351261e-04, +4.364029184e+00 + 2.435814e+00, +1.920186059e-02, +3.225495985e-04, +3.138426025e-03, +1.500145359e-04, +4.319424711e+00 + 2.473288e+00, +1.906306149e-02, +1.840331757e-04, +3.055918232e-03, +1.489301679e-04, +4.296324704e+00 + 2.510762e+00, +1.905052602e-02, +2.217725829e-04, +3.046207985e-03, +1.488322345e-04, +4.319068012e+00 + 2.548236e+00, +1.922732383e-02, +1.821453144e-04, +3.098865948e-03, +1.502134674e-04, +4.391888403e+00 + 2.585710e+00, +1.953172619e-02, +8.506756807e-05, +2.933659325e-03, +1.525916109e-04, +4.481531511e+00 + 2.623184e+00, +1.974732686e-02, +1.004249256e-04, +3.113361480e-03, +1.542759911e-04, +4.519891310e+00 + 2.660658e+00, +1.956384336e-02, +1.196983100e-04, +3.005984915e-03, +1.528425262e-04, +4.429739436e+00 + 2.698132e+00, +1.871919389e-02, +1.071559565e-04, +2.901948751e-03, +1.462437022e-04, +4.164358438e+00 + 2.735606e+00, +1.718960116e-02, +1.320053754e-04, +2.803797700e-03, +1.342937591e-04, +3.755791737e+00 + 2.773080e+00, +1.546491642e-02, +9.509285112e-05, +2.408371066e-03, +1.208196596e-04, +3.369663722e+00 + 2.810554e+00, +1.468011525e-02, +7.315500309e-05, +2.441928087e-03, +1.146884004e-04, +3.300120354e+00 + 2.848028e+00, +1.586922846e-02, +4.353556071e-05, +2.665555862e-03, +1.239783474e-04, +3.715710458e+00 + 2.885502e+00, +1.862615405e-02, +2.018520484e-05, +3.373944154e-03, +1.455168285e-04, +4.412752860e+00 + 2.922976e+00, +2.144870957e-02, +7.479911126e-06, +4.038673924e-03, +1.675680435e-04, +5.035126169e+00 + 2.960451e+00, +2.292343187e-02, +9.559526315e-06, +4.563613742e-03, +1.790893115e-04, +5.296819140e+00 + 2.997925e+00, +2.211029675e-02, +1.216757197e-05, +4.513889015e-03, +1.727366933e-04, +5.025358059e+00 + 3.035399e+00, +1.863331026e-02, +1.063158449e-05, +3.843556397e-03, +1.455727364e-04, +4.165761053e+00 + 3.072873e+00, +1.272017108e-02, +8.184202988e-06, +2.614012446e-03, +9.937633654e-05, +2.782770985e+00 + 3.110347e+00, +5.289924605e-03, +4.602990863e-06, +9.892105078e-04, +4.132753598e-05, +1.091922648e+00 + 3.147821e+00, +4.602799904e-03, +1.691676549e-06, +8.722525258e-04, +3.595937425e-05, +1.087911650e+00 + 3.185295e+00, +1.215215559e-02, +2.265588631e-06, +2.536343724e-03, +9.493871558e-05, +2.795375974e+00 + 3.222769e+00, +1.838959288e-02, +3.425649809e-06, +3.876930278e-03, +1.436686944e-04, +4.198258824e+00 + 3.260243e+00, +2.210912947e-02, +3.716661553e-06, +4.665403675e-03, +1.727275740e-04, +5.054847159e+00 + 3.297717e+00, +2.287279659e-02, +5.465969625e-06, +4.800014100e-03, +1.786937234e-04, +5.271021749e+00 + 3.335191e+00, +2.087582236e-02, +1.094747830e-05, +4.296501282e-03, +1.630923622e-04, +4.875726472e+00 + 3.372665e+00, +1.693740095e-02, +1.957205488e-05, +3.274370536e-03, +1.323234449e-04, +4.026175767e+00 + 3.410139e+00, +1.265391041e-02, +3.033725029e-05, +2.351759897e-03, +9.885867509e-05, +3.029504715e+00 + 3.447613e+00, +1.056944488e-02, +3.699160296e-05, +1.868764045e-03, +8.257378816e-05, +2.402834715e+00 + 3.485087e+00, +1.203465459e-02, +5.157331520e-05, +1.987519572e-03, +9.402073900e-05, +2.582889975e+00 + 3.522561e+00, +1.483191160e-02, +4.264807631e-05, +2.431130292e-03, +1.158743094e-04, +3.207270944e+00 + 3.560035e+00, +1.694364854e-02, +4.348712933e-05, +2.651516775e-03, +1.323722542e-04, +3.758945722e+00 + 3.597509e+00, +1.777812511e-02, +3.816772941e-05, +2.830632885e-03, +1.388916024e-04, +4.038955569e+00 + 3.634984e+00, +1.749152616e-02, +2.610873059e-05, +2.685137941e-03, +1.366525481e-04, +4.041150977e+00 + 3.672458e+00, +1.659354032e-02, +6.478738042e-05, +2.504827607e-03, +1.296370337e-04, +3.857101060e+00 + 3.709932e+00, +1.567966849e-02, +1.324957095e-04, +2.523973396e-03, +1.224974101e-04, +3.618583613e+00 + 3.747406e+00, +1.515242977e-02, +1.837750403e-04, +2.479688076e-03, +1.183783576e-04, +3.441317861e+00 + 3.784880e+00, +1.505241218e-02, +2.589867086e-04, +2.413883235e-03, +1.175969702e-04, +3.374168903e+00 + 3.822354e+00, +1.517594323e-02, +2.648845439e-04, +2.482655271e-03, +1.185620565e-04, +3.390825944e+00 + 3.859828e+00, +1.530534707e-02, +3.460441840e-04, +2.325030154e-03, +1.195730240e-04, +3.433304933e+00 + 3.897302e+00, +1.533180643e-02, +2.591782733e-04, +2.448645571e-03, +1.197797378e-04, +3.458360447e+00 + 3.934776e+00, +1.525209327e-02, +3.263088590e-04, +2.386365319e-03, +1.191569787e-04, +3.452521374e+00 + 3.972250e+00, +1.511733834e-02, +2.656776671e-04, +2.367449999e-03, +1.181042058e-04, +3.423956689e+00 + 4.009724e+00, +1.498169885e-02, +1.893347598e-04, +2.402930182e-03, +1.170445223e-04, +3.388114585e+00 + 4.047198e+00, +1.486760910e-02, +8.599169429e-05, +2.247589227e-03, +1.161531961e-04, +3.356524519e+00 + 4.084672e+00, +1.476249589e-02, +7.080243992e-05, +2.377243442e-03, +1.153319991e-04, +3.331853567e+00 + 4.122146e+00, +1.464115772e-02, +1.002760363e-04, +2.290380547e-03, +1.143840447e-04, +3.308603435e+00 + 4.159620e+00, +1.448100038e-02, +1.127755126e-04, +2.309758265e-03, +1.131328154e-04, +3.277119137e+00 + 4.197094e+00, +1.425845180e-02, +1.046536840e-04, +2.317095465e-03, +1.113941547e-04, +3.228713172e+00 + 4.234568e+00, +1.397280756e-02, +8.171047692e-05, +2.200893917e-03, +1.091625591e-04, +3.160502439e+00 + 4.272043e+00, +1.365800502e-02, +5.594416747e-05, +2.299295490e-03, +1.067031642e-04, +3.078660427e+00 + 4.309517e+00, +1.335595262e-02, +3.262293196e-05, +2.188198744e-03, +1.043433798e-04, +2.998121297e+00 + 4.346991e+00, +1.310147749e-02, +1.743681123e-05, +2.244566050e-03, +1.023552929e-04, +2.936669456e+00 + 4.384465e+00, +1.291316405e-02, +1.318337128e-05, +2.228431206e-03, +1.008840942e-04, +2.903997885e+00 + 4.421939e+00, +1.276376144e-02, +1.435774115e-05, +2.182482661e-03, +9.971688628e-05, +2.891883014e+00 + 4.459413e+00, +1.257886362e-02, +1.385785904e-05, +2.254463041e-03, +9.827237205e-05, +2.873950817e+00 + 4.496887e+00, +1.227354453e-02, +1.139016125e-05, +2.117954973e-03, +9.588706666e-05, +2.816794882e+00 + 4.534361e+00, +1.178139290e-02, +8.224353168e-06, +2.159095857e-03, +9.204213207e-05, +2.695716179e+00 + 4.571835e+00, +1.108386626e-02, +4.702489008e-06, +2.144208193e-03, +8.659270512e-05, +2.507677639e+00 + 4.609309e+00, +1.023691016e-02, +2.918639729e-06, +1.911635775e-03, +7.997586064e-05, +2.277833009e+00 + 4.646783e+00, +9.371323060e-03, +1.545409438e-06, +1.700487260e-03, +7.321346140e-05, +2.056659349e+00 + 4.684257e+00, +8.657719207e-03, +2.177147857e-06, +1.788518736e-03, +6.763843131e-05, +1.900934905e+00 + 4.721731e+00, +8.217440504e-03, +1.601040994e-07, +1.709076280e-03, +6.419875393e-05, +1.836493022e+00 + 4.759205e+00, +8.019652879e-03, +1.376983126e-06, +1.786480820e-03, +6.265353812e-05, +1.832790721e+00 + 4.796679e+00, +7.890364250e-03, +1.486445732e-06, +1.768891640e-03, +6.164347071e-05, +1.825328220e+00 + 4.834153e+00, +7.621359210e-03, +9.019618284e-07, +1.685597957e-03, +5.954186883e-05, +1.758585885e+00 + 4.871627e+00, +7.068989772e-03, +9.009942214e-07, +1.773969886e-03, +5.522648259e-05, +1.608502805e+00 + 4.909101e+00, +6.213005286e-03, +2.594281588e-07, +1.651811891e-03, +4.853910380e-05, +1.384459441e+00 + 4.946576e+00, +5.162372600e-03, +1.085336926e-06, +1.352451292e-03, +4.033103594e-05, +1.123478548e+00 + 4.984050e+00, +4.122759940e-03, +6.012849127e-07, +9.429225306e-04, +3.220906203e-05, +8.826005714e-01 + 5.021524e+00, +3.350707607e-03, +6.078339164e-07, +7.200088085e-04, +2.617740318e-05, +7.236678618e-01 + 5.058998e+00, +3.013498513e-03, +6.067707923e-07, +7.103653175e-04, +2.354295713e-05, +6.701157353e-01 + 5.096472e+00, +2.982577267e-03, +6.275290444e-08, +7.400640006e-04, +2.330138490e-05, +6.737270463e-01 + 5.133946e+00, +2.967454273e-03, +7.366753069e-07, +6.898839589e-04, +2.318323651e-05, +6.688583217e-01 + 5.171420e+00, +2.790092469e-03, +5.208321575e-07, +7.384504272e-04, +2.179759741e-05, +6.231475505e-01 + 5.208894e+00, +2.418548983e-03, +9.276751686e-07, +7.000383268e-04, +1.889491393e-05, +5.351074185e-01 + 5.246368e+00, +1.911062359e-03, +1.344568641e-06, +5.773793291e-04, +1.493017468e-05, +4.200268783e-01 + 5.283842e+00, +1.372440607e-03, +2.056363581e-06, +4.096268542e-04, +1.072219224e-05, +3.000708009e-01 + 5.321316e+00, +9.213288732e-04, +2.114823140e-06, +2.372579002e-04, +7.197881822e-06, +1.991713402e-01 + 5.358790e+00, +6.722993623e-04, +2.927023180e-06, +1.457815470e-04, +5.252338768e-06, +1.401144692e-01 + 5.396264e+00, +6.283225289e-04, +2.643605723e-06, +1.625307580e-04, +4.908769757e-06, +1.268544987e-01 + 5.433738e+00, +6.348530847e-04, +1.837386900e-06, +1.550414848e-04, +4.959789724e-06, +1.297308749e-01 + 5.471212e+00, +6.001491642e-04, +2.469214797e-06, +1.570842279e-04, +4.688665345e-06, +1.254275991e-01 + 5.508686e+00, +5.139588842e-04, +6.781782666e-07, +1.472964294e-04, +4.015303783e-06, +1.099729019e-01 + 5.546160e+00, +3.974099555e-04, +2.282771322e-06, +1.184808923e-04, +3.104765278e-06, +8.751673075e-02 + 5.583635e+00, +2.814766597e-04, +4.142124635e-06, +8.237874515e-05, +2.199036404e-06, +6.404830704e-02 + 5.621109e+00, +1.971418371e-04, +5.379287255e-06, +4.879963190e-05, +1.540170602e-06, +4.499940303e-02 + 5.658583e+00, +1.566562902e-04, +5.397792904e-06, +2.429803286e-05, +1.223877267e-06, +3.378394289e-02 + 5.696057e+00, +1.405994048e-04, +2.383391840e-06, +2.103885736e-05, +1.098432850e-06, +2.942426165e-02 + 5.733531e+00, +1.313588362e-04, +1.776514710e-06, +1.911272259e-05, +1.026240908e-06, +2.730267726e-02 + 5.771005e+00, +1.162490253e-04, +2.192974420e-06, +1.780103948e-05, +9.081955100e-07, +2.433964679e-02 + 5.808479e+00, +9.660978201e-05, +1.832208009e-06, +1.546361781e-05, +7.547639219e-07, +2.048189674e-02 + 5.845953e+00, +7.972386380e-05, +2.032041682e-06, +1.277530060e-05, +6.228426860e-07, +1.785059556e-02 + 5.883427e+00, +8.075440717e-05, +9.629804893e-07, +1.405092362e-05, +6.308938060e-07, +1.905071629e-02 + 5.920901e+00, +9.847125170e-05, +1.617457848e-06, +1.897478289e-05, +7.693066539e-07, +2.355183009e-02 + 5.958375e+00, +1.201247755e-04, +1.250282629e-06, +2.394888242e-05, +9.384748086e-07, +2.860407565e-02 + 5.995849e+00, +1.353459673e-04, +4.093061083e-07, +2.700896460e-05, +1.057390370e-06, +3.211115798e-02 + 6.033323e+00, +1.393064150e-04, +1.148347939e-06, +2.893048920e-05, +1.088331367e-06, +3.289020287e-02 + 6.070797e+00, +1.306427216e-04, +8.982552787e-07, +2.770021710e-05, +1.020646262e-06, +3.044147732e-02 + 6.108271e+00, +1.061818817e-04, +7.339137606e-07, +2.278162369e-05, +8.295459508e-07, +2.486915720e-02 + 6.145745e+00, +7.209552669e-05, +3.375268173e-07, +1.566264719e-05, +5.632463022e-07, +1.688222621e-02 + 6.183219e+00, +3.514370026e-05, +6.910550007e-07, +6.938825756e-06, +2.745601583e-07, +8.222569600e-03 + 6.220694e+00, +3.332434420e-05, +8.008707418e-07, +5.858394799e-06, +2.603464391e-07, +7.112443417e-03 + 6.258168e+00, +6.596901518e-05, +4.918351718e-07, +1.173531420e-05, +5.153829311e-07, +1.487473373e-02 + 6.295642e+00, +9.667915042e-05, +3.027018996e-07, +1.881777970e-05, +7.553058626e-07, +2.195995150e-02 + 6.333116e+00, +1.151311727e-04, +1.042992248e-06, +2.293042411e-05, +8.994622865e-07, +2.658857944e-02 + 6.370590e+00, +1.207366953e-04, +7.528801127e-07, +2.380623796e-05, +9.432554324e-07, +2.836724125e-02 + 6.408064e+00, +1.164009914e-04, +2.748897153e-07, +2.213552833e-05, +9.093827450e-07, +2.754771847e-02 + 6.445538e+00, +1.035095292e-04, +1.131867278e-06, +1.908048887e-05, +8.086681970e-07, +2.492432935e-02 + 6.483012e+00, +9.055022958e-05, +1.313210723e-06, +1.572048713e-05, +7.074236686e-07, +2.173397508e-02 + 6.520486e+00, +8.224274449e-05, +7.030109454e-07, +1.364569357e-05, +6.425214413e-07, +1.939641242e-02 + 6.557960e+00, +8.240698910e-05, +1.148313876e-06, +1.353813680e-05, +6.438046023e-07, +1.881287963e-02 + 6.595434e+00, +8.663895923e-05, +1.631803635e-06, +1.483972930e-05, +6.768668690e-07, +1.965705398e-02 + 6.632908e+00, +9.150938695e-05, +8.959336189e-07, +1.500001364e-05, +7.149170856e-07, +2.089087372e-02 + 6.670382e+00, +9.411556011e-05, +7.071727640e-07, +1.471136654e-05, +7.352778134e-07, +2.174361592e-02 + 6.707856e+00, +9.427881789e-05, +2.232688753e-06, +1.527592352e-05, +7.365532648e-07, +2.197690465e-02 + 6.745330e+00, +9.283419598e-05, +1.674328225e-06, +1.475470626e-05, +7.252671561e-07, +2.172213826e-02 + 6.782804e+00, +9.091313119e-05, +2.172433946e-06, +1.494130966e-05, +7.102588375e-07, +2.125742296e-02 + 6.820278e+00, +8.947559314e-05, +1.208738989e-06, +1.467196179e-05, +6.990280714e-07, +2.082523460e-02 + 6.857752e+00, +8.849046096e-05, +2.183297733e-07, +1.422391043e-05, +6.913317262e-07, +2.053535812e-02 + 6.895227e+00, +8.869124726e-05, +1.872610662e-06, +1.475311712e-05, +6.929003692e-07, +2.037002683e-02 + 6.932701e+00, +8.703583658e-05, +1.303860761e-06, +1.403557491e-05, +6.799674733e-07, +2.025706959e-02 + 6.970175e+00, +8.705236106e-05, +1.310170098e-06, +1.440134747e-05, +6.800965708e-07, +2.014118630e-02 + 7.007649e+00, +8.649188885e-05, +1.642909337e-06, +1.413717954e-05, +6.757178817e-07, +2.000795493e-02 + 7.045123e+00, +8.557114224e-05, +8.070387401e-07, +1.402625378e-05, +6.685245487e-07, +1.987188346e-02 + 7.082597e+00, +8.556574405e-05, +6.303920028e-07, +1.437910365e-05, +6.684823754e-07, +1.974815441e-02 + 7.120071e+00, +8.395754839e-05, +2.492708417e-07, +1.333948635e-05, +6.559183468e-07, +1.962815933e-02 + 7.157545e+00, +8.469831726e-05, +4.394357425e-07, +1.399445266e-05, +6.617056036e-07, +1.947236585e-02 + 7.195019e+00, +8.219518931e-05, +2.016475059e-07, +1.374633426e-05, +6.421499165e-07, +1.922019946e-02 + 7.232493e+00, +8.060990044e-05, +7.126283209e-07, +1.375634865e-05, +6.297648471e-07, +1.881533865e-02 + 7.269967e+00, +7.975417972e-05, +2.119913433e-07, +1.379736957e-05, +6.230795291e-07, +1.823277535e-02 + 7.307441e+00, +7.470325724e-05, +1.169668973e-07, +1.285632329e-05, +5.836191972e-07, +1.750653819e-02 + 7.344915e+00, +7.330184966e-05, +2.570909515e-07, +1.365659277e-05, +5.726707005e-07, +1.673934977e-02 + 7.382389e+00, +7.020344015e-05, +4.248389728e-07, +1.310218281e-05, +5.484643762e-07, +1.607601725e-02 + 7.419863e+00, +6.896813460e-05, +4.112240516e-07, +1.338802711e-05, +5.388135515e-07, +1.564435535e-02 + 7.457337e+00, +6.715833151e-05, +1.872800848e-07, +1.323199478e-05, +5.246744649e-07, +1.547011399e-02 + 7.494811e+00, +6.573000789e-05, +1.923953202e-07, +1.301561971e-05, +5.135156866e-07, +1.543460452e-02 + Mean, +7.685673980e-03, +2.479807676e-04, +5.882818741e-04, +6.004432797e-05, +0.000000000e+00 diff --git a/test/ref/cpw/lumped_adaptive/error-indicators.csv b/test/ref/cpw/lumped_adaptive/error-indicators.csv new file mode 100644 index 000000000..463b36465 --- /dev/null +++ b/test/ref/cpw/lumped_adaptive/error-indicators.csv @@ -0,0 +1,11 @@ + f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization + 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +6.373526308e-06, +2.186423623e-01 + 3.000000e+01, +7.048573140e-01, +1.801733890e-06, +1.911288392e-02, +6.372109948e-06, +4.552939315e+00 + 1.820000e+01, +7.087438254e-01, +1.180056136e-06, +1.962899718e-02, +6.407245113e-06, +2.525522343e+00 + 1.370000e+01, +7.056684565e-01, +1.203625733e-06, +1.815263574e-02, +6.379442906e-06, +2.145476899e+00 + 2.540000e+01, +7.062740050e-01, +1.129132128e-06, +1.917725181e-02, +6.384917236e-06, +3.611335581e+00 + 6.200000e+00, +7.029687080e-01, +7.051383242e-07, +1.772868227e-02, +6.355036414e-06, +7.852578706e-01 + 2.810000e+01, +7.051964244e-01, +9.430618882e-07, +1.902937537e-02, +6.375175602e-06, +4.282992773e+00 + 9.900000e+00, +7.033807914e-01, +7.345028171e-07, +1.792395353e-02, +6.358761765e-06, +1.430053254e+00 + 2.090000e+01, +7.085921616e-01, +7.822419312e-07, +1.961007783e-02, +6.405874029e-06, +2.739108293e+00 + Mean, +7.056328525e-01, +1.160955956e-06, +1.695571829e-02, +6.379121036e-06, +2.186423623e-01 diff --git a/test/ref/cpw/lumped_uniform/error-indicators.csv b/test/ref/cpw/lumped_uniform/error-indicators.csv new file mode 100644 index 000000000..0c718788d --- /dev/null +++ b/test/ref/cpw/lumped_uniform/error-indicators.csv @@ -0,0 +1,17 @@ + f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization + 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +6.373526308e-06, +2.186423623e-01 + 4.000000e+00, +7.037534929e-01, +8.348638678e-07, +1.716164791e-02, +6.362131092e-06, +4.676837686e-01 + 6.000000e+00, +7.030062911e-01, +7.118689649e-07, +1.767838766e-02, +6.355376176e-06, +7.544551055e-01 + 8.000000e+00, +7.029165326e-01, +6.832337447e-07, +1.800232414e-02, +6.354564734e-06, +1.081001045e+00 + 1.000000e+01, +7.034192125e-01, +7.418014192e-07, +1.790795027e-02, +6.359109103e-06, +1.449355685e+00 + 1.200000e+01, +7.044599247e-01, +9.086471535e-07, +1.751260139e-02, +6.368517436e-06, +1.842149227e+00 + 1.400000e+01, +7.058985472e-01, +1.260914522e-06, +1.833900466e-02, +6.381522991e-06, +2.190892577e+00 + 1.600000e+01, +7.074564924e-01, +1.093986277e-06, +1.938386319e-02, +6.395607258e-06, +2.406792037e+00 + 1.800000e+01, +7.086746633e-01, +1.209024854e-06, +1.965872658e-02, +6.406619868e-06, +2.515530177e+00 + 2.000000e+01, +7.088448209e-01, +8.917186070e-07, +1.963851961e-02, +6.408158141e-06, +2.646279233e+00 + 2.200000e+01, +7.081043245e-01, +7.960606263e-07, +1.946623509e-02, +6.401463844e-06, +2.889633318e+00 + 2.400000e+01, +7.070103246e-01, +9.980955322e-07, +1.928974677e-02, +6.391573774e-06, +3.271855291e+00 + 2.600000e+01, +7.059908728e-01, +1.195377481e-06, +1.913395059e-02, +6.382357641e-06, +3.767453335e+00 + 2.800000e+01, +7.052267075e-01, +9.841023725e-07, +1.902600821e-02, +6.375449370e-06, +4.261952660e+00 + 3.000000e+01, +7.048573097e-01, +1.801758126e-06, +1.911288376e-02, +6.372109909e-06, +4.552939157e+00 + Mean, +7.056422335e-01, +1.101163768e-06, +1.672465772e-02, +6.379205843e-06, +2.186423623e-01 diff --git a/test/ref/cpw/wave_adaptive/error-indicators.csv b/test/ref/cpw/wave_adaptive/error-indicators.csv new file mode 100644 index 000000000..d5bfae979 --- /dev/null +++ b/test/ref/cpw/wave_adaptive/error-indicators.csv @@ -0,0 +1,11 @@ + f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization + 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +6.731376290e-06, +2.614559004e-01 + 3.000000e+01, +6.802918578e-01, +5.681633835e-06, +1.845184430e-02, +6.548319901e-06, +4.266998699e+00 + 1.810000e+01, +6.988982656e-01, +1.353606358e-06, +1.749494134e-02, +6.727420545e-06, +2.409892387e+00 + 2.750000e+01, +6.989649048e-01, +3.352201085e-06, +1.700252668e-02, +6.728061998e-06, +3.692517431e+00 + 2.920000e+01, +6.936265525e-01, +3.739780438e-06, +1.745465144e-02, +6.676676349e-06, +4.005812881e+00 + 2.450000e+01, +6.989754646e-01, +2.953814391e-06, +1.726940726e-02, +6.728163644e-06, +3.271877994e+00 + 1.680000e+01, +6.971976662e-01, +4.668231523e-06, +1.732341704e-02, +6.711050999e-06, +2.246634753e+00 + 1.350000e+01, +6.996120816e-01, +1.992113352e-06, +1.706069694e-02, +6.734291561e-06, +1.810579073e+00 + 1.540000e+01, +6.990129264e-01, +4.297760845e-06, +1.727653180e-02, +6.728524242e-06, +2.063625604e+00 + Mean, +6.962098822e-01, +4.894505955e-06, +1.703956245e-02, +6.701542836e-06, +2.614559004e-01 diff --git a/test/ref/cpw/wave_uniform/error-indicators.csv b/test/ref/cpw/wave_uniform/error-indicators.csv new file mode 100644 index 000000000..84504cef3 --- /dev/null +++ b/test/ref/cpw/wave_uniform/error-indicators.csv @@ -0,0 +1,17 @@ + f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization + 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +6.731376290e-06, +2.614559004e-01 + 4.000000e+00, +6.992890992e-01, +1.736035539e-06, +1.680419113e-02, +6.731182612e-06, +5.258892594e-01 + 6.000000e+00, +6.992881535e-01, +2.021152752e-06, +1.676125294e-02, +6.731173509e-06, +7.947256627e-01 + 8.000000e+00, +6.993312910e-01, +2.448759169e-06, +1.675682757e-02, +6.731588740e-06, +1.067084152e+00 + 1.000000e+01, +6.994243806e-01, +2.399252549e-06, +1.681663205e-02, +6.732484797e-06, +1.340172770e+00 + 1.200000e+01, +6.995448511e-01, +2.366910745e-06, +1.694287832e-02, +6.733644416e-06, +1.610652070e+00 + 1.400000e+01, +6.995942415e-01, +1.874364476e-06, +1.709174983e-02, +6.734119836e-06, +1.876886099e+00 + 1.600000e+01, +6.980880025e-01, +5.469569257e-06, +1.721279606e-02, +6.719621155e-06, +2.144504886e+00 + 1.800000e+01, +6.988641967e-01, +1.805639359e-06, +1.748858870e-02, +6.727092606e-06, +2.396980543e+00 + 2.000000e+01, +6.990669364e-01, +2.883025514e-06, +1.755245634e-02, +6.729044128e-06, +2.658455702e+00 + 2.200000e+01, +6.990046675e-01, +3.191459315e-06, +1.749703902e-02, +6.728444743e-06, +2.926893770e+00 + 2.400000e+01, +6.989710885e-01, +2.953254947e-06, +1.732841600e-02, +6.728121520e-06, +3.202248951e+00 + 2.600000e+01, +6.990090439e-01, +3.218910262e-06, +1.709437140e-02, +6.728486869e-06, +3.481668528e+00 + 2.800000e+01, +6.988244472e-01, +3.387084517e-06, +1.706546999e-02, +6.726709988e-06, +3.764267665e+00 + 3.000000e+01, +6.802918601e-01, +5.681637479e-06, +1.845184332e-02, +6.548319923e-06, +4.266998672e+00 + Mean, +6.978600986e-01, +4.485174026e-06, +1.714127920e-02, +6.717427409e-06, +2.614559004e-01 diff --git a/test/ref/rings/error-indicators.csv b/test/ref/rings/error-indicators.csv new file mode 100644 index 000000000..bf92e8ad6 --- /dev/null +++ b/test/ref/rings/error-indicators.csv @@ -0,0 +1,4 @@ + i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization + 1.000000e+00, +3.546095276e-01, +4.103265502e-09, +2.980197519e-02, +6.790554137e-06, +1.115530397e-01 + 2.000000e+00, +2.764870097e-01, +1.023265220e-07, +7.245396387e-03, +5.294556016e-06, +4.557585167e-01 + Mean, +3.155482686e-01, +5.321489377e-08, +1.495441011e-02, +6.042555076e-06, +1.115530397e-01 diff --git a/test/ref/spheres/error-indicators.csv b/test/ref/spheres/error-indicators.csv new file mode 100644 index 000000000..512c94d6e --- /dev/null +++ b/test/ref/spheres/error-indicators.csv @@ -0,0 +1,4 @@ + i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization + 1.000000e+00, +7.827775744e-03, +7.728353753e-07, +5.589618554e-04, +7.474244003e-07, +3.052392581e-01 + 2.000000e+00, +7.529030499e-03, +8.146986865e-07, +7.245677921e-04, +7.188991214e-07, +4.319876077e-01 + Mean, +7.678403122e-03, +1.617085360e-06, +4.783727878e-04, +7.331617609e-07, +3.052392581e-01 diff --git a/test/testcase.jl b/test/testcase.jl index 32e4e7f1c..adff1141f 100644 --- a/test/testcase.jl +++ b/test/testcase.jl @@ -52,6 +52,29 @@ function testcase( @test proc.exitcode == 0 end + # Convert variables to Float64, or replace with zeros + function asfloat(x::Float64) + return x + end + function asfloat(str) + try + return parse(Float64, str) + catch + return 0.0 + end + end + # Convert variables to String15, or replace with empty + function asstring(x::String15) + return x + end + function asstring(str) + try + return parse(String15, str) + catch + return "" + end + end + @testset "Results" begin # Test that directories were created @test isdir(postprodir) @@ -68,7 +91,15 @@ function testcase( data = CSV.File(joinpath(postprodir, file); header=1) |> DataFrame data = data[1:size(dataref, 1), :] - test = isapprox.(data, dataref; rtol=rtol, atol=atol) + fdata = asfloat.(data) + fdataref = asfloat.(dataref) + sdata = asstring.(data) + sdataref = asstring.(dataref) + + test = isapprox.(fdata, fdataref; rtol=rtol, atol=atol) + if (atol < Inf && rtol < Inf) + test .&= (sdata .== sdataref) + end for (row, rowdataref, rowdata) in zip(eachrow(test), eachrow(dataref), eachrow(data)) for (rowcol, rowcoldataref, rowcoldata) in From 97372d7f72303c0f0be2e19352f84d521de2f812 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Tue, 3 Oct 2023 11:07:37 -0400 Subject: [PATCH 05/39] Cleanup debris and formatting fixes --- palace/drivers/drivensolver.cpp | 14 +++++++++----- palace/drivers/eigensolver.cpp | 1 - palace/drivers/electrostaticsolver.cpp | 2 -- palace/drivers/transientsolver.hpp | 2 +- palace/fem/coefficient.hpp | 2 -- palace/fem/integrator.hpp | 1 - palace/linalg/errorestimator.cpp | 12 ------------ palace/linalg/errorestimator.hpp | 8 ++++---- palace/linalg/fluxprojector.hpp | 14 +++++++------- palace/main.cpp | 2 +- palace/models/postoperator.cpp | 12 ++++++------ palace/models/postoperator.hpp | 3 +-- palace/utils/timer.hpp | 1 + 13 files changed, 30 insertions(+), 44 deletions(-) diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 0c944413c..1155218e8 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -120,6 +120,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator // Because the Dirichlet BC is always homogenous, no special elimination is required on // the RHS. Assemble the linear system for the initial frequency (so we can call // KspSolver::SetOperators). Compute everything at the first frequency step. + BlockTimer bt0(Timer::CONSTRUCT); auto K = spaceop.GetStiffnessMatrix(Operator::DIAG_ONE); auto C = spaceop.GetDampingMatrix(Operator::DIAG_ZERO); auto M = spaceop.GetMassMatrix(Operator::DIAG_ZERO); @@ -165,11 +166,12 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator // Main frequency sweep loop. double omega = omega0; + auto t0 = Timer::Now(); for (int step = step0; step < nstep; step++) { - // const double freq = iodata.DimensionalizeValue(IoData::ValueType::FREQUENCY, omega); - // Mpi::Print("\nIt {:d}/{:d}: ω/2π = {:.3e} GHz (elapsed time = {:.2e} s)\n", step + 1, - // nstep, freq, Timer::Duration(Timer::Now() - t0).count()); + const double freq = iodata.DimensionalizeValue(IoData::ValueType::FREQUENCY, omega); + Mpi::Print("\nIt {:d}/{:d}: ω/2π = {:.3e} GHz (elapsed time = {:.2e} s)\n", step + 1, + nstep, freq, Timer::Duration(Timer::Now() - t0).count()); // Assemble the linear system. if (step > step0) @@ -191,8 +193,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator ksp.Mult(RHS, E); // Compute the error indicators, and post process the indicator field. - UpdateErrorIndicators(E, step, - iodata.DimensionalizeValue(IoData::ValueType::FREQUENCY, omega)); + UpdateErrorIndicators(E, step, freq); // Compute B = -1/(iω) ∇ x E on the true dofs, and set the internal GridFunctions in // PostOperator for all postprocessing operations. @@ -233,6 +234,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator double delta_omega) const { // Configure default parameters if not specified. + BlockTimer bt0(Timer::CONSTRUCT); double offline_tol = iodata.solver.driven.adaptive_tol; int nmax = iodata.solver.driven.adaptive_nmax; int ncand = iodata.solver.driven.adaptive_ncand; @@ -339,6 +341,8 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator (iter == nmax) ? " reached maximum" : " converged with", iter, prom.GetReducedDimension(), max_error, offline_tol); utils::PrettyPrint(prom.GetSampleFrequencies(), f0, " Sampled frequencies (GHz):"); + Mpi::Print(" Total offline phase elapsed time: {:.2e} s\n", + Timer::Duration(Timer::Now() - t0).count()); // Timing on root SaveMetadata(prom.GetLinearSolver()); // Set the indicator field to the combined field for postprocessing. diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 6e34bcfac..0b085c188 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -251,7 +251,6 @@ EigenSolver::Solve(const std::vector> &mesh) cons ksp->SetOperators(*A, *P); eigen->SetLinearSolver(*ksp); - // TODO: Add a block timer for estimate construction auto estimator = [&]() { BlockTimer bt(Timer::ESTCONSTRUCT); diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 3f9fcb4c6..cdb025b17 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -70,8 +70,6 @@ ElectrostaticSolver::Solve(const std::vector> &me // Next terminal. step++; } - - // Construct estimator and reducer for error indicators. auto estimator = [&]() { BlockTimer bt(Timer::ESTCONSTRUCT); diff --git a/palace/drivers/transientsolver.hpp b/palace/drivers/transientsolver.hpp index 85dba778d..20a17ac5b 100644 --- a/palace/drivers/transientsolver.hpp +++ b/palace/drivers/transientsolver.hpp @@ -51,7 +51,7 @@ class TransientSolver : public BaseSolver using BaseSolver::BaseSolver; ErrorIndicators - Solve(const std::vector> &mesh) const override; + Solve(const std::vector> &mesh) const final; }; } // namespace palace diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index 464c9e536..3ad9e5db2 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -484,7 +484,6 @@ class CurlFluxCoefficient : public mfem::VectorCoefficient private: const mfem::ParGridFunction &X; MaterialPropertyCoefficient coef; - mfem::DenseMatrix muinv; mfem::Vector curl; @@ -511,7 +510,6 @@ class GradFluxCoefficient : public mfem::VectorCoefficient private: const mfem::ParGridFunction φ MaterialPropertyCoefficient coef; - mfem::Vector grad; mfem::DenseMatrix eps; diff --git a/palace/fem/integrator.hpp b/palace/fem/integrator.hpp index 367dfa2fb..6e068f32c 100644 --- a/palace/fem/integrator.hpp +++ b/palace/fem/integrator.hpp @@ -66,7 +66,6 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator fe.CalcVShape(ip, vshape); Q.Eval(f_loc, Tr, ip); - Tr.InverseJacobian().Mult(f_loc, f_hat); f_hat *= ip.weight * Tr.Weight(); vshape.AddMult(f_hat, elvect); diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index f65808136..813b7b363 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -53,7 +53,6 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVecto mfem::ParComplexGridFunction field(&fes); field.real().SetFromTrueDofs(v.Real()); field.imag().SetFromTrueDofs(v.Imag()); - const int nelem = smooth_flux_fes.GetFinestFESpace().GetNE(); // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). @@ -97,11 +96,8 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVecto flux.imag().ExchangeFaceNbrData(); return flux; }; - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fes.GetFinestFESpace()); - mfem::ParComplexGridFunction coarse_flux_func(&coarse_flux_fes); - coarse_flux_func.real().ProjectCoefficient(real_coef); coarse_flux_func.imag().ProjectCoefficient(imag_coef); @@ -166,7 +162,6 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v, { mfem::ParGridFunction field(&fes); field.SetFromTrueDofs(v); - const int nelem = smooth_flux_fes.GetFinestFESpace().GetNE(); // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). @@ -183,7 +178,6 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v, return RHS; }; - const auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fes.GetFinestFESpace(), coef); // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass @@ -207,7 +201,6 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v, return flux; }; auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fes.GetFinestFESpace()); - mfem::ParGridFunction coarse_flux_func(&coarse_flux_fes); coarse_flux_func.ProjectCoefficient(coef); @@ -289,7 +282,6 @@ IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v, { mfem::ParGridFunction field(&fes); field.SetFromTrueDofs(v); - const int nelem = smooth_flux_fes.GetNE(); // Coefficients for computing the discontinuous flux., i.e. (V, ϵ ∇ ϕ). @@ -328,7 +320,6 @@ IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v, rhs_comp.MakeRef(rhs, i * stride, stride); proj.Mult(rhs_comp, flux_comp); } - return flux; }; auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); @@ -362,7 +353,6 @@ IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v, const int ndof = coarse_vec.Size() / 3; coarse_sub_vec.SetSize(ndof); smooth_sub_vec.SetSize(ndof); - for (int c = 0; c < 3; c++) { coarse_sub_vec.MakeRef(coarse_vec, c * ndof); @@ -373,7 +363,6 @@ IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v, estimates[e] += scalar_mass_matrices[e].InnerProduct(coarse_sub_vec, coarse_sub_vec); // Integrate } - estimates[e] = std::sqrt(estimates[e]); } Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); @@ -394,7 +383,6 @@ IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v, est_field.SetFromTrueDofs(estimates); paraview.RegisterField("ErrorIndicator", &est_field); - paraview.Save(); } if (normalize) diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 8d9b9a672..17d948ff9 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -21,8 +21,8 @@ class PetscParVector; } // namespace petsc // Class used for computing curl flux error estimate, -// i.e. || μ⁻¹∇ × V - F ||_K -// where F denotes a smooth reconstruction of μ⁻¹∇ × V. +// i.e. || μ⁻¹∇ × Vₕ - F ||_K +// where F denotes a smooth reconstruction of μ⁻¹∇ × Vₕ. class CurlFluxErrorEstimator { const MaterialOperator &mat_op; @@ -54,8 +54,8 @@ class CurlFluxErrorEstimator }; // Class used for computing grad flux error estimate, -// i.e. || ϵ ∇ ϕ - F ||_K -// where F denotes a smooth reconstruction of ϵ ∇ ϕ. +// i.e. || ϵ ∇ ϕₕ - F ||_K +// where F denotes a smooth reconstruction of ϵ ∇ ϕₕ. class GradFluxErrorEstimator { const MaterialOperator &mat_op; diff --git a/palace/linalg/fluxprojector.hpp b/palace/linalg/fluxprojector.hpp index eccda9fc0..24194b615 100644 --- a/palace/linalg/fluxprojector.hpp +++ b/palace/linalg/fluxprojector.hpp @@ -16,20 +16,20 @@ namespace palace class MaterialOperator; // -// This solver implements a solver to compute a smooth reconstruction of a -// discontinuous flux. The difference between this resulting smooth flux and the -// original non-smooth flux provides a localizable error estimate. An instance -// of FluxProjector can be reused across solutions, thus the construction of the -// operator is separated from the construction of the flux RHS. +// This solver computes a smooth reconstruction of a discontinuous flux. The difference +// between this resulting smooth flux and the original non-smooth flux provides a +// localizable error estimate. An instance of FluxProjector can be reused across solutions, +// thus the construction of the operator is separated from the construction of the flux RHS. class FluxProjector { private: - // Operator for the mass matrix inversion + // Operator for the mass matrix inversion. std::unique_ptr M; - // Linear solver and preconditioner for the projected linear system M σ = σ̂ + // Linear solver and preconditioner for the projected linear system M σ = σ̂. std::unique_ptr ksp; + // Intermediate storage vector used in Mult. mutable Vector tmp; public: diff --git a/palace/main.cpp b/palace/main.cpp index a19902b58..7be814a39 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -155,7 +155,7 @@ int main(int argc, char *argv[]) #endif // Initialize the problem driver. - const std::unique_ptr solver = [&]() -> std::unique_ptr + const auto solver = [&]() -> std::unique_ptr { switch (iodata.problem.type) { diff --git a/palace/models/postoperator.cpp b/palace/models/postoperator.cpp index a49fc6978..49e7c2f5b 100644 --- a/palace/models/postoperator.cpp +++ b/palace/models/postoperator.cpp @@ -47,7 +47,7 @@ PostOperator::PostOperator(const IoData &iodata, SpaceOperator &spaceop, E(&spaceop.GetNDSpace()), B(&spaceop.GetRTSpace()), V(std::nullopt), A(std::nullopt), indicator_fec(0, spaceop.GetNDSpace().GetParMesh()->Dimension()), indicator_fes(spaceop.GetNDSpace().GetParMesh(), &indicator_fec), - indicator_field(std::nullopt), lumped_port_init(false), wave_port_init(false), + indicator_field(&indicator_fes), lumped_port_init(false), wave_port_init(false), paraview(CreateParaviewPath(iodata, name), spaceop.GetNDSpace().GetParMesh()), paraview_bdr(CreateParaviewPath(iodata, name) + "_boundary", spaceop.GetNDSpace().GetParMesh()), @@ -104,7 +104,7 @@ PostOperator::PostOperator(const IoData &iodata, LaplaceOperator &laplaceop, V(&laplaceop.GetH1Space()), A(std::nullopt), indicator_fec(0, laplaceop.GetH1Space().GetParMesh()->Dimension()), indicator_fes(laplaceop.GetH1Space().GetParMesh(), &indicator_fec), - indicator_field(std::nullopt), lumped_port_init(false), wave_port_init(false), + indicator_field(&indicator_fes), lumped_port_init(false), wave_port_init(false), paraview(CreateParaviewPath(iodata, name), laplaceop.GetNDSpace().GetParMesh()), paraview_bdr(CreateParaviewPath(iodata, name) + "_boundary", laplaceop.GetNDSpace().GetParMesh()), @@ -134,7 +134,7 @@ PostOperator::PostOperator(const IoData &iodata, CurlCurlOperator &curlcurlop, A(&curlcurlop.GetNDSpace()), indicator_fec(0, curlcurlop.GetNDSpace().GetParMesh()->Dimension()), indicator_fes(curlcurlop.GetNDSpace().GetParMesh(), &indicator_fec), - indicator_field(std::nullopt), lumped_port_init(false), wave_port_init(false), + indicator_field(&indicator_fes), lumped_port_init(false), wave_port_init(false), paraview(CreateParaviewPath(iodata, name), curlcurlop.GetNDSpace().GetParMesh()), paraview_bdr(CreateParaviewPath(iodata, name) + "_boundary", curlcurlop.GetNDSpace().GetParMesh()), @@ -231,6 +231,7 @@ void PostOperator::InitializeDataCollection(const IoData &iodata) paraview.RegisterField("A", &*A); paraview_bdr.RegisterVCoeffField("A", As.get()); } + paraview.RegisterField("ErrorIndicator", &*indicator_field); // Extract surface charge from normally discontinuous ND E-field. Also extract surface // currents from tangentially discontinuous RT B-field The surface charge and surface @@ -340,10 +341,9 @@ void PostOperator::SetAGridFunction(const Vector &a) void PostOperator::SetIndicatorGridFunction(const mfem::Vector &i) { - indicator_field = mfem::ParGridFunction(&indicator_fes); + MFEM_VERIFY(indicator_field, + "Incorrect usage of PostOperator::SetIndicatorGridFunction!"); indicator_field->SetFromTrueDofs(i); - // Reregistration overwrites the underlying. - paraview.RegisterField("ErrorIndicator", std::addressof(indicator_field.value())); } void PostOperator::UpdatePorts(const LumpedPortOperator &lumped_port_op, double omega) diff --git a/palace/models/postoperator.hpp b/palace/models/postoperator.hpp index 943a1dfa2..ff0e6b898 100644 --- a/palace/models/postoperator.hpp +++ b/palace/models/postoperator.hpp @@ -168,9 +168,8 @@ class PostOperator // Write to disk the E- and B-fields extracted from the solution vectors. Note that fields // are not redimensionalized, to do so one needs to compute: B <= B * (μ₀ H₀), E <= E * - // (Z₀ H₀), V <= V * (Z₀ H₀ L₀), etc. Optionally also write error indicator field. + // (Z₀ H₀), V <= V * (Z₀ H₀ L₀), etc. void WriteFields(int step, double time) const; - void WriteFields(int step, double time, ErrorIndicators &indicators) const; // Probe the E- and B-fields for their vector-values at speceified locations in space. // Locations of probes are set up in constructor from configuration file data. If diff --git a/palace/utils/timer.hpp b/palace/utils/timer.hpp index 15a305164..e55d341d1 100644 --- a/palace/utils/timer.hpp +++ b/palace/utils/timer.hpp @@ -81,6 +81,7 @@ class Timer // Return the time elapsed since timer creation. Duration TimeFromStart() const { return Now() - start_time; } + // Save a timing step by adding a duration, without lapping; optionally, count it. Duration SaveTime(Index idx, Duration time, bool count_it = true) { From d161003a1d1ac30152bb14c4e77d6e8b3a3ad845 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Tue, 3 Oct 2023 15:08:24 -0400 Subject: [PATCH 06/39] Add option to exclude columns of reference data from regression testing. Updated CHANGELOG --- CHANGELOG.md | 4 ++++ test/runtests.jl | 24 ++++++++++++++++-------- test/testcase.jl | 18 +++++++++++++----- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4f82c504..d182f9f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,10 @@ The format of this changelog is based on based for now). - Added improved OpenMP support in `palace` wrapper script and CI tests. - Added Apptainer/Singularity container build definition for Palace. + - Added flux-based error estimation, reported in `error-estimate.csv`. This computes the + difference between the numerical gradient (electrostatics) or curl (otherwise) of the + solution, and a smoother approximation obtained through a global mass matrix inversion. + The results are reported in `error-estimates.csv` within the `"Output"` folder. ## [0.11.2] - 2023-07-14 diff --git a/test/runtests.jl b/test/runtests.jl index 0d6c05be1..f4c7fb906 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -33,7 +33,8 @@ abstol = 1.0e-16 palace=palace, np=numprocs, rtol=reltol, - atol=abstol + atol=abstol, + excluded_columns=["Max Elem", "Min Elem"] ) @info "Testing rings..." @@ -44,7 +45,8 @@ abstol = 1.0e-16 palace=palace, np=numprocs, rtol=reltol, - atol=abstol + atol=abstol, + excluded_columns=["Max Elem", "Min Elem"] ) @info "Testing cavity (PEC)..." @@ -55,7 +57,8 @@ abstol = 1.0e-16 palace=palace, np=numprocs, rtol=reltol, - atol=abstol + atol=abstol, + excluded_columns=["Max Elem", "Min Elem"] ) @info "Testing cavity (impedance)..." @@ -66,7 +69,8 @@ abstol = 1.0e-16 palace=palace, np=numprocs, rtol=reltol, - atol=abstol + atol=abstol, + excluded_columns=["Max Elem", "Min Elem"] ) # Coarser test tolerances for driven simulations with ports @@ -81,7 +85,8 @@ abstol = 2.0e-12 palace=palace, np=numprocs, rtol=reltol, - atol=abstol + atol=abstol, + excluded_columns=["Max Elem", "Min Elem"] ) @info "Testing coaxial (matched)..." @@ -92,7 +97,8 @@ abstol = 2.0e-12 palace=palace, np=numprocs, rtol=reltol, - atol=abstol + atol=abstol, + excluded_columns=["Max Elem", "Min Elem"] ) @info "Testing CPW (lumped ports)" @@ -103,7 +109,8 @@ abstol = 2.0e-12 palace=palace, np=numprocs, rtol=reltol, - atol=abstol + atol=abstol, + excluded_columns=["Max Elem", "Min Elem"] ) @info "Testing CPW (wave ports)" @@ -114,7 +121,8 @@ abstol = 2.0e-12 palace=palace, np=numprocs, rtol=reltol, - atol=abstol + atol=abstol, + excluded_columns=["Max Elem", "Min Elem"] ) # Don't check accuracy for adaptive frequency sweep simulations diff --git a/test/testcase.jl b/test/testcase.jl index adff1141f..f74bc25db 100644 --- a/test/testcase.jl +++ b/test/testcase.jl @@ -8,7 +8,8 @@ function testcase( palace="palace", np=1, rtol=1.0e-6, - atol=1.0e-18 + atol=1.0e-18, + excluded_columns=[] ) if isempty(testdir) @info "$testdir/ is empty, skipping tests" @@ -52,22 +53,22 @@ function testcase( @test proc.exitcode == 0 end - # Convert variables to Float64, or replace with zeros + # Convert variables to Float64, or replace with zeros. function asfloat(x::Float64) return x end - function asfloat(str) + function asfloat(str)::Float64 try return parse(Float64, str) catch return 0.0 end end - # Convert variables to String15, or replace with empty + # Convert variables to String15, or replace with empty. function asstring(x::String15) return x end - function asstring(str) + function asstring(str)::String15 try return parse(String15, str) catch @@ -91,6 +92,13 @@ function testcase( data = CSV.File(joinpath(postprodir, file); header=1) |> DataFrame data = data[1:size(dataref, 1), :] + # Check the number of columns matches, before removing any excluded_columns + @test ncol(data) == ncol(dataref) + for col ∈ excluded_columns + select!(data, Not(Cols(contains(col)))) + select!(dataref, Not(Cols(contains(col)))) + end + fdata = asfloat.(data) fdataref = asfloat.(dataref) sdata = asstring.(data) From 8abe8b52ba5c72f35621d5177241e080114b4402 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Tue, 3 Oct 2023 20:50:09 -0400 Subject: [PATCH 07/39] Some first PR comment fixes --- palace/drivers/basesolver.hpp | 4 +++- palace/fem/coefficient.hpp | 32 +++++++++++++++----------------- palace/linalg/errorestimator.cpp | 4 ++-- palace/linalg/errorestimator.hpp | 5 ----- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index e411c0f24..42c6571e1 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -11,8 +11,10 @@ namespace mfem { + class ParFiniteElementSpaceHierarchy; class ParMesh; + } // namespace mfem namespace palace @@ -80,7 +82,7 @@ class BaseSolver const char *git_tag = nullptr); virtual ~BaseSolver() = default; - // Performs a solve using the mesh sequence, and report error indicators. + // Performs a solve using the mesh sequence, and reports error indicators. virtual ErrorIndicators Solve(const std::vector> &mesh) const = 0; diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index 3ad9e5db2..83756b706 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -477,20 +477,19 @@ inline double DielectricInterfaceCoefficient:: return 0.5 * ts * epsilon * (V * V); } -// Computes the flux, μ⁻¹ ∇ × X, of a field, X, where X can be the electric field E, or the +// Computes the flux, μ⁻¹ ∇ × U, of a field, U, where U can be the electric field E, or the // magnetic vector potential A. class CurlFluxCoefficient : public mfem::VectorCoefficient { private: - const mfem::ParGridFunction &X; - MaterialPropertyCoefficient coef; - mfem::DenseMatrix muinv; + const mfem::ParGridFunction &U; + const MaterialOperator &mat_op; mfem::Vector curl; public: - CurlFluxCoefficient(const mfem::ParGridFunction &pgf, const MaterialOperator &op) - : mfem::VectorCoefficient(pgf.ParFESpace()->GetParMesh()->SpaceDimension()), X(pgf), - coef(op, 1.0), muinv(3), curl(3) + CurlFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) + : mfem::VectorCoefficient(gf.ParFESpace()->GetParMesh()->SpaceDimension()), U(gf), + mat_op(mat_op), curl(3) { } @@ -498,9 +497,8 @@ class CurlFluxCoefficient : public mfem::VectorCoefficient const mfem::IntegrationPoint &ip) override { V.SetSize(3); - coef.Eval(muinv, T, ip); - X.GetCurl(T, curl); - muinv.Mult(curl, V); + U.GetCurl(T, curl); + mat_op.GetInvPermeability(T.Attribute).Mult(curl, V); } }; @@ -509,14 +507,13 @@ class GradFluxCoefficient : public mfem::VectorCoefficient { private: const mfem::ParGridFunction φ - MaterialPropertyCoefficient coef; + const MaterialOperator &mat_op; mfem::Vector grad; - mfem::DenseMatrix eps; public: - GradFluxCoefficient(const mfem::ParGridFunction &pgf, const MaterialOperator &op) - : mfem::VectorCoefficient(pgf.ParFESpace()->GetParMesh()->SpaceDimension()), phi(pgf), - coef(op, 1.0), grad(3), eps(3) + GradFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) + : mfem::VectorCoefficient(gf.ParFESpace()->GetParMesh()->SpaceDimension()), phi(gf), + mat_op(mat_op), grad(3) { } @@ -524,9 +521,8 @@ class GradFluxCoefficient : public mfem::VectorCoefficient const mfem::IntegrationPoint &ip) override { V.SetSize(3); - coef.Eval(eps, T, ip); phi.GetGradient(T, grad); - eps.Mult(grad, V); + mat_op.GetPermittivityReal(T.Attribute).Mult(grad, V); } }; @@ -792,6 +788,8 @@ class SumCoefficient : public mfem::Coefficient } public: + SumCoefficient() : mfem::Coefficient() {} + bool empty() const { return c.empty(); } void AddCoefficient(std::unique_ptr &&coef) diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 813b7b363..5f37c92b4 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -85,7 +85,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVecto }; auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); - // Given a complex solution represented with a PetscParVector, build a ComplexGridFunction + // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction // for evaluation. auto build_func = [](const ComplexVector &f, mfem::ParFiniteElementSpace &fes) { @@ -191,7 +191,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v, }; auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); - // Given a complex solution represented with a PetscParVector, build a ComplexGridFunction + // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction // for evaluation. auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fes) { diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 17d948ff9..dcd17713a 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -15,11 +15,6 @@ namespace palace class IoData; class MaterialOperator; -namespace petsc -{ -class PetscParVector; -} // namespace petsc - // Class used for computing curl flux error estimate, // i.e. || μ⁻¹∇ × Vₕ - F ||_K // where F denotes a smooth reconstruction of μ⁻¹∇ × Vₕ. From 700d047dacb05f47608edd6322130102197d8755 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Tue, 3 Oct 2023 21:33:16 -0400 Subject: [PATCH 08/39] More PR comment fixes --- palace/drivers/drivensolver.cpp | 15 +++--- palace/drivers/eigensolver.cpp | 11 ++-- palace/drivers/electrostaticsolver.cpp | 9 ++-- palace/drivers/magnetostaticsolver.cpp | 7 ++- palace/drivers/transientsolver.cpp | 7 ++- palace/linalg/errorestimator.cpp | 6 +-- palace/linalg/errorestimator.hpp | 8 +-- palace/utils/errorindicators.cpp | 73 +++++++++++++++----------- palace/utils/errorindicators.hpp | 39 ++++++++------ 9 files changed, 90 insertions(+), 85 deletions(-) diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 1155218e8..8ca8e8312 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -147,13 +147,12 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); - const ErrorReductionOperator ErrorReducer; - auto UpdateErrorIndicators = [this, &estimator, &indicators, &ErrorReducer, + auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &E, int step, double f) { BlockTimer bt0(Timer::ESTSOLVE); constexpr bool normalized = true; - auto estimate = estimator(E, normalized); + auto estimate = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. postop.SetIndicatorGridFunction(estimate.indicators); @@ -161,7 +160,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator "f (GHz)", step, f, ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, normalized); - ErrorReducer(indicators, std::move(estimate)); + indicators.AddEstimates(estimate.indicators, estimate.normalization); }; // Main frequency sweep loop. @@ -284,20 +283,18 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // The error indicators will be calculated for each HDM sample rather than for // the online stage. ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); - ErrorReductionOperator ErrorReducer; - auto UpdateErrorIndicators = [this, &estimator, &indicators, - &ErrorReducer](const auto &E, int step, double frequency) + auto UpdateErrorIndicators = [this, &estimator, &indicators](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTSOLVE); constexpr bool normalized = true; - auto estimate = estimator(E, normalized); + auto estimate = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. PostprocessErrorIndicators( "f (GHz)", step, frequency, ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, normalized); - ErrorReducer(indicators, std::move(estimate)); + indicators.AddEstimates(estimate.indicators, estimate.normalization); }; // Initialize the basis with samples from the top and bottom of the frequency diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 0b085c188..e8b091333 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -266,21 +266,20 @@ EigenSolver::Solve(const std::vector> &mesh) cons // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); - const ErrorReductionOperator ErrorReducer; auto UpdateErrorIndicators = - [this, &estimator, &indicators, &ErrorReducer, &postop](const auto &E, int i) + [this, &estimator, &indicators, &postop](const auto &E, int i) { BlockTimer bt0(Timer::ESTSOLVE); constexpr bool normalized = true; - auto ind = estimator(E, normalized); + auto estimate = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); - postop.SetIndicatorGridFunction(ind.indicators); + postop.SetIndicatorGridFunction(estimate.indicators); // Write the indicator for this mode. PostprocessErrorIndicators( "m", i, i + 1, - ErrorIndicators{ind, indicators.GlobalTrueVSize(), indicators.GetComm()}, + ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, normalized); - ErrorReducer(indicators, std::move(ind)); + indicators.AddEstimates(estimate.indicators, estimate.normalization); }; // Postprocess the results. diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index cdb025b17..c98e94738 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -101,14 +101,11 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, mfem::DenseMatrix C(nstep), Cm(nstep); Vector E(Grad->Height()), Vij(Grad->Width()); ErrorIndicators indicators(laplaceop.GlobalTrueVSize(), laplaceop.GetComm()); - const ErrorReductionOperator ErrorReducer; - auto UpdateErrorIndicators = [this, &estimator, &indicators, &ErrorReducer, - &postop](const auto &V, int i, double idx) + auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &V, int i, double idx) { BlockTimer bt0(Timer::ESTSOLVE); constexpr bool normalized = true; - auto estimate = estimator(V, normalized); - ErrorReducer(indicators, estimate); + auto estimate = estimator.ComputeIndicators(V, normalized); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. postop.SetIndicatorGridFunction(estimate.indicators); @@ -116,7 +113,7 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, "i", i, idx, ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, normalized); - ErrorReducer(indicators, std::move(estimate)); + indicators.AddEstimates(estimate.indicators, estimate.normalization); }; if (iodata.solver.electrostatic.n_post > 0) { diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index b19389fe1..982c8f0bc 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -105,13 +105,12 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators indicators(curlcurlop.GlobalTrueVSize(), curlcurlop.GetComm()); - ErrorReductionOperator ErrorReducer; - auto UpdateErrorIndicators = [this, &estimator, &indicators, &ErrorReducer, + auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &A, int i, double idx) { BlockTimer bt0(Timer::ESTSOLVE); constexpr bool normalized = true; - auto estimate = estimator(A, normalized); + auto estimate = estimator.ComputeIndicators(A, normalized); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. postop.SetIndicatorGridFunction(estimate.indicators); @@ -119,7 +118,7 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, "i", i, idx, ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, normalized); - ErrorReducer(indicators, std::move(estimate)); + indicators.AddEstimates(estimate.indicators, estimate.normalization); }; if (iodata.solver.magnetostatic.n_post > 0) { diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 568720de5..60af27e73 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -85,21 +85,20 @@ TransientSolver::Solve(const std::vector> &mesh) spaceop.GetNDSpace()); }(); ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); - ErrorReductionOperator ErrorReducer; - auto UpdateErrorIndicators = [this, &estimator, &indicators, &ErrorReducer, + auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &E, int step, double time) { BlockTimer bt0(Timer::ESTSOLVE); // Initial flux of zero would return nan. bool constexpr normalized = false; - auto estimate = estimator(E, normalized); + auto estimate = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(estimate.indicators); PostprocessErrorIndicators( "t (ns)", step, time, ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, normalized); - ErrorReducer(indicators, std::move(estimate)); + indicators.AddEstimates(estimate.indicators, estimate.normalization); }; // Main time integration loop. diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 5f37c92b4..c7cf12f69 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -47,7 +47,7 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( } } -IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVector &v, +IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, bool normalize) const { mfem::ParComplexGridFunction field(&fes); @@ -157,7 +157,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const ComplexVecto return {estimates, normalization}; } -IndicatorsAndNormalization CurlFluxErrorEstimator::operator()(const Vector &v, +IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const { mfem::ParGridFunction field(&fes); @@ -277,7 +277,7 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( } } -IndicatorsAndNormalization GradFluxErrorEstimator::operator()(const Vector &v, +IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const { mfem::ParGridFunction field(&fes); diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index dcd17713a..9d4194507 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -41,11 +41,11 @@ class CurlFluxErrorEstimator mfem::ParFiniteElementSpace &fes); // Compute elemental error indicators given a complex vector of true DOF. - IndicatorsAndNormalization operator()(const ComplexVector &v, - bool normalize = true) const; + IndicatorsAndNormalization ComputeIndicators(const ComplexVector &v, + bool normalize) const; // Compute elemental error indicators given a vector of true DOF. - IndicatorsAndNormalization operator()(const Vector &v, bool normalize = true) const; + IndicatorsAndNormalization ComputeIndicators(const Vector &v, bool normalize ) const; }; // Class used for computing grad flux error estimate, @@ -78,7 +78,7 @@ class GradFluxErrorEstimator mfem::ParFiniteElementSpace &fes); // Compute elemental error indicators given a vector of true DOF. - IndicatorsAndNormalization operator()(const Vector &v, bool normalize = true) const; + IndicatorsAndNormalization ComputeIndicators(const Vector &v, bool normalize) const; }; } // namespace palace diff --git a/palace/utils/errorindicators.cpp b/palace/utils/errorindicators.cpp index 409fdb5cd..d2823ecde 100644 --- a/palace/utils/errorindicators.cpp +++ b/palace/utils/errorindicators.cpp @@ -11,13 +11,13 @@ namespace palace ErrorIndicators::ErrorIndicators(const IndicatorsAndNormalization &indicators, int global_true_v_size, MPI_Comm comm) : local_error_indicators(indicators.indicators), global_true_v_size(global_true_v_size), - comm(comm), normalization(indicators.normalization) + comm(comm), mean_normalization(indicators.normalization) { // Compute the global indicator across all processors. constexpr int p = 2; global_error_indicator = std::transform_reduce(local_error_indicators.begin(), local_error_indicators.end(), - 0.0, std::plus(), [&p](auto val) { return std::pow(val, p); }); + 0.0, std::plus(), [](auto val) { return std::pow(val, p); }); Mpi::GlobalSum(1, &global_error_indicator, comm); global_error_indicator = std::pow(global_error_indicator, 1.0 / p); min = local_error_indicators.Min(); @@ -29,24 +29,23 @@ ErrorIndicators::ErrorIndicators(const IndicatorsAndNormalization &indicators, mean = global_error_indicator / size; } -void ErrorReductionOperator::operator()(ErrorIndicators &ebar, - const IndicatorsAndNormalization &ind, - double p) const +void +ErrorIndicators::Reset() { - if (n == 0) - { - // No direct reduction necessary. - ebar = ErrorIndicators(ind, ebar.global_true_v_size, ebar.comm); - n++; - return; - } - - const auto comm = ebar.GetComm(); + n = 0; + local_error_indicators = Vector(); + global_error_indicator = std::numeric_limits::max(); + mean_normalization = 0.0; +} +void +ErrorIndicators::AddEstimates(const Vector &indicators, double normalization) +{ // Compute the global indicator across all processors. + constexpr int p = 2; double candidate_global_error_indicator = - std::transform_reduce(ind.indicators.begin(), ind.indicators.end(), 0.0, std::plus(), - [&p](auto val) { return std::pow(val, p); }); + std::transform_reduce(indicators.begin(), indicators.end(), 0.0, std::plus(), + [](auto val) { return std::pow(val, p); }); Mpi::GlobalSum(1, &candidate_global_error_indicator, comm); candidate_global_error_indicator = std::pow(candidate_global_error_indicator, 1.0 / p); @@ -56,26 +55,36 @@ void ErrorReductionOperator::operator()(ErrorIndicators &ebar, // important to many solves, rather than only large in one solve. // TODO: Could alternatively consider the maximum. - ebar.global_error_indicator = - (n * ebar.global_error_indicator + candidate_global_error_indicator) / (n + 1); - auto running_average = [this](const auto &xbar, const auto &x) - { return (xbar * n + x) / (n + 1); }; - MFEM_VERIFY(ebar.local_error_indicators.Size() == ind.indicators.Size(), - "Local error indicator vectors mismatch."); - // Combine these error indicators into the current average. - std::transform(ebar.local_error_indicators.begin(), ebar.local_error_indicators.end(), - ind.indicators.begin(), ebar.local_error_indicators.begin(), - running_average); + if (n > 0) + { + global_error_indicator = + (n * global_error_indicator + candidate_global_error_indicator) / (n + 1); + mean_normalization = (n * mean_normalization + normalization) / (n + 1); + auto running_average = [this](const auto &xbar, const auto &x) + { return (xbar * n + x) / (n + 1); }; + MFEM_VERIFY(local_error_indicators.Size() == indicators.Size(), + "Local error indicator vectors mismatch."); + // Combine these error indicators into the current average. + std::transform(local_error_indicators.begin(), local_error_indicators.end(), + indicators.begin(), local_error_indicators.begin(), + running_average); + } + else + { + global_error_indicator = candidate_global_error_indicator; + local_error_indicators = indicators; + mean_normalization = normalization; + } // Assumes that the global error indicator is already reduced across processors. - ebar.min = ebar.local_error_indicators.Min(); - ebar.max = ebar.local_error_indicators.Max(); - int size = ebar.local_error_indicators.Size(); - Mpi::GlobalMin(1, &ebar.min, comm); - Mpi::GlobalMax(1, &ebar.max, comm); + min = local_error_indicators.Min(); + max = local_error_indicators.Max(); + int size = local_error_indicators.Size(); + Mpi::GlobalMin(1, &min, comm); + Mpi::GlobalMax(1, &max, comm); Mpi::GlobalSum(1, &size, comm); - ebar.mean = ebar.global_error_indicator / size; + mean = global_error_indicator / size; // Another sample has been added, increment for the running average lambda. n++; diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index 646ea9da8..15d3b4b8f 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -50,14 +50,17 @@ class ErrorIndicators // Return the mean normalized local error indicator. auto GetMeanErrorIndicator() const { return mean; } // Return the normalization constant for the absolute error. - auto GetNormalization() const { return normalization; } + auto GetNormalization() const { return mean_normalization; } // The communicator used in any reductions over processors. const MPI_Comm &GetComm() { return comm; } // Return the global number of true dofs associated with this set of error indicators. auto GlobalTrueVSize() const { return global_true_v_size; } + // Add a set of indicators to the running totals. + void AddEstimates(const Vector &indicators, double normalization); + // Reset a running total of error indicators ready for computing a new running average. + void Reset(); protected: - friend class ErrorReductionOperator; // Elemental localized error indicators. Used for marking elements for // refinement and coarsening. Vector local_error_indicators; @@ -71,25 +74,27 @@ class ErrorIndicators // Statistics, updated simultaneously with the global error indicator. double min, max, mean; // Mean normalization constant. - double normalization; -}; - -// Operator for performing reduction of a vector of local indicators into a -// global running total for use in adaptation. -class ErrorReductionOperator -{ + double mean_normalization; // Number of samples. Mutability required to guarantee operation. mutable int n = 0; - -public: - // Reduce a vector indicators, i, computed with norm p, into the combined - // error indicator e. - void operator()(ErrorIndicators &e, const IndicatorsAndNormalization &i, - double p = 2) const; - // Resets the internal counter for number of samples. - void Reset() { n = 0; } }; +// // Operator for performing reduction of a vector of local indicators into a +// // global running total for use in adaptation. +// class ErrorReductionOperator +// { +// // Number of samples. Mutability required to guarantee operation. +// mutable int n = 0; + +// public: +// // Reduce a vector indicators, i, computed with norm p, into the combined +// // error indicator e. +// void operator()(ErrorIndicators &e, const IndicatorsAndNormalization &i, +// double p = 2) const; +// // Resets the internal counter for number of samples. +// void Reset() { n = 0; } +// }; + } // namespace palace #endif // PALACE_UTILS_ERROR_INDICATORS_HPP From 24c800dfe71b565e2f7cb4ac16cc3bead754a11c Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Tue, 3 Oct 2023 21:52:57 -0400 Subject: [PATCH 09/39] Revert integrator changes --- palace/fem/integrator.hpp | 60 ++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/palace/fem/integrator.hpp b/palace/fem/integrator.hpp index 6e068f32c..b96938d9f 100644 --- a/palace/fem/integrator.hpp +++ b/palace/fem/integrator.hpp @@ -9,34 +9,33 @@ namespace palace { -namespace fem -{ -// Helper functions for creating an integration rule to exactly integrate 2p + q -// polynomials. order_increment can be used to raise or lower the order, e.g. in -// the case of derivative fes. -inline const mfem::IntegrationRule *GetDefaultRule(const mfem::FiniteElement &trial_fe, - const mfem::FiniteElement &test_fe, - mfem::ElementTransformation &Tr, - int order_increment = 0) -{ - const int ir_order = - trial_fe.GetOrder() + test_fe.GetOrder() + Tr.OrderW() + order_increment; - MFEM_ASSERT(ir_order >= 0, "Negative integration order not allowed"); - return &mfem::IntRules.Get(trial_fe.GetGeomType(), ir_order); -} -inline const mfem::IntegrationRule *GetDefaultRule(const mfem::FiniteElement &fe, - mfem::ElementTransformation &Tr, - int order_increment = 0) +// +// Derived integrator classes extending the linear and bilinear form integrators of MFEM. +// + +class DefaultIntegrationRule { - return GetDefaultRule(fe, fe, Tr, order_increment); -} +protected: + static const mfem::IntegrationRule *GetDefaultRule(const mfem::FiniteElement &trial_fe, + const mfem::FiniteElement &test_fe, + mfem::ElementTransformation &Tr) + { + const int ir_order = trial_fe.GetOrder() + test_fe.GetOrder() + Tr.OrderW(); + return &mfem::IntRules.Get(trial_fe.GetGeomType(), ir_order); + } -} // namespace fem + static const mfem::IntegrationRule *GetDefaultRule(const mfem::FiniteElement &fe, + mfem::ElementTransformation &Tr) + { + return GetDefaultRule(fe, fe, Tr); + } +}; // Similar to MFEM's VectorFEBoundaryTangentLFIntegrator for ND spaces, but instead of // computing (n x f, v), this just computes (f, v). Also eliminates the a and b quadrature -// parameters and uses fem::GetDefaultRule instead. -class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator +// parameters and uses GetDefaultRule instead. +class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator, + public DefaultIntegrationRule { private: mfem::VectorCoefficient &Q; @@ -44,7 +43,10 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator mfem::Vector f_loc, f_hat; public: - VectorFEBoundaryLFIntegrator(mfem::VectorCoefficient &QG) : Q(QG), f_loc(QG.GetVDim()) {} + VectorFEBoundaryLFIntegrator(mfem::VectorCoefficient &QG) + : Q(QG), f_loc(QG.GetVDim()), f_hat(QG.GetVDim() - 1) + { + } void AssembleRHSElementVect(const mfem::FiniteElement &fe, mfem::ElementTransformation &Tr, @@ -53,11 +55,10 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator const int dof = fe.GetDof(); const int dim = fe.GetDim(); const mfem::IntegrationRule *ir = - (IntRule != nullptr) ? IntRule : fem::GetDefaultRule(fe, Tr); + (IntRule != nullptr) ? IntRule : GetDefaultRule(fe, Tr); vshape.SetSize(dof, dim); elvect.SetSize(dof); elvect = 0.0; - f_hat.SetSize(dim); for (int i = 0; i < ir->GetNPoints(); i++) { @@ -74,8 +75,9 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator }; // Similar to MFEM's BoundaryLFIntegrator for H1 spaces, but eliminates the a and b -// quadrature parameters and uses fem::GetDefaultRule instead. -class BoundaryLFIntegrator : public mfem::LinearFormIntegrator +// quadrature parameters and uses GetDefaultRule instead. +class BoundaryLFIntegrator : public mfem::LinearFormIntegrator, + public DefaultIntegrationRule { private: mfem::Coefficient &Q; @@ -90,7 +92,7 @@ class BoundaryLFIntegrator : public mfem::LinearFormIntegrator { const int dof = fe.GetDof(); const mfem::IntegrationRule *ir = - (IntRule != nullptr) ? IntRule : fem::GetDefaultRule(fe, Tr); + (IntRule != nullptr) ? IntRule : GetDefaultRule(fe, Tr); shape.SetSize(dof); elvect.SetSize(dof); elvect = 0.0; From c7eb850ce50c7284602c72a96d9aa07625eab1ac Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 09:14:45 -0400 Subject: [PATCH 10/39] More PR fixes. Redo fix on integrator, regenerate the error indicator csvs as the mean normalization was incorrect, also move flux projector into the error estimator files --- palace/fem/integrator.hpp | 3 +- palace/linalg/CMakeLists.txt | 1 - palace/linalg/errorestimator.cpp | 211 ++++++--- palace/linalg/errorestimator.hpp | 67 ++- palace/linalg/fluxprojector.cpp | 83 ---- palace/linalg/fluxprojector.hpp | 60 --- palace/utils/errorindicators.cpp | 3 - palace/utils/timer.hpp | 10 +- .../ref/cavity/impedance/error-indicators.csv | 32 +- test/ref/cavity/pec/error-indicators.csv | 32 +- test/ref/coaxial/matched/error-indicators.csv | 402 +++++++++--------- test/ref/coaxial/open/error-indicators.csv | 402 +++++++++--------- .../cpw/lumped_adaptive/error-indicators.csv | 20 +- .../cpw/lumped_uniform/error-indicators.csv | 32 +- .../cpw/wave_adaptive/error-indicators.csv | 20 +- .../ref/cpw/wave_uniform/error-indicators.csv | 30 +- test/ref/rings/error-indicators.csv | 6 +- test/ref/spheres/error-indicators.csv | 6 +- 18 files changed, 699 insertions(+), 721 deletions(-) delete mode 100644 palace/linalg/fluxprojector.cpp delete mode 100644 palace/linalg/fluxprojector.hpp diff --git a/palace/fem/integrator.hpp b/palace/fem/integrator.hpp index b96938d9f..aadf1f02a 100644 --- a/palace/fem/integrator.hpp +++ b/palace/fem/integrator.hpp @@ -44,7 +44,7 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator, public: VectorFEBoundaryLFIntegrator(mfem::VectorCoefficient &QG) - : Q(QG), f_loc(QG.GetVDim()), f_hat(QG.GetVDim() - 1) + : Q(QG), f_loc(QG.GetVDim()) { } @@ -59,6 +59,7 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator, vshape.SetSize(dof, dim); elvect.SetSize(dof); elvect = 0.0; + f_hat.SetSize(dim); for (int i = 0; i < ir->GetNPoints(); i++) { diff --git a/palace/linalg/CMakeLists.txt b/palace/linalg/CMakeLists.txt index bc6e476fc..b20b3fb0d 100644 --- a/palace/linalg/CMakeLists.txt +++ b/palace/linalg/CMakeLists.txt @@ -14,7 +14,6 @@ target_sources(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/distrelaxation.cpp ${CMAKE_CURRENT_SOURCE_DIR}/divfree.cpp ${CMAKE_CURRENT_SOURCE_DIR}/errorestimator.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/fluxprojector.cpp ${CMAKE_CURRENT_SOURCE_DIR}/gmg.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hcurl.cpp ${CMAKE_CURRENT_SOURCE_DIR}/jacobi.cpp diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index c7cf12f69..ee5159f9e 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -1,6 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +#include #include "errorestimator.hpp" #include "fem/coefficient.hpp" #include "fem/integrator.hpp" @@ -9,40 +10,111 @@ #include "utils/communication.hpp" #include "utils/errorindicators.hpp" #include "utils/iodata.hpp" +#include "linalg/amg.hpp" +#include "linalg/gmg.hpp" +#include "linalg/iterative.hpp" +#include "linalg/rap.hpp" namespace palace { using namespace fem; +// Given a finite element space hierarchy, construct a vector of mass matrix +// operators corresponding to each level. +std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHierarchy &h, + int pa_order_threshold) +{ + constexpr int skip_zeros = 0; + const bool is_scalar_FE_space = + h.GetFESpaceAtLevel(0).GetFE(0)->GetRangeType() == mfem::FiniteElement::SCALAR; + + // Assemble the bilinear form operator + auto M = std::make_unique(h.GetNumLevels()); + for (int l = 0; l < h.GetNumLevels(); l++) + { + auto &h_l = h.GetFESpaceAtLevel(l); + auto m = std::make_unique(&h_l); + + if (is_scalar_FE_space) + { + MFEM_ASSERT(h_l.GetVDim() == 1, + "Scalar mass matrix hierarchy assumes a component-wise solve."); + m->AddDomainIntegrator(new mfem::MassIntegrator); + } + else + { + m->AddDomainIntegrator(new mfem::VectorFEMassIntegrator); + } + + auto M_l = std::make_unique( + fem::AssembleOperator(std::move(m), true, (l > 0) ? pa_order_threshold : 100, + skip_zeros), + h_l); + + // Set the essential dofs (none). + M->AddOperator(std::move(M_l)); + } + return M; +} + +FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fespace, + double tol, int max_it, int print, + int pa_order_threshold) + : M(BuildMassMatrixOperator(smooth_flux_fespace, pa_order_threshold)) +{ + // The system matrix for the projection is real and SPD. For the coarse-level AMG solve, + // we don't use an exact solve on the coarsest level. + auto amg = + std::make_unique>(std::make_unique(1, 1, 0)); + auto gmg = std::make_unique>( + std::move(amg), smooth_flux_fespace, nullptr, 1, 1, 2, 1.0, 0.0, true, + pa_order_threshold); + + auto pcg = std::make_unique>( + smooth_flux_fespace.GetFinestFESpace().GetComm(), print); + + pcg->SetInitialGuess(false); + pcg->SetRelTol(tol); + pcg->SetAbsTol(std::numeric_limits::epsilon()); + pcg->SetMaxIter(max_it); + + ksp = std::make_unique(std::move(pcg), std::move(gmg)); + ksp->SetOperators(*M, *M); + + tmp.SetSize(smooth_flux_fespace.GetFinestFESpace().GetTrueVSize()); +} + + + CurlFluxErrorEstimator::CurlFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, const std::vector> &mesh, - mfem::ParFiniteElementSpace &fes) - : mat_op(mat_op), fes(fes), + mfem::ParFiniteElementSpace &fespace) + : mat_op(mat_op), fespace(fespace), smooth_flux_fecs(ConstructFECollections( iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_coarsen_type, iodata.solver.linear.pc_mat_lor)), - smooth_flux_fes(ConstructFiniteElementSpaceHierarchy( + smooth_flux_fespace(ConstructFiniteElementSpaceHierarchy( iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), - smooth_projector(smooth_flux_fes, iodata.solver.linear.tol, 200, 0, + smooth_projector(smooth_flux_fespace, iodata.solver.linear.tol, 200, 0, iodata.solver.pa_order_threshold), coarse_flux_fec(iodata.solver.order, mesh.back()->Dimension(), mfem::BasisType::GaussLobatto), - coarse_flux_fes(mesh.back().get(), &coarse_flux_fec, mesh.back()->Dimension()), - scalar_mass_matrices(fes.GetNE()), smooth_to_coarse_embed(fes.GetNE()) + coarse_flux_fespace(mesh.back().get(), &coarse_flux_fec, mesh.back()->Dimension()), + scalar_mass_matrices(fespace.GetNE()), smooth_to_coarse_embed(fespace.GetNE()) { mfem::MassIntegrator mass_integrator; - for (int e = 0; e < fes.GetNE(); e++) + for (int e = 0; e < fespace.GetNE(); e++) { // Loop over each element, and save an elemental mass matrix. // Will exploit the fact that vector L2 mass matrix components are independent. - const auto &coarse_fe = *coarse_flux_fes.GetFE(e); - auto &T = *coarse_flux_fes.GetElementTransformation(e); + const auto &coarse_fe = *coarse_flux_fespace.GetFE(e); + auto &T = *coarse_flux_fespace.GetElementTransformation(e); mass_integrator.AssembleElementMatrix(coarse_fe, T, scalar_mass_matrices[e]); - const auto &smooth_fe = *smooth_flux_fes.GetFinestFESpace().GetFE(e); + const auto &smooth_fe = *smooth_flux_fespace.GetFinestFESpace().GetFE(e); coarse_fe.Project(smooth_fe, T, smooth_to_coarse_embed[e]); } } @@ -50,29 +122,30 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, bool normalize) const { - mfem::ParComplexGridFunction field(&fes); + const int sdim = fespace.GetMesh()->SpaceDimension(); + mfem::ParComplexGridFunction field(&fespace); field.real().SetFromTrueDofs(v.Real()); field.imag().SetFromTrueDofs(v.Imag()); - const int nelem = smooth_flux_fes.GetFinestFESpace().GetNE(); + const int nelem = smooth_flux_fespace.GetFinestFESpace().GetNE(); // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). CurlFluxCoefficient real_coef(field.real(), mat_op), imag_coef(field.imag(), mat_op); - auto rhs_from_coef = [](mfem::ParFiniteElementSpace &flux_fes, auto &coef) + auto rhs_from_coef = [](mfem::ParFiniteElementSpace &flux_fespace, auto &coef) { - Vector RHS(flux_fes.GetTrueVSize()); + Vector RHS(flux_fespace.GetTrueVSize()); - mfem::LinearForm rhs(&flux_fes); + mfem::LinearForm rhs(&flux_fespace); rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); rhs.UseFastAssembly(false); rhs.Assemble(); - flux_fes.GetProlongationMatrix()->MultTranspose(rhs, RHS); + flux_fespace.GetProlongationMatrix()->MultTranspose(rhs, RHS); return RHS; }; const auto smooth_flux_rhs = - ComplexVector(rhs_from_coef(smooth_flux_fes.GetFinestFESpace(), real_coef), - rhs_from_coef(smooth_flux_fes.GetFinestFESpace(), imag_coef)); + ComplexVector(rhs_from_coef(smooth_flux_fespace.GetFinestFESpace(), real_coef), + rhs_from_coef(smooth_flux_fespace.GetFinestFESpace(), imag_coef)); // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. @@ -87,17 +160,17 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction // for evaluation. - auto build_func = [](const ComplexVector &f, mfem::ParFiniteElementSpace &fes) + auto build_func = [](const ComplexVector &f, mfem::ParFiniteElementSpace &fespace) { - mfem::ParComplexGridFunction flux(&fes); + mfem::ParComplexGridFunction flux(&fespace); flux.real().SetFromTrueDofs(f.Real()); flux.imag().SetFromTrueDofs(f.Imag()); flux.real().ExchangeFaceNbrData(); flux.imag().ExchangeFaceNbrData(); return flux; }; - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fes.GetFinestFESpace()); - mfem::ParComplexGridFunction coarse_flux_func(&coarse_flux_fes); + auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fespace.GetFinestFESpace()); + mfem::ParComplexGridFunction coarse_flux_func(&coarse_flux_fespace); coarse_flux_func.real().ProjectCoefficient(real_coef); coarse_flux_func.imag().ProjectCoefficient(imag_coef); @@ -106,13 +179,13 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl Vector smooth_vec, coarse_vec, sub_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - for (int e = 0; e < fes.GetNE(); e++) + for (int e = 0; e < fespace.GetNE(); e++) { // real smooth_flux_func.real().GetElementDofValues(e, smooth_vec); coarse_flux_func.real().GetElementDofValues(e, coarse_vec); - const int ndof = coarse_vec.Size() / 3; + const int ndof = coarse_vec.Size() / sdim; sub_vec.SetSize(ndof); for (int c = 0; c < 3; c++) { @@ -160,25 +233,26 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const { - mfem::ParGridFunction field(&fes); + const int sdim = fespace.GetMesh()->SpaceDimension(); + mfem::ParGridFunction field(&fespace); field.SetFromTrueDofs(v); - const int nelem = smooth_flux_fes.GetFinestFESpace().GetNE(); + const int nelem = smooth_flux_fespace.GetFinestFESpace().GetNE(); // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). CurlFluxCoefficient coef(field, mat_op); - auto rhs_from_coef = [](mfem::ParFiniteElementSpace &flux_fes, auto &coef) + auto rhs_from_coef = [](mfem::ParFiniteElementSpace &flux_fespace, auto &coef) { - Vector RHS(flux_fes.GetTrueVSize()); + Vector RHS(flux_fespace.GetTrueVSize()); - mfem::LinearForm rhs(&flux_fes); + mfem::LinearForm rhs(&flux_fespace); rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); rhs.UseFastAssembly(false); rhs.Assemble(); - flux_fes.GetProlongationMatrix()->MultTranspose(rhs, RHS); + flux_fespace.GetProlongationMatrix()->MultTranspose(rhs, RHS); return RHS; }; - const auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fes.GetFinestFESpace(), coef); + const auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fespace.GetFinestFESpace(), coef); // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. @@ -193,15 +267,15 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vecto // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction // for evaluation. - auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fes) + auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fespace) { - mfem::ParGridFunction flux(&fes); + mfem::ParGridFunction flux(&fespace); flux.SetFromTrueDofs(f); flux.ExchangeFaceNbrData(); return flux; }; - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fes.GetFinestFESpace()); - mfem::ParGridFunction coarse_flux_func(&coarse_flux_fes); + auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fespace.GetFinestFESpace()); + mfem::ParGridFunction coarse_flux_func(&coarse_flux_fespace); coarse_flux_func.ProjectCoefficient(coef); // Loop over elements, embed the smooth flux into the coarse flux space, then compute @@ -209,12 +283,12 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vecto Vector smooth_vec, coarse_vec, sub_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - for (int e = 0; e < fes.GetNE(); e++) + for (int e = 0; e < fespace.GetNE(); e++) { smooth_flux_func.GetElementDofValues(e, smooth_vec); coarse_flux_func.GetElementDofValues(e, coarse_vec); - const int ndof = coarse_vec.Size() / 3; + const int ndof = coarse_vec.Size() / sdim; sub_vec.SetSize(ndof); for (int c = 0; c < 3; c++) { @@ -245,34 +319,34 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vecto GradFluxErrorEstimator::GradFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, const std::vector> &mesh, - mfem::ParFiniteElementSpace &fes) - : mat_op(mat_op), fes(fes), + mfem::ParFiniteElementSpace &fespace) + : mat_op(mat_op), fespace(fespace), smooth_flux_fecs(ConstructFECollections( iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_coarsen_type, false)), - smooth_flux_component_fes(ConstructFiniteElementSpaceHierarchy( + smooth_flux_component_fespace(ConstructFiniteElementSpaceHierarchy( iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), - smooth_flux_fes(mesh.back().get(), smooth_flux_fecs.back().get(), + smooth_flux_fespace(mesh.back().get(), smooth_flux_fecs.back().get(), mesh.back()->Dimension()), - smooth_projector(smooth_flux_component_fes, iodata.solver.linear.tol, 200, 0, + smooth_projector(smooth_flux_component_fespace, iodata.solver.linear.tol, 200, 0, iodata.solver.pa_order_threshold), coarse_flux_fec(iodata.solver.order, mesh.back()->Dimension(), mfem::BasisType::GaussLobatto), - coarse_flux_fes(mesh.back().get(), &coarse_flux_fec, mesh.back()->Dimension()), - scalar_mass_matrices(fes.GetNE()), smooth_to_coarse_embed(fes.GetNE()) + coarse_flux_fespace(mesh.back().get(), &coarse_flux_fec, mesh.back()->Dimension()), + scalar_mass_matrices(fespace.GetNE()), smooth_to_coarse_embed(fespace.GetNE()) { mfem::MassIntegrator mass_integrator; - for (int e = 0; e < fes.GetNE(); e++) + for (int e = 0; e < fespace.GetNE(); e++) { // Loop over each element, and save an elemental mass matrix. // Will exploit the fact that vector L2 mass matrix components are independent. - const auto &coarse_fe = *coarse_flux_fes.GetFE(e); - auto &T = *fes.GetElementTransformation(e); + const auto &coarse_fe = *coarse_flux_fespace.GetFE(e); + auto &T = *fespace.GetElementTransformation(e); mass_integrator.AssembleElementMatrix(coarse_fe, T, scalar_mass_matrices[e]); - const auto &smooth_fe = *smooth_flux_component_fes.GetFinestFESpace().GetFE(e); + const auto &smooth_fe = *smooth_flux_component_fespace.GetFinestFESpace().GetFE(e); coarse_fe.Project(smooth_fe, T, smooth_to_coarse_embed[e]); } } @@ -280,29 +354,30 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const { - mfem::ParGridFunction field(&fes); + const int sdim = fespace.GetMesh()->SpaceDimension(); + mfem::ParGridFunction field(&fespace); field.SetFromTrueDofs(v); - const int nelem = smooth_flux_fes.GetNE(); + const int nelem = smooth_flux_fespace.GetNE(); // Coefficients for computing the discontinuous flux., i.e. (V, ϵ ∇ ϕ). GradFluxCoefficient coef(field, mat_op); - auto rhs_from_coef = [](mfem::ParFiniteElementSpace &fes, auto &coef) + auto rhs_from_coef = [](mfem::ParFiniteElementSpace &fespace, auto &coef) { - Vector RHS(fes.GetTrueVSize()); + Vector RHS(fespace.GetTrueVSize()); - mfem::LinearForm rhs(&fes); + mfem::LinearForm rhs(&fespace); rhs.AddDomainIntegrator(new mfem::VectorDomainLFIntegrator(coef)); rhs.UseFastAssembly(false); rhs.Assemble(); - fes.GetProlongationMatrix()->MultTranspose(rhs, RHS); + fespace.GetProlongationMatrix()->MultTranspose(rhs, RHS); return RHS; }; - auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fes, coef); + auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fespace, coef); // Given the RHS vector of non-smooth flux, construct a flux projector and perform // component wise mass matrix inversion in the appropriate space, giving fᵢ = M⁻¹ f̂ᵢ. - auto build_flux = [](const FluxProjector &proj, Vector &rhs) + auto build_flux = [sdim](const FluxProjector &proj, Vector &rhs) { // Use a copy construction to match appropriate size. Vector flux(rhs.Size()); @@ -310,7 +385,7 @@ IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vecto // Apply the flux projector component wise. const int ndof = flux.Size(); - const int stride = ndof / 3; + const int stride = ndof / sdim; MFEM_ASSERT(ndof % 3 == 0, "!"); Vector flux_comp, rhs_comp; @@ -325,22 +400,22 @@ IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vecto auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); // Given a solution represented with a Vector, build a GridFunction for evaluation. - auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fes) + auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fespace) { - mfem::ParGridFunction flux(&fes); + mfem::ParGridFunction flux(&fespace); flux.SetFromTrueDofs(f); flux.ExchangeFaceNbrData(); return flux; }; - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fes); + auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fespace); - mfem::ParGridFunction coarse_flux(&coarse_flux_fes); + mfem::ParGridFunction coarse_flux(&coarse_flux_fespace); coarse_flux.ProjectCoefficient(coef); Vector coarse_vec, smooth_vec, coarse_sub_vec, smooth_sub_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - for (int e = 0; e < fes.GetNE(); e++) + for (int e = 0; e < fespace.GetNE(); e++) { coarse_flux.GetElementDofValues(e, coarse_vec); smooth_flux_func.GetElementDofValues(e, smooth_vec); @@ -350,7 +425,7 @@ IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vecto "and should be exactly divisible by 3: " << coarse_vec.Size() << " " << smooth_vec.Size()); - const int ndof = coarse_vec.Size() / 3; + const int ndof = coarse_vec.Size() / sdim; coarse_sub_vec.SetSize(ndof); smooth_sub_vec.SetSize(ndof); for (int c = 0; c < 3; c++) @@ -371,15 +446,15 @@ IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vecto if constexpr (false) { // Debugging branch generates some intermediate fields for paraview. - mfem::ParaViewDataCollection paraview("debug", fes.GetParMesh()); + mfem::ParaViewDataCollection paraview("debug", fespace.GetParMesh()); paraview.RegisterVCoeffField("Flux", &coef); - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fes); + auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fespace); paraview.RegisterField("SmoothFlux", &smooth_flux_func); mfem::L2_FECollection est_fec(0, 3); - mfem::ParFiniteElementSpace est_fes(fes.GetParMesh(), &est_fec); - mfem::ParGridFunction est_field(&est_fes); + mfem::ParFiniteElementSpace est_fespace(fespace.GetParMesh(), &est_fec); + mfem::ParGridFunction est_field(&est_fespace); est_field.SetFromTrueDofs(estimates); paraview.RegisterField("ErrorIndicator", &est_field); @@ -392,4 +467,6 @@ IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vecto } return {estimates, normalization}; } + + } // namespace palace diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 9d4194507..1e659b270 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -4,9 +4,11 @@ #ifndef PALACE_LINALG_ERROR_ESTIMATOR_HPP #define PALACE_LINALG_ERROR_ESTIMATOR_HPP +#include #include - -#include "linalg/fluxprojector.hpp" +#include "linalg/ksp.hpp" +#include "linalg/operator.hpp" +#include "linalg/vector.hpp" #include "utils/errorindicators.hpp" namespace palace @@ -15,6 +17,46 @@ namespace palace class IoData; class MaterialOperator; +// +// This solver computes a smooth reconstruction of a discontinuous flux. The difference +// between this resulting smooth flux and the original non-smooth flux provides a +// localizable error estimate. An instance of FluxProjector can be reused across solutions, +// thus the construction of the operator is separated from the construction of the flux RHS. +class FluxProjector +{ +private: + // Operator for the mass matrix inversion. + std::unique_ptr M; + + // Linear solver and preconditioner for the projected linear system M σ = σ̂. + std::unique_ptr ksp; + + // Intermediate storage vector used in Mult. + mutable Vector tmp; + +public: + FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, double tol = 1e-12, + int max_it = 200, int print_level = 1, int pa_order_threshold = 1); + + // Given a vector of dof defining the flux, compute the smooth flux IN PLACE. + void Mult(Vector &x) const + { + tmp = x; + Mult(tmp, x); + } + void Mult(const Vector &x, Vector &y) const { ksp->Mult(x, y); } + void Mult(ComplexVector &x) const + { + Mult(x.Real()); + Mult(x.Imag()); + } + void Mult(const ComplexVector &x, ComplexVector &y) const + { + y = x; + Mult(y); + } +}; + // Class used for computing curl flux error estimate, // i.e. || μ⁻¹∇ × Vₕ - F ||_K // where F denotes a smooth reconstruction of μ⁻¹∇ × Vₕ. @@ -22,23 +64,26 @@ class CurlFluxErrorEstimator { const MaterialOperator &mat_op; // The finite element space used to represent V. - mfem::ParFiniteElementSpace &fes; + mfem::ParFiniteElementSpace &fespace; std::vector> smooth_flux_fecs; - mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_fes; + mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_fespace; mutable FluxProjector smooth_projector; mfem::L2_FECollection coarse_flux_fec; - mutable mfem::ParFiniteElementSpace coarse_flux_fes; + mutable mfem::ParFiniteElementSpace coarse_flux_fespace; std::vector scalar_mass_matrices; std::vector smooth_to_coarse_embed; + mutable ComplexVector complex_flux; + mutable Vector real_flux; + public: // Constructor for using geometric and p multigrid. CurlFluxErrorEstimator(const IoData &iodata, const MaterialOperator &mat_op, const std::vector> &mesh, - mfem::ParFiniteElementSpace &fes); + mfem::ParFiniteElementSpace &fespace); // Compute elemental error indicators given a complex vector of true DOF. IndicatorsAndNormalization ComputeIndicators(const ComplexVector &v, @@ -55,18 +100,18 @@ class GradFluxErrorEstimator { const MaterialOperator &mat_op; // The finite element space used to represent ϕ. - mfem::ParFiniteElementSpace &fes; + mfem::ParFiniteElementSpace &fespace; // Collections and spaces for the smooth flux. Note the hierarchy uses the // SCALAR finite element space, whilst the true flux is in the VECTOR finite // element space. This allows for a component wise inversion. std::vector> smooth_flux_fecs; - mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_component_fes; - mutable mfem::ParFiniteElementSpace smooth_flux_fes; + mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_component_fespace; + mutable mfem::ParFiniteElementSpace smooth_flux_fespace; mutable FluxProjector smooth_projector; mfem::L2_FECollection coarse_flux_fec; - mutable mfem::ParFiniteElementSpace coarse_flux_fes; + mutable mfem::ParFiniteElementSpace coarse_flux_fespace; std::vector scalar_mass_matrices; std::vector smooth_to_coarse_embed; @@ -75,7 +120,7 @@ class GradFluxErrorEstimator // Constructor for using geometric and p multigrid. GradFluxErrorEstimator(const IoData &iodata, const MaterialOperator &mat_op, const std::vector> &mesh, - mfem::ParFiniteElementSpace &fes); + mfem::ParFiniteElementSpace &fespace); // Compute elemental error indicators given a vector of true DOF. IndicatorsAndNormalization ComputeIndicators(const Vector &v, bool normalize) const; diff --git a/palace/linalg/fluxprojector.cpp b/palace/linalg/fluxprojector.cpp deleted file mode 100644 index e688cb529..000000000 --- a/palace/linalg/fluxprojector.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#include "fluxprojector.hpp" - -#include -#include "fem/coefficient.hpp" -#include "fem/multigrid.hpp" -#include "linalg/amg.hpp" -#include "linalg/gmg.hpp" -#include "linalg/iterative.hpp" -#include "linalg/rap.hpp" -#include "models/materialoperator.hpp" - -namespace palace -{ - -// Given a finite element space hierarchy, construct a vector of mass matrix -// operators corresponding to each level. -std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHierarchy &h, - int pa_order_threshold) -{ - constexpr int skip_zeros = 0; - const bool is_scalar_FE_space = - h.GetFESpaceAtLevel(0).GetFE(0)->GetRangeType() == mfem::FiniteElement::SCALAR; - - // Assemble the bilinear form operator - auto M = std::make_unique(h.GetNumLevels()); - for (int l = 0; l < h.GetNumLevels(); l++) - { - auto &h_l = h.GetFESpaceAtLevel(l); - auto m = std::make_unique(&h_l); - - if (is_scalar_FE_space) - { - MFEM_ASSERT(h_l.GetVDim() == 1, - "Scalar mass matrix hierarchy assumes a component-wise solve."); - m->AddDomainIntegrator(new mfem::MassIntegrator); - } - else - { - m->AddDomainIntegrator(new mfem::VectorFEMassIntegrator); - } - - auto M_l = std::make_unique( - fem::AssembleOperator(std::move(m), true, (l > 0) ? pa_order_threshold : 100, - skip_zeros), - h_l); - - // Set the essential dofs (none). - M->AddOperator(std::move(M_l)); - } - return M; -} - -FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, - double tol, int max_it, int print_level, - int pa_order_threshold) - : M(BuildMassMatrixOperator(smooth_flux_fes, pa_order_threshold)) -{ - // The system matrix for the projection is real and SPD. For the coarse-level AMG solve, - // we don't use an exact solve on the coarsest level. - auto amg = - std::make_unique>(std::make_unique(1, 1, 0)); - auto gmg = std::make_unique>( - std::move(amg), smooth_flux_fes, nullptr, 1, 1, 2, 1.0, 0.0, true, - pa_order_threshold); - - auto pcg = std::make_unique>( - smooth_flux_fes.GetFinestFESpace().GetComm(), print_level); - - pcg->SetInitialGuess(false); - pcg->SetRelTol(tol); - pcg->SetAbsTol(std::numeric_limits::epsilon()); - pcg->SetMaxIter(max_it); - - ksp = std::make_unique(std::move(pcg), std::move(gmg)); - ksp->SetOperators(*M, *M); - - tmp.SetSize(smooth_flux_fes.GetFinestFESpace().GetTrueVSize()); -} - -} // namespace palace diff --git a/palace/linalg/fluxprojector.hpp b/palace/linalg/fluxprojector.hpp deleted file mode 100644 index 24194b615..000000000 --- a/palace/linalg/fluxprojector.hpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#ifndef PALACE_LINALG_FLUX_PROJECTOR_HPP -#define PALACE_LINALG_FLUX_PROJECTOR_HPP - -#include -#include -#include "linalg/ksp.hpp" -#include "linalg/operator.hpp" -#include "linalg/vector.hpp" - -namespace palace -{ - -class MaterialOperator; - -// -// This solver computes a smooth reconstruction of a discontinuous flux. The difference -// between this resulting smooth flux and the original non-smooth flux provides a -// localizable error estimate. An instance of FluxProjector can be reused across solutions, -// thus the construction of the operator is separated from the construction of the flux RHS. -class FluxProjector -{ -private: - // Operator for the mass matrix inversion. - std::unique_ptr M; - - // Linear solver and preconditioner for the projected linear system M σ = σ̂. - std::unique_ptr ksp; - - // Intermediate storage vector used in Mult. - mutable Vector tmp; - -public: - FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, double tol = 1e-12, - int max_it = 200, int print_level = 1, int pa_order_threshold = 1); - - // Given a vector of dof defining the flux, compute the smooth flux IN PLACE. - void Mult(Vector &x) const - { - tmp = x; - Mult(tmp, x); - } - void Mult(const Vector &x, Vector &y) const { ksp->Mult(x, y); } - void Mult(ComplexVector &x) const - { - Mult(x.Real()); - Mult(x.Imag()); - } - void Mult(const ComplexVector &x, ComplexVector &y) const - { - y = x; - Mult(y); - } -}; - -} // namespace palace - -#endif // PALACE_LINALG_FLUX_PROJECTOR_HPP diff --git a/palace/utils/errorindicators.cpp b/palace/utils/errorindicators.cpp index d2823ecde..13caa99c1 100644 --- a/palace/utils/errorindicators.cpp +++ b/palace/utils/errorindicators.cpp @@ -53,9 +53,6 @@ ErrorIndicators::AddEstimates(const Vector &indicators, double normalization) // values. The average local indicator is used rather than the indicator for the maximum // error to drive the adaptation, to account for a local error that might be marginally // important to many solves, rather than only large in one solve. - - // TODO: Could alternatively consider the maximum. - if (n > 0) { global_error_indicator = diff --git a/palace/utils/timer.hpp b/palace/utils/timer.hpp index e55d341d1..81bd1d692 100644 --- a/palace/utils/timer.hpp +++ b/palace/utils/timer.hpp @@ -29,11 +29,12 @@ class Timer INIT = 0, CONSTRUCT, WAVEPORT, // Wave port solver - ESTCONSTRUCT, // Construction of estimate operator SOLVE, PRECONDITIONER, // Linear solver COARSESOLVE, // Linear solver - ESTSOLVE, // Estimation calculation + ESTIMATION, // Estimation + ESTCONSTRUCT, // Construction of estimator + ESTSOLVE, // Evaluation of estimator CONSTRUCTPROM, // Adaptive frequency sweep SOLVEPROM, // Adaptive frequency sweep POSTPRO, @@ -45,11 +46,12 @@ class Timer inline static const std::vector descriptions{"Initialization", "Operator Construction", " Wave Ports", - " Error Estimate", "Solve", " Preconditioner", " Coarse Solve", - " Error Estimate", + "Estimation", + " Construction", + " Evaluation", "PROM Construction", "PROM Solve", "Postprocessing", diff --git a/test/ref/cavity/impedance/error-indicators.csv b/test/ref/cavity/impedance/error-indicators.csv index c6bf6809c..1a34e270a 100644 --- a/test/ref/cavity/impedance/error-indicators.csv +++ b/test/ref/cavity/impedance/error-indicators.csv @@ -1,17 +1,17 @@ m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +8.211772836e-03, +9.214906601e-05, +6.517143357e-04, +1.425655006e-05, +6.964432246e-01 - 2.000000e+00, +7.376205407e-03, +1.120202798e-04, +5.122686573e-04, +1.280591217e-05, +4.545414625e-01 - 3.000000e+00, +7.965439262e-03, +1.093565445e-04, +6.153862588e-04, +1.382888761e-05, +5.168938560e-01 - 4.000000e+00, +8.289455502e-03, +6.591010796e-05, +8.975703487e-04, +1.439141580e-05, +7.254717341e-01 - 5.000000e+00, +1.680165239e-02, +2.687010660e-04, +1.173134621e-03, +2.916953541e-05, +7.021945474e-01 - 6.000000e+00, +1.724222225e-02, +2.679598776e-04, +1.285586152e-03, +2.993441363e-05, +6.917663295e-01 - 7.000000e+00, +1.541389559e-02, +2.982919759e-04, +1.088771718e-03, +2.676023539e-05, +6.836966993e-01 - 8.000000e+00, +1.589306594e-02, +2.709869621e-04, +1.182747694e-03, +2.759212837e-05, +7.774852285e-01 - 9.000000e+00, +2.161744447e-02, +3.939248466e-04, +1.590949614e-03, +3.753028555e-05, +1.082707044e+00 - 1.000000e+01, +2.568013719e-02, +3.749226902e-04, +2.345062994e-03, +4.458357151e-05, +1.143427856e+00 - 1.100000e+01, +1.171936131e-02, +2.548287031e-04, +8.771786578e-04, +2.034611339e-05, +8.880851774e-01 - 1.200000e+01, +2.424099505e-02, +3.311784727e-04, +2.230405100e-03, +4.208506085e-05, +8.563666051e-01 - 1.300000e+01, +2.164706035e-02, +1.439791555e-04, +2.192101984e-03, +3.758170199e-05, +1.123784326e+00 - 1.400000e+01, +2.570506911e-02, +1.191879068e-04, +3.229781992e-03, +4.462685610e-05, +1.125048653e+00 - 1.500000e+01, +2.690501452e-02, +4.630239998e-04, +1.771500567e-03, +4.671009465e-05, +8.954818243e-01 - Mean, +1.698058608e-02, +3.754014884e-04, +1.180747207e-03, +2.948018416e-05, +6.964432246e-01 + 1.000000e+00, +8.211772854e-03, +9.214906491e-05, +6.517143501e-04, +1.425655009e-05, +6.964432246e-01 + 2.000000e+00, +7.376205362e-03, +1.120203634e-04, +5.122692155e-04, +1.280591209e-05, +4.545414510e-01 + 3.000000e+00, +7.965439217e-03, +1.093565748e-04, +6.153863198e-04, +1.382888753e-05, +5.168938583e-01 + 4.000000e+00, +8.289455761e-03, +6.591008815e-05, +8.975703459e-04, +1.439141625e-05, +7.254717341e-01 + 5.000000e+00, +1.680165062e-02, +2.686897967e-04, +1.173118394e-03, +2.916953233e-05, +7.021942152e-01 + 6.000000e+00, +1.724222117e-02, +2.679646989e-04, +1.285588040e-03, +2.993441175e-05, +6.917661151e-01 + 7.000000e+00, +1.541389064e-02, +2.982023357e-04, +1.088582974e-03, +2.676022681e-05, +6.836991197e-01 + 8.000000e+00, +1.589306497e-02, +2.709931996e-04, +1.182754190e-03, +2.759212668e-05, +7.774857333e-01 + 9.000000e+00, +2.161744473e-02, +3.939248464e-04, +1.590949752e-03, +3.753028600e-05, +1.082707043e+00 + 1.000000e+01, +2.568013712e-02, +3.749227031e-04, +2.345062909e-03, +4.458357139e-05, +1.143427854e+00 + 1.100000e+01, +1.171936065e-02, +2.548287325e-04, +8.771786122e-04, +2.034611225e-05, +8.880851779e-01 + 1.200000e+01, +2.424099577e-02, +3.311786201e-04, +2.230405154e-03, +4.208506210e-05, +8.563665976e-01 + 1.300000e+01, +2.164706045e-02, +1.439794677e-04, +2.192103020e-03, +3.758170218e-05, +1.123784317e+00 + 1.400000e+01, +2.570506964e-02, +1.191879269e-04, +3.229783266e-03, +4.462685701e-05, +1.125048654e+00 + 1.500000e+01, +2.690499163e-02, +4.630246596e-04, +1.771494907e-03, +4.671005492e-05, +8.954809561e-01 + Mean, +1.698058404e-02, +3.754059600e-04, +1.180751362e-03, +2.948018062e-05, +8.242264034e-01 diff --git a/test/ref/cavity/pec/error-indicators.csv b/test/ref/cavity/pec/error-indicators.csv index 43f7be263..e1f56438e 100644 --- a/test/ref/cavity/pec/error-indicators.csv +++ b/test/ref/cavity/pec/error-indicators.csv @@ -1,17 +1,17 @@ m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +8.211772151e-03, +9.214886120e-05, +6.517144290e-04, +1.425654887e-05, +6.964432123e-01 - 2.000000e+00, +7.376205453e-03, +1.120202072e-04, +5.122678696e-04, +1.280591225e-05, +4.545414720e-01 - 3.000000e+00, +7.965439326e-03, +1.093559864e-04, +6.153852901e-04, +1.382888772e-05, +5.168938061e-01 - 4.000000e+00, +8.289455200e-03, +6.591004762e-05, +8.975704258e-04, +1.439141528e-05, +7.254717195e-01 - 5.000000e+00, +1.680164760e-02, +2.686712679e-04, +1.173091559e-03, +2.916952709e-05, +7.021936550e-01 - 6.000000e+00, +1.724222427e-02, +2.679523802e-04, +1.285583367e-03, +2.993441713e-05, +6.917666416e-01 - 7.000000e+00, +1.541388964e-02, +2.981725401e-04, +1.088519843e-03, +2.676022508e-05, +6.836999314e-01 - 8.000000e+00, +1.589306788e-02, +2.709682463e-04, +1.182728017e-03, +2.759213174e-05, +7.774837036e-01 - 9.000000e+00, +2.161744444e-02, +3.939247063e-04, +1.590950711e-03, +3.753028548e-05, +1.082707023e+00 - 1.000000e+01, +2.568013806e-02, +3.749227088e-04, +2.345063718e-03, +4.458357302e-05, +1.143427857e+00 - 1.100000e+01, +1.171936187e-02, +2.548287393e-04, +8.771787445e-04, +2.034611435e-05, +8.880851653e-01 - 1.200000e+01, +2.424099394e-02, +3.311788874e-04, +2.230405084e-03, +4.208505892e-05, +8.563665736e-01 - 1.300000e+01, +2.164706073e-02, +1.439799500e-04, +2.192105601e-03, +3.758170266e-05, +1.123784343e+00 - 1.400000e+01, +2.570507021e-02, +1.191879086e-04, +3.229783591e-03, +4.462685800e-05, +1.125048641e+00 - 1.500000e+01, +2.690494118e-02, +4.630260842e-04, +1.771482822e-03, +4.670996732e-05, +8.954790492e-01 - Mean, +1.698058080e-02, +3.754082954e-04, +1.180749794e-03, +2.948017499e-05, +6.964432123e-01 + 1.000000e+00, +8.211772523e-03, +9.214886535e-05, +6.517144226e-04, +1.425654952e-05, +6.964432122e-01 + 2.000000e+00, +7.376205452e-03, +1.120202197e-04, +5.122680096e-04, +1.280591224e-05, +4.545414691e-01 + 3.000000e+00, +7.965439333e-03, +1.093560248e-04, +6.153853635e-04, +1.382888773e-05, +5.168938091e-01 + 4.000000e+00, +8.289455219e-03, +6.591005173e-05, +8.975704252e-04, +1.439141531e-05, +7.254717195e-01 + 5.000000e+00, +1.680164766e-02, +2.686717116e-04, +1.173092213e-03, +2.916952718e-05, +7.021936684e-01 + 6.000000e+00, +1.724222477e-02, +2.679503077e-04, +1.285582570e-03, +2.993441800e-05, +6.917667336e-01 + 7.000000e+00, +1.541389133e-02, +2.982045184e-04, +1.088587389e-03, +2.676022801e-05, +6.836990559e-01 + 8.000000e+00, +1.589306779e-02, +2.709696912e-04, +1.182729517e-03, +2.759213158e-05, +7.774838203e-01 + 9.000000e+00, +2.161744463e-02, +3.939247378e-04, +1.590950569e-03, +3.753028582e-05, +1.082707024e+00 + 1.000000e+01, +2.568013828e-02, +3.749227076e-04, +2.345063736e-03, +4.458357340e-05, +1.143427857e+00 + 1.100000e+01, +1.171936185e-02, +2.548286854e-04, +8.771787530e-04, +2.034611433e-05, +8.880851654e-01 + 1.200000e+01, +2.424099407e-02, +3.311788996e-04, +2.230405073e-03, +4.208505915e-05, +8.563665731e-01 + 1.300000e+01, +2.164706072e-02, +1.439800054e-04, +2.192106108e-03, +3.758170264e-05, +1.123784347e+00 + 1.400000e+01, +2.570507035e-02, +1.191878736e-04, +3.229783780e-03, +4.462685825e-05, +1.125048642e+00 + 1.500000e+01, +2.690494192e-02, +4.630260577e-04, +1.771483008e-03, +4.670996860e-05, +8.954790792e-01 + Mean, +1.698058106e-02, +3.754074357e-04, +1.180748376e-03, +2.948017545e-05, +8.242261450e-01 diff --git a/test/ref/coaxial/matched/error-indicators.csv b/test/ref/coaxial/matched/error-indicators.csv index ae715ec78..c3f574b19 100644 --- a/test/ref/coaxial/matched/error-indicators.csv +++ b/test/ref/coaxial/matched/error-indicators.csv @@ -1,203 +1,203 @@ t (ns), Sum Abs., Min Elem Abs., Max Elem Abs., Mean Elem Abs., Normalization 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 - 3.747406e-02, +5.656919329e-06, +4.843867778e-19, +1.429146542e-06, +4.419468225e-08, +1.015026012e-04 - 7.494811e-02, +1.331394786e-05, +1.254533371e-18, +3.354187386e-06, +1.040152177e-07, +1.286467416e-04 - 1.124222e-01, +4.533166152e-06, +8.678124436e-19, +1.144058873e-06, +3.541536056e-08, +1.363432445e-04 - 1.498962e-01, +5.316539312e-06, +5.368957501e-19, +1.352378049e-06, +4.153546338e-08, +2.237453773e-04 - 1.873703e-01, +2.287184397e-05, +3.014012761e-18, +5.789590613e-06, +1.786862810e-07, +5.988251413e-04 - 2.248443e-01, +1.598438319e-05, +2.227475254e-18, +4.071906511e-06, +1.248779937e-07, +1.231318974e-03 - 2.623184e-01, +1.150013517e-05, +5.207497192e-18, +3.089062447e-06, +8.984480602e-08, +2.183514588e-03 - 2.997925e-01, +2.092441719e-05, +1.273586617e-17, +5.172537100e-06, +1.634720093e-07, +3.443881315e-03 - 3.372665e-01, +3.023129661e-05, +1.998967366e-17, +8.595200612e-06, +2.361820048e-07, +4.896471117e-03 - 3.747406e-01, +2.966149891e-05, +5.653700090e-17, +8.160478469e-06, +2.317304602e-07, +6.288238192e-03 - 4.122146e-01, +3.934747640e-05, +1.413811959e-17, +1.043816140e-05, +3.074021594e-07, +7.236958436e-03 - 4.496887e-01, +5.496375443e-05, +1.412220851e-16, +1.376602993e-05, +4.294043315e-07, +7.438657288e-03 - 4.871627e-01, +5.317715171e-05, +1.765081252e-16, +1.345278693e-05, +4.154464977e-07, +7.655736081e-03 - 5.246368e-01, +6.345987516e-05, +3.583095408e-16, +1.616235728e-05, +4.957802747e-07, +1.144507368e-02 - 5.621109e-01, +1.000188545e-04, +6.686901513e-16, +2.890344809e-05, +7.813973009e-07, +2.129749051e-02 - 5.995849e-01, +1.569853883e-04, +6.894682576e-16, +5.054827882e-05, +1.226448346e-06, +3.669225071e-02 - 6.370590e-01, +2.371414637e-04, +2.987326546e-15, +7.811657098e-05, +1.852667685e-06, +5.641515864e-02 - 6.745330e-01, +3.436988518e-04, +2.164626179e-15, +1.084030298e-04, +2.685147279e-06, +7.828737440e-02 - 7.120071e-01, +4.503449422e-04, +8.429520619e-15, +1.334532774e-04, +3.518319861e-06, +9.885303849e-02 - 7.494811e-01, +5.387201987e-04, +2.207111440e-14, +1.460192421e-04, +4.208751553e-06, +1.137546488e-01 - 7.869552e-01, +5.854634658e-04, +3.677392161e-14, +1.426211788e-04, +4.573933327e-06, +1.195362049e-01 - 8.244293e-01, +5.945252634e-04, +1.530577540e-13, +1.553115760e-04, +4.644728621e-06, +1.188356113e-01 - 8.619033e-01, +6.277794936e-04, +8.375922406e-13, +1.446548827e-04, +4.904527294e-06, +1.309570623e-01 - 8.993774e-01, +8.196023636e-04, +3.513223930e-12, +2.064570783e-04, +6.403143466e-06, +1.857784636e-01 - 9.368514e-01, +1.245063027e-03, +1.222917751e-11, +3.796113154e-04, +9.727054900e-06, +2.867343617e-01 - 9.743255e-01, +1.824265443e-03, +3.836943631e-11, +5.734398443e-04, +1.425207377e-05, +4.167726757e-01 - 1.011800e+00, +2.450890627e-03, +1.109093309e-10, +7.521115017e-04, +1.914758302e-05, +5.549120055e-01 - 1.049274e+00, +3.008970317e-03, +2.905634904e-10, +8.708173546e-04, +2.350758060e-05, +6.782193157e-01 - 1.086748e+00, +3.391209593e-03, +6.708727454e-10, +8.861180161e-04, +2.649382494e-05, +7.649055092e-01 - 1.124222e+00, +3.537834071e-03, +1.316367029e-09, +8.504123136e-04, +2.763932868e-05, +8.022591780e-01 - 1.161696e+00, +3.511779316e-03, +2.035245965e-09, +8.840350012e-04, +2.743577591e-05, +8.009806200e-01 - 1.199170e+00, +3.584287871e-03, +1.923005472e-09, +8.159752392e-04, +2.800224899e-05, +8.143332817e-01 - 1.236644e+00, +4.160738357e-03, +3.636929129e-09, +8.694210879e-04, +3.250576841e-05, +9.294499013e-01 - 1.274118e+00, +5.352965496e-03, +1.872212829e-08, +1.361582719e-03, +4.182004294e-05, +1.185780064e+00 - 1.311592e+00, +6.884036966e-03, +5.579982808e-08, +1.934136740e-03, +5.378153880e-05, +1.534661136e+00 - 1.349066e+00, +8.393574758e-03, +1.270371513e-07, +2.361243463e-03, +6.557480279e-05, +1.896361173e+00 - 1.386540e+00, +9.587144869e-03, +2.368634260e-07, +2.543459607e-03, +7.489956929e-05, +2.199964362e+00 - 1.424014e+00, +1.030847844e-02, +3.669558013e-07, +2.421466310e-03, +8.053498780e-05, +2.397709121e+00 - 1.461488e+00, +1.058479827e-02, +4.699423782e-07, +2.459542459e-03, +8.269373645e-05, +2.477248853e+00 - 1.498962e+00, +1.065114462e-02, +5.188096886e-07, +2.478084260e-03, +8.321206733e-05, +2.474045346e+00 - 1.536436e+00, +1.088707747e-02, +7.069806642e-07, +2.347785517e-03, +8.505529270e-05, +2.473655996e+00 - 1.573910e+00, +1.161350503e-02, +1.366657071e-06, +2.453010503e-03, +9.073050803e-05, +2.579211451e+00 - 1.611384e+00, +1.284893913e-02, +2.472581913e-06, +2.368957242e-03, +1.003823369e-04, +2.837696929e+00 - 1.648859e+00, +1.431859015e-02, +3.857804555e-06, +3.068473841e-03, +1.118639856e-04, +3.200566296e+00 - 1.686333e+00, +1.568466897e-02, +5.309872328e-06, +3.469083241e-03, +1.225364763e-04, +3.569988955e+00 - 1.723807e+00, +1.672073192e-02, +6.527746680e-06, +3.499417907e-03, +1.306307181e-04, +3.860977237e+00 - 1.761281e+00, +1.735917503e-02, +7.046758600e-06, +3.289526113e-03, +1.356185549e-04, +4.031108681e+00 - 1.798755e+00, +1.767118125e-02, +6.301192058e-06, +3.511818706e-03, +1.380561035e-04, +4.086235159e+00 - 1.836229e+00, +1.781514385e-02, +5.066557197e-06, +3.414133307e-03, +1.391808113e-04, +4.072115006e+00 - 1.873703e+00, +1.795734365e-02, +9.785529399e-06, +3.394013663e-03, +1.402917473e-04, +4.053080663e+00 - 1.911177e+00, +1.820297964e-02, +2.153285398e-05, +3.420849872e-03, +1.422107784e-04, +4.080884955e+00 - 1.948651e+00, +1.856531991e-02, +3.797781532e-05, +3.234892491e-03, +1.450415618e-04, +4.169137462e+00 - 1.986125e+00, +1.899015404e-02, +5.721899093e-05, +3.396745163e-03, +1.483605785e-04, +4.292812394e+00 - 2.023599e+00, +1.939524081e-02, +6.688605498e-05, +3.250581882e-03, +1.515253188e-04, +4.411264745e+00 - 2.061073e+00, +1.970534620e-02, +8.499883944e-05, +3.312961586e-03, +1.539480172e-04, +4.493132813e+00 - 2.098547e+00, +1.988283045e-02, +6.241843571e-05, +3.301605857e-03, +1.553346129e-04, +4.527959658e+00 - 2.136021e+00, +1.993607356e-02, +7.807678532e-05, +3.172906476e-03, +1.557505747e-04, +4.524294614e+00 - 2.173495e+00, +1.989960314e-02, +4.892651018e-05, +3.293718894e-03, +1.554656495e-04, +4.500332857e+00 - 2.210969e+00, +1.981531470e-02, +6.512326289e-05, +3.115928939e-03, +1.548071461e-04, +4.473391974e+00 - 2.248443e+00, +1.972658289e-02, +1.542799260e-04, +3.228336522e-03, +1.541139288e-04, +4.453059505e+00 - 2.285917e+00, +1.965740166e-02, +2.721087012e-04, +3.180446561e-03, +1.535734504e-04, +4.440114615e+00 - 2.323392e+00, +1.959904710e-02, +3.980665603e-04, +3.107500278e-03, +1.531175555e-04, +4.429876188e+00 - 2.360866e+00, +1.953282422e-02, +4.003191673e-04, +3.189015057e-03, +1.526001892e-04, +4.416676540e+00 - 2.398340e+00, +1.944869644e-02, +5.464091449e-04, +2.981980322e-03, +1.519429410e-04, +4.396869067e+00 - 2.435814e+00, +1.933806042e-02, +4.155885733e-04, +3.140904648e-03, +1.510785971e-04, +4.369773349e+00 - 2.473288e+00, +1.920073395e-02, +4.446796335e-04, +3.059026781e-03, +1.500057340e-04, +4.337298959e+00 - 2.510762e+00, +1.905522216e-02, +3.406503094e-04, +3.038745856e-03, +1.488689231e-04, +4.303041277e+00 - 2.548236e+00, +1.892141280e-02, +1.821447779e-04, +3.082892837e-03, +1.478235375e-04, +4.271004959e+00 - 2.585710e+00, +1.879970377e-02, +8.506630768e-05, +2.887937022e-03, +1.468726857e-04, +4.243717289e+00 - 2.623184e+00, +1.867373516e-02, +1.004256193e-04, +3.051640523e-03, +1.458885559e-04, +4.219927993e+00 - 2.660658e+00, +1.852070404e-02, +1.196737664e-04, +2.938772398e-03, +1.446930003e-04, +4.193080930e+00 - 2.698132e+00, +1.830775799e-02, +1.071572521e-04, +2.968150322e-03, +1.430293593e-04, +4.152255197e+00 - 2.735606e+00, +1.800218944e-02, +1.320213032e-04, +2.975736659e-03, +1.406421050e-04, +4.086458524e+00 - 2.773080e+00, +1.760560892e-02, +9.506619049e-05, +2.829530695e-03, +1.375438197e-04, +3.991233526e+00 - 2.810554e+00, +1.716131434e-02, +7.317905711e-05, +2.954310998e-03, +1.340727683e-04, +3.874560228e+00 - 2.848028e+00, +1.672503706e-02, +4.353864457e-05, +2.810145825e-03, +1.306643520e-04, +3.757798136e+00 - 2.885502e+00, +1.634861093e-02, +2.010916948e-05, +2.884632875e-03, +1.277235229e-04, +3.667738464e+00 - 2.922976e+00, +1.607034327e-02, +7.273872449e-06, +2.862475829e-03, +1.255495568e-04, +3.620214926e+00 - 2.960451e+00, +1.586218226e-02, +9.724124591e-06, +2.809723296e-03, +1.239232989e-04, +3.604829965e+00 - 2.997925e+00, +1.562061453e-02, +1.277801775e-05, +2.899257047e-03, +1.220360510e-04, +3.584446446e+00 - 3.035399e+00, +1.522782614e-02, +1.233271900e-05, +2.721416546e-03, +1.189673917e-04, +3.511909116e+00 - 3.072873e+00, +1.458718436e-02, +1.012356779e-05, +2.782997074e-03, +1.139623778e-04, +3.352720251e+00 - 3.110347e+00, +1.366320273e-02, +7.294721134e-06, +2.762079550e-03, +1.067437713e-04, +3.102486911e+00 - 3.147821e+00, +1.251768483e-02, +4.420404497e-06, +2.455956528e-03, +9.779441270e-05, +2.794887923e+00 - 3.185295e+00, +1.133578878e-02, +2.426618937e-06, +2.045393417e-03, +8.856084987e-05, +2.497506493e+00 - 3.222769e+00, +1.036779337e-02, +1.089662298e-06, +2.156643879e-03, +8.099838569e-05, +2.287032135e+00 - 3.260243e+00, +9.775726200e-03, +7.775481997e-07, +2.063789951e-03, +7.637286094e-05, +2.199308734e+00 - 3.297717e+00, +9.532734097e-03, +6.108364044e-07, +2.153749292e-03, +7.447448513e-05, +2.194284318e+00 - 3.335191e+00, +9.398661259e-03, +8.216045578e-07, +2.135688547e-03, +7.342704109e-05, +2.187377027e+00 - 3.372665e+00, +9.084908004e-03, +4.990304560e-07, +2.040467598e-03, +7.097584378e-05, +2.107636508e+00 - 3.410139e+00, +8.406765257e-03, +2.134421440e-07, +2.145048813e-03, +6.567785357e-05, +1.925405968e+00 - 3.447613e+00, +7.345670737e-03, +2.218387340e-07, +1.993262694e-03, +5.738805263e-05, +1.652352466e+00 - 3.485087e+00, +6.047746915e-03, +3.358002123e-07, +1.633531207e-03, +4.724802278e-05, +1.332422196e+00 - 3.522561e+00, +4.746403684e-03, +5.213325624e-07, +1.144017220e-03, +3.708127878e-05, +1.031986493e+00 - 3.560035e+00, +3.769640682e-03, +7.310928528e-07, +8.020516437e-04, +2.945031782e-05, +8.248254263e-01 - 3.597509e+00, +3.333383968e-03, +7.229543071e-07, +7.732301888e-04, +2.604206225e-05, +7.467641311e-01 - 3.634984e+00, +3.288196120e-03, +5.789534976e-07, +8.209433323e-04, +2.568903219e-05, +7.476238539e-01 - 3.672458e+00, +3.293284151e-03, +8.017459876e-07, +7.778578450e-04, +2.572878243e-05, +7.473541982e-01 - 3.709932e+00, +3.117292905e-03, +4.714346501e-07, +8.140977754e-04, +2.435385082e-05, +7.038828045e-01 - 3.747406e+00, +2.729593477e-03, +5.298794400e-07, +7.857588551e-04, +2.132494904e-05, +6.126782673e-01 - 3.784880e+00, +2.180764435e-03, +1.216180507e-06, +6.625316701e-04, +1.703722215e-05, +4.890491612e-01 - 3.822354e+00, +1.586088428e-03, +2.384923310e-06, +4.875486123e-04, +1.239131585e-05, +3.558862685e-01 - 3.859828e+00, +1.066809700e-03, +3.511481065e-06, +3.033193261e-04, +8.334450780e-06, +2.372274680e-01 - 3.897302e+00, +7.233506887e-04, +4.708543875e-06, +1.463309998e-04, +5.651177256e-06, +1.561842423e-01 - 3.934776e+00, +6.203759622e-04, +3.573109435e-06, +1.545835338e-04, +4.846687204e-06, +1.265111525e-01 - 3.972250e+00, +6.226640636e-04, +4.851805508e-06, +1.561429812e-04, +4.864562997e-06, +1.273690025e-01 - 4.009724e+00, +6.087857616e-04, +4.093612728e-06, +1.485704094e-04, +4.756138763e-06, +1.276030578e-01 - 4.047198e+00, +5.400191444e-04, +3.742942640e-06, +1.478591037e-04, +4.218899565e-06, +1.171779781e-01 - 4.084672e+00, +4.401182610e-04, +4.379634926e-06, +1.273842804e-04, +3.438423914e-06, +9.815150341e-02 - 4.122146e+00, +3.324057867e-04, +3.390263835e-06, +9.627154190e-05, +2.596920209e-06, +7.584800868e-02 - 4.159620e+00, +2.370075783e-04, +4.831244850e-06, +6.338579248e-05, +1.851621706e-06, +5.553016806e-02 - 4.197094e+00, +1.811418739e-04, +5.641649752e-06, +3.631936955e-05, +1.415170890e-06, +4.116690846e-02 - 4.234568e+00, +1.557249175e-04, +2.668727064e-06, +2.221487415e-05, +1.216600918e-06, +3.415513459e-02 - 4.272043e+00, +1.465187122e-04, +1.674662472e-06, +2.170675982e-05, +1.144677439e-06, +3.215710958e-02 - 4.309517e+00, +1.405423708e-04, +1.924531635e-06, +2.149418194e-05, +1.097987272e-06, +3.160248470e-02 - 4.346991e+00, +1.357847027e-04, +1.605849913e-06, +2.192148955e-05, +1.060817990e-06, +3.065472344e-02 - 4.384465e+00, +1.258550769e-04, +1.284603768e-06, +2.046648332e-05, +9.832427880e-07, +2.904522959e-02 - 4.421939e+00, +1.169869496e-04, +1.278082689e-06, +2.159284942e-05, +9.139605438e-07, +2.713926010e-02 - 4.459413e+00, +1.110703423e-04, +1.509710125e-06, +2.079338969e-05, +8.677370496e-07, +2.542558185e-02 - 4.496887e+00, +1.057701136e-04, +5.790173697e-07, +2.100616047e-05, +8.263290126e-07, +2.427050930e-02 - 4.534361e+00, +1.026251685e-04, +7.810732718e-07, +2.116271293e-05, +8.017591292e-07, +2.376362919e-02 - 4.571835e+00, +1.011917516e-04, +5.963699975e-07, +2.046230884e-05, +7.905605596e-07, +2.366920631e-02 - 4.609309e+00, +1.001634828e-04, +8.947493041e-07, +2.134477255e-05, +7.825272090e-07, +2.354631490e-02 - 4.646783e+00, +9.707341248e-05, +3.485090573e-07, +2.031439706e-05, +7.583860350e-07, +2.295738668e-02 - 4.684257e+00, +9.167802949e-05, +3.892732641e-07, +2.066985033e-05, +7.162346054e-07, +2.162696749e-02 - 4.721731e+00, +8.383810226e-05, +2.180435689e-07, +2.059725781e-05, +6.549851739e-07, +1.951430734e-02 - 4.759205e+00, +7.380265099e-05, +2.824981367e-07, +1.856655811e-05, +5.765832109e-07, +1.682504287e-02 - 4.796679e+00, +6.231475191e-05, +3.941097394e-07, +1.489212560e-05, +4.868339993e-07, +1.398756018e-02 - 4.834153e+00, +5.205405315e-05, +9.286816014e-08, +1.031777329e-05, +4.066722902e-07, +1.158631725e-02 - 4.871627e+00, +4.563461801e-05, +2.947284352e-07, +9.944042992e-06, +3.565204532e-07, +1.014939239e-02 - 4.909101e+00, +4.281885346e-05, +2.428193082e-07, +9.489779373e-06, +3.345222927e-07, +9.727415693e-03 - 4.946576e+00, +4.221795835e-05, +3.153021840e-07, +1.010970714e-05, +3.298277996e-07, +9.783286724e-03 - 4.984050e+00, +4.193062278e-05, +1.303308212e-07, +9.664688337e-06, +3.275829905e-07, +9.697488727e-03 - 5.021524e+00, +3.983627118e-05, +2.205996708e-07, +9.929565063e-06, +3.112208686e-07, +9.137274998e-03 - 5.058998e+00, +3.557555069e-05, +4.469304141e-07, +9.812803111e-06, +2.779339898e-07, +8.056891307e-03 - 5.096472e+00, +2.974014514e-05, +3.707371569e-07, +8.638593197e-06, +2.323448839e-07, +6.593883084e-03 - 5.133946e+00, +2.283060266e-05, +1.698975080e-07, +6.683001397e-06, +1.783640832e-07, +4.979337600e-03 - 5.171420e+00, +1.666241488e-05, +3.152853125e-07, +4.477294455e-06, +1.301751163e-07, +3.482333541e-03 - 5.208894e+00, +1.250577556e-05, +1.567320721e-07, +2.385719075e-06, +9.770137157e-08, +2.384295551e-03 - 5.246368e+00, +1.120766321e-05, +2.127374541e-07, +2.042794930e-06, +8.755986884e-08, +1.893867546e-03 - 5.283842e+00, +1.074669371e-05, +1.995526713e-07, +2.278776543e-06, +8.395854458e-08, +1.857462365e-03 - 5.321316e+00, +1.005005277e-05, +3.020728485e-07, +2.048453621e-06, +7.851603724e-08, +1.884026962e-03 - 5.358790e+00, +1.027674820e-05, +1.968047729e-07, +2.146664014e-06, +8.028709530e-08, +1.781458878e-03 - 5.396264e+00, +9.992556903e-06, +2.897865156e-07, +2.019372889e-06, +7.806685080e-08, +1.539662271e-03 - 5.433738e+00, +8.130179588e-06, +9.113607665e-08, +1.644339079e-06, +6.351702803e-08, +1.215813603e-03 - 5.471212e+00, +6.073721937e-06, +1.478699844e-07, +1.146645281e-06, +4.745095264e-08, +8.794652134e-04 - 5.508686e+00, +6.698146250e-06, +9.692032714e-08, +9.740127564e-07, +5.232926758e-08, +5.925767029e-04 - 5.546160e+00, +7.402378271e-06, +3.073582384e-07, +1.094274341e-06, +5.783108024e-08, +4.025875389e-04 - 5.583635e+00, +6.171372548e-06, +1.350664345e-07, +9.761640377e-07, +4.821384803e-08, +3.250012544e-04 - 5.621109e+00, +5.253380349e-06, +9.249781991e-08, +7.532475805e-07, +4.104203398e-08, +3.144552651e-04 - 5.658583e+00, +6.034991990e-06, +1.524640444e-07, +8.379384916e-07, +4.714837492e-08, +3.143233654e-04 - 5.696057e+00, +6.954997371e-06, +1.738910829e-07, +1.032800032e-06, +5.433591696e-08, +3.053781280e-04 - 5.733531e+00, +6.618311823e-06, +1.393653595e-07, +9.706910964e-07, +5.170556112e-08, +2.905274734e-04 - 5.771005e+00, +4.983685205e-06, +7.755106256e-08, +7.959431861e-07, +3.893504067e-08, +2.767846518e-04 - 5.808479e+00, +5.498601999e-06, +5.420219015e-08, +8.069512442e-07, +4.295782812e-08, +2.672840112e-04 - 5.845953e+00, +6.952197828e-06, +1.517744765e-07, +1.004761387e-06, +5.431404553e-08, +2.606043537e-04 - 5.883427e+00, +6.329200774e-06, +9.133367320e-08, +8.427280265e-07, +4.944688105e-08, +2.537151852e-04 - 5.920901e+00, +5.215815525e-06, +2.025761161e-08, +8.189533922e-07, +4.074855879e-08, +2.439748930e-04 - 5.958375e+00, +5.305475054e-06, +1.698310034e-07, +6.730686943e-07, +4.144902386e-08, +2.288006244e-04 - 5.995849e+00, +6.176556969e-06, +1.308471229e-07, +9.587659988e-07, +4.825435132e-08, +2.091970502e-04 - 6.033323e+00, +6.645688022e-06, +8.608171152e-09, +8.996642786e-07, +5.191943767e-08, +1.868959008e-04 - 6.070797e+00, +5.398007444e-06, +1.508347448e-07, +8.539658993e-07, +4.217193316e-08, +1.639107998e-04 - 6.108271e+00, +4.639738945e-06, +1.461885578e-07, +5.846045529e-07, +3.624796051e-08, +1.453359455e-04 - 6.145745e+00, +6.145503917e-06, +8.241360256e-08, +9.145264944e-07, +4.801174935e-08, +1.340081739e-04 - 6.183219e+00, +6.485408052e-06, +6.256044891e-08, +9.129625606e-07, +5.066725041e-08, +1.300350501e-04 - 6.220694e+00, +5.542152865e-06, +8.574375172e-08, +7.821942712e-07, +4.329806926e-08, +1.300871660e-04 - 6.258168e+00, +4.985301527e-06, +1.082258939e-07, +7.100283107e-07, +3.894766818e-08, +1.297254971e-04 - 6.295642e+00, +5.334225761e-06, +4.704333371e-08, +7.980112259e-07, +4.167363876e-08, +1.253586089e-04 - 6.333116e+00, +6.421230064e-06, +2.899044502e-07, +9.204389046e-07, +5.016585987e-08, +1.158965803e-04 - 6.370590e+00, +6.073729696e-06, +8.837745352e-08, +7.781934516e-07, +4.745101325e-08, +1.015407485e-04 - 6.408064e+00, +4.266464125e-06, +7.836990080e-08, +6.328354800e-07, +3.333175098e-08, +8.295426644e-05 - 6.445538e+00, +5.024336673e-06, +9.562368737e-08, +6.546446017e-07, +3.925263026e-08, +6.413712406e-05 - 6.483012e+00, +6.379473131e-06, +2.230146922e-07, +9.421359087e-07, +4.983963384e-08, +4.719626400e-05 - 6.520486e+00, +5.693647315e-06, +1.453003864e-07, +7.169231018e-07, +4.448161965e-08, +3.428721749e-05 - 6.557960e+00, +4.451825872e-06, +6.643434571e-08, +7.111725148e-07, +3.477988963e-08, +2.871636923e-05 - 6.595434e+00, +4.899534092e-06, +1.455463377e-07, +7.337732989e-07, +3.827761010e-08, +2.830713259e-05 - 6.632908e+00, +5.715247683e-06, +1.362646790e-07, +8.893278507e-07, +4.465037252e-08, +2.868930397e-05 - 6.670382e+00, +5.627211459e-06, +1.506603539e-07, +7.797612740e-07, +4.396258952e-08, +2.757669223e-05 - 6.707856e+00, +4.948318503e-06, +1.411310733e-07, +7.615922799e-07, +3.865873830e-08, +2.478379048e-05 - 6.745330e+00, +4.504619390e-06, +8.895057287e-08, +7.798763972e-07, +3.519233898e-08, +2.028553556e-05 - 6.782804e+00, +4.993991381e-06, +8.728597097e-08, +7.967551049e-07, +3.901555766e-08, +1.596593978e-05 - 6.820278e+00, +5.747900143e-06, +1.409301742e-07, +8.337881468e-07, +4.490546987e-08, +1.262078790e-05 - 6.857752e+00, +5.247854702e-06, +1.813322114e-07, +7.726407318e-07, +4.099886486e-08, +9.582347785e-06 - 6.895227e+00, +3.985177625e-06, +3.643398367e-09, +6.312565334e-07, +3.113420020e-08, +7.451330914e-06 - 6.932701e+00, +4.586522913e-06, +8.433043328e-08, +6.862489896e-07, +3.583221026e-08, +7.714034018e-06 - 6.970175e+00, +5.649889468e-06, +9.980671974e-08, +8.434616317e-07, +4.413976147e-08, +8.153445503e-06 - 7.007649e+00, +5.253206883e-06, +6.315370615e-08, +6.957073006e-07, +4.104067878e-08, +8.013246854e-06 - 7.045123e+00, +3.957428597e-06, +3.892861524e-08, +6.417155614e-07, +3.091741091e-08, +7.103149370e-06 - 7.082597e+00, +4.067556090e-06, +1.030158119e-07, +5.483078992e-07, +3.177778195e-08, +6.251988925e-06 - 7.120071e+00, +5.339246881e-06, +3.932281422e-08, +8.433152884e-07, +4.171286626e-08, +7.632283077e-06 - 7.157545e+00, +5.327917643e-06, +1.177532816e-07, +6.938631104e-07, +4.162435658e-08, +7.329495899e-06 - 7.195019e+00, +3.969772189e-06, +7.950843774e-08, +6.036690615e-07, +3.101384523e-08, +6.045781511e-06 - 7.232493e+00, +3.715635311e-06, +1.404290934e-07, +5.594075907e-07, +2.902840086e-08, +5.792378480e-06 - 7.269967e+00, +5.014967369e-06, +1.612835104e-07, +7.753574291e-07, +3.917943257e-08, +6.622438552e-06 - 7.307441e+00, +5.352771106e-06, +1.184136830e-07, +7.375169940e-07, +4.181852426e-08, +7.298828834e-06 - 7.344915e+00, +4.153470584e-06, +4.480562392e-08, +6.123684096e-07, +3.244898894e-08, +5.538450050e-06 - 7.382389e+00, +3.599983803e-06, +3.650956171e-08, +5.446721100e-07, +2.812487346e-08, +5.496706240e-06 - 7.419863e+00, +4.830766906e-06, +5.884627608e-08, +7.174872509e-07, +3.774036645e-08, +6.712351610e-06 - 7.457337e+00, +5.146870332e-06, +1.163179384e-07, +7.555277815e-07, +4.020992447e-08, +6.428644479e-06 - 7.494811e+00, +4.397400790e-06, +1.080429328e-07, +5.798177323e-07, +3.435469367e-08, +6.238567259e-06 - Mean, +4.855456549e-03, +2.700723228e-04, +3.584548856e-04, +3.793325429e-05, +0.000000000e+00 + 3.747406e-02, +5.656919329e-06, +4.877275943e-19, +1.429146542e-06, +4.419468225e-08, +1.015026012e-04 + 7.494811e-02, +1.331394786e-05, +1.262176118e-18, +3.354187386e-06, +1.040152177e-07, +1.286467416e-04 + 1.124222e-01, +4.533166152e-06, +8.797629984e-19, +1.144058873e-06, +3.541536056e-08, +1.363432445e-04 + 1.498962e-01, +5.316539312e-06, +5.339102739e-19, +1.352378049e-06, +4.153546337e-08, +2.237453773e-04 + 1.873703e-01, +2.287184397e-05, +2.999088821e-18, +5.789590609e-06, +1.786862810e-07, +5.988251413e-04 + 2.248443e-01, +1.598438319e-05, +2.232109810e-18, +4.071906508e-06, +1.248779937e-07, +1.231318974e-03 + 2.623184e-01, +1.150013517e-05, +5.196882665e-18, +3.089062447e-06, +8.984480601e-08, +2.183514588e-03 + 2.997925e-01, +2.092441721e-05, +1.274335458e-17, +5.172537104e-06, +1.634720094e-07, +3.443881315e-03 + 3.372665e-01, +3.023129660e-05, +2.000023090e-17, +8.595200606e-06, +2.361820047e-07, +4.896471117e-03 + 3.747406e-01, +2.966149896e-05, +5.645231840e-17, +8.160478469e-06, +2.317304606e-07, +6.288238192e-03 + 4.122146e-01, +3.934747639e-05, +1.414197098e-17, +1.043816137e-05, +3.074021593e-07, +7.236958436e-03 + 4.496887e-01, +5.496375443e-05, +1.412135240e-16, +1.376602992e-05, +4.294043315e-07, +7.438657288e-03 + 4.871627e-01, +5.317715171e-05, +1.765052700e-16, +1.345278687e-05, +4.154464978e-07, +7.655736081e-03 + 5.246368e-01, +6.345987510e-05, +3.578882920e-16, +1.616235667e-05, +4.957802742e-07, +1.144507368e-02 + 5.621109e-01, +1.000188544e-04, +6.683863674e-16, +2.890344790e-05, +7.813973001e-07, +2.129749051e-02 + 5.995849e-01, +1.569853883e-04, +6.894686202e-16, +5.054827882e-05, +1.226448346e-06, +3.669225071e-02 + 6.370590e-01, +2.371414640e-04, +2.988956019e-15, +7.811657109e-05, +1.852667687e-06, +5.641515864e-02 + 6.745330e-01, +3.436988525e-04, +2.171452447e-15, +1.084030298e-04, +2.685147285e-06, +7.828737440e-02 + 7.120071e-01, +4.503449432e-04, +8.422790023e-15, +1.334532772e-04, +3.518319868e-06, +9.885303849e-02 + 7.494811e-01, +5.387201999e-04, +2.207064524e-14, +1.460192415e-04, +4.208751561e-06, +1.137546488e-01 + 7.869552e-01, +5.854634666e-04, +3.678749831e-14, +1.426211774e-04, +4.573933333e-06, +1.195362049e-01 + 8.244293e-01, +5.945252636e-04, +1.530621531e-13, +1.553115755e-04, +4.644728622e-06, +1.188356113e-01 + 8.619033e-01, +6.277794930e-04, +8.375528408e-13, +1.446548827e-04, +4.904527289e-06, +1.309570623e-01 + 8.993774e-01, +8.196023618e-04, +3.513126401e-12, +2.064570778e-04, +6.403143451e-06, +1.857784636e-01 + 9.368514e-01, +1.245063026e-03, +1.222913601e-11, +3.796113147e-04, +9.727054889e-06, +2.867343617e-01 + 9.743255e-01, +1.824265444e-03, +3.836946290e-11, +5.734398444e-04, +1.425207378e-05, +4.167726757e-01 + 1.011800e+00, +2.450890631e-03, +1.109091742e-10, +7.521115026e-04, +1.914758306e-05, +5.549120054e-01 + 1.049274e+00, +3.008970324e-03, +2.905630779e-10, +8.708173545e-04, +2.350758066e-05, +6.782193157e-01 + 1.086748e+00, +3.391209599e-03, +6.708721725e-10, +8.861180142e-04, +2.649382500e-05, +7.649055092e-01 + 1.124222e+00, +3.537834075e-03, +1.316366440e-09, +8.504123104e-04, +2.763932871e-05, +8.022591780e-01 + 1.161696e+00, +3.511779316e-03, +2.035246014e-09, +8.840349986e-04, +2.743577591e-05, +8.009806200e-01 + 1.199170e+00, +3.584287867e-03, +1.923006182e-09, +8.159752264e-04, +2.800224896e-05, +8.143332817e-01 + 1.236644e+00, +4.160738346e-03, +3.636930649e-09, +8.694210875e-04, +3.250576833e-05, +9.294499013e-01 + 1.274118e+00, +5.352965483e-03, +1.872213299e-08, +1.361582714e-03, +4.182004284e-05, +1.185780064e+00 + 1.311592e+00, +6.884036964e-03, +5.579983321e-08, +1.934136736e-03, +5.378153878e-05, +1.534661136e+00 + 1.349066e+00, +8.393574770e-03, +1.270371506e-07, +2.361243466e-03, +6.557480289e-05, +1.896361173e+00 + 1.386540e+00, +9.587144888e-03, +2.368634115e-07, +2.543459610e-03, +7.489956944e-05, +2.199964362e+00 + 1.424014e+00, +1.030847846e-02, +3.669557621e-07, +2.421466309e-03, +8.053498795e-05, +2.397709121e+00 + 1.461488e+00, +1.058479828e-02, +4.699423030e-07, +2.459542440e-03, +8.269373653e-05, +2.477248853e+00 + 1.498962e+00, +1.065114462e-02, +5.188095796e-07, +2.478084232e-03, +8.321206732e-05, +2.474045346e+00 + 1.536436e+00, +1.088707745e-02, +7.069806391e-07, +2.347785514e-03, +8.505529260e-05, +2.473655996e+00 + 1.573910e+00, +1.161350500e-02, +1.366657274e-06, +2.453010484e-03, +9.073050782e-05, +2.579211451e+00 + 1.611384e+00, +1.284893909e-02, +2.472582291e-06, +2.368957236e-03, +1.003823366e-04, +2.837696929e+00 + 1.648859e+00, +1.431859013e-02, +3.857804985e-06, +3.068473826e-03, +1.118639854e-04, +3.200566296e+00 + 1.686333e+00, +1.568466897e-02, +5.309872634e-06, +3.469083243e-03, +1.225364764e-04, +3.569988955e+00 + 1.723807e+00, +1.672073194e-02, +6.527746720e-06, +3.499417909e-03, +1.306307183e-04, +3.860977237e+00 + 1.761281e+00, +1.735917505e-02, +7.046758211e-06, +3.289526062e-03, +1.356185551e-04, +4.031108681e+00 + 1.798755e+00, +1.767118126e-02, +6.301190978e-06, +3.511818671e-03, +1.380561036e-04, +4.086235159e+00 + 1.836229e+00, +1.781514385e-02, +5.066555573e-06, +3.414133251e-03, +1.391808113e-04, +4.072115006e+00 + 1.873703e+00, +1.795734364e-02, +9.785530080e-06, +3.394013652e-03, +1.402917472e-04, +4.053080663e+00 + 1.911177e+00, +1.820297960e-02, +2.153285729e-05, +3.420849841e-03, +1.422107781e-04, +4.080884955e+00 + 1.948651e+00, +1.856531984e-02, +3.797782051e-05, +3.234892450e-03, +1.450415613e-04, +4.169137462e+00 + 1.986125e+00, +1.899015399e-02, +5.721899660e-05, +3.396745120e-03, +1.483605781e-04, +4.292812394e+00 + 2.023599e+00, +1.939524079e-02, +6.688604613e-05, +3.250581869e-03, +1.515253187e-04, +4.411264745e+00 + 2.061073e+00, +1.970534621e-02, +8.499884299e-05, +3.312961563e-03, +1.539480172e-04, +4.493132813e+00 + 2.098547e+00, +1.988283047e-02, +6.241843847e-05, +3.301605834e-03, +1.553346131e-04, +4.527959658e+00 + 2.136021e+00, +1.993607358e-02, +7.807678218e-05, +3.172906433e-03, +1.557505749e-04, +4.524294614e+00 + 2.173495e+00, +1.989960316e-02, +4.892650876e-05, +3.293718876e-03, +1.554656497e-04, +4.500332857e+00 + 2.210969e+00, +1.981531470e-02, +6.512328567e-05, +3.115928915e-03, +1.548071461e-04, +4.473391974e+00 + 2.248443e+00, +1.972658286e-02, +1.542799595e-04, +3.228336502e-03, +1.541139286e-04, +4.453059505e+00 + 2.285917e+00, +1.965740159e-02, +2.721087405e-04, +3.180446523e-03, +1.535734500e-04, +4.440114615e+00 + 2.323392e+00, +1.959904705e-02, +3.980665951e-04, +3.107500308e-03, +1.531175551e-04, +4.429876188e+00 + 2.360866e+00, +1.953282420e-02, +4.003191631e-04, +3.189015088e-03, +1.526001891e-04, +4.416676540e+00 + 2.398340e+00, +1.944869644e-02, +5.464091330e-04, +2.981980375e-03, +1.519429410e-04, +4.396869067e+00 + 2.435814e+00, +1.933806044e-02, +4.155885738e-04, +3.140904615e-03, +1.510785972e-04, +4.369773349e+00 + 2.473288e+00, +1.920073398e-02, +4.446796367e-04, +3.059026759e-03, +1.500057342e-04, +4.337298959e+00 + 2.510762e+00, +1.905522218e-02, +3.406503422e-04, +3.038745874e-03, +1.488689233e-04, +4.303041277e+00 + 2.548236e+00, +1.892141281e-02, +1.821448184e-04, +3.082892831e-03, +1.478235376e-04, +4.271004959e+00 + 2.585710e+00, +1.879970376e-02, +8.506633659e-05, +2.887937012e-03, +1.468726856e-04, +4.243717289e+00 + 2.623184e+00, +1.867373515e-02, +1.004256191e-04, +3.051640505e-03, +1.458885559e-04, +4.219927993e+00 + 2.660658e+00, +1.852070404e-02, +1.196737672e-04, +2.938772388e-03, +1.446930003e-04, +4.193080930e+00 + 2.698132e+00, +1.830775800e-02, +1.071572532e-04, +2.968150277e-03, +1.430293593e-04, +4.152255197e+00 + 2.735606e+00, +1.800218945e-02, +1.320213036e-04, +2.975736631e-03, +1.406421051e-04, +4.086458524e+00 + 2.773080e+00, +1.760560892e-02, +9.506618993e-05, +2.829530709e-03, +1.375438197e-04, +3.991233526e+00 + 2.810554e+00, +1.716131434e-02, +7.317906244e-05, +2.954310998e-03, +1.340727683e-04, +3.874560228e+00 + 2.848028e+00, +1.672503705e-02, +4.353865043e-05, +2.810145815e-03, +1.306643519e-04, +3.757798136e+00 + 2.885502e+00, +1.634861092e-02, +2.010917331e-05, +2.884632857e-03, +1.277235228e-04, +3.667738464e+00 + 2.922976e+00, +1.607034328e-02, +7.273872482e-06, +2.862475827e-03, +1.255495569e-04, +3.620214926e+00 + 2.960451e+00, +1.586218229e-02, +9.724124820e-06, +2.809723238e-03, +1.239232991e-04, +3.604829965e+00 + 2.997925e+00, +1.562061457e-02, +1.277801777e-05, +2.899257020e-03, +1.220360513e-04, +3.584446446e+00 + 3.035399e+00, +1.522782617e-02, +1.233271869e-05, +2.721416525e-03, +1.189673920e-04, +3.511909116e+00 + 3.072873e+00, +1.458718439e-02, +1.012356765e-05, +2.782997075e-03, +1.139623780e-04, +3.352720251e+00 + 3.110347e+00, +1.366320273e-02, +7.294721417e-06, +2.762079543e-03, +1.067437714e-04, +3.102486911e+00 + 3.147821e+00, +1.251768481e-02, +4.420404970e-06, +2.455956523e-03, +9.779441258e-05, +2.794887923e+00 + 3.185295e+00, +1.133578876e-02, +2.426619315e-06, +2.045393409e-03, +8.856084967e-05, +2.497506493e+00 + 3.222769e+00, +1.036779336e-02, +1.089662333e-06, +2.156643882e-03, +8.099838562e-05, +2.287032135e+00 + 3.260243e+00, +9.775726216e-03, +7.775481631e-07, +2.063789978e-03, +7.637286106e-05, +2.199308734e+00 + 3.297717e+00, +9.532734129e-03, +6.108364501e-07, +2.153749277e-03, +7.447448538e-05, +2.194284318e+00 + 3.335191e+00, +9.398661297e-03, +8.216045826e-07, +2.135688544e-03, +7.342704138e-05, +2.187377027e+00 + 3.372665e+00, +9.084908041e-03, +4.990304566e-07, +2.040467603e-03, +7.097584407e-05, +2.107636508e+00 + 3.410139e+00, +8.406765284e-03, +2.134421476e-07, +2.145048814e-03, +6.567785378e-05, +1.925405968e+00 + 3.447613e+00, +7.345670750e-03, +2.218387414e-07, +1.993262691e-03, +5.738805273e-05, +1.652352466e+00 + 3.485087e+00, +6.047746911e-03, +3.358003217e-07, +1.633531203e-03, +4.724802274e-05, +1.332422196e+00 + 3.522561e+00, +4.746403671e-03, +5.213325765e-07, +1.144017234e-03, +3.708127868e-05, +1.031986493e+00 + 3.560035e+00, +3.769640679e-03, +7.310929636e-07, +8.020516541e-04, +2.945031781e-05, +8.248254263e-01 + 3.597509e+00, +3.333383976e-03, +7.229544209e-07, +7.732301786e-04, +2.604206231e-05, +7.467641311e-01 + 3.634984e+00, +3.288196133e-03, +5.789536836e-07, +8.209433331e-04, +2.568903229e-05, +7.476238539e-01 + 3.672458e+00, +3.293284166e-03, +8.017460625e-07, +7.778578434e-04, +2.572878255e-05, +7.473541982e-01 + 3.709932e+00, +3.117292919e-03, +4.714346928e-07, +8.140977771e-04, +2.435385093e-05, +7.038828045e-01 + 3.747406e+00, +2.729593487e-03, +5.298796474e-07, +7.857588558e-04, +2.132494912e-05, +6.126782673e-01 + 3.784880e+00, +2.180764441e-03, +1.216180682e-06, +6.625316699e-04, +1.703722219e-05, +4.890491612e-01 + 3.822354e+00, +1.586088429e-03, +2.384923708e-06, +4.875486122e-04, +1.239131585e-05, +3.558862685e-01 + 3.859828e+00, +1.066809698e-03, +3.511481400e-06, +3.033193280e-04, +8.334450763e-06, +2.372274680e-01 + 3.897302e+00, +7.233506887e-04, +4.708544179e-06, +1.463310009e-04, +5.651177256e-06, +1.561842423e-01 + 3.934776e+00, +6.203759634e-04, +3.573109786e-06, +1.545835346e-04, +4.846687214e-06, +1.265111525e-01 + 3.972250e+00, +6.226640658e-04, +4.851805755e-06, +1.561429812e-04, +4.864563014e-06, +1.273690025e-01 + 4.009724e+00, +6.087857640e-04, +4.093612810e-06, +1.485704092e-04, +4.756138781e-06, +1.276030578e-01 + 4.047198e+00, +5.400191464e-04, +3.742942575e-06, +1.478591034e-04, +4.218899581e-06, +1.171779781e-01 + 4.084672e+00, +4.401182623e-04, +4.379635266e-06, +1.273842800e-04, +3.438423924e-06, +9.815150341e-02 + 4.122146e+00, +3.324057871e-04, +3.390264825e-06, +9.627154161e-05, +2.596920212e-06, +7.584800868e-02 + 4.159620e+00, +2.370075780e-04, +4.831246141e-06, +6.338579234e-05, +1.851621704e-06, +5.553016806e-02 + 4.197094e+00, +1.811418738e-04, +5.641649803e-06, +3.631936953e-05, +1.415170889e-06, +4.116690846e-02 + 4.234568e+00, +1.557249175e-04, +2.668727426e-06, +2.221487393e-05, +1.216600918e-06, +3.415513459e-02 + 4.272043e+00, +1.465187123e-04, +1.674662891e-06, +2.170675953e-05, +1.144677440e-06, +3.215710958e-02 + 4.309517e+00, +1.405423711e-04, +1.924531308e-06, +2.149418176e-05, +1.097987274e-06, +3.160248470e-02 + 4.346991e+00, +1.357847028e-04, +1.605850099e-06, +2.192148957e-05, +1.060817991e-06, +3.065472344e-02 + 4.384465e+00, +1.258550769e-04, +1.284603555e-06, +2.046648335e-05, +9.832427884e-07, +2.904522959e-02 + 4.421939e+00, +1.169869494e-04, +1.278083034e-06, +2.159284921e-05, +9.139605420e-07, +2.713926010e-02 + 4.459413e+00, +1.110703419e-04, +1.509709848e-06, +2.079338937e-05, +8.677370462e-07, +2.542558185e-02 + 4.496887e+00, +1.057701132e-04, +5.790175045e-07, +2.100616009e-05, +8.263290094e-07, +2.427050930e-02 + 4.534361e+00, +1.026251684e-04, +7.810732683e-07, +2.116271305e-05, +8.017591281e-07, +2.376362919e-02 + 4.571835e+00, +1.011917515e-04, +5.963700164e-07, +2.046230847e-05, +7.905605584e-07, +2.366920631e-02 + 4.609309e+00, +1.001634828e-04, +8.947496930e-07, +2.134477231e-05, +7.825272092e-07, +2.354631490e-02 + 4.646783e+00, +9.707341240e-05, +3.485090968e-07, +2.031439678e-05, +7.583860344e-07, +2.295738668e-02 + 4.684257e+00, +9.167802975e-05, +3.892734021e-07, +2.066985033e-05, +7.162346074e-07, +2.162696749e-02 + 4.721731e+00, +8.383810233e-05, +2.180439299e-07, +2.059725780e-05, +6.549851744e-07, +1.951430734e-02 + 4.759205e+00, +7.380265088e-05, +2.824979897e-07, +1.856655807e-05, +5.765832100e-07, +1.682504287e-02 + 4.796679e+00, +6.231475173e-05, +3.941094597e-07, +1.489212553e-05, +4.868339979e-07, +1.398756018e-02 + 4.834153e+00, +5.205405290e-05, +9.286779095e-08, +1.031777326e-05, +4.066722883e-07, +1.158631725e-02 + 4.871627e+00, +4.563461790e-05, +2.947284984e-07, +9.944042898e-06, +3.565204523e-07, +1.014939239e-02 + 4.909101e+00, +4.281885319e-05, +2.428192628e-07, +9.489779107e-06, +3.345222905e-07, +9.727415693e-03 + 4.946576e+00, +4.221795820e-05, +3.153020264e-07, +1.010970700e-05, +3.298277985e-07, +9.783286724e-03 + 4.984050e+00, +4.193062269e-05, +1.303311765e-07, +9.664688198e-06, +3.275829898e-07, +9.697488727e-03 + 5.021524e+00, +3.983627123e-05, +2.205995585e-07, +9.929565065e-06, +3.112208689e-07, +9.137274998e-03 + 5.058998e+00, +3.557555053e-05, +4.469302524e-07, +9.812803061e-06, +2.779339885e-07, +8.056891307e-03 + 5.096472e+00, +2.974014513e-05, +3.707369784e-07, +8.638593246e-06, +2.323448838e-07, +6.593883084e-03 + 5.133946e+00, +2.283060209e-05, +1.698980880e-07, +6.683001259e-06, +1.783640788e-07, +4.979337600e-03 + 5.171420e+00, +1.666241488e-05, +3.152850598e-07, +4.477294414e-06, +1.301751163e-07, +3.482333541e-03 + 5.208894e+00, +1.250577530e-05, +1.567322049e-07, +2.385718952e-06, +9.770136956e-08, +2.384295551e-03 + 5.246368e+00, +1.120766256e-05, +2.127376472e-07, +2.042794895e-06, +8.755986377e-08, +1.893867546e-03 + 5.283842e+00, +1.074669282e-05, +1.995524573e-07, +2.278776422e-06, +8.395853768e-08, +1.857462365e-03 + 5.321316e+00, +1.005005248e-05, +3.020729087e-07, +2.048453694e-06, +7.851603503e-08, +1.884026962e-03 + 5.358790e+00, +1.027674791e-05, +1.968048130e-07, +2.146663943e-06, +8.028709303e-08, +1.781458878e-03 + 5.396264e+00, +9.992556393e-06, +2.897864871e-07, +2.019372838e-06, +7.806684682e-08, +1.539662271e-03 + 5.433738e+00, +8.130178520e-06, +9.113637532e-08, +1.644339013e-06, +6.351701969e-08, +1.215813603e-03 + 5.471212e+00, +6.073720860e-06, +1.478698592e-07, +1.146645185e-06, +4.745094422e-08, +8.794652133e-04 + 5.508686e+00, +6.698145957e-06, +9.692050058e-08, +9.740123784e-07, +5.232926529e-08, +5.925767028e-04 + 5.546160e+00, +7.402377781e-06, +3.073583532e-07, +1.094274187e-06, +5.783107641e-08, +4.025875390e-04 + 5.583635e+00, +6.171371328e-06, +1.350665052e-07, +9.761632436e-07, +4.821383850e-08, +3.250012544e-04 + 5.621109e+00, +5.253378796e-06, +9.249799012e-08, +7.532469469e-07, +4.104202184e-08, +3.144552650e-04 + 5.658583e+00, +6.034991375e-06, +1.524639589e-07, +8.379380711e-07, +4.714837011e-08, +3.143233654e-04 + 5.696057e+00, +6.954997063e-06, +1.738911611e-07, +1.032799808e-06, +5.433591456e-08, +3.053781279e-04 + 5.733531e+00, +6.618310969e-06, +1.393653114e-07, +9.706904912e-07, +5.170555445e-08, +2.905274734e-04 + 5.771005e+00, +4.983683461e-06, +7.755124598e-08, +7.959423977e-07, +3.893502704e-08, +2.767846518e-04 + 5.808479e+00, +5.498600911e-06, +5.420209390e-08, +8.069513404e-07, +4.295781962e-08, +2.672840112e-04 + 5.845953e+00, +6.952197529e-06, +1.517748078e-07, +1.004761064e-06, +5.431404319e-08, +2.606043537e-04 + 5.883427e+00, +6.329200324e-06, +9.133362427e-08, +8.427275928e-07, +4.944687753e-08, +2.537151853e-04 + 5.920901e+00, +5.215814043e-06, +2.025757389e-08, +8.189526702e-07, +4.074854721e-08, +2.439748930e-04 + 5.958375e+00, +5.305473522e-06, +1.698308984e-07, +6.730686945e-07, +4.144901189e-08, +2.288006244e-04 + 5.995849e+00, +6.176556538e-06, +1.308471106e-07, +9.587655854e-07, +4.825434796e-08, +2.091970502e-04 + 6.033323e+00, +6.645687902e-06, +8.608255961e-09, +8.996641328e-07, +5.191943673e-08, +1.868959007e-04 + 6.070797e+00, +5.398006420e-06, +1.508346841e-07, +8.539653084e-07, +4.217192516e-08, +1.639107998e-04 + 6.108271e+00, +4.639737004e-06, +1.461880318e-07, +5.846045576e-07, +3.624794534e-08, +1.453359455e-04 + 6.145745e+00, +6.145503127e-06, +8.241347046e-08, +9.145260134e-07, +4.801174318e-08, +1.340081738e-04 + 6.183219e+00, +6.485408075e-06, +6.256052170e-08, +9.129623215e-07, +5.066725059e-08, +1.300350502e-04 + 6.220694e+00, +5.542152354e-06, +8.574382827e-08, +7.821938023e-07, +4.329806526e-08, +1.300871660e-04 + 6.258168e+00, +4.985299793e-06, +1.082258410e-07, +7.100280797e-07, +3.894765463e-08, +1.297254971e-04 + 6.295642e+00, +5.334224443e-06, +4.704382370e-08, +7.980106490e-07, +4.167362846e-08, +1.253586088e-04 + 6.333116e+00, +6.421229966e-06, +2.899044795e-07, +9.204385162e-07, +5.016585911e-08, +1.158965803e-04 + 6.370590e+00, +6.073729617e-06, +8.837766608e-08, +7.781931780e-07, +4.745101264e-08, +1.015407485e-04 + 6.408064e+00, +4.266462607e-06, +7.837010701e-08, +6.328355156e-07, +3.333173912e-08, +8.295426639e-05 + 6.445538e+00, +5.024334961e-06, +9.562404129e-08, +6.546439061e-07, +3.925261688e-08, +6.413712390e-05 + 6.483012e+00, +6.379472633e-06, +2.230152814e-07, +9.421354668e-07, +4.983962995e-08, +4.719626387e-05 + 6.520486e+00, +5.693647492e-06, +1.453002947e-07, +7.169230581e-07, +4.448162103e-08, +3.428721761e-05 + 6.557960e+00, +4.451825136e-06, +6.643439179e-08, +7.111721174e-07, +3.477988388e-08, +2.871636911e-05 + 6.595434e+00, +4.899532173e-06, +1.455459486e-07, +7.337733172e-07, +3.827759510e-08, +2.830713222e-05 + 6.632908e+00, +5.715246554e-06, +1.362644650e-07, +8.893272823e-07, +4.465036370e-08, +2.868930373e-05 + 6.670382e+00, +5.627211642e-06, +1.506604533e-07, +7.797610449e-07, +4.396259095e-08, +2.757669232e-05 + 6.707856e+00, +4.948318403e-06, +1.411311683e-07, +7.615920776e-07, +3.865873753e-08, +2.478379040e-05 + 6.745330e+00, +4.504617494e-06, +8.895024618e-08, +7.798763415e-07, +3.519232417e-08, +2.028553515e-05 + 6.782804e+00, +4.993989569e-06, +8.728616039e-08, +7.967544777e-07, +3.901554351e-08, +1.596593919e-05 + 6.820278e+00, +5.747899974e-06, +1.409305399e-07, +8.337878046e-07, +4.490546855e-08, +1.262078777e-05 + 6.857752e+00, +5.247855022e-06, +1.813322600e-07, +7.726407085e-07, +4.099886736e-08, +9.582348020e-06 + 6.895227e+00, +3.985176119e-06, +3.643501908e-09, +6.312561916e-07, +3.113418843e-08, +7.451330020e-06 + 6.932701e+00, +4.586520602e-06, +8.433082430e-08, +6.862483344e-07, +3.583219221e-08, +7.714032597e-06 + 6.970175e+00, +5.649888792e-06, +9.980674145e-08, +8.434611994e-07, +4.413975619e-08, +8.153444926e-06 + 7.007649e+00, +5.253207318e-06, +6.315408219e-08, +6.957074664e-07, +4.104068217e-08, +8.013247188e-06 + 7.045123e+00, +3.957427885e-06, +3.892856681e-08, +6.417153927e-07, +3.091740535e-08, +7.103148867e-06 + 7.082597e+00, +4.067553643e-06, +1.030158487e-07, +5.483071957e-07, +3.177776283e-08, +6.251987237e-06 + 7.120071e+00, +5.339245594e-06, +3.932283532e-08, +8.433147156e-07, +4.171285620e-08, +7.632282159e-06 + 7.157545e+00, +5.327917860e-06, +1.177531273e-07, +6.938631171e-07, +4.162435828e-08, +7.329495938e-06 + 7.195019e+00, +3.969772302e-06, +7.950854911e-08, +6.036690244e-07, +3.101384611e-08, +6.045781632e-06 + 7.232493e+00, +3.715633257e-06, +1.404288927e-07, +5.594073805e-07, +2.902838482e-08, +5.792377053e-06 + 7.269967e+00, +5.014965578e-06, +1.612842163e-07, +7.753567255e-07, +3.917941858e-08, +6.622437127e-06 + 7.307441e+00, +5.352770855e-06, +1.184136808e-07, +7.375166156e-07, +4.181852231e-08, +7.298828655e-06 + 7.344915e+00, +4.153471107e-06, +4.480572672e-08, +6.123685349e-07, +3.244899303e-08, +5.538450355e-06 + 7.382389e+00, +3.599982642e-06, +3.650950617e-08, +5.446719495e-07, +2.812486439e-08, +5.496705485e-06 + 7.419863e+00, +4.830764950e-06, +5.884614929e-08, +7.174872599e-07, +3.774035117e-08, +6.712350142e-06 + 7.457337e+00, +5.146869399e-06, +1.163178472e-07, +7.555272227e-07, +4.020991718e-08, +6.428643692e-06 + 7.494811e+00, +4.397401248e-06, +1.080430387e-07, +5.798179600e-07, +3.435469725e-08, +6.238567531e-06 + Mean, +4.855456550e-03, +2.700723318e-04, +3.584548722e-04, +3.793325429e-05, +1.099258140e+00 diff --git a/test/ref/coaxial/open/error-indicators.csv b/test/ref/coaxial/open/error-indicators.csv index 692e67174..ab63216db 100644 --- a/test/ref/coaxial/open/error-indicators.csv +++ b/test/ref/coaxial/open/error-indicators.csv @@ -1,203 +1,203 @@ t (ns), Sum Abs., Min Elem Abs., Max Elem Abs., Mean Elem Abs., Normalization 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 - 3.747406e-02, +5.656919329e-06, +4.856284755e-19, +1.429146542e-06, +4.419468226e-08, +1.015026012e-04 - 7.494811e-02, +1.331394786e-05, +1.256204662e-18, +3.354187386e-06, +1.040152177e-07, +1.286467416e-04 - 1.124222e-01, +4.533166150e-06, +8.691346112e-19, +1.144058873e-06, +3.541536055e-08, +1.363432444e-04 - 1.498962e-01, +5.316539313e-06, +5.386495332e-19, +1.352378050e-06, +4.153546339e-08, +2.237453773e-04 - 1.873703e-01, +2.287184397e-05, +3.012395724e-18, +5.789590612e-06, +1.786862810e-07, +5.988251413e-04 - 2.248443e-01, +1.598438319e-05, +2.221781868e-18, +4.071906511e-06, +1.248779937e-07, +1.231318974e-03 - 2.623184e-01, +1.150013517e-05, +5.210625999e-18, +3.089062447e-06, +8.984480601e-08, +2.183514588e-03 - 2.997925e-01, +2.092441719e-05, +1.272709259e-17, +5.172537100e-06, +1.634720093e-07, +3.443881315e-03 - 3.372665e-01, +3.023129661e-05, +1.999099091e-17, +8.595200611e-06, +2.361820048e-07, +4.896471117e-03 - 3.747406e-01, +2.966149891e-05, +5.653422694e-17, +8.160478469e-06, +2.317304602e-07, +6.288238192e-03 - 4.122146e-01, +3.934747640e-05, +1.412915615e-17, +1.043816139e-05, +3.074021593e-07, +7.236958436e-03 - 4.496887e-01, +5.496375444e-05, +1.412118829e-16, +1.376602993e-05, +4.294043315e-07, +7.438657288e-03 - 4.871627e-01, +5.317715171e-05, +1.764915993e-16, +1.345278693e-05, +4.154464977e-07, +7.655736081e-03 - 5.246368e-01, +6.345987516e-05, +3.583062602e-16, +1.616235728e-05, +4.957802747e-07, +1.144507368e-02 - 5.621109e-01, +1.000188545e-04, +6.686919397e-16, +2.890344809e-05, +7.813973009e-07, +2.129749051e-02 - 5.995849e-01, +1.569853883e-04, +6.894487631e-16, +5.054827882e-05, +1.226448346e-06, +3.669225071e-02 - 6.370590e-01, +2.371414637e-04, +2.987336867e-15, +7.811657098e-05, +1.852667685e-06, +5.641515864e-02 - 6.745330e-01, +3.436988518e-04, +2.164168027e-15, +1.084030298e-04, +2.685147279e-06, +7.828737440e-02 - 7.120071e-01, +4.503449422e-04, +8.434257674e-15, +1.334532774e-04, +3.518319861e-06, +9.885303849e-02 - 7.494811e-01, +5.387201988e-04, +2.210553102e-14, +1.460192421e-04, +4.208751553e-06, +1.137546488e-01 - 7.869552e-01, +5.854634659e-04, +3.699123455e-14, +1.426211788e-04, +4.573933327e-06, +1.195362049e-01 - 8.244293e-01, +5.945252635e-04, +1.542889987e-13, +1.553115760e-04, +4.644728621e-06, +1.188356113e-01 - 8.619033e-01, +6.277794936e-04, +8.438991416e-13, +1.446548827e-04, +4.904527294e-06, +1.309570623e-01 - 8.993774e-01, +8.196023636e-04, +3.542625876e-12, +2.064570783e-04, +6.403143466e-06, +1.857784636e-01 - 9.368514e-01, +1.245063027e-03, +1.235452852e-11, +3.796113156e-04, +9.727054902e-06, +2.867343617e-01 - 9.743255e-01, +1.824265442e-03, +3.885931837e-11, +5.734398442e-04, +1.425207377e-05, +4.167726757e-01 - 1.011800e+00, +2.450890627e-03, +1.126668431e-10, +7.521115016e-04, +1.914758302e-05, +5.549120054e-01 - 1.049274e+00, +3.008970318e-03, +2.963478149e-10, +8.708173547e-04, +2.350758061e-05, +6.782193157e-01 - 1.086748e+00, +3.391209592e-03, +6.882830451e-10, +8.861180159e-04, +2.649382494e-05, +7.649055092e-01 - 1.124222e+00, +3.537834071e-03, +1.363904689e-09, +8.504123137e-04, +2.763932868e-05, +8.022591780e-01 - 1.161696e+00, +3.511779316e-03, +2.150129889e-09, +8.840350013e-04, +2.743577591e-05, +8.009806200e-01 - 1.199170e+00, +3.584287871e-03, +2.125157867e-09, +8.159752391e-04, +2.800224899e-05, +8.143332817e-01 - 1.236644e+00, +4.160738356e-03, +3.088716716e-09, +8.694210875e-04, +3.250576840e-05, +9.294499013e-01 - 1.274118e+00, +5.352965496e-03, +1.778269645e-08, +1.361582719e-03, +4.182004293e-05, +1.185780064e+00 - 1.311592e+00, +6.884036966e-03, +5.482403424e-08, +1.934136740e-03, +5.378153880e-05, +1.534661136e+00 - 1.349066e+00, +8.393574758e-03, +1.274739062e-07, +2.361243463e-03, +6.557480280e-05, +1.896361173e+00 - 1.386540e+00, +9.587144870e-03, +2.427922285e-07, +2.543459607e-03, +7.489956930e-05, +2.199964363e+00 - 1.424014e+00, +1.030847845e-02, +3.870793288e-07, +2.421466310e-03, +8.053498792e-05, +2.397709121e+00 - 1.461488e+00, +1.058479831e-02, +5.172682974e-07, +2.459542459e-03, +8.269373679e-05, +2.477248854e+00 - 1.498962e+00, +1.065114467e-02, +5.900129004e-07, +2.478084260e-03, +8.321206774e-05, +2.474045348e+00 - 1.536436e+00, +1.088707737e-02, +6.933528164e-07, +2.347785517e-03, +8.505529193e-05, +2.473655990e+00 - 1.573910e+00, +1.161350430e-02, +1.128992608e-06, +2.453010503e-03, +9.073050232e-05, +2.579211406e+00 - 1.611384e+00, +1.284893703e-02, +1.982498898e-06, +2.368957242e-03, +1.003823205e-04, +2.837696762e+00 - 1.648859e+00, +1.431858623e-02, +3.112369243e-06, +3.068473841e-03, +1.118639549e-04, +3.200565871e+00 - 1.686333e+00, +1.568466437e-02, +4.376324705e-06, +3.469083241e-03, +1.225364404e-04, +3.569988173e+00 - 1.723807e+00, +1.672073184e-02, +5.660135647e-06, +3.499417907e-03, +1.306307175e-04, +3.860976369e+00 - 1.761281e+00, +1.735919124e-02, +6.836616888e-06, +3.289526113e-03, +1.356186815e-04, +4.031109202e+00 - 1.798755e+00, +1.767123055e-02, +7.821966341e-06, +3.511818707e-03, +1.380564887e-04, +4.086241163e+00 - 1.836229e+00, +1.781523617e-02, +9.246723861e-06, +3.414133306e-03, +1.391815326e-04, +4.072134164e+00 - 1.873703e+00, +1.795745681e-02, +1.328997691e-05, +3.394013662e-03, +1.402926314e-04, +4.053121061e+00 - 1.911177e+00, +1.820302848e-02, +2.161284355e-05, +3.420849873e-03, +1.422111600e-04, +4.080942379e+00 - 1.948651e+00, +1.856514788e-02, +3.356781169e-05, +3.234892490e-03, +1.450402178e-04, +4.169173455e+00 - 1.986125e+00, +1.898957180e-02, +4.723137868e-05, +3.396745165e-03, +1.483560297e-04, +4.292737520e+00 - 2.023599e+00, +1.939414578e-02, +5.977348853e-05, +3.250581883e-03, +1.515167639e-04, +4.410955044e+00 - 2.061073e+00, +1.970394331e-02, +6.737231934e-05, +3.312961614e-03, +1.539370571e-04, +4.492517857e+00 - 2.098547e+00, +1.988190622e-02, +6.610344670e-05, +3.301605892e-03, +1.553273923e-04, +4.527215199e+00 - 2.136021e+00, +1.993731499e-02, +5.671443237e-05, +3.172906528e-03, +1.557602734e-04, +4.524104599e+00 - 2.173495e+00, +1.990562097e-02, +6.101946084e-05, +3.293718936e-03, +1.555126638e-04, +4.502015291e+00 - 2.210969e+00, +1.982893776e-02, +1.088878841e-04, +3.115929788e-03, +1.549135762e-04, +4.478491192e+00 - 2.248443e+00, +1.974876101e-02, +1.873801406e-04, +3.228307132e-03, +1.542871954e-04, +4.462065326e+00 - 2.285917e+00, +1.968329021e-02, +2.772981557e-04, +3.180410477e-03, +1.537757048e-04, +4.450543317e+00 - 2.323392e+00, +1.961439724e-02, +3.585051575e-04, +3.107072876e-03, +1.532374784e-04, +4.434922062e+00 - 2.360866e+00, +1.951363989e-02, +4.075955424e-04, +3.188334830e-03, +1.524503116e-04, +4.406444465e+00 - 2.398340e+00, +1.937089613e-02, +4.015208980e-04, +2.980831052e-03, +1.513351261e-04, +4.364029184e+00 - 2.435814e+00, +1.920186059e-02, +3.225495985e-04, +3.138426025e-03, +1.500145359e-04, +4.319424711e+00 - 2.473288e+00, +1.906306149e-02, +1.840331757e-04, +3.055918232e-03, +1.489301679e-04, +4.296324704e+00 - 2.510762e+00, +1.905052602e-02, +2.217725829e-04, +3.046207985e-03, +1.488322345e-04, +4.319068012e+00 - 2.548236e+00, +1.922732383e-02, +1.821453144e-04, +3.098865948e-03, +1.502134674e-04, +4.391888403e+00 - 2.585710e+00, +1.953172619e-02, +8.506756807e-05, +2.933659325e-03, +1.525916109e-04, +4.481531511e+00 - 2.623184e+00, +1.974732686e-02, +1.004249256e-04, +3.113361480e-03, +1.542759911e-04, +4.519891310e+00 - 2.660658e+00, +1.956384336e-02, +1.196983100e-04, +3.005984915e-03, +1.528425262e-04, +4.429739436e+00 - 2.698132e+00, +1.871919389e-02, +1.071559565e-04, +2.901948751e-03, +1.462437022e-04, +4.164358438e+00 - 2.735606e+00, +1.718960116e-02, +1.320053754e-04, +2.803797700e-03, +1.342937591e-04, +3.755791737e+00 - 2.773080e+00, +1.546491642e-02, +9.509285112e-05, +2.408371066e-03, +1.208196596e-04, +3.369663722e+00 - 2.810554e+00, +1.468011525e-02, +7.315500309e-05, +2.441928087e-03, +1.146884004e-04, +3.300120354e+00 - 2.848028e+00, +1.586922846e-02, +4.353556071e-05, +2.665555862e-03, +1.239783474e-04, +3.715710458e+00 - 2.885502e+00, +1.862615405e-02, +2.018520484e-05, +3.373944154e-03, +1.455168285e-04, +4.412752860e+00 - 2.922976e+00, +2.144870957e-02, +7.479911126e-06, +4.038673924e-03, +1.675680435e-04, +5.035126169e+00 - 2.960451e+00, +2.292343187e-02, +9.559526315e-06, +4.563613742e-03, +1.790893115e-04, +5.296819140e+00 - 2.997925e+00, +2.211029675e-02, +1.216757197e-05, +4.513889015e-03, +1.727366933e-04, +5.025358059e+00 - 3.035399e+00, +1.863331026e-02, +1.063158449e-05, +3.843556397e-03, +1.455727364e-04, +4.165761053e+00 - 3.072873e+00, +1.272017108e-02, +8.184202988e-06, +2.614012446e-03, +9.937633654e-05, +2.782770985e+00 - 3.110347e+00, +5.289924605e-03, +4.602990863e-06, +9.892105078e-04, +4.132753598e-05, +1.091922648e+00 - 3.147821e+00, +4.602799904e-03, +1.691676549e-06, +8.722525258e-04, +3.595937425e-05, +1.087911650e+00 - 3.185295e+00, +1.215215559e-02, +2.265588631e-06, +2.536343724e-03, +9.493871558e-05, +2.795375974e+00 - 3.222769e+00, +1.838959288e-02, +3.425649809e-06, +3.876930278e-03, +1.436686944e-04, +4.198258824e+00 - 3.260243e+00, +2.210912947e-02, +3.716661553e-06, +4.665403675e-03, +1.727275740e-04, +5.054847159e+00 - 3.297717e+00, +2.287279659e-02, +5.465969625e-06, +4.800014100e-03, +1.786937234e-04, +5.271021749e+00 - 3.335191e+00, +2.087582236e-02, +1.094747830e-05, +4.296501282e-03, +1.630923622e-04, +4.875726472e+00 - 3.372665e+00, +1.693740095e-02, +1.957205488e-05, +3.274370536e-03, +1.323234449e-04, +4.026175767e+00 - 3.410139e+00, +1.265391041e-02, +3.033725029e-05, +2.351759897e-03, +9.885867509e-05, +3.029504715e+00 - 3.447613e+00, +1.056944488e-02, +3.699160296e-05, +1.868764045e-03, +8.257378816e-05, +2.402834715e+00 - 3.485087e+00, +1.203465459e-02, +5.157331520e-05, +1.987519572e-03, +9.402073900e-05, +2.582889975e+00 - 3.522561e+00, +1.483191160e-02, +4.264807631e-05, +2.431130292e-03, +1.158743094e-04, +3.207270944e+00 - 3.560035e+00, +1.694364854e-02, +4.348712933e-05, +2.651516775e-03, +1.323722542e-04, +3.758945722e+00 - 3.597509e+00, +1.777812511e-02, +3.816772941e-05, +2.830632885e-03, +1.388916024e-04, +4.038955569e+00 - 3.634984e+00, +1.749152616e-02, +2.610873059e-05, +2.685137941e-03, +1.366525481e-04, +4.041150977e+00 - 3.672458e+00, +1.659354032e-02, +6.478738042e-05, +2.504827607e-03, +1.296370337e-04, +3.857101060e+00 - 3.709932e+00, +1.567966849e-02, +1.324957095e-04, +2.523973396e-03, +1.224974101e-04, +3.618583613e+00 - 3.747406e+00, +1.515242977e-02, +1.837750403e-04, +2.479688076e-03, +1.183783576e-04, +3.441317861e+00 - 3.784880e+00, +1.505241218e-02, +2.589867086e-04, +2.413883235e-03, +1.175969702e-04, +3.374168903e+00 - 3.822354e+00, +1.517594323e-02, +2.648845439e-04, +2.482655271e-03, +1.185620565e-04, +3.390825944e+00 - 3.859828e+00, +1.530534707e-02, +3.460441840e-04, +2.325030154e-03, +1.195730240e-04, +3.433304933e+00 - 3.897302e+00, +1.533180643e-02, +2.591782733e-04, +2.448645571e-03, +1.197797378e-04, +3.458360447e+00 - 3.934776e+00, +1.525209327e-02, +3.263088590e-04, +2.386365319e-03, +1.191569787e-04, +3.452521374e+00 - 3.972250e+00, +1.511733834e-02, +2.656776671e-04, +2.367449999e-03, +1.181042058e-04, +3.423956689e+00 - 4.009724e+00, +1.498169885e-02, +1.893347598e-04, +2.402930182e-03, +1.170445223e-04, +3.388114585e+00 - 4.047198e+00, +1.486760910e-02, +8.599169429e-05, +2.247589227e-03, +1.161531961e-04, +3.356524519e+00 - 4.084672e+00, +1.476249589e-02, +7.080243992e-05, +2.377243442e-03, +1.153319991e-04, +3.331853567e+00 - 4.122146e+00, +1.464115772e-02, +1.002760363e-04, +2.290380547e-03, +1.143840447e-04, +3.308603435e+00 - 4.159620e+00, +1.448100038e-02, +1.127755126e-04, +2.309758265e-03, +1.131328154e-04, +3.277119137e+00 - 4.197094e+00, +1.425845180e-02, +1.046536840e-04, +2.317095465e-03, +1.113941547e-04, +3.228713172e+00 - 4.234568e+00, +1.397280756e-02, +8.171047692e-05, +2.200893917e-03, +1.091625591e-04, +3.160502439e+00 - 4.272043e+00, +1.365800502e-02, +5.594416747e-05, +2.299295490e-03, +1.067031642e-04, +3.078660427e+00 - 4.309517e+00, +1.335595262e-02, +3.262293196e-05, +2.188198744e-03, +1.043433798e-04, +2.998121297e+00 - 4.346991e+00, +1.310147749e-02, +1.743681123e-05, +2.244566050e-03, +1.023552929e-04, +2.936669456e+00 - 4.384465e+00, +1.291316405e-02, +1.318337128e-05, +2.228431206e-03, +1.008840942e-04, +2.903997885e+00 - 4.421939e+00, +1.276376144e-02, +1.435774115e-05, +2.182482661e-03, +9.971688628e-05, +2.891883014e+00 - 4.459413e+00, +1.257886362e-02, +1.385785904e-05, +2.254463041e-03, +9.827237205e-05, +2.873950817e+00 - 4.496887e+00, +1.227354453e-02, +1.139016125e-05, +2.117954973e-03, +9.588706666e-05, +2.816794882e+00 - 4.534361e+00, +1.178139290e-02, +8.224353168e-06, +2.159095857e-03, +9.204213207e-05, +2.695716179e+00 - 4.571835e+00, +1.108386626e-02, +4.702489008e-06, +2.144208193e-03, +8.659270512e-05, +2.507677639e+00 - 4.609309e+00, +1.023691016e-02, +2.918639729e-06, +1.911635775e-03, +7.997586064e-05, +2.277833009e+00 - 4.646783e+00, +9.371323060e-03, +1.545409438e-06, +1.700487260e-03, +7.321346140e-05, +2.056659349e+00 - 4.684257e+00, +8.657719207e-03, +2.177147857e-06, +1.788518736e-03, +6.763843131e-05, +1.900934905e+00 - 4.721731e+00, +8.217440504e-03, +1.601040994e-07, +1.709076280e-03, +6.419875393e-05, +1.836493022e+00 - 4.759205e+00, +8.019652879e-03, +1.376983126e-06, +1.786480820e-03, +6.265353812e-05, +1.832790721e+00 - 4.796679e+00, +7.890364250e-03, +1.486445732e-06, +1.768891640e-03, +6.164347071e-05, +1.825328220e+00 - 4.834153e+00, +7.621359210e-03, +9.019618284e-07, +1.685597957e-03, +5.954186883e-05, +1.758585885e+00 - 4.871627e+00, +7.068989772e-03, +9.009942214e-07, +1.773969886e-03, +5.522648259e-05, +1.608502805e+00 - 4.909101e+00, +6.213005286e-03, +2.594281588e-07, +1.651811891e-03, +4.853910380e-05, +1.384459441e+00 - 4.946576e+00, +5.162372600e-03, +1.085336926e-06, +1.352451292e-03, +4.033103594e-05, +1.123478548e+00 - 4.984050e+00, +4.122759940e-03, +6.012849127e-07, +9.429225306e-04, +3.220906203e-05, +8.826005714e-01 - 5.021524e+00, +3.350707607e-03, +6.078339164e-07, +7.200088085e-04, +2.617740318e-05, +7.236678618e-01 - 5.058998e+00, +3.013498513e-03, +6.067707923e-07, +7.103653175e-04, +2.354295713e-05, +6.701157353e-01 - 5.096472e+00, +2.982577267e-03, +6.275290444e-08, +7.400640006e-04, +2.330138490e-05, +6.737270463e-01 - 5.133946e+00, +2.967454273e-03, +7.366753069e-07, +6.898839589e-04, +2.318323651e-05, +6.688583217e-01 - 5.171420e+00, +2.790092469e-03, +5.208321575e-07, +7.384504272e-04, +2.179759741e-05, +6.231475505e-01 - 5.208894e+00, +2.418548983e-03, +9.276751686e-07, +7.000383268e-04, +1.889491393e-05, +5.351074185e-01 - 5.246368e+00, +1.911062359e-03, +1.344568641e-06, +5.773793291e-04, +1.493017468e-05, +4.200268783e-01 - 5.283842e+00, +1.372440607e-03, +2.056363581e-06, +4.096268542e-04, +1.072219224e-05, +3.000708009e-01 - 5.321316e+00, +9.213288732e-04, +2.114823140e-06, +2.372579002e-04, +7.197881822e-06, +1.991713402e-01 - 5.358790e+00, +6.722993623e-04, +2.927023180e-06, +1.457815470e-04, +5.252338768e-06, +1.401144692e-01 - 5.396264e+00, +6.283225289e-04, +2.643605723e-06, +1.625307580e-04, +4.908769757e-06, +1.268544987e-01 - 5.433738e+00, +6.348530847e-04, +1.837386900e-06, +1.550414848e-04, +4.959789724e-06, +1.297308749e-01 - 5.471212e+00, +6.001491642e-04, +2.469214797e-06, +1.570842279e-04, +4.688665345e-06, +1.254275991e-01 - 5.508686e+00, +5.139588842e-04, +6.781782666e-07, +1.472964294e-04, +4.015303783e-06, +1.099729019e-01 - 5.546160e+00, +3.974099555e-04, +2.282771322e-06, +1.184808923e-04, +3.104765278e-06, +8.751673075e-02 - 5.583635e+00, +2.814766597e-04, +4.142124635e-06, +8.237874515e-05, +2.199036404e-06, +6.404830704e-02 - 5.621109e+00, +1.971418371e-04, +5.379287255e-06, +4.879963190e-05, +1.540170602e-06, +4.499940303e-02 - 5.658583e+00, +1.566562902e-04, +5.397792904e-06, +2.429803286e-05, +1.223877267e-06, +3.378394289e-02 - 5.696057e+00, +1.405994048e-04, +2.383391840e-06, +2.103885736e-05, +1.098432850e-06, +2.942426165e-02 - 5.733531e+00, +1.313588362e-04, +1.776514710e-06, +1.911272259e-05, +1.026240908e-06, +2.730267726e-02 - 5.771005e+00, +1.162490253e-04, +2.192974420e-06, +1.780103948e-05, +9.081955100e-07, +2.433964679e-02 - 5.808479e+00, +9.660978201e-05, +1.832208009e-06, +1.546361781e-05, +7.547639219e-07, +2.048189674e-02 - 5.845953e+00, +7.972386380e-05, +2.032041682e-06, +1.277530060e-05, +6.228426860e-07, +1.785059556e-02 - 5.883427e+00, +8.075440717e-05, +9.629804893e-07, +1.405092362e-05, +6.308938060e-07, +1.905071629e-02 - 5.920901e+00, +9.847125170e-05, +1.617457848e-06, +1.897478289e-05, +7.693066539e-07, +2.355183009e-02 - 5.958375e+00, +1.201247755e-04, +1.250282629e-06, +2.394888242e-05, +9.384748086e-07, +2.860407565e-02 - 5.995849e+00, +1.353459673e-04, +4.093061083e-07, +2.700896460e-05, +1.057390370e-06, +3.211115798e-02 - 6.033323e+00, +1.393064150e-04, +1.148347939e-06, +2.893048920e-05, +1.088331367e-06, +3.289020287e-02 - 6.070797e+00, +1.306427216e-04, +8.982552787e-07, +2.770021710e-05, +1.020646262e-06, +3.044147732e-02 - 6.108271e+00, +1.061818817e-04, +7.339137606e-07, +2.278162369e-05, +8.295459508e-07, +2.486915720e-02 - 6.145745e+00, +7.209552669e-05, +3.375268173e-07, +1.566264719e-05, +5.632463022e-07, +1.688222621e-02 - 6.183219e+00, +3.514370026e-05, +6.910550007e-07, +6.938825756e-06, +2.745601583e-07, +8.222569600e-03 - 6.220694e+00, +3.332434420e-05, +8.008707418e-07, +5.858394799e-06, +2.603464391e-07, +7.112443417e-03 - 6.258168e+00, +6.596901518e-05, +4.918351718e-07, +1.173531420e-05, +5.153829311e-07, +1.487473373e-02 - 6.295642e+00, +9.667915042e-05, +3.027018996e-07, +1.881777970e-05, +7.553058626e-07, +2.195995150e-02 - 6.333116e+00, +1.151311727e-04, +1.042992248e-06, +2.293042411e-05, +8.994622865e-07, +2.658857944e-02 - 6.370590e+00, +1.207366953e-04, +7.528801127e-07, +2.380623796e-05, +9.432554324e-07, +2.836724125e-02 - 6.408064e+00, +1.164009914e-04, +2.748897153e-07, +2.213552833e-05, +9.093827450e-07, +2.754771847e-02 - 6.445538e+00, +1.035095292e-04, +1.131867278e-06, +1.908048887e-05, +8.086681970e-07, +2.492432935e-02 - 6.483012e+00, +9.055022958e-05, +1.313210723e-06, +1.572048713e-05, +7.074236686e-07, +2.173397508e-02 - 6.520486e+00, +8.224274449e-05, +7.030109454e-07, +1.364569357e-05, +6.425214413e-07, +1.939641242e-02 - 6.557960e+00, +8.240698910e-05, +1.148313876e-06, +1.353813680e-05, +6.438046023e-07, +1.881287963e-02 - 6.595434e+00, +8.663895923e-05, +1.631803635e-06, +1.483972930e-05, +6.768668690e-07, +1.965705398e-02 - 6.632908e+00, +9.150938695e-05, +8.959336189e-07, +1.500001364e-05, +7.149170856e-07, +2.089087372e-02 - 6.670382e+00, +9.411556011e-05, +7.071727640e-07, +1.471136654e-05, +7.352778134e-07, +2.174361592e-02 - 6.707856e+00, +9.427881789e-05, +2.232688753e-06, +1.527592352e-05, +7.365532648e-07, +2.197690465e-02 - 6.745330e+00, +9.283419598e-05, +1.674328225e-06, +1.475470626e-05, +7.252671561e-07, +2.172213826e-02 - 6.782804e+00, +9.091313119e-05, +2.172433946e-06, +1.494130966e-05, +7.102588375e-07, +2.125742296e-02 - 6.820278e+00, +8.947559314e-05, +1.208738989e-06, +1.467196179e-05, +6.990280714e-07, +2.082523460e-02 - 6.857752e+00, +8.849046096e-05, +2.183297733e-07, +1.422391043e-05, +6.913317262e-07, +2.053535812e-02 - 6.895227e+00, +8.869124726e-05, +1.872610662e-06, +1.475311712e-05, +6.929003692e-07, +2.037002683e-02 - 6.932701e+00, +8.703583658e-05, +1.303860761e-06, +1.403557491e-05, +6.799674733e-07, +2.025706959e-02 - 6.970175e+00, +8.705236106e-05, +1.310170098e-06, +1.440134747e-05, +6.800965708e-07, +2.014118630e-02 - 7.007649e+00, +8.649188885e-05, +1.642909337e-06, +1.413717954e-05, +6.757178817e-07, +2.000795493e-02 - 7.045123e+00, +8.557114224e-05, +8.070387401e-07, +1.402625378e-05, +6.685245487e-07, +1.987188346e-02 - 7.082597e+00, +8.556574405e-05, +6.303920028e-07, +1.437910365e-05, +6.684823754e-07, +1.974815441e-02 - 7.120071e+00, +8.395754839e-05, +2.492708417e-07, +1.333948635e-05, +6.559183468e-07, +1.962815933e-02 - 7.157545e+00, +8.469831726e-05, +4.394357425e-07, +1.399445266e-05, +6.617056036e-07, +1.947236585e-02 - 7.195019e+00, +8.219518931e-05, +2.016475059e-07, +1.374633426e-05, +6.421499165e-07, +1.922019946e-02 - 7.232493e+00, +8.060990044e-05, +7.126283209e-07, +1.375634865e-05, +6.297648471e-07, +1.881533865e-02 - 7.269967e+00, +7.975417972e-05, +2.119913433e-07, +1.379736957e-05, +6.230795291e-07, +1.823277535e-02 - 7.307441e+00, +7.470325724e-05, +1.169668973e-07, +1.285632329e-05, +5.836191972e-07, +1.750653819e-02 - 7.344915e+00, +7.330184966e-05, +2.570909515e-07, +1.365659277e-05, +5.726707005e-07, +1.673934977e-02 - 7.382389e+00, +7.020344015e-05, +4.248389728e-07, +1.310218281e-05, +5.484643762e-07, +1.607601725e-02 - 7.419863e+00, +6.896813460e-05, +4.112240516e-07, +1.338802711e-05, +5.388135515e-07, +1.564435535e-02 - 7.457337e+00, +6.715833151e-05, +1.872800848e-07, +1.323199478e-05, +5.246744649e-07, +1.547011399e-02 - 7.494811e+00, +6.573000789e-05, +1.923953202e-07, +1.301561971e-05, +5.135156866e-07, +1.543460452e-02 - Mean, +7.685673980e-03, +2.479807676e-04, +5.882818741e-04, +6.004432797e-05, +0.000000000e+00 + 3.747406e-02, +5.656919329e-06, +4.859397461e-19, +1.429146542e-06, +4.419468226e-08, +1.015026012e-04 + 7.494811e-02, +1.331394786e-05, +1.261528861e-18, +3.354187386e-06, +1.040152177e-07, +1.286467416e-04 + 1.124222e-01, +4.533166150e-06, +8.827124714e-19, +1.144058873e-06, +3.541536055e-08, +1.363432444e-04 + 1.498962e-01, +5.316539313e-06, +5.373308021e-19, +1.352378049e-06, +4.153546339e-08, +2.237453773e-04 + 1.873703e-01, +2.287184397e-05, +3.000946784e-18, +5.789590609e-06, +1.786862810e-07, +5.988251413e-04 + 2.248443e-01, +1.598438319e-05, +2.230531364e-18, +4.071906507e-06, +1.248779937e-07, +1.231318974e-03 + 2.623184e-01, +1.150013517e-05, +5.191338644e-18, +3.089062447e-06, +8.984480600e-08, +2.183514588e-03 + 2.997925e-01, +2.092441721e-05, +1.273595463e-17, +5.172537104e-06, +1.634720094e-07, +3.443881315e-03 + 3.372665e-01, +3.023129660e-05, +1.998560108e-17, +8.595200606e-06, +2.361820047e-07, +4.896471117e-03 + 3.747406e-01, +2.966149896e-05, +5.643586447e-17, +8.160478469e-06, +2.317304606e-07, +6.288238192e-03 + 4.122146e-01, +3.934747639e-05, +1.415540740e-17, +1.043816137e-05, +3.074021593e-07, +7.236958436e-03 + 4.496887e-01, +5.496375443e-05, +1.412705025e-16, +1.376602992e-05, +4.294043315e-07, +7.438657288e-03 + 4.871627e-01, +5.317715172e-05, +1.765597600e-16, +1.345278688e-05, +4.154464978e-07, +7.655736081e-03 + 5.246368e-01, +6.345987510e-05, +3.578921520e-16, +1.616235667e-05, +4.957802742e-07, +1.144507368e-02 + 5.621109e-01, +1.000188544e-04, +6.683968190e-16, +2.890344790e-05, +7.813973001e-07, +2.129749051e-02 + 5.995849e-01, +1.569853883e-04, +6.893744780e-16, +5.054827882e-05, +1.226448346e-06, +3.669225071e-02 + 6.370590e-01, +2.371414640e-04, +2.988883429e-15, +7.811657109e-05, +1.852667687e-06, +5.641515864e-02 + 6.745330e-01, +3.436988525e-04, +2.170694938e-15, +1.084030298e-04, +2.685147285e-06, +7.828737440e-02 + 7.120071e-01, +4.503449432e-04, +8.427839505e-15, +1.334532772e-04, +3.518319869e-06, +9.885303849e-02 + 7.494811e-01, +5.387201999e-04, +2.210536808e-14, +1.460192415e-04, +4.208751561e-06, +1.137546488e-01 + 7.869552e-01, +5.854634666e-04, +3.700525228e-14, +1.426211774e-04, +4.573933333e-06, +1.195362049e-01 + 8.244293e-01, +5.945252636e-04, +1.542935864e-13, +1.553115756e-04, +4.644728622e-06, +1.188356113e-01 + 8.619033e-01, +6.277794930e-04, +8.438596361e-13, +1.446548827e-04, +4.904527289e-06, +1.309570623e-01 + 8.993774e-01, +8.196023618e-04, +3.542527947e-12, +2.064570778e-04, +6.403143451e-06, +1.857784636e-01 + 9.368514e-01, +1.245063026e-03, +1.235451610e-11, +3.796113148e-04, +9.727054891e-06, +2.867343617e-01 + 9.743255e-01, +1.824265444e-03, +3.885931987e-11, +5.734398443e-04, +1.425207378e-05, +4.167726757e-01 + 1.011800e+00, +2.450890631e-03, +1.126666778e-10, +7.521115025e-04, +1.914758306e-05, +5.549120054e-01 + 1.049274e+00, +3.008970325e-03, +2.963474220e-10, +8.708173546e-04, +2.350758066e-05, +6.782193157e-01 + 1.086748e+00, +3.391209599e-03, +6.882824319e-10, +8.861180141e-04, +2.649382499e-05, +7.649055092e-01 + 1.124222e+00, +3.537834075e-03, +1.363904131e-09, +8.504123105e-04, +2.763932871e-05, +8.022591780e-01 + 1.161696e+00, +3.511779317e-03, +2.150129927e-09, +8.840349987e-04, +2.743577591e-05, +8.009806200e-01 + 1.199170e+00, +3.584287866e-03, +2.125158607e-09, +8.159752263e-04, +2.800224896e-05, +8.143332817e-01 + 1.236644e+00, +4.160738345e-03, +3.088718318e-09, +8.694210871e-04, +3.250576832e-05, +9.294499013e-01 + 1.274118e+00, +5.352965483e-03, +1.778270100e-08, +1.361582714e-03, +4.182004283e-05, +1.185780064e+00 + 1.311592e+00, +6.884036964e-03, +5.482403929e-08, +1.934136735e-03, +5.378153878e-05, +1.534661136e+00 + 1.349066e+00, +8.393574770e-03, +1.274739058e-07, +2.361243467e-03, +6.557480289e-05, +1.896361173e+00 + 1.386540e+00, +9.587144890e-03, +2.427922136e-07, +2.543459610e-03, +7.489956945e-05, +2.199964363e+00 + 1.424014e+00, +1.030847847e-02, +3.870792896e-07, +2.421466309e-03, +8.053498806e-05, +2.397709121e+00 + 1.461488e+00, +1.058479832e-02, +5.172682213e-07, +2.459542440e-03, +8.269373687e-05, +2.477248854e+00 + 1.498962e+00, +1.065114467e-02, +5.900127780e-07, +2.478084232e-03, +8.321206773e-05, +2.474045348e+00 + 1.536436e+00, +1.088707735e-02, +6.933527190e-07, +2.347785514e-03, +8.505529183e-05, +2.473655990e+00 + 1.573910e+00, +1.161350427e-02, +1.128992731e-06, +2.453010483e-03, +9.073050211e-05, +2.579211406e+00 + 1.611384e+00, +1.284893699e-02, +1.982499253e-06, +2.368957236e-03, +1.003823202e-04, +2.837696762e+00 + 1.648859e+00, +1.431858621e-02, +3.112369697e-06, +3.068473826e-03, +1.118639547e-04, +3.200565871e+00 + 1.686333e+00, +1.568466438e-02, +4.376325047e-06, +3.469083243e-03, +1.225364404e-04, +3.569988173e+00 + 1.723807e+00, +1.672073186e-02, +5.660135676e-06, +3.499417909e-03, +1.306307177e-04, +3.860976369e+00 + 1.761281e+00, +1.735919126e-02, +6.836616389e-06, +3.289526062e-03, +1.356186817e-04, +4.031109202e+00 + 1.798755e+00, +1.767123056e-02, +7.821965102e-06, +3.511818671e-03, +1.380564888e-04, +4.086241163e+00 + 1.836229e+00, +1.781523617e-02, +9.246722069e-06, +3.414133250e-03, +1.391815326e-04, +4.072134164e+00 + 1.873703e+00, +1.795745680e-02, +1.328997588e-05, +3.394013651e-03, +1.402926312e-04, +4.053121061e+00 + 1.911177e+00, +1.820302843e-02, +2.161284504e-05, +3.420849841e-03, +1.422111596e-04, +4.080942379e+00 + 1.948651e+00, +1.856514782e-02, +3.356781622e-05, +3.234892449e-03, +1.450402173e-04, +4.169173455e+00 + 1.986125e+00, +1.898957175e-02, +4.723138491e-05, +3.396745121e-03, +1.483560293e-04, +4.292737520e+00 + 2.023599e+00, +1.939414576e-02, +5.977349433e-05, +3.250581870e-03, +1.515167637e-04, +4.410955044e+00 + 2.061073e+00, +1.970394332e-02, +6.737232249e-05, +3.312961591e-03, +1.539370572e-04, +4.492517857e+00 + 2.098547e+00, +1.988190624e-02, +6.610345079e-05, +3.301605870e-03, +1.553273925e-04, +4.527215199e+00 + 2.136021e+00, +1.993731501e-02, +5.671442401e-05, +3.172906485e-03, +1.557602735e-04, +4.524104599e+00 + 2.173495e+00, +1.990562099e-02, +6.101944972e-05, +3.293718918e-03, +1.555126640e-04, +4.502015291e+00 + 2.210969e+00, +1.982893776e-02, +1.088878828e-04, +3.115929763e-03, +1.549135762e-04, +4.478491192e+00 + 2.248443e+00, +1.974876097e-02, +1.873801567e-04, +3.228307111e-03, +1.542871951e-04, +4.462065326e+00 + 2.285917e+00, +1.968329015e-02, +2.772981887e-04, +3.180410440e-03, +1.537757043e-04, +4.450543317e+00 + 2.323392e+00, +1.961439719e-02, +3.585051976e-04, +3.107072907e-03, +1.532374780e-04, +4.434922062e+00 + 2.360866e+00, +1.951363987e-02, +4.075955778e-04, +3.188334861e-03, +1.524503115e-04, +4.406444465e+00 + 2.398340e+00, +1.937089614e-02, +4.015209202e-04, +2.980831106e-03, +1.513351261e-04, +4.364029184e+00 + 2.435814e+00, +1.920186060e-02, +3.225496007e-04, +3.138425993e-03, +1.500145360e-04, +4.319424711e+00 + 2.473288e+00, +1.906306151e-02, +1.840331470e-04, +3.055918212e-03, +1.489301681e-04, +4.296324704e+00 + 2.510762e+00, +1.905052604e-02, +2.217725636e-04, +3.046208004e-03, +1.488322347e-04, +4.319068012e+00 + 2.548236e+00, +1.922732384e-02, +1.821453542e-04, +3.098865945e-03, +1.502134675e-04, +4.391888403e+00 + 2.585710e+00, +1.953172619e-02, +8.506759637e-05, +2.933659318e-03, +1.525916109e-04, +4.481531511e+00 + 2.623184e+00, +1.974732686e-02, +1.004249255e-04, +3.113361469e-03, +1.542759911e-04, +4.519891310e+00 + 2.660658e+00, +1.956384336e-02, +1.196983105e-04, +3.005984908e-03, +1.528425263e-04, +4.429739436e+00 + 2.698132e+00, +1.871919390e-02, +1.071559579e-04, +2.901948705e-03, +1.462437023e-04, +4.164358438e+00 + 2.735606e+00, +1.718960117e-02, +1.320053755e-04, +2.803797671e-03, +1.342937591e-04, +3.755791737e+00 + 2.773080e+00, +1.546491643e-02, +9.509285091e-05, +2.408371021e-03, +1.208196596e-04, +3.369663722e+00 + 2.810554e+00, +1.468011525e-02, +7.315500821e-05, +2.441928107e-03, +1.146884004e-04, +3.300120354e+00 + 2.848028e+00, +1.586922847e-02, +4.353556598e-05, +2.665555836e-03, +1.239783474e-04, +3.715710458e+00 + 2.885502e+00, +1.862615408e-02, +2.018520823e-05, +3.373944166e-03, +1.455168288e-04, +4.412752860e+00 + 2.922976e+00, +2.144870961e-02, +7.479911328e-06, +4.038673871e-03, +1.675680439e-04, +5.035126169e+00 + 2.960451e+00, +2.292343193e-02, +9.559526669e-06, +4.563613681e-03, +1.790893120e-04, +5.296819140e+00 + 2.997925e+00, +2.211029681e-02, +1.216757195e-05, +4.513888960e-03, +1.727366938e-04, +5.025358059e+00 + 3.035399e+00, +1.863331031e-02, +1.063158688e-05, +3.843556356e-03, +1.455727368e-04, +4.165761053e+00 + 3.072873e+00, +1.272017111e-02, +8.184202773e-06, +2.614012422e-03, +9.937633682e-05, +2.782770985e+00 + 3.110347e+00, +5.289924617e-03, +4.602990989e-06, +9.892105005e-04, +4.132753607e-05, +1.091922648e+00 + 3.147821e+00, +4.602799913e-03, +1.691676596e-06, +8.722525324e-04, +3.595937432e-05, +1.087911650e+00 + 3.185295e+00, +1.215215563e-02, +2.265588759e-06, +2.536343693e-03, +9.493871585e-05, +2.795375974e+00 + 3.222769e+00, +1.838959293e-02, +3.425650080e-06, +3.876930233e-03, +1.436686948e-04, +4.198258824e+00 + 3.260243e+00, +2.210912954e-02, +3.716661787e-06, +4.665403620e-03, +1.727275745e-04, +5.054847159e+00 + 3.297717e+00, +2.287279666e-02, +5.465969699e-06, +4.800014042e-03, +1.786937239e-04, +5.271021749e+00 + 3.335191e+00, +2.087582241e-02, +1.094747916e-05, +4.296501227e-03, +1.630923626e-04, +4.875726472e+00 + 3.372665e+00, +1.693740099e-02, +1.957205640e-05, +3.274370495e-03, +1.323234452e-04, +4.026175767e+00 + 3.410139e+00, +1.265391042e-02, +3.033725212e-05, +2.351759896e-03, +9.885867518e-05, +3.029504715e+00 + 3.447613e+00, +1.056944488e-02, +3.699160287e-05, +1.868764054e-03, +8.257378814e-05, +2.402834715e+00 + 3.485087e+00, +1.203465460e-02, +5.157331508e-05, +1.987519538e-03, +9.402073902e-05, +2.582889975e+00 + 3.522561e+00, +1.483191162e-02, +4.264808044e-05, +2.431130279e-03, +1.158743095e-04, +3.207270944e+00 + 3.560035e+00, +1.694364856e-02, +4.348712938e-05, +2.651516726e-03, +1.323722544e-04, +3.758945722e+00 + 3.597509e+00, +1.777812513e-02, +3.816772972e-05, +2.830632832e-03, +1.388916026e-04, +4.038955569e+00 + 3.634984e+00, +1.749152618e-02, +2.610873465e-05, +2.685137944e-03, +1.366525483e-04, +4.041150977e+00 + 3.672458e+00, +1.659354033e-02, +6.478739420e-05, +2.504827606e-03, +1.296370338e-04, +3.857101060e+00 + 3.709932e+00, +1.567966850e-02, +1.324957253e-04, +2.523973389e-03, +1.224974102e-04, +3.618583613e+00 + 3.747406e+00, +1.515242979e-02, +1.837750152e-04, +2.479688081e-03, +1.183783577e-04, +3.441317861e+00 + 3.784880e+00, +1.505241220e-02, +2.589866907e-04, +2.413883219e-03, +1.175969703e-04, +3.374168903e+00 + 3.822354e+00, +1.517594324e-02, +2.648845550e-04, +2.482655253e-03, +1.185620565e-04, +3.390825944e+00 + 3.859828e+00, +1.530534707e-02, +3.460441858e-04, +2.325030107e-03, +1.195730240e-04, +3.433304933e+00 + 3.897302e+00, +1.533180643e-02, +2.591782740e-04, +2.448645600e-03, +1.197797377e-04, +3.458360447e+00 + 3.934776e+00, +1.525209324e-02, +3.263088666e-04, +2.386365345e-03, +1.191569785e-04, +3.452521374e+00 + 3.972250e+00, +1.511733830e-02, +2.656776913e-04, +2.367449970e-03, +1.181042055e-04, +3.423956689e+00 + 4.009724e+00, +1.498169883e-02, +1.893347752e-04, +2.402930160e-03, +1.170445221e-04, +3.388114585e+00 + 4.047198e+00, +1.486760911e-02, +8.599168463e-05, +2.247589218e-03, +1.161531962e-04, +3.356524519e+00 + 4.084672e+00, +1.476249591e-02, +7.080242635e-05, +2.377243440e-03, +1.153319993e-04, +3.331853567e+00 + 4.122146e+00, +1.464115774e-02, +1.002760332e-04, +2.290380531e-03, +1.143840448e-04, +3.308603435e+00 + 4.159620e+00, +1.448100039e-02, +1.127755160e-04, +2.309758242e-03, +1.131328156e-04, +3.277119137e+00 + 4.197094e+00, +1.425845180e-02, +1.046536919e-04, +2.317095442e-03, +1.113941547e-04, +3.228713172e+00 + 4.234568e+00, +1.397280754e-02, +8.171048574e-05, +2.200893910e-03, +1.091625589e-04, +3.160502439e+00 + 4.272043e+00, +1.365800498e-02, +5.594417538e-05, +2.299295461e-03, +1.067031639e-04, +3.078660427e+00 + 4.309517e+00, +1.335595257e-02, +3.262293522e-05, +2.188198722e-03, +1.043433795e-04, +2.998121297e+00 + 4.346991e+00, +1.310147746e-02, +1.743680856e-05, +2.244566034e-03, +1.023552927e-04, +2.936669456e+00 + 4.384465e+00, +1.291316404e-02, +1.318336764e-05, +2.228431201e-03, +1.008840941e-04, +2.903997885e+00 + 4.421939e+00, +1.276376145e-02, +1.435774037e-05, +2.182482625e-03, +9.971688630e-05, +2.891883014e+00 + 4.459413e+00, +1.257886363e-02, +1.385785895e-05, +2.254463017e-03, +9.827237213e-05, +2.873950817e+00 + 4.496887e+00, +1.227354455e-02, +1.139016156e-05, +2.117954940e-03, +9.588706676e-05, +2.816794882e+00 + 4.534361e+00, +1.178139292e-02, +8.224354336e-06, +2.159095858e-03, +9.204213215e-05, +2.695716179e+00 + 4.571835e+00, +1.108386625e-02, +4.702489471e-06, +2.144208192e-03, +8.659270510e-05, +2.507677639e+00 + 4.609309e+00, +1.023691014e-02, +2.918640342e-06, +1.911635765e-03, +7.997586047e-05, +2.277833009e+00 + 4.646783e+00, +9.371323031e-03, +1.545409171e-06, +1.700487237e-03, +7.321346118e-05, +2.056659349e+00 + 4.684257e+00, +8.657719189e-03, +2.177147837e-06, +1.788518719e-03, +6.763843116e-05, +1.900934905e+00 + 4.721731e+00, +8.217440495e-03, +1.601037153e-07, +1.709076276e-03, +6.419875387e-05, +1.836493022e+00 + 4.759205e+00, +8.019652879e-03, +1.376983597e-06, +1.786480796e-03, +6.265353811e-05, +1.832790721e+00 + 4.796679e+00, +7.890364257e-03, +1.486445721e-06, +1.768891622e-03, +6.164347076e-05, +1.825328220e+00 + 4.834153e+00, +7.621359223e-03, +9.019622716e-07, +1.685597957e-03, +5.954186893e-05, +1.758585885e+00 + 4.871627e+00, +7.068989784e-03, +9.009948019e-07, +1.773969888e-03, +5.522648269e-05, +1.608502805e+00 + 4.909101e+00, +6.213005292e-03, +2.594278515e-07, +1.651811893e-03, +4.853910384e-05, +1.384459441e+00 + 4.946576e+00, +5.162372595e-03, +1.085336870e-06, +1.352451286e-03, +4.033103590e-05, +1.123478548e+00 + 4.984050e+00, +4.122759929e-03, +6.012850877e-07, +9.429225286e-04, +3.220906194e-05, +8.826005714e-01 + 5.021524e+00, +3.350707601e-03, +6.078344522e-07, +7.200088076e-04, +2.617740313e-05, +7.236678618e-01 + 5.058998e+00, +3.013498510e-03, +6.067704731e-07, +7.103653051e-04, +2.354295711e-05, +6.701157353e-01 + 5.096472e+00, +2.982577268e-03, +6.275275747e-08, +7.400639948e-04, +2.330138491e-05, +6.737270462e-01 + 5.133946e+00, +2.967454278e-03, +7.366752407e-07, +6.898839516e-04, +2.318323655e-05, +6.688583217e-01 + 5.171420e+00, +2.790092474e-03, +5.208324035e-07, +7.384504272e-04, +2.179759745e-05, +6.231475505e-01 + 5.208894e+00, +2.418548988e-03, +9.276752457e-07, +7.000383276e-04, +1.889491397e-05, +5.351074185e-01 + 5.246368e+00, +1.911062361e-03, +1.344568805e-06, +5.773793295e-04, +1.493017470e-05, +4.200268783e-01 + 5.283842e+00, +1.372440606e-03, +2.056363962e-06, +4.096268533e-04, +1.072219224e-05, +3.000708009e-01 + 5.321316e+00, +9.213288711e-04, +2.114823211e-06, +2.372578999e-04, +7.197881805e-06, +1.991713402e-01 + 5.358790e+00, +6.722993615e-04, +2.927023810e-06, +1.457815447e-04, +5.252338761e-06, +1.401144692e-01 + 5.396264e+00, +6.283225291e-04, +2.643605712e-06, +1.625307574e-04, +4.908769758e-06, +1.268544987e-01 + 5.433738e+00, +6.348530854e-04, +1.837387106e-06, +1.550414836e-04, +4.959789730e-06, +1.297308749e-01 + 5.471212e+00, +6.001491653e-04, +2.469214990e-06, +1.570842275e-04, +4.688665354e-06, +1.254275991e-01 + 5.508686e+00, +5.139588852e-04, +6.781779592e-07, +1.472964293e-04, +4.015303791e-06, +1.099729019e-01 + 5.546160e+00, +3.974099564e-04, +2.282771424e-06, +1.184808925e-04, +3.104765285e-06, +8.751673075e-02 + 5.583635e+00, +2.814766597e-04, +4.142124481e-06, +8.237874526e-05, +2.199036404e-06, +6.404830704e-02 + 5.621109e+00, +1.971418372e-04, +5.379288482e-06, +4.879963190e-05, +1.540170604e-06, +4.499940303e-02 + 5.658583e+00, +1.566562903e-04, +5.397792871e-06, +2.429803265e-05, +1.223877268e-06, +3.378394289e-02 + 5.696057e+00, +1.405994051e-04, +2.383392345e-06, +2.103885737e-05, +1.098432853e-06, +2.942426165e-02 + 5.733531e+00, +1.313588361e-04, +1.776515077e-06, +1.911272236e-05, +1.026240907e-06, +2.730267726e-02 + 5.771005e+00, +1.162490257e-04, +2.192974526e-06, +1.780103944e-05, +9.081955133e-07, +2.433964679e-02 + 5.808479e+00, +9.660978209e-05, +1.832208312e-06, +1.546361775e-05, +7.547639226e-07, +2.048189674e-02 + 5.845953e+00, +7.972386355e-05, +2.032041548e-06, +1.277530080e-05, +6.228426840e-07, +1.785059556e-02 + 5.883427e+00, +8.075440741e-05, +9.629804944e-07, +1.405092356e-05, +6.308938079e-07, +1.905071629e-02 + 5.920901e+00, +9.847125181e-05, +1.617458010e-06, +1.897478279e-05, +7.693066548e-07, +2.355183009e-02 + 5.958375e+00, +1.201247757e-04, +1.250282725e-06, +2.394888269e-05, +9.384748101e-07, +2.860407565e-02 + 5.995849e+00, +1.353459675e-04, +4.093061374e-07, +2.700896494e-05, +1.057390371e-06, +3.211115798e-02 + 6.033323e+00, +1.393064156e-04, +1.148348275e-06, +2.893048894e-05, +1.088331372e-06, +3.289020287e-02 + 6.070797e+00, +1.306427217e-04, +8.982553633e-07, +2.770021651e-05, +1.020646263e-06, +3.044147732e-02 + 6.108271e+00, +1.061818821e-04, +7.339142826e-07, +2.278162355e-05, +8.295459542e-07, +2.486915720e-02 + 6.145745e+00, +7.209552695e-05, +3.375269493e-07, +1.566264693e-05, +5.632463043e-07, +1.688222621e-02 + 6.183219e+00, +3.514370025e-05, +6.910549004e-07, +6.938825859e-06, +2.745601582e-07, +8.222569600e-03 + 6.220694e+00, +3.332434404e-05, +8.008708289e-07, +5.858394718e-06, +2.603464378e-07, +7.112443417e-03 + 6.258168e+00, +6.596901561e-05, +4.918351346e-07, +1.173531422e-05, +5.153829345e-07, +1.487473373e-02 + 6.295642e+00, +9.667915064e-05, +3.027019372e-07, +1.881777929e-05, +7.553058644e-07, +2.195995150e-02 + 6.333116e+00, +1.151311728e-04, +1.042992568e-06, +2.293042378e-05, +8.994622877e-07, +2.658857944e-02 + 6.370590e+00, +1.207366960e-04, +7.528801062e-07, +2.380623776e-05, +9.432554375e-07, +2.836724125e-02 + 6.408064e+00, +1.164009914e-04, +2.748897183e-07, +2.213552786e-05, +9.093827456e-07, +2.754771847e-02 + 6.445538e+00, +1.035095295e-04, +1.131867308e-06, +1.908048890e-05, +8.086681991e-07, +2.492432935e-02 + 6.483012e+00, +9.055022966e-05, +1.313210738e-06, +1.572048715e-05, +7.074236692e-07, +2.173397508e-02 + 6.520486e+00, +8.224274468e-05, +7.030108963e-07, +1.364569358e-05, +6.425214429e-07, +1.939641242e-02 + 6.557960e+00, +8.240698886e-05, +1.148314156e-06, +1.353813700e-05, +6.438046004e-07, +1.881287963e-02 + 6.595434e+00, +8.663895944e-05, +1.631803893e-06, +1.483972908e-05, +6.768668706e-07, +1.965705398e-02 + 6.632908e+00, +9.150938696e-05, +8.959336730e-07, +1.500001344e-05, +7.149170856e-07, +2.089087372e-02 + 6.670382e+00, +9.411555999e-05, +7.071728276e-07, +1.471136650e-05, +7.352778124e-07, +2.174361592e-02 + 6.707856e+00, +9.427881810e-05, +2.232689138e-06, +1.527592348e-05, +7.365532664e-07, +2.197690465e-02 + 6.745330e+00, +9.283419611e-05, +1.674328273e-06, +1.475470634e-05, +7.252671571e-07, +2.172213826e-02 + 6.782804e+00, +9.091313126e-05, +2.172434071e-06, +1.494130972e-05, +7.102588379e-07, +2.125742296e-02 + 6.820278e+00, +8.947559322e-05, +1.208738908e-06, +1.467196182e-05, +6.990280721e-07, +2.082523460e-02 + 6.857752e+00, +8.849046137e-05, +2.183296734e-07, +1.422391044e-05, +6.913317295e-07, +2.053535812e-02 + 6.895227e+00, +8.869124720e-05, +1.872610390e-06, +1.475311687e-05, +6.929003687e-07, +2.037002683e-02 + 6.932701e+00, +8.703583659e-05, +1.303861073e-06, +1.403557469e-05, +6.799674734e-07, +2.025706959e-02 + 6.970175e+00, +8.705236101e-05, +1.310170334e-06, +1.440134753e-05, +6.800965704e-07, +2.014118630e-02 + 7.007649e+00, +8.649188868e-05, +1.642909372e-06, +1.413717956e-05, +6.757178803e-07, +2.000795493e-02 + 7.045123e+00, +8.557114199e-05, +8.070387273e-07, +1.402625353e-05, +6.685245468e-07, +1.987188346e-02 + 7.082597e+00, +8.556574398e-05, +6.303919740e-07, +1.437910359e-05, +6.684823748e-07, +1.974815441e-02 + 7.120071e+00, +8.395754835e-05, +2.492708841e-07, +1.333948625e-05, +6.559183464e-07, +1.962815933e-02 + 7.157545e+00, +8.469831735e-05, +4.394354772e-07, +1.399445249e-05, +6.617056043e-07, +1.947236585e-02 + 7.195019e+00, +8.219518954e-05, +2.016472612e-07, +1.374633414e-05, +6.421499183e-07, +1.922019946e-02 + 7.232493e+00, +8.060990055e-05, +7.126280650e-07, +1.375634861e-05, +6.297648481e-07, +1.881533865e-02 + 7.269967e+00, +7.975417976e-05, +2.119912937e-07, +1.379736945e-05, +6.230795294e-07, +1.823277535e-02 + 7.307441e+00, +7.470325725e-05, +1.169669241e-07, +1.285632322e-05, +5.836191972e-07, +1.750653819e-02 + 7.344915e+00, +7.330184956e-05, +2.570909088e-07, +1.365659262e-05, +5.726706997e-07, +1.673934977e-02 + 7.382389e+00, +7.020343991e-05, +4.248388020e-07, +1.310218265e-05, +5.484643743e-07, +1.607601725e-02 + 7.419863e+00, +6.896813444e-05, +4.112239141e-07, +1.338802699e-05, +5.388135503e-07, +1.564435535e-02 + 7.457337e+00, +6.715833136e-05, +1.872803050e-07, +1.323199471e-05, +5.246744638e-07, +1.547011399e-02 + 7.494811e+00, +6.573000791e-05, +1.923954388e-07, +1.301561943e-05, +5.135156868e-07, +1.543460452e-02 + Mean, +7.685673982e-03, +2.479807792e-04, +5.882818519e-04, +6.004432799e-05, +1.742443487e+00 diff --git a/test/ref/cpw/lumped_adaptive/error-indicators.csv b/test/ref/cpw/lumped_adaptive/error-indicators.csv index 463b36465..9a627cf4a 100644 --- a/test/ref/cpw/lumped_adaptive/error-indicators.csv +++ b/test/ref/cpw/lumped_adaptive/error-indicators.csv @@ -1,11 +1,11 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +6.373526308e-06, +2.186423623e-01 - 3.000000e+01, +7.048573140e-01, +1.801733890e-06, +1.911288392e-02, +6.372109948e-06, +4.552939315e+00 - 1.820000e+01, +7.087438254e-01, +1.180056136e-06, +1.962899718e-02, +6.407245113e-06, +2.525522343e+00 - 1.370000e+01, +7.056684565e-01, +1.203625733e-06, +1.815263574e-02, +6.379442906e-06, +2.145476899e+00 - 2.540000e+01, +7.062740050e-01, +1.129132128e-06, +1.917725181e-02, +6.384917236e-06, +3.611335581e+00 - 6.200000e+00, +7.029687080e-01, +7.051383242e-07, +1.772868227e-02, +6.355036414e-06, +7.852578706e-01 - 2.810000e+01, +7.051964244e-01, +9.430618882e-07, +1.902937537e-02, +6.375175602e-06, +4.282992773e+00 - 9.900000e+00, +7.033807914e-01, +7.345028171e-07, +1.792395353e-02, +6.358761765e-06, +1.430053254e+00 - 2.090000e+01, +7.085921616e-01, +7.822419312e-07, +1.961007783e-02, +6.405874029e-06, +2.739108293e+00 - Mean, +7.056328525e-01, +1.160955956e-06, +1.695571829e-02, +6.379121036e-06, +2.186423623e-01 + 2.000000e+00, +7.050139872e-01, +1.089715623e-06, +1.591585952e-02, +6.373526318e-06, +2.186423623e-01 + 3.000000e+01, +7.048573159e-01, +1.801749017e-06, +1.911288427e-02, +6.372109965e-06, +4.552939305e+00 + 1.780000e+01, +7.085941898e-01, +1.239011768e-06, +1.967802015e-02, +6.405892364e-06, +2.505814500e+00 + 1.280000e+01, +7.050022016e-01, +1.023153730e-06, +1.779303491e-02, +6.373419773e-06, +1.992681983e+00 + 2.510000e+01, +7.064238116e-01, +9.104840972e-07, +1.919993236e-02, +6.386271531e-06, +3.534944010e+00 + 5.200000e+00, +7.032239220e-01, +7.481839787e-07, +1.760116584e-02, +6.357343621e-06, +6.351597186e-01 + 2.790000e+01, +7.052578369e-01, +1.050803997e-06, +1.902258442e-02, +6.375730789e-06, +4.240374356e+00 + 1.540000e+01, +7.069908612e-01, +1.449849369e-06, +1.912291719e-02, +6.391397819e-06, +2.357765408e+00 + 8.300000e+00, +7.029555592e-01, +6.862523461e-07, +1.801637153e-02, +6.354917545e-06, +1.133642571e+00 + Mean, +7.053688539e-01, +1.196501131e-06, +1.658438367e-02, +6.376734414e-06, +2.352440468e+00 diff --git a/test/ref/cpw/lumped_uniform/error-indicators.csv b/test/ref/cpw/lumped_uniform/error-indicators.csv index 0c718788d..2cd6296ed 100644 --- a/test/ref/cpw/lumped_uniform/error-indicators.csv +++ b/test/ref/cpw/lumped_uniform/error-indicators.csv @@ -1,17 +1,17 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +6.373526308e-06, +2.186423623e-01 - 4.000000e+00, +7.037534929e-01, +8.348638678e-07, +1.716164791e-02, +6.362131092e-06, +4.676837686e-01 - 6.000000e+00, +7.030062911e-01, +7.118689649e-07, +1.767838766e-02, +6.355376176e-06, +7.544551055e-01 - 8.000000e+00, +7.029165326e-01, +6.832337447e-07, +1.800232414e-02, +6.354564734e-06, +1.081001045e+00 - 1.000000e+01, +7.034192125e-01, +7.418014192e-07, +1.790795027e-02, +6.359109103e-06, +1.449355685e+00 - 1.200000e+01, +7.044599247e-01, +9.086471535e-07, +1.751260139e-02, +6.368517436e-06, +1.842149227e+00 - 1.400000e+01, +7.058985472e-01, +1.260914522e-06, +1.833900466e-02, +6.381522991e-06, +2.190892577e+00 - 1.600000e+01, +7.074564924e-01, +1.093986277e-06, +1.938386319e-02, +6.395607258e-06, +2.406792037e+00 - 1.800000e+01, +7.086746633e-01, +1.209024854e-06, +1.965872658e-02, +6.406619868e-06, +2.515530177e+00 - 2.000000e+01, +7.088448209e-01, +8.917186070e-07, +1.963851961e-02, +6.408158141e-06, +2.646279233e+00 - 2.200000e+01, +7.081043245e-01, +7.960606263e-07, +1.946623509e-02, +6.401463844e-06, +2.889633318e+00 - 2.400000e+01, +7.070103246e-01, +9.980955322e-07, +1.928974677e-02, +6.391573774e-06, +3.271855291e+00 - 2.600000e+01, +7.059908728e-01, +1.195377481e-06, +1.913395059e-02, +6.382357641e-06, +3.767453335e+00 - 2.800000e+01, +7.052267075e-01, +9.841023725e-07, +1.902600821e-02, +6.375449370e-06, +4.261952660e+00 - 3.000000e+01, +7.048573097e-01, +1.801758126e-06, +1.911288376e-02, +6.372109909e-06, +4.552939157e+00 - Mean, +7.056422335e-01, +1.101163768e-06, +1.672465772e-02, +6.379205843e-06, +2.186423623e-01 + 2.000000e+00, +7.050139872e-01, +1.089715622e-06, +1.591585952e-02, +6.373526318e-06, +2.186423623e-01 + 4.000000e+00, +7.037534941e-01, +8.348646390e-07, +1.716164813e-02, +6.362131103e-06, +4.676837690e-01 + 6.000000e+00, +7.030062925e-01, +7.118692553e-07, +1.767838746e-02, +6.355376189e-06, +7.544551064e-01 + 8.000000e+00, +7.029165341e-01, +6.832340564e-07, +1.800232389e-02, +6.354564748e-06, +1.081001046e+00 + 1.000000e+01, +7.034192143e-01, +7.417840765e-07, +1.790794998e-02, +6.359109119e-06, +1.449355687e+00 + 1.200000e+01, +7.044599266e-01, +9.086861419e-07, +1.751260195e-02, +6.368517453e-06, +1.842149229e+00 + 1.400000e+01, +7.058985483e-01, +1.260916409e-06, +1.833900406e-02, +6.381523001e-06, +2.190892578e+00 + 1.600000e+01, +7.074564926e-01, +1.093990747e-06, +1.938386316e-02, +6.395607259e-06, +2.406792037e+00 + 1.800000e+01, +7.086746629e-01, +1.209021510e-06, +1.965872720e-02, +6.406619864e-06, +2.515530177e+00 + 2.000000e+01, +7.088448208e-01, +8.917049253e-07, +1.963851901e-02, +6.408158140e-06, +2.646279233e+00 + 2.200000e+01, +7.081043249e-01, +7.960618462e-07, +1.946623450e-02, +6.401463847e-06, +2.889633320e+00 + 2.400000e+01, +7.070103252e-01, +8.888321469e-07, +1.928974719e-02, +6.391573780e-06, +3.271855293e+00 + 2.600000e+01, +7.059908737e-01, +1.025100239e-06, +1.913395101e-02, +6.382357649e-06, +3.767453334e+00 + 2.800000e+01, +7.052267089e-01, +9.840962024e-07, +1.902600862e-02, +6.375449382e-06, +4.261952652e+00 + 3.000000e+01, +7.048573116e-01, +1.801773182e-06, +1.911288411e-02, +6.372109927e-06, +4.552939148e+00 + Mean, +7.056422345e-01, +1.101172703e-06, +1.672465773e-02, +6.379205852e-06, +2.287774331e+00 diff --git a/test/ref/cpw/wave_adaptive/error-indicators.csv b/test/ref/cpw/wave_adaptive/error-indicators.csv index d5bfae979..19728fc81 100644 --- a/test/ref/cpw/wave_adaptive/error-indicators.csv +++ b/test/ref/cpw/wave_adaptive/error-indicators.csv @@ -1,11 +1,11 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +6.731376290e-06, +2.614559004e-01 - 3.000000e+01, +6.802918578e-01, +5.681633835e-06, +1.845184430e-02, +6.548319901e-06, +4.266998699e+00 - 1.810000e+01, +6.988982656e-01, +1.353606358e-06, +1.749494134e-02, +6.727420545e-06, +2.409892387e+00 - 2.750000e+01, +6.989649048e-01, +3.352201085e-06, +1.700252668e-02, +6.728061998e-06, +3.692517431e+00 - 2.920000e+01, +6.936265525e-01, +3.739780438e-06, +1.745465144e-02, +6.676676349e-06, +4.005812881e+00 - 2.450000e+01, +6.989754646e-01, +2.953814391e-06, +1.726940726e-02, +6.728163644e-06, +3.271877994e+00 - 1.680000e+01, +6.971976662e-01, +4.668231523e-06, +1.732341704e-02, +6.711050999e-06, +2.246634753e+00 - 1.350000e+01, +6.996120816e-01, +1.992113352e-06, +1.706069694e-02, +6.734291561e-06, +1.810579073e+00 - 1.540000e+01, +6.990129264e-01, +4.297760845e-06, +1.727653180e-02, +6.728524242e-06, +2.063625604e+00 - Mean, +6.962098822e-01, +4.894505955e-06, +1.703956245e-02, +6.701542836e-06, +2.614559004e-01 + 2.000000e+00, +6.993092200e-01, +1.566265933e-06, +1.685105942e-02, +6.731376290e-06, +2.614559004e-01 + 3.000000e+01, +6.802918578e-01, +5.681633843e-06, +1.845184430e-02, +6.548319901e-06, +4.266998699e+00 + 1.740000e+01, +6.984728390e-01, +3.015199052e-06, +1.743761926e-02, +6.723325495e-06, +2.320170390e+00 + 2.670000e+01, +6.990163126e-01, +3.326374904e-06, +1.698618907e-02, +6.728556836e-06, +3.579808172e+00 + 2.030000e+01, +6.990616766e-01, +2.913715164e-06, +1.755160089e-02, +6.728993498e-06, +2.698250801e+00 + 1.640000e+01, +6.962139900e-01, +4.324939119e-06, +1.717774037e-02, +6.701582378e-06, +2.200470547e+00 + 2.960000e+01, +6.281346087e-01, +8.180986750e-06, +1.580175763e-02, +6.046267217e-06, +5.361105316e+00 + 1.530000e+01, +6.988269093e-01, +4.708649999e-06, +1.727962179e-02, +6.726733687e-06, +2.051893344e+00 + 8.200000e+00, +6.993384974e-01, +2.436893373e-06, +1.675967485e-02, +6.731658107e-06, +1.094416733e+00 + Mean, +6.887406568e-01, +5.788786385e-06, +1.631021899e-02, +6.629645934e-06, +2.648285545e+00 diff --git a/test/ref/cpw/wave_uniform/error-indicators.csv b/test/ref/cpw/wave_uniform/error-indicators.csv index 84504cef3..7347e6188 100644 --- a/test/ref/cpw/wave_uniform/error-indicators.csv +++ b/test/ref/cpw/wave_uniform/error-indicators.csv @@ -1,17 +1,17 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +6.731376290e-06, +2.614559004e-01 - 4.000000e+00, +6.992890992e-01, +1.736035539e-06, +1.680419113e-02, +6.731182612e-06, +5.258892594e-01 - 6.000000e+00, +6.992881535e-01, +2.021152752e-06, +1.676125294e-02, +6.731173509e-06, +7.947256627e-01 - 8.000000e+00, +6.993312910e-01, +2.448759169e-06, +1.675682757e-02, +6.731588740e-06, +1.067084152e+00 - 1.000000e+01, +6.994243806e-01, +2.399252549e-06, +1.681663205e-02, +6.732484797e-06, +1.340172770e+00 - 1.200000e+01, +6.995448511e-01, +2.366910745e-06, +1.694287832e-02, +6.733644416e-06, +1.610652070e+00 + 2.000000e+00, +6.993092200e-01, +1.566265933e-06, +1.685105942e-02, +6.731376290e-06, +2.614559004e-01 + 4.000000e+00, +6.992890992e-01, +1.736035553e-06, +1.680419113e-02, +6.731182612e-06, +5.258892594e-01 + 6.000000e+00, +6.992881535e-01, +2.021152765e-06, +1.676125294e-02, +6.731173509e-06, +7.947256627e-01 + 8.000000e+00, +6.993312910e-01, +2.448759168e-06, +1.675682757e-02, +6.731588740e-06, +1.067084152e+00 + 1.000000e+01, +6.994243806e-01, +2.399252545e-06, +1.681663205e-02, +6.732484797e-06, +1.340172770e+00 + 1.200000e+01, +6.995448511e-01, +2.366910744e-06, +1.694287832e-02, +6.733644416e-06, +1.610652070e+00 1.400000e+01, +6.995942415e-01, +1.874364476e-06, +1.709174983e-02, +6.734119836e-06, +1.876886099e+00 - 1.600000e+01, +6.980880025e-01, +5.469569257e-06, +1.721279606e-02, +6.719621155e-06, +2.144504886e+00 - 1.800000e+01, +6.988641967e-01, +1.805639359e-06, +1.748858870e-02, +6.727092606e-06, +2.396980543e+00 - 2.000000e+01, +6.990669364e-01, +2.883025514e-06, +1.755245634e-02, +6.729044128e-06, +2.658455702e+00 - 2.200000e+01, +6.990046675e-01, +3.191459315e-06, +1.749703902e-02, +6.728444743e-06, +2.926893770e+00 - 2.400000e+01, +6.989710885e-01, +2.953254947e-06, +1.732841600e-02, +6.728121520e-06, +3.202248951e+00 - 2.600000e+01, +6.990090439e-01, +3.218910262e-06, +1.709437140e-02, +6.728486869e-06, +3.481668528e+00 - 2.800000e+01, +6.988244472e-01, +3.387084517e-06, +1.706546999e-02, +6.726709988e-06, +3.764267665e+00 - 3.000000e+01, +6.802918601e-01, +5.681637479e-06, +1.845184332e-02, +6.548319923e-06, +4.266998672e+00 - Mean, +6.978600986e-01, +4.485174026e-06, +1.714127920e-02, +6.717427409e-06, +2.614559004e-01 + 1.600000e+01, +6.980880025e-01, +5.469569269e-06, +1.721279606e-02, +6.719621155e-06, +2.144504886e+00 + 1.800000e+01, +6.988641967e-01, +1.805639364e-06, +1.748858870e-02, +6.727092606e-06, +2.396980543e+00 + 2.000000e+01, +6.990669364e-01, +2.883025512e-06, +1.755245634e-02, +6.729044128e-06, +2.658455702e+00 + 2.200000e+01, +6.990046675e-01, +3.191459322e-06, +1.749703902e-02, +6.728444743e-06, +2.926893770e+00 + 2.400000e+01, +6.989710885e-01, +2.953254976e-06, +1.732841600e-02, +6.728121520e-06, +3.202248951e+00 + 2.600000e+01, +6.990090439e-01, +3.218910288e-06, +1.709437140e-02, +6.728486869e-06, +3.481668528e+00 + 2.800000e+01, +6.988244472e-01, +3.387084530e-06, +1.706546999e-02, +6.726709988e-06, +3.764267665e+00 + 3.000000e+01, +6.802918601e-01, +5.681637487e-06, +1.845184332e-02, +6.548319923e-06, +4.266998672e+00 + Mean, +6.978600986e-01, +4.485174041e-06, +1.714127920e-02, +6.717427409e-06, +2.154592309e+00 diff --git a/test/ref/rings/error-indicators.csv b/test/ref/rings/error-indicators.csv index bf92e8ad6..189aa400c 100644 --- a/test/ref/rings/error-indicators.csv +++ b/test/ref/rings/error-indicators.csv @@ -1,4 +1,4 @@ i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +3.546095276e-01, +4.103265502e-09, +2.980197519e-02, +6.790554137e-06, +1.115530397e-01 - 2.000000e+00, +2.764870097e-01, +1.023265220e-07, +7.245396387e-03, +5.294556016e-06, +4.557585167e-01 - Mean, +3.155482686e-01, +5.321489377e-08, +1.495441011e-02, +6.042555076e-06, +1.115530397e-01 + 1.000000e+00, +3.546095276e-01, +4.103305750e-09, +2.980197521e-02, +6.790554136e-06, +1.115530397e-01 + 2.000000e+00, +2.764870097e-01, +1.023265200e-07, +7.245396381e-03, +5.294556016e-06, +4.557585167e-01 + Mean, +3.155482686e-01, +5.321491286e-08, +1.495441012e-02, +6.042555076e-06, +2.836557782e-01 diff --git a/test/ref/spheres/error-indicators.csv b/test/ref/spheres/error-indicators.csv index 512c94d6e..29704f4a0 100644 --- a/test/ref/spheres/error-indicators.csv +++ b/test/ref/spheres/error-indicators.csv @@ -1,4 +1,4 @@ i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +7.827775744e-03, +7.728353753e-07, +5.589618554e-04, +7.474244003e-07, +3.052392581e-01 - 2.000000e+00, +7.529030499e-03, +8.146986865e-07, +7.245677921e-04, +7.188991214e-07, +4.319876077e-01 - Mean, +7.678403122e-03, +1.617085360e-06, +4.783727878e-04, +7.331617609e-07, +3.052392581e-01 + 1.000000e+00, +7.827776488e-03, +7.729127800e-07, +5.589600332e-04, +7.474244713e-07, +3.052392581e-01 + 2.000000e+00, +7.529026549e-03, +8.147247392e-07, +7.245667936e-04, +7.188987443e-07, +4.319876077e-01 + Mean, +7.678401519e-03, +1.617075494e-06, +4.783729070e-04, +7.331616078e-07, +3.686134329e-01 From a33bd0d93d0d91a1ca0db36a5455f2c40311ebdf Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 09:43:18 -0400 Subject: [PATCH 11/39] Small change to have flux projector templated on FEC and use that to figure out scalar fe or not --- palace/linalg/errorestimator.cpp | 23 +++++++++-------- palace/linalg/errorestimator.hpp | 44 +++++++++++++++----------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index ee5159f9e..dff8490a1 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -22,12 +22,14 @@ using namespace fem; // Given a finite element space hierarchy, construct a vector of mass matrix // operators corresponding to each level. +template std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHierarchy &h, int pa_order_threshold) { constexpr int skip_zeros = 0; - const bool is_scalar_FE_space = - h.GetFESpaceAtLevel(0).GetFE(0)->GetRangeType() == mfem::FiniteElement::SCALAR; + + constexpr bool ScalarFESpace = std::is_same::value + || std::is_same::value; // Assemble the bilinear form operator auto M = std::make_unique(h.GetNumLevels()); @@ -36,7 +38,7 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie auto &h_l = h.GetFESpaceAtLevel(l); auto m = std::make_unique(&h_l); - if (is_scalar_FE_space) + if constexpr (ScalarFESpace) { MFEM_ASSERT(h_l.GetVDim() == 1, "Scalar mass matrix hierarchy assumes a component-wise solve."); @@ -46,7 +48,6 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie { m->AddDomainIntegrator(new mfem::VectorFEMassIntegrator); } - auto M_l = std::make_unique( fem::AssembleOperator(std::move(m), true, (l > 0) ? pa_order_threshold : 100, skip_zeros), @@ -58,10 +59,11 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie return M; } -FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fespace, +template +FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fespace, double tol, int max_it, int print, int pa_order_threshold) - : M(BuildMassMatrixOperator(smooth_flux_fespace, pa_order_threshold)) + : M(BuildMassMatrixOperator(smooth_flux_fespace, pa_order_threshold)) { // The system matrix for the projection is real and SPD. For the coarse-level AMG solve, // we don't use an exact solve on the coarsest level. @@ -86,7 +88,6 @@ FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_f } - CurlFluxErrorEstimator::CurlFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, const std::vector> &mesh, @@ -119,6 +120,7 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( } } +template <> IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, bool normalize) const { @@ -149,7 +151,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - auto build_flux = [](const FluxProjector &proj, const ComplexVector &flux_coef) + auto build_flux = [](const FluxProjector&proj, const ComplexVector &flux_coef) { // Use a copy construction to match appropriate size. ComplexVector flux(flux_coef); @@ -230,6 +232,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl return {estimates, normalization}; } +template <> IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const { @@ -256,7 +259,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vecto // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - auto build_flux = [](const FluxProjector &proj, const Vector &flux_coef) + auto build_flux = [](const FluxProjector &proj, const Vector &flux_coef) { // Use a copy construction to match appropriate size. Vector flux(flux_coef); @@ -377,7 +380,7 @@ IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vecto // Given the RHS vector of non-smooth flux, construct a flux projector and perform // component wise mass matrix inversion in the appropriate space, giving fᵢ = M⁻¹ f̂ᵢ. - auto build_flux = [sdim](const FluxProjector &proj, Vector &rhs) + auto build_flux = [sdim](const FluxProjector &proj, Vector &rhs) { // Use a copy construction to match appropriate size. Vector flux(rhs.Size()); diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 1e659b270..32f4fc0c1 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -22,6 +22,8 @@ class MaterialOperator; // between this resulting smooth flux and the original non-smooth flux provides a // localizable error estimate. An instance of FluxProjector can be reused across solutions, // thus the construction of the operator is separated from the construction of the flux RHS. + +template class FluxProjector { private: @@ -35,40 +37,38 @@ class FluxProjector mutable Vector tmp; public: - FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, double tol = 1e-12, - int max_it = 200, int print_level = 1, int pa_order_threshold = 1); + FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, double tol, + int max_it, int print_level, int pa_order_threshold); - // Given a vector of dof defining the flux, compute the smooth flux IN PLACE. - void Mult(Vector &x) const + inline void Mult(Vector &x) const { tmp = x; Mult(tmp, x); } - void Mult(const Vector &x, Vector &y) const { ksp->Mult(x, y); } - void Mult(ComplexVector &x) const + inline void Mult(const Vector &x, Vector &y) const { ksp->Mult(x, y); } + inline void Mult(ComplexVector &x) const { Mult(x.Real()); Mult(x.Imag()); } - void Mult(const ComplexVector &x, ComplexVector &y) const + inline void Mult(const ComplexVector &x, ComplexVector &y) const { y = x; Mult(y); } }; -// Class used for computing curl flux error estimate, -// i.e. || μ⁻¹∇ × Vₕ - F ||_K -// where F denotes a smooth reconstruction of μ⁻¹∇ × Vₕ. +// Class used for computing curl flux error estimate, i.e. || μ⁻¹∇ × Vₕ - F ||_K where F +// denotes a smooth reconstruction of μ⁻¹∇ × Vₕ. class CurlFluxErrorEstimator { const MaterialOperator &mat_op; - // The finite element space used to represent V. + // The finite element space used to represent V, and F. mfem::ParFiniteElementSpace &fespace; std::vector> smooth_flux_fecs; mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_fespace; - mutable FluxProjector smooth_projector; + mutable FluxProjector smooth_projector; mfem::L2_FECollection coarse_flux_fec; mutable mfem::ParFiniteElementSpace coarse_flux_fespace; @@ -86,29 +86,25 @@ class CurlFluxErrorEstimator mfem::ParFiniteElementSpace &fespace); // Compute elemental error indicators given a complex vector of true DOF. - IndicatorsAndNormalization ComputeIndicators(const ComplexVector &v, - bool normalize) const; - - // Compute elemental error indicators given a vector of true DOF. - IndicatorsAndNormalization ComputeIndicators(const Vector &v, bool normalize ) const; + template + IndicatorsAndNormalization ComputeIndicators(const VectorType &v, bool normalize) const; }; -// Class used for computing grad flux error estimate, -// i.e. || ϵ ∇ ϕₕ - F ||_K -// where F denotes a smooth reconstruction of ϵ ∇ ϕₕ. +// Class used for computing grad flux error estimate, i.e. || ϵ ∇ ϕₕ - F ||_K where F +// denotes a smooth reconstruction of ϵ ∇ ϕₕ. class GradFluxErrorEstimator { const MaterialOperator &mat_op; // The finite element space used to represent ϕ. mfem::ParFiniteElementSpace &fespace; - // Collections and spaces for the smooth flux. Note the hierarchy uses the - // SCALAR finite element space, whilst the true flux is in the VECTOR finite - // element space. This allows for a component wise inversion. + // Collections and spaces for the smooth flux. Note the hierarchy uses the SCALAR finite + // element space, whilst the true flux is in the VECTOR finite element space. This allows + // for a component wise inversion. std::vector> smooth_flux_fecs; mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_component_fespace; mutable mfem::ParFiniteElementSpace smooth_flux_fespace; - mutable FluxProjector smooth_projector; + mutable FluxProjectorsmooth_projector; mfem::L2_FECollection coarse_flux_fec; mutable mfem::ParFiniteElementSpace coarse_flux_fespace; From 802b4466b26a4c5358b4f329a28d9e15edbe5fd6 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 10:37:10 -0400 Subject: [PATCH 12/39] Use GetVectorValue rather than precomputed components --- palace/drivers/drivensolver.cpp | 9 +- palace/drivers/drivensolver.hpp | 4 +- palace/drivers/eigensolver.cpp | 5 +- palace/drivers/magnetostaticsolver.cpp | 8 +- palace/drivers/magnetostaticsolver.hpp | 2 +- palace/drivers/transientsolver.cpp | 3 +- palace/linalg/errorestimator.cpp | 235 ++++++++----------------- palace/linalg/errorestimator.hpp | 20 +-- 8 files changed, 93 insertions(+), 193 deletions(-) diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 8ca8e8312..2b384d80a 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -100,8 +100,7 @@ DrivenSolver::Solve(const std::vector> &mesh) con auto estimator = [&]() { BlockTimer bt(Timer::ESTCONSTRUCT); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), mesh, - spaceop.GetNDSpace()); + return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); // Main frequency sweep loop. @@ -111,7 +110,7 @@ DrivenSolver::Solve(const std::vector> &mesh) con } ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &postop, - const CurlFluxErrorEstimator &estimator, + CurlFluxErrorEstimator &estimator, int nstep, int step0, double omega0, double delta_omega) const { @@ -150,7 +149,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &E, int step, double f) { - BlockTimer bt0(Timer::ESTSOLVE); + BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto estimate = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); @@ -228,7 +227,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator } ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, - const CurlFluxErrorEstimator &estimator, + CurlFluxErrorEstimator &estimator, int nstep, int step0, double omega0, double delta_omega) const { diff --git a/palace/drivers/drivensolver.hpp b/palace/drivers/drivensolver.hpp index 7e5a92a5c..458786639 100644 --- a/palace/drivers/drivensolver.hpp +++ b/palace/drivers/drivensolver.hpp @@ -37,10 +37,10 @@ class DrivenSolver : public BaseSolver int GetNumSteps(double start, double end, double delta) const; ErrorIndicators SweepUniform(SpaceOperator &spaceop, PostOperator &postop, - const CurlFluxErrorEstimator &estimator, int nstep, + CurlFluxErrorEstimator &estimator, int nstep, int step0, double omega0, double delta_omega) const; ErrorIndicators SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, - const CurlFluxErrorEstimator &estimator, int nstep, + CurlFluxErrorEstimator &estimator, int nstep, int step0, double omega0, double delta_omega) const; void Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index e8b091333..790f8ff50 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -254,8 +254,7 @@ EigenSolver::Solve(const std::vector> &mesh) cons auto estimator = [&]() { BlockTimer bt(Timer::ESTCONSTRUCT); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), mesh, - spaceop.GetNDSpace()); + return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); // Eigenvalue problem solve. @@ -269,7 +268,7 @@ EigenSolver::Solve(const std::vector> &mesh) cons auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &E, int i) { - BlockTimer bt0(Timer::ESTSOLVE); + BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto estimate = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index 982c8f0bc..c288ebed8 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -74,8 +74,8 @@ MagnetostaticSolver::Solve(const std::vector> &me auto estimator = [&]() { BlockTimer bt(Timer::ESTCONSTRUCT); - return CurlFluxErrorEstimator(iodata, curlcurlop.GetMaterialOp(), mesh, - curlcurlop.GetNDSpace()); + return CurlFluxErrorEstimator(iodata, curlcurlop.GetMaterialOp(), + curlcurlop.GetNDSpaces()); }(); // Postprocess the capacitance matrix from the computed field solutions. @@ -86,7 +86,7 @@ MagnetostaticSolver::Solve(const std::vector> &me ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, - const CurlFluxErrorEstimator &estimator, + CurlFluxErrorEstimator &estimator, const std::vector &A) const { // Postprocess the Maxwell inductance matrix. See p. 97 of the COMSOL AC/DC Module manual @@ -108,7 +108,7 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &A, int i, double idx) { - BlockTimer bt0(Timer::ESTSOLVE); + BlockTimer bt(Timer::ESTIMATION); constexpr bool normalized = true; auto estimate = estimator.ComputeIndicators(A, normalized); BlockTimer bt1(Timer::POSTPRO); diff --git a/palace/drivers/magnetostaticsolver.hpp b/palace/drivers/magnetostaticsolver.hpp index b224c2538..29150f2cf 100644 --- a/palace/drivers/magnetostaticsolver.hpp +++ b/palace/drivers/magnetostaticsolver.hpp @@ -35,7 +35,7 @@ class MagnetostaticSolver : public BaseSolver { private: ErrorIndicators Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, - const CurlFluxErrorEstimator &estimator, + CurlFluxErrorEstimator &estimator, const std::vector &A) const; void PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 60af27e73..43588b6d7 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -81,8 +81,7 @@ TransientSolver::Solve(const std::vector> &mesh) auto estimator = [&]() { BlockTimer bt(Timer::ESTCONSTRUCT); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), mesh, - spaceop.GetNDSpace()); + return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); auto UpdateErrorIndicators = [this, &estimator, &indicators, diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index dff8490a1..f14bc6df9 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -14,6 +14,7 @@ #include "linalg/gmg.hpp" #include "linalg/iterative.hpp" #include "linalg/rap.hpp" +#include "utils/timer.hpp" namespace palace { @@ -60,9 +61,7 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie } template -FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fespace, - double tol, int max_it, int print, - int pa_order_threshold) +FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fespace, double tol, int max_it, int print, int pa_order_threshold) : M(BuildMassMatrixOperator(smooth_flux_fespace, pa_order_threshold)) { // The system matrix for the projection is real and SPD. For the coarse-level AMG solve, @@ -87,139 +86,76 @@ FluxProjector::FluxProjector(mfem::ParFiniteE tmp.SetSize(smooth_flux_fespace.GetFinestFESpace().GetTrueVSize()); } - CurlFluxErrorEstimator::CurlFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, - const std::vector> &mesh, - mfem::ParFiniteElementSpace &fespace) - : mat_op(mat_op), fespace(fespace), - smooth_flux_fecs(ConstructFECollections( - iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, - iodata.solver.linear.mg_coarsen_type, iodata.solver.linear.pc_mat_lor)), - smooth_flux_fespace(ConstructFiniteElementSpaceHierarchy( - iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), - smooth_projector(smooth_flux_fespace, iodata.solver.linear.tol, 200, 0, + mfem::ParFiniteElementSpaceHierarchy &nd_fespaces) + : mat_op(mat_op), nd_fespaces(nd_fespaces), + smooth_projector(nd_fespaces, iodata.solver.linear.tol, 200, 0, iodata.solver.pa_order_threshold), - coarse_flux_fec(iodata.solver.order, mesh.back()->Dimension(), - mfem::BasisType::GaussLobatto), - coarse_flux_fespace(mesh.back().get(), &coarse_flux_fec, mesh.back()->Dimension()), - scalar_mass_matrices(fespace.GetNE()), smooth_to_coarse_embed(fespace.GetNE()) + smooth_flux(nd_fespaces.GetFinestFESpace().GetTrueVSize()), + flux_rhs(nd_fespaces.GetFinestFESpace().GetTrueVSize()) { - mfem::MassIntegrator mass_integrator; - for (int e = 0; e < fespace.GetNE(); e++) - { - // Loop over each element, and save an elemental mass matrix. - // Will exploit the fact that vector L2 mass matrix components are independent. - const auto &coarse_fe = *coarse_flux_fespace.GetFE(e); - auto &T = *coarse_flux_fespace.GetElementTransformation(e); - mass_integrator.AssembleElementMatrix(coarse_fe, T, scalar_mass_matrices[e]); - - const auto &smooth_fe = *smooth_flux_fespace.GetFinestFESpace().GetFE(e); - coarse_fe.Project(smooth_fe, T, smooth_to_coarse_embed[e]); - } } template <> -IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, - bool normalize) const +IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators( + const ComplexVector& v, bool normalize) { - const int sdim = fespace.GetMesh()->SpaceDimension(); - mfem::ParComplexGridFunction field(&fespace); + auto &nd_fespace = nd_fespaces.GetFinestFESpace(); + mfem::ParComplexGridFunction field(&nd_fespace); field.real().SetFromTrueDofs(v.Real()); field.imag().SetFromTrueDofs(v.Imag()); - const int nelem = smooth_flux_fespace.GetFinestFESpace().GetNE(); - - // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). - CurlFluxCoefficient real_coef(field.real(), mat_op), imag_coef(field.imag(), mat_op); - auto rhs_from_coef = [](mfem::ParFiniteElementSpace &flux_fespace, auto &coef) - { - Vector RHS(flux_fespace.GetTrueVSize()); - - mfem::LinearForm rhs(&flux_fespace); - rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); - rhs.UseFastAssembly(false); - rhs.Assemble(); - flux_fespace.GetProlongationMatrix()->MultTranspose(rhs, RHS); - - return RHS; - }; + const int nelem = nd_fespace.GetNE(); - const auto smooth_flux_rhs = - ComplexVector(rhs_from_coef(smooth_flux_fespace.GetFinestFESpace(), real_coef), - rhs_from_coef(smooth_flux_fespace.GetFinestFESpace(), imag_coef)); - - // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass - // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - auto build_flux = [](const FluxProjector&proj, const ComplexVector &flux_coef) - { - // Use a copy construction to match appropriate size. - ComplexVector flux(flux_coef); - proj.Mult(flux_coef, flux); - return flux; - }; - auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); - - // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction - // for evaluation. - auto build_func = [](const ComplexVector &f, mfem::ParFiniteElementSpace &fespace) - { - mfem::ParComplexGridFunction flux(&fespace); - flux.real().SetFromTrueDofs(f.Real()); - flux.imag().SetFromTrueDofs(f.Imag()); - flux.real().ExchangeFaceNbrData(); - flux.imag().ExchangeFaceNbrData(); - return flux; - }; - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fespace.GetFinestFESpace()); - mfem::ParComplexGridFunction coarse_flux_func(&coarse_flux_fespace); - coarse_flux_func.real().ProjectCoefficient(real_coef); - coarse_flux_func.imag().ProjectCoefficient(imag_coef); - - // Loop over elements, embed the smooth flux into the coarse flux space, then compute - // squared integral using a component-wise mass matrix. - Vector smooth_vec, coarse_vec, sub_vec, estimates(nelem); + Vector smooth_vec, coarse_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - for (int e = 0; e < fespace.GetNE(); e++) - { - // real - smooth_flux_func.real().GetElementDofValues(e, smooth_vec); - coarse_flux_func.real().GetElementDofValues(e, coarse_vec); - const int ndof = coarse_vec.Size() / sdim; - sub_vec.SetSize(ndof); - for (int c = 0; c < 3; c++) - { - sub_vec.MakeRef(coarse_vec, c * ndof); - normalization += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); - } + mfem::ParGridFunction flux_func(&nd_fespaces.GetFinestFESpace()); + for (bool real : {true, false}) + { + auto &field_component = real ? field.real() : field.imag(); - smooth_to_coarse_embed[e].AddMult(smooth_vec, coarse_vec, -1.0); - for (int c = 0; c < 3; c++) + // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). + CurlFluxCoefficient coef(field_component, mat_op); { - sub_vec.MakeRef(coarse_vec, c * ndof); - estimates[e] += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); + mfem::LinearForm rhs(&nd_fespace); + rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); + rhs.UseFastAssembly(false); + rhs.Assemble(); + nd_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); } - // imag - smooth_flux_func.imag().GetElementDofValues(e, smooth_vec); - coarse_flux_func.imag().GetElementDofValues(e, coarse_vec); + // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass + // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. + smooth_projector.Mult(flux_rhs, smooth_flux); + flux_func.SetFromTrueDofs(smooth_flux); + flux_func.ExchangeFaceNbrData(); - for (int c = 0; c < 3; c++) - { - sub_vec.MakeRef(coarse_vec, c * ndof); - normalization += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); - } - - smooth_to_coarse_embed[e].AddMult(smooth_vec, coarse_vec, -1.0); - for (int c = 0; c < 3; c++) + // Loop over elements and accumulate the estimates from this component + for (int e = 0; e < nd_fespace.GetNE(); e++) { - sub_vec.MakeRef(coarse_vec, c * ndof); - estimates[e] += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); + auto &T = *nd_fespace.GetElementTransformation(e); + // integration order 2p + q + const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), 2 * nd_fespace.GetFE(e)->GetOrder() + T.Order()); + for (const auto &ip : ir) + { + T.SetIntPoint(&ip); + + flux_func.GetVectorValue(e, ip, smooth_vec); + coef.Eval(coarse_vec, T, ip); + double w_i = ip.weight * T.Weight(); + for (int c = 0; c < 3; c++) + { + estimates[e] += w_i * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); + normalization += w_i * coarse_vec[c] * coarse_vec[c]; + } + } } - - estimates[e] = std::sqrt(estimates[e]); + } + for (auto &e : estimates) + { + e = std::sqrt(e); } Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); @@ -233,40 +169,27 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl } template <> -IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, - bool normalize) const +IndicatorsAndNormalization +CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) { - const int sdim = fespace.GetMesh()->SpaceDimension(); - mfem::ParGridFunction field(&fespace); + auto &nd_fespace = nd_fespaces.GetFinestFESpace(); + mfem::ParGridFunction field(&nd_fespace); field.SetFromTrueDofs(v); - const int nelem = smooth_flux_fespace.GetFinestFESpace().GetNE(); + const int nelem = nd_fespaces.GetFinestFESpace().GetNE(); // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). CurlFluxCoefficient coef(field, mat_op); - auto rhs_from_coef = [](mfem::ParFiniteElementSpace &flux_fespace, auto &coef) { - Vector RHS(flux_fespace.GetTrueVSize()); - - mfem::LinearForm rhs(&flux_fespace); + mfem::LinearForm rhs(&nd_fespace); rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); rhs.UseFastAssembly(false); rhs.Assemble(); - flux_fespace.GetProlongationMatrix()->MultTranspose(rhs, RHS); - - return RHS; - }; - const auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fespace.GetFinestFESpace(), coef); + nd_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); + } // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - auto build_flux = [](const FluxProjector &proj, const Vector &flux_coef) - { - // Use a copy construction to match appropriate size. - Vector flux(flux_coef); - proj.Mult(flux_coef, flux); - return flux; - }; - auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); + smooth_projector.Mult(flux_rhs, smooth_flux); // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction // for evaluation. @@ -277,35 +200,27 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vecto flux.ExchangeFaceNbrData(); return flux; }; - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fespace.GetFinestFESpace()); - mfem::ParGridFunction coarse_flux_func(&coarse_flux_fespace); - coarse_flux_func.ProjectCoefficient(coef); + auto smooth_flux_func = build_func(smooth_flux, nd_fespaces.GetFinestFESpace()); - // Loop over elements, embed the smooth flux into the coarse flux space, then compute - // squared integral using a component-wise mass matrix. - Vector smooth_vec, coarse_vec, sub_vec, estimates(nelem); + Vector smooth_vec, coarse_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - for (int e = 0; e < fespace.GetNE(); e++) + for (int e = 0; e < nd_fespace.GetNE(); e++) { - smooth_flux_func.GetElementDofValues(e, smooth_vec); - coarse_flux_func.GetElementDofValues(e, coarse_vec); - - const int ndof = coarse_vec.Size() / sdim; - sub_vec.SetSize(ndof); - for (int c = 0; c < 3; c++) + auto &T = *nd_fespace.GetElementTransformation(e); + // integration order 2p + q + const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), 2 * nd_fespace.GetFE(e)->GetOrder() + T.Order()); + for (const auto &ip : ir) { - sub_vec.MakeRef(coarse_vec, c * ndof); - normalization += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); + T.SetIntPoint(&ip); + smooth_flux_func.GetVectorValue(e, ip, smooth_vec); + coef.Eval(coarse_vec, T, ip); + for (int c = 0; c < 3; c++) + { + estimates[e] += ip.weight * T.Weight() * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); + normalization += ip.weight * T.Weight() * coarse_vec[c] * coarse_vec[c]; + } } - - smooth_to_coarse_embed[e].AddMult(smooth_vec, coarse_vec, -1.0); - for (int c = 0; c < 3; c++) - { - sub_vec.MakeRef(coarse_vec, c * ndof); - estimates[e] += scalar_mass_matrices[e].InnerProduct(sub_vec, sub_vec); - } - estimates[e] = std::sqrt(estimates[e]); } diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 32f4fc0c1..aadaae876 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -64,30 +64,18 @@ class CurlFluxErrorEstimator { const MaterialOperator &mat_op; // The finite element space used to represent V, and F. - mfem::ParFiniteElementSpace &fespace; - - std::vector> smooth_flux_fecs; - mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_fespace; + mfem::ParFiniteElementSpaceHierarchy &nd_fespaces; mutable FluxProjector smooth_projector; - - mfem::L2_FECollection coarse_flux_fec; - mutable mfem::ParFiniteElementSpace coarse_flux_fespace; - - std::vector scalar_mass_matrices; - std::vector smooth_to_coarse_embed; - - mutable ComplexVector complex_flux; - mutable Vector real_flux; + mutable Vector smooth_flux, flux_rhs; public: // Constructor for using geometric and p multigrid. CurlFluxErrorEstimator(const IoData &iodata, const MaterialOperator &mat_op, - const std::vector> &mesh, - mfem::ParFiniteElementSpace &fespace); + mfem::ParFiniteElementSpaceHierarchy &nd_fespaces); // Compute elemental error indicators given a complex vector of true DOF. template - IndicatorsAndNormalization ComputeIndicators(const VectorType &v, bool normalize) const; + IndicatorsAndNormalization ComputeIndicators(const VectorType &v, bool normalize); }; // Class used for computing grad flux error estimate, i.e. || ϵ ∇ ϕₕ - F ||_K where F From 3d65f399d4d80fba4e2d386a48e2d85e36a1de4d Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 13:33:31 -0400 Subject: [PATCH 13/39] style fixes --- palace/drivers/drivensolver.cpp | 15 +++---- palace/drivers/drivensolver.hpp | 8 ++-- palace/drivers/electrostaticsolver.cpp | 3 +- palace/drivers/magnetostaticsolver.cpp | 6 +-- palace/drivers/transientsolver.cpp | 4 +- palace/fem/integrator.hpp | 5 +-- palace/linalg/errorestimator.cpp | 55 +++++++++++++++----------- palace/linalg/errorestimator.hpp | 4 +- palace/utils/errorindicators.cpp | 9 ++--- palace/utils/timer.hpp | 24 +++++------ 10 files changed, 65 insertions(+), 68 deletions(-) diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 2b384d80a..6fa2938a8 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -110,8 +110,8 @@ DrivenSolver::Solve(const std::vector> &mesh) con } ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, - int nstep, int step0, double omega0, + CurlFluxErrorEstimator &estimator, int nstep, + int step0, double omega0, double delta_omega) const { // Construct the system matrices defining the linear operator. PEC boundaries are handled @@ -146,8 +146,8 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); - auto UpdateErrorIndicators = [this, &estimator, &indicators, - &postop](const auto &E, int step, double f) + auto UpdateErrorIndicators = + [this, &estimator, &indicators, &postop](const auto &E, int step, double f) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; @@ -227,8 +227,8 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator } ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, - int nstep, int step0, double omega0, + CurlFluxErrorEstimator &estimator, int nstep, + int step0, double omega0, double delta_omega) const { // Configure default parameters if not specified. @@ -282,7 +282,8 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // The error indicators will be calculated for each HDM sample rather than for // the online stage. ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); - auto UpdateErrorIndicators = [this, &estimator, &indicators](const auto &E, int step, double frequency) + auto UpdateErrorIndicators = + [this, &estimator, &indicators](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTSOLVE); constexpr bool normalized = true; diff --git a/palace/drivers/drivensolver.hpp b/palace/drivers/drivensolver.hpp index 458786639..ff82bd684 100644 --- a/palace/drivers/drivensolver.hpp +++ b/palace/drivers/drivensolver.hpp @@ -37,11 +37,11 @@ class DrivenSolver : public BaseSolver int GetNumSteps(double start, double end, double delta) const; ErrorIndicators SweepUniform(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, - int step0, double omega0, double delta_omega) const; + CurlFluxErrorEstimator &estimator, int nstep, int step0, + double omega0, double delta_omega) const; ErrorIndicators SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, - int step0, double omega0, double delta_omega) const; + CurlFluxErrorEstimator &estimator, int nstep, int step0, + double omega0, double delta_omega) const; void Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, const WavePortOperator &wave_port_op, diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index c98e94738..8b9d25fcb 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -101,7 +101,8 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, mfem::DenseMatrix C(nstep), Cm(nstep); Vector E(Grad->Height()), Vij(Grad->Width()); ErrorIndicators indicators(laplaceop.GlobalTrueVSize(), laplaceop.GetComm()); - auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &V, int i, double idx) + auto UpdateErrorIndicators = + [this, &estimator, &indicators, &postop](const auto &V, int i, double idx) { BlockTimer bt0(Timer::ESTSOLVE); constexpr bool normalized = true; diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index c288ebed8..c8000d1a7 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -105,10 +105,10 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators indicators(curlcurlop.GlobalTrueVSize(), curlcurlop.GetComm()); - auto UpdateErrorIndicators = [this, &estimator, &indicators, - &postop](const auto &A, int i, double idx) + auto UpdateErrorIndicators = + [this, &estimator, &indicators, &postop](const auto &A, int i, double idx) { - BlockTimer bt(Timer::ESTIMATION); + BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto estimate = estimator.ComputeIndicators(A, normalized); BlockTimer bt1(Timer::POSTPRO); diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 43588b6d7..5f5010455 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -84,8 +84,8 @@ TransientSolver::Solve(const std::vector> &mesh) return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); - auto UpdateErrorIndicators = [this, &estimator, &indicators, - &postop](const auto &E, int step, double time) + auto UpdateErrorIndicators = + [this, &estimator, &indicators, &postop](const auto &E, int step, double time) { BlockTimer bt0(Timer::ESTSOLVE); // Initial flux of zero would return nan. diff --git a/palace/fem/integrator.hpp b/palace/fem/integrator.hpp index aadf1f02a..5cd6734b4 100644 --- a/palace/fem/integrator.hpp +++ b/palace/fem/integrator.hpp @@ -43,10 +43,7 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator, mfem::Vector f_loc, f_hat; public: - VectorFEBoundaryLFIntegrator(mfem::VectorCoefficient &QG) - : Q(QG), f_loc(QG.GetVDim()) - { - } + VectorFEBoundaryLFIntegrator(mfem::VectorCoefficient &QG) : Q(QG), f_loc(QG.GetVDim()) {} void AssembleRHSElementVect(const mfem::FiniteElement &fe, mfem::ElementTransformation &Tr, diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index f14bc6df9..a4725f7c8 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -1,19 +1,19 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -#include #include "errorestimator.hpp" +#include #include "fem/coefficient.hpp" #include "fem/integrator.hpp" #include "fem/multigrid.hpp" -#include "models/materialoperator.hpp" -#include "utils/communication.hpp" -#include "utils/errorindicators.hpp" -#include "utils/iodata.hpp" #include "linalg/amg.hpp" #include "linalg/gmg.hpp" #include "linalg/iterative.hpp" #include "linalg/rap.hpp" +#include "models/materialoperator.hpp" +#include "utils/communication.hpp" +#include "utils/errorindicators.hpp" +#include "utils/iodata.hpp" #include "utils/timer.hpp" namespace palace @@ -29,8 +29,9 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie { constexpr int skip_zeros = 0; - constexpr bool ScalarFESpace = std::is_same::value - || std::is_same::value; + constexpr bool ScalarFESpace = + std::is_same::value || + std::is_same::value; // Assemble the bilinear form operator auto M = std::make_unique(h.GetNumLevels()); @@ -61,8 +62,11 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie } template -FluxProjector::FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fespace, double tol, int max_it, int print, int pa_order_threshold) - : M(BuildMassMatrixOperator(smooth_flux_fespace, pa_order_threshold)) +FluxProjector::FluxProjector( + mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fespace, double tol, int max_it, + int print, int pa_order_threshold) + : M(BuildMassMatrixOperator(smooth_flux_fespace, + pa_order_threshold)) { // The system matrix for the projection is real and SPD. For the coarse-level AMG solve, // we don't use an exact solve on the coarsest level. @@ -98,8 +102,8 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( } template <> -IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators( - const ComplexVector& v, bool normalize) +IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, + bool normalize) { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); mfem::ParComplexGridFunction field(&nd_fespace); @@ -116,7 +120,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators( { auto &field_component = real ? field.real() : field.imag(); - // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). + // Coefficients for computing the discontinuous flux component, i.e. (W, μ⁻¹∇ × V). CurlFluxCoefficient coef(field_component, mat_op); { mfem::LinearForm rhs(&nd_fespace); @@ -137,7 +141,8 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators( { auto &T = *nd_fespace.GetElementTransformation(e); // integration order 2p + q - const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), 2 * nd_fespace.GetFE(e)->GetOrder() + T.Order()); + const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), + 2 * nd_fespace.GetFE(e)->GetOrder() + T.Order()); for (const auto &ip : ir) { T.SetIntPoint(&ip); @@ -169,8 +174,8 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators( } template <> -IndicatorsAndNormalization -CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) +IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, + bool normalize) { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); mfem::ParGridFunction field(&nd_fespace); @@ -209,16 +214,18 @@ CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) { auto &T = *nd_fespace.GetElementTransformation(e); // integration order 2p + q - const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), 2 * nd_fespace.GetFE(e)->GetOrder() + T.Order()); + const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), + 2 * nd_fespace.GetFE(e)->GetOrder() + T.Order()); for (const auto &ip : ir) { T.SetIntPoint(&ip); smooth_flux_func.GetVectorValue(e, ip, smooth_vec); coef.Eval(coarse_vec, T, ip); + const double w_i = ip.weight * T.Weight(); for (int c = 0; c < 3; c++) { - estimates[e] += ip.weight * T.Weight() * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); - normalization += ip.weight * T.Weight() * coarse_vec[c] * coarse_vec[c]; + estimates[e] += w_i * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); + normalization += w_i * coarse_vec[c] * coarse_vec[c]; } } estimates[e] = std::sqrt(estimates[e]); @@ -242,11 +249,12 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( smooth_flux_fecs(ConstructFECollections( iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_coarsen_type, false)), - smooth_flux_component_fespace(ConstructFiniteElementSpaceHierarchy( - iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), + smooth_flux_component_fespace( + ConstructFiniteElementSpaceHierarchy( + iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, + iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), smooth_flux_fespace(mesh.back().get(), smooth_flux_fecs.back().get(), - mesh.back()->Dimension()), + mesh.back()->Dimension()), smooth_projector(smooth_flux_component_fespace, iodata.solver.linear.tol, 200, 0, iodata.solver.pa_order_threshold), coarse_flux_fec(iodata.solver.order, mesh.back()->Dimension(), @@ -270,7 +278,7 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( } IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vector &v, - bool normalize) const + bool normalize) const { const int sdim = fespace.GetMesh()->SpaceDimension(); mfem::ParGridFunction field(&fespace); @@ -386,5 +394,4 @@ IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vecto return {estimates, normalization}; } - } // namespace palace diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index aadaae876..2a03eb546 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -71,7 +71,7 @@ class CurlFluxErrorEstimator public: // Constructor for using geometric and p multigrid. CurlFluxErrorEstimator(const IoData &iodata, const MaterialOperator &mat_op, - mfem::ParFiniteElementSpaceHierarchy &nd_fespaces); + mfem::ParFiniteElementSpaceHierarchy &nd_fespaces); // Compute elemental error indicators given a complex vector of true DOF. template @@ -92,7 +92,7 @@ class GradFluxErrorEstimator std::vector> smooth_flux_fecs; mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_component_fespace; mutable mfem::ParFiniteElementSpace smooth_flux_fespace; - mutable FluxProjectorsmooth_projector; + mutable FluxProjector smooth_projector; mfem::L2_FECollection coarse_flux_fec; mutable mfem::ParFiniteElementSpace coarse_flux_fespace; diff --git a/palace/utils/errorindicators.cpp b/palace/utils/errorindicators.cpp index 13caa99c1..d7deb9e72 100644 --- a/palace/utils/errorindicators.cpp +++ b/palace/utils/errorindicators.cpp @@ -29,8 +29,7 @@ ErrorIndicators::ErrorIndicators(const IndicatorsAndNormalization &indicators, mean = global_error_indicator / size; } -void -ErrorIndicators::Reset() +void ErrorIndicators::Reset() { n = 0; local_error_indicators = Vector(); @@ -38,8 +37,7 @@ ErrorIndicators::Reset() mean_normalization = 0.0; } -void -ErrorIndicators::AddEstimates(const Vector &indicators, double normalization) +void ErrorIndicators::AddEstimates(const Vector &indicators, double normalization) { // Compute the global indicator across all processors. constexpr int p = 2; @@ -64,8 +62,7 @@ ErrorIndicators::AddEstimates(const Vector &indicators, double normalization) "Local error indicator vectors mismatch."); // Combine these error indicators into the current average. std::transform(local_error_indicators.begin(), local_error_indicators.end(), - indicators.begin(), local_error_indicators.begin(), - running_average); + indicators.begin(), local_error_indicators.begin(), running_average); } else { diff --git a/palace/utils/timer.hpp b/palace/utils/timer.hpp index 81bd1d692..30f686ee3 100644 --- a/palace/utils/timer.hpp +++ b/palace/utils/timer.hpp @@ -28,7 +28,7 @@ class Timer { INIT = 0, CONSTRUCT, - WAVEPORT, // Wave port solver + WAVEPORT, // Wave port solver SOLVE, PRECONDITIONER, // Linear solver COARSESOLVE, // Linear solver @@ -43,20 +43,14 @@ class Timer NUMTIMINGS }; - inline static const std::vector descriptions{"Initialization", - "Operator Construction", - " Wave Ports", - "Solve", - " Preconditioner", - " Coarse Solve", - "Estimation", - " Construction", - " Evaluation", - "PROM Construction", - "PROM Solve", - "Postprocessing", - "Disk IO", - "Total"}; + inline static const std::vector descriptions{ + "Initialization", "Operator Construction", + " Wave Ports", "Solve", + " Preconditioner", " Coarse Solve", + "Estimation", " Construction", + " Evaluation", "PROM Construction", + "PROM Solve", "Postprocessing", + "Disk IO", "Total"}; private: const TimePoint start_time; From 512bce414eeb88b4944ef626ab62fedd37a230c7 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 15:29:15 -0400 Subject: [PATCH 14/39] Change grad flux estimation to use matrix free. A little slower than the previous version, but much less memory --- palace/drivers/electrostaticsolver.cpp | 8 +- palace/drivers/electrostaticsolver.hpp | 2 +- palace/fem/coefficient.hpp | 27 ++- palace/linalg/errorestimator.cpp | 217 +++++++++---------------- palace/linalg/errorestimator.hpp | 22 +-- palace/utils/errorindicators.hpp | 16 -- 6 files changed, 106 insertions(+), 186 deletions(-) diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 8b9d25fcb..f61d2121f 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -73,8 +73,7 @@ ElectrostaticSolver::Solve(const std::vector> &me auto estimator = [&]() { BlockTimer bt(Timer::ESTCONSTRUCT); - return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), mesh, - laplaceop.GetH1Space()); + return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), laplaceop.GetH1Spaces()); }(); // Postprocess the capacitance matrix from the computed field solutions. @@ -85,7 +84,7 @@ ElectrostaticSolver::Solve(const std::vector> &me ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, - const GradFluxErrorEstimator &estimator, + GradFluxErrorEstimator &estimator, const std::vector &V) const { // Postprocess the Maxwell capacitance matrix. See p. 97 of the COMSOL AC/DC Module manual @@ -104,7 +103,7 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop](const auto &V, int i, double idx) { - BlockTimer bt0(Timer::ESTSOLVE); + BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto estimate = estimator.ComputeIndicators(V, normalized); BlockTimer bt1(Timer::POSTPRO); @@ -133,6 +132,7 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostprocessDomains(postop, "i", i, idx, Ue, 0.0, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, Ue, 0.0, 1.0, 0.0); PostprocessProbes(postop, "i", i, idx); + Mpi::Print("Computing error estimates for terminal {:d}\n", idx); UpdateErrorIndicators(V[i], i, idx); if (i < iodata.solver.electrostatic.n_post) { diff --git a/palace/drivers/electrostaticsolver.hpp b/palace/drivers/electrostaticsolver.hpp index 5891aa293..f8070553f 100644 --- a/palace/drivers/electrostaticsolver.hpp +++ b/palace/drivers/electrostaticsolver.hpp @@ -37,7 +37,7 @@ class ElectrostaticSolver : public BaseSolver { private: ErrorIndicators Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, - const GradFluxErrorEstimator &estimator, + GradFluxErrorEstimator &estimator, const std::vector &V) const; void PostprocessTerminals(const std::map> &terminal_sources, diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index 83756b706..574e4051f 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -503,26 +503,35 @@ class CurlFluxCoefficient : public mfem::VectorCoefficient }; // Computes the flux, ϵ ∇ ϕ, of the electrostatic potential ϕ. -class GradFluxCoefficient : public mfem::VectorCoefficient +class GradFluxCoefficient : public mfem::Coefficient { private: - const mfem::ParGridFunction φ + const mfem::ParGridFunction &gf; const MaterialOperator &mat_op; - mfem::Vector grad; + mfem::Vector grad, V; + int component; public: GradFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) - : mfem::VectorCoefficient(gf.ParFESpace()->GetParMesh()->SpaceDimension()), phi(gf), - mat_op(mat_op), grad(3) + : mfem::Coefficient(), gf(gf), mat_op(mat_op), + grad(gf.ParFESpace()->GetParMesh()->SpaceDimension()), + V(gf.ParFESpace()->GetParMesh()->SpaceDimension()), component(-1) { } - void Eval(mfem::Vector &V, mfem::ElementTransformation &T, - const mfem::IntegrationPoint &ip) override + // Specify the component + void SetComponent(int i) { - V.SetSize(3); - phi.GetGradient(T, grad); + MFEM_ASSERT(i >= 0 && i < gf.ParFESpace()->GetParMesh()->SpaceDimension(), "Invalid component index!"); + component = i; + } + + double Eval(mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip) override + { + MFEM_ASSERT(component >= 0 && component < gf.ParFESpace()->GetParMesh()->SpaceDimension(), "Invalid component index, try calling SetComponent(int)!"); + gf.GetGradient(T, grad); mat_op.GetPermittivityReal(T.Attribute).Mult(grad, V); + return V(component); } }; diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index a4725f7c8..90c38a1b8 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -42,7 +42,7 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie if constexpr (ScalarFESpace) { - MFEM_ASSERT(h_l.GetVDim() == 1, + MFEM_VERIFY(h_l.GetVDim() == 1, "Scalar mass matrix hierarchy assumes a component-wise solve."); m->AddDomainIntegrator(new mfem::MassIntegrator); } @@ -94,8 +94,7 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &nd_fespaces) : mat_op(mat_op), nd_fespaces(nd_fespaces), - smooth_projector(nd_fespaces, iodata.solver.linear.tol, 200, 0, - iodata.solver.pa_order_threshold), + smooth_projector(nd_fespaces, iodata.solver.linear.tol, 200, 1, iodata.solver.pa_order_threshold), smooth_flux(nd_fespaces.GetFinestFESpace().GetTrueVSize()), flux_rhs(nd_fespaces.GetFinestFESpace().GetTrueVSize()) { @@ -115,7 +114,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl estimates = 0.0; double normalization = 0.0; - mfem::ParGridFunction flux_func(&nd_fespaces.GetFinestFESpace()); + mfem::ParGridFunction flux_func(&nd_fespace); for (bool real : {true, false}) { auto &field_component = real ? field.real() : field.imag(); @@ -123,16 +122,17 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl // Coefficients for computing the discontinuous flux component, i.e. (W, μ⁻¹∇ × V). CurlFluxCoefficient coef(field_component, mat_op); { + // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass + // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. + Mpi::Print("Computing smooth flux approximation of {} component\n", real ? "real" : "imaginary" ); + BlockTimer bt(Timer::ESTSOLVE); mfem::LinearForm rhs(&nd_fespace); rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); rhs.UseFastAssembly(false); rhs.Assemble(); nd_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); + smooth_projector.Mult(flux_rhs, smooth_flux); } - - // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass - // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - smooth_projector.Mult(flux_rhs, smooth_flux); flux_func.SetFromTrueDofs(smooth_flux); flux_func.ExchangeFaceNbrData(); @@ -180,22 +180,23 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vecto auto &nd_fespace = nd_fespaces.GetFinestFESpace(); mfem::ParGridFunction field(&nd_fespace); field.SetFromTrueDofs(v); - const int nelem = nd_fespaces.GetFinestFESpace().GetNE(); + const int nelem = nd_fespace.GetNE(); // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). CurlFluxCoefficient coef(field, mat_op); { + // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass + // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. + Mpi::Print("Computing smooth flux approximation\n"); + BlockTimer bt(Timer::ESTSOLVE); mfem::LinearForm rhs(&nd_fespace); rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); rhs.UseFastAssembly(false); rhs.Assemble(); nd_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); + smooth_projector.Mult(flux_rhs, smooth_flux); } - // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass - // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - smooth_projector.Mult(flux_rhs, smooth_flux); - // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction // for evaluation. auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fespace) @@ -205,7 +206,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vecto flux.ExchangeFaceNbrData(); return flux; }; - auto smooth_flux_func = build_func(smooth_flux, nd_fespaces.GetFinestFESpace()); + auto smooth_flux_func = build_func(smooth_flux, nd_fespace); Vector smooth_vec, coarse_vec, estimates(nelem); estimates = 0.0; @@ -243,149 +244,87 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vecto GradFluxErrorEstimator::GradFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, - const std::vector> &mesh, - mfem::ParFiniteElementSpace &fespace) - : mat_op(mat_op), fespace(fespace), - smooth_flux_fecs(ConstructFECollections( - iodata.solver.order, mesh.back()->Dimension(), iodata.solver.linear.mg_max_levels, - iodata.solver.linear.mg_coarsen_type, false)), - smooth_flux_component_fespace( - ConstructFiniteElementSpaceHierarchy( - iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - iodata.solver.pa_order_threshold, mesh, smooth_flux_fecs)), - smooth_flux_fespace(mesh.back().get(), smooth_flux_fecs.back().get(), - mesh.back()->Dimension()), - smooth_projector(smooth_flux_component_fespace, iodata.solver.linear.tol, 200, 0, - iodata.solver.pa_order_threshold), - coarse_flux_fec(iodata.solver.order, mesh.back()->Dimension(), - mfem::BasisType::GaussLobatto), - coarse_flux_fespace(mesh.back().get(), &coarse_flux_fec, mesh.back()->Dimension()), - scalar_mass_matrices(fespace.GetNE()), smooth_to_coarse_embed(fespace.GetNE()) + mfem::ParFiniteElementSpaceHierarchy &h1_fespaces) + : mat_op(mat_op), h1_fespaces(h1_fespaces), + smooth_projector(h1_fespaces, iodata.solver.linear.tol, 200, 1, iodata.solver.pa_order_threshold), + smooth_flux(h1_fespaces.GetFinestFESpace().GetTrueVSize()), + flux_rhs(h1_fespaces.GetFinestFESpace().GetTrueVSize()) { - mfem::MassIntegrator mass_integrator; - - for (int e = 0; e < fespace.GetNE(); e++) - { - // Loop over each element, and save an elemental mass matrix. - // Will exploit the fact that vector L2 mass matrix components are independent. - const auto &coarse_fe = *coarse_flux_fespace.GetFE(e); - auto &T = *fespace.GetElementTransformation(e); - mass_integrator.AssembleElementMatrix(coarse_fe, T, scalar_mass_matrices[e]); - - const auto &smooth_fe = *smooth_flux_component_fespace.GetFinestFESpace().GetFE(e); - coarse_fe.Project(smooth_fe, T, smooth_to_coarse_embed[e]); - } } IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const { - const int sdim = fespace.GetMesh()->SpaceDimension(); - mfem::ParGridFunction field(&fespace); + auto &h1_fespace = h1_fespaces.GetFinestFESpace(); + const int sdim = h1_fespace.GetMesh()->SpaceDimension(); + mfem::ParGridFunction field(&h1_fespace); field.SetFromTrueDofs(v); - const int nelem = smooth_flux_fespace.GetNE(); + const int nelem = h1_fespace.GetNE(); - // Coefficients for computing the discontinuous flux., i.e. (V, ϵ ∇ ϕ). - GradFluxCoefficient coef(field, mat_op); - auto rhs_from_coef = [](mfem::ParFiniteElementSpace &fespace, auto &coef) - { - Vector RHS(fespace.GetTrueVSize()); - - mfem::LinearForm rhs(&fespace); - rhs.AddDomainIntegrator(new mfem::VectorDomainLFIntegrator(coef)); - rhs.UseFastAssembly(false); - rhs.Assemble(); - fespace.GetProlongationMatrix()->MultTranspose(rhs, RHS); - - return RHS; - }; - auto smooth_flux_rhs = rhs_from_coef(smooth_flux_fespace, coef); + Vector estimates(nelem); + estimates = 0.0; + double normalization = 0.0; - // Given the RHS vector of non-smooth flux, construct a flux projector and perform - // component wise mass matrix inversion in the appropriate space, giving fᵢ = M⁻¹ f̂ᵢ. - auto build_flux = [sdim](const FluxProjector &proj, Vector &rhs) + // Coefficient for computing the discontinuous flux., i.e. (Vᵢ, (ϵ ∇ ϕ)ᵢ). + GradFluxCoefficient coef(field, mat_op); + mfem::ParGridFunction smooth_flux_func(&h1_fespace); + for (int c = 0; c < 3; c++) { - // Use a copy construction to match appropriate size. - Vector flux(rhs.Size()); - flux = 0.0; - - // Apply the flux projector component wise. - const int ndof = flux.Size(); - const int stride = ndof / sdim; - MFEM_ASSERT(ndof % 3 == 0, "!"); - - Vector flux_comp, rhs_comp; - for (int i = 0; i < 3; i++) + coef.SetComponent(c); { - flux_comp.MakeRef(flux, i * stride, stride); - rhs_comp.MakeRef(rhs, i * stride, stride); - proj.Mult(rhs_comp, flux_comp); + // Given the RHS vector of non-smooth flux component, compute fᵢ = M⁻¹ f̂ᵢ. + Mpi::Print("Computing smooth flux approximation of component {}\n", c); + BlockTimer bt0(Timer::ESTSOLVE); + mfem::LinearForm rhs(&h1_fespace); + rhs.AddDomainIntegrator(new mfem::DomainLFIntegrator(coef)); + rhs.UseFastAssembly(false); + rhs.Assemble(); + h1_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); + smooth_projector.Mult(flux_rhs, smooth_flux); } - return flux; - }; - auto smooth_flux = build_flux(smooth_projector, smooth_flux_rhs); - - // Given a solution represented with a Vector, build a GridFunction for evaluation. - auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fespace) - { - mfem::ParGridFunction flux(&fespace); - flux.SetFromTrueDofs(f); - flux.ExchangeFaceNbrData(); - return flux; - }; - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fespace); - - mfem::ParGridFunction coarse_flux(&coarse_flux_fespace); - coarse_flux.ProjectCoefficient(coef); - - Vector coarse_vec, smooth_vec, coarse_sub_vec, smooth_sub_vec, estimates(nelem); - estimates = 0.0; - double normalization = 0.0; - for (int e = 0; e < fespace.GetNE(); e++) - { - coarse_flux.GetElementDofValues(e, coarse_vec); - smooth_flux_func.GetElementDofValues(e, smooth_vec); - - MFEM_ASSERT(coarse_vec.Size() == smooth_vec.Size() && coarse_vec.Size() % 3 == 0, - "H1d and L2d spaces should have the same number of elemental dofs, " - "and should be exactly divisible by 3: " - << coarse_vec.Size() << " " << smooth_vec.Size()); - - const int ndof = coarse_vec.Size() / sdim; - coarse_sub_vec.SetSize(ndof); - smooth_sub_vec.SetSize(ndof); - for (int c = 0; c < 3; c++) + smooth_flux_func.SetFromTrueDofs(smooth_flux); + smooth_flux_func.ExchangeFaceNbrData(); + for (int e = 0; e < h1_fespace.GetNE(); e++) { - coarse_sub_vec.MakeRef(coarse_vec, c * ndof); - smooth_sub_vec.MakeRef(smooth_vec, c * ndof); - - normalization += scalar_mass_matrices[e].InnerProduct(coarse_sub_vec, coarse_sub_vec); - smooth_to_coarse_embed[e].AddMult(smooth_sub_vec, coarse_sub_vec, -1.0); // Embed - estimates[e] += scalar_mass_matrices[e].InnerProduct(coarse_sub_vec, - coarse_sub_vec); // Integrate + auto &T = *h1_fespace.GetElementTransformation(e); + // integration order 2p + q + const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), + 2 * h1_fespace.GetFE(e)->GetOrder() + T.Order()); + for (const auto &ip : ir) + { + T.SetIntPoint(&ip); + double smooth_val = smooth_flux_func.GetValue(e, ip); + double coarse_val = coef.Eval(T, ip); + const double w_i = ip.weight * T.Weight(); + estimates[e] += w_i * std::pow(smooth_val - coarse_val, 2.0); + // std::cout << "smooth_val " << smooth_val << " coarse_val " << coarse_val << std::endl; + normalization += w_i * std::pow(coarse_val, 2.0); + } + } + if constexpr (false) + { + // Debugging branch generates some intermediate fields for paraview. + mfem::ParaViewDataCollection paraview("debug_coeff" + std::to_string(c), h1_fespace.GetParMesh()); + paraview.RegisterCoeffField("Flux", &coef); + paraview.RegisterField("SmoothFlux", &smooth_flux_func); + + mfem::L2_FECollection est_fec(0, sdim); + mfem::ParFiniteElementSpace est_fespace(h1_fespace.GetParMesh(), &est_fec); + mfem::ParGridFunction est_field(&est_fespace); + est_field.SetFromTrueDofs(estimates); + + paraview.RegisterField("ErrorIndicator", &est_field); + paraview.Save(); } - estimates[e] = std::sqrt(estimates[e]); } - Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); - normalization = std::sqrt(normalization); - - if constexpr (false) + for (auto &e : estimates) { - // Debugging branch generates some intermediate fields for paraview. - mfem::ParaViewDataCollection paraview("debug", fespace.GetParMesh()); - paraview.RegisterVCoeffField("Flux", &coef); - - auto smooth_flux_func = build_func(smooth_flux, smooth_flux_fespace); - paraview.RegisterField("SmoothFlux", &smooth_flux_func); + e = std::sqrt(e); + } - mfem::L2_FECollection est_fec(0, 3); - mfem::ParFiniteElementSpace est_fespace(fespace.GetParMesh(), &est_fec); - mfem::ParGridFunction est_field(&est_fespace); - est_field.SetFromTrueDofs(estimates); + Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); + normalization = std::sqrt(normalization); - paraview.RegisterField("ErrorIndicator", &est_field); - paraview.Save(); - } if (normalize) { std::for_each(estimates.begin(), estimates.end(), diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 2a03eb546..4c0ce7201 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -83,28 +83,16 @@ class CurlFluxErrorEstimator class GradFluxErrorEstimator { const MaterialOperator &mat_op; - // The finite element space used to represent ϕ. - mfem::ParFiniteElementSpace &fespace; - - // Collections and spaces for the smooth flux. Note the hierarchy uses the SCALAR finite - // element space, whilst the true flux is in the VECTOR finite element space. This allows - // for a component wise inversion. - std::vector> smooth_flux_fecs; - mutable mfem::ParFiniteElementSpaceHierarchy smooth_flux_component_fespace; - mutable mfem::ParFiniteElementSpace smooth_flux_fespace; - mutable FluxProjector smooth_projector; - - mfem::L2_FECollection coarse_flux_fec; - mutable mfem::ParFiniteElementSpace coarse_flux_fespace; + // The finite element space used to represent ϕ, and components of F + mfem::ParFiniteElementSpaceHierarchy &h1_fespaces; - std::vector scalar_mass_matrices; - std::vector smooth_to_coarse_embed; + mutable FluxProjector smooth_projector; + mutable Vector smooth_flux, flux_rhs; public: // Constructor for using geometric and p multigrid. GradFluxErrorEstimator(const IoData &iodata, const MaterialOperator &mat_op, - const std::vector> &mesh, - mfem::ParFiniteElementSpace &fespace); + mfem::ParFiniteElementSpaceHierarchy &h1_fespaces); // Compute elemental error indicators given a vector of true DOF. IndicatorsAndNormalization ComputeIndicators(const Vector &v, bool normalize) const; diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index 15d3b4b8f..61457b161 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -79,22 +79,6 @@ class ErrorIndicators mutable int n = 0; }; -// // Operator for performing reduction of a vector of local indicators into a -// // global running total for use in adaptation. -// class ErrorReductionOperator -// { -// // Number of samples. Mutability required to guarantee operation. -// mutable int n = 0; - -// public: -// // Reduce a vector indicators, i, computed with norm p, into the combined -// // error indicator e. -// void operator()(ErrorIndicators &e, const IndicatorsAndNormalization &i, -// double p = 2) const; -// // Resets the internal counter for number of samples. -// void Reset() { n = 0; } -// }; - } // namespace palace #endif // PALACE_UTILS_ERROR_INDICATORS_HPP From 3051448c5f36492ae25ba2d991043c44acdd0cf2 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 16:51:50 -0400 Subject: [PATCH 15/39] Refactor to combine many elements in the ErrorIndicator capability - Drop carry the TrueDof around. This will be added in with Adaptivity when it is needed. - Add printing of useful information to the log, including the run time. - Redefine the reduction operator so that the global indicator computed from combining the reduced local indicators, is equal to the average of the global estimates from each sample. --- palace/drivers/basesolver.cpp | 24 +++---- palace/drivers/basesolver.hpp | 10 ++- palace/drivers/drivensolver.cpp | 57 +++++++++++------ palace/drivers/eigensolver.cpp | 27 +++++--- palace/drivers/electrostaticsolver.cpp | 26 +++++--- palace/drivers/magnetostaticsolver.cpp | 26 +++++--- palace/drivers/transientsolver.cpp | 25 +++++--- palace/linalg/errorestimator.cpp | 6 +- palace/linalg/errorestimator.hpp | 4 +- palace/main.cpp | 2 +- palace/utils/errorindicators.cpp | 89 ++++++++------------------ palace/utils/errorindicators.hpp | 87 ++++++++++++------------- 12 files changed, 200 insertions(+), 183 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index be62bced6..a67adf5d2 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -567,7 +567,7 @@ void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double } void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, double time, - const ErrorIndicators &indicators, + double global, double min, double max, double mean, double normalization, bool normalized) const { if (post_dir.length() == 0) @@ -596,17 +596,17 @@ void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, d // clang-format off output.print("{:{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", time, table.w1, table.p1, - indicators.GetGlobalErrorIndicator(), table.w, table.p, - indicators.GetMinErrorIndicator(), table.w, table.p, - indicators.GetMaxErrorIndicator(), table.w, table.p, - indicators.GetMeanErrorIndicator(), table.w, table.p, - indicators.GetNormalization(), table.w, table.p); + global, table.w, table.p, + min, table.w, table.p, + max, table.w, table.p, + mean, table.w, table.p, + normalization, table.w, table.p); // clang-format on } } void BaseSolver::PostprocessErrorIndicators(const std::string &name, - const ErrorIndicators &indicators) const + double global, double min, double max, double mean, double normalization) const { if (post_dir.length() == 0) { @@ -621,11 +621,11 @@ void BaseSolver::PostprocessErrorIndicators(const std::string &name, // clang-format off output.print("{:>{}s},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", name, table.w1, - indicators.GetGlobalErrorIndicator(), table.w, table.p, - indicators.GetMinErrorIndicator(), table.w, table.p, - indicators.GetMaxErrorIndicator(), table.w, table.p, - indicators.GetMeanErrorIndicator(), table.w, table.p, - indicators.GetNormalization(), table.w, table.p); + global, table.w, table.p, + min, table.w, table.p, + max, table.w, table.p, + mean, table.w, table.p, + normalization, table.w, table.p); // clang-format on } } diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index 42c6571e1..cfc9fc426 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -72,19 +72,17 @@ class BaseSolver // Postprocess granular error indicator to file. The argument normalized indicates if // supplied indicators have already been normalized. void PostprocessErrorIndicators(const std::string &name, int step, double time, - const ErrorIndicators &indicators, bool normalized) const; + double global, double min, double max, double mean, double normalization, bool normalized) const; // Write a string labeled error indicator. Used for writing statistics. void PostprocessErrorIndicators(const std::string &name, - const ErrorIndicators &indicators) const; - + double global, double min, double max, double mean, double normalization) const; public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, const char *git_tag = nullptr); virtual ~BaseSolver() = default; - // Performs a solve using the mesh sequence, and reports error indicators. - virtual ErrorIndicators - Solve(const std::vector> &mesh) const = 0; + // Performs a solve using the mesh sequence, then reports error indicators. + virtual ErrorIndicators Solve(const std::vector> &mesh) const = 0; // These methods write different simulation metadata to a JSON file in post_dir. void SaveMetadata(const mfem::ParFiniteElementSpaceHierarchy &fespaces) const; diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 6fa2938a8..067bde5af 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -145,21 +145,24 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator B = 0.0; // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); + ErrorIndicators combined_indicators; auto UpdateErrorIndicators = - [this, &estimator, &indicators, &postop](const auto &E, int step, double f) + [this, &estimator, &combined_indicators, &postop, &spaceop](const auto &E, int step, double f) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto estimate = estimator.ComputeIndicators(E, normalized); + auto indicators = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); - // Write the indicator for this mode. - postop.SetIndicatorGridFunction(estimate.indicators); + postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators( "f (GHz)", step, f, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + indicators.GetMinErrorIndicator(spaceop.GetComm()), + indicators.GetMaxErrorIndicator(spaceop.GetComm()), + indicators.GetMeanErrorIndicator(spaceop.GetComm()), + indicators.GetNormalization(), normalized); - indicators.AddEstimates(estimate.indicators, estimate.normalization); + combined_indicators.AddIndicators(indicators); }; // Main frequency sweep loop. @@ -222,8 +225,13 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator omega += delta_omega; } SaveMetadata(ksp); - PostprocessErrorIndicators("Mean", indicators); - return indicators; + PostprocessErrorIndicators("Mean", + combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), + combined_indicators.GetNormalization()); + return combined_indicators; } ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, @@ -281,20 +289,23 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // The error indicators will be calculated for each HDM sample rather than for // the online stage. - ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); + ErrorIndicators combined_indicators; auto UpdateErrorIndicators = - [this, &estimator, &indicators](const auto &E, int step, double frequency) + [this, &estimator, &combined_indicators, &spaceop](const auto &E, int step, double frequency) { - BlockTimer bt0(Timer::ESTSOLVE); + BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto estimate = estimator.ComputeIndicators(E, normalized); + auto indicators = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); - // Write the indicator for this mode. PostprocessErrorIndicators( "f (GHz)", step, frequency, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + indicators.GetMinErrorIndicator(spaceop.GetComm()), + indicators.GetMaxErrorIndicator(spaceop.GetComm()), + indicators.GetMeanErrorIndicator(spaceop.GetComm()), + indicators.GetNormalization(), normalized); - indicators.AddEstimates(estimate.indicators, estimate.normalization); + combined_indicators.AddIndicators(indicators); }; // Initialize the basis with samples from the top and bottom of the frequency @@ -303,9 +314,11 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // of the RomOperator. BlockTimer bt1(Timer::CONSTRUCTPROM); prom.SolveHDM(omega0, E); // Print matrix stats at first HDM solve + Mpi::Print("Computing error estimates for frequency {:d} (GHz)\n", omega0 * f0); UpdateErrorIndicators(E, 0, omega0 * f0); prom.AddHDMSample(omega0, E); prom.SolveHDM(omega0 + (nstep - step0 - 1) * delta_omega, E); + Mpi::Print("Computing error estimates for frequency {:d} (GHz)\n", (omega0 + (nstep - step0 - 1) * delta_omega) * f0); UpdateErrorIndicators(E, 1, (omega0 + (nstep - step0 - 1) * delta_omega) * f0); prom.AddHDMSample(omega0 + (nstep - step0 - 1) * delta_omega, E); @@ -329,6 +342,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator iter - iter0 + 1, prom.GetReducedDimension(), omega_star * f0, omega_star, max_error); prom.SolveHDM(omega_star, E); + Mpi::Print("Computing error estimates\n"); UpdateErrorIndicators(E, iter - iter0 + 1, omega_star * f0); prom.AddHDMSample(omega_star, E); iter++; @@ -343,8 +357,13 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator SaveMetadata(prom.GetLinearSolver()); // Set the indicator field to the combined field for postprocessing. - PostprocessErrorIndicators("Mean", indicators); // Report the mean value - postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); + PostprocessErrorIndicators("Mean", + combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), + combined_indicators.GetNormalization()); + postop.SetIndicatorGridFunction(combined_indicators.GetLocalErrorIndicators()); // Main fast frequency sweep loop (online phase). BlockTimer bt2(Timer::CONSTRUCT); @@ -393,7 +412,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator step++; omega += delta_omega; } - return indicators; + return combined_indicators; } int DrivenSolver::GetNumSteps(double start, double end, double delta) const diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 790f8ff50..c44478e0b 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -264,21 +264,24 @@ EigenSolver::Solve(const std::vector> &mesh) cons SaveMetadata(*ksp); // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); + ErrorIndicators combined_indicators; auto UpdateErrorIndicators = - [this, &estimator, &indicators, &postop](const auto &E, int i) + [this, &estimator, &combined_indicators, &postop, &spaceop](const auto &E, int i) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto estimate = estimator.ComputeIndicators(E, normalized); + auto indicators = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); - postop.SetIndicatorGridFunction(estimate.indicators); - // Write the indicator for this mode. + postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators( "m", i, i + 1, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + indicators.GetMinErrorIndicator(spaceop.GetComm()), + indicators.GetMaxErrorIndicator(spaceop.GetComm()), + indicators.GetMeanErrorIndicator(spaceop.GetComm()), + indicators.GetNormalization(), normalized); - indicators.AddEstimates(estimate.indicators, estimate.normalization); + combined_indicators.AddIndicators(indicators); }; // Postprocess the results. @@ -315,6 +318,7 @@ EigenSolver::Solve(const std::vector> &mesh) cons if (i < iodata.solver.eigenmode.n) { // Only update the error indicator for targeted modes. + Mpi::Print("Computing error estimates for mode {:d}\n", i); UpdateErrorIndicators(E, i); } @@ -325,8 +329,13 @@ EigenSolver::Solve(const std::vector> &mesh) cons // Postprocess the mode. Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv); } - PostprocessErrorIndicators("Mean", indicators); - return indicators; + PostprocessErrorIndicators("Mean", + combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), + combined_indicators.GetNormalization()); + return combined_indicators; } void EigenSolver::Postprocess(const PostOperator &postop, diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index f61d2121f..9477760d3 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -99,21 +99,24 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, int nstep = static_cast(terminal_sources.size()); mfem::DenseMatrix C(nstep), Cm(nstep); Vector E(Grad->Height()), Vij(Grad->Width()); - ErrorIndicators indicators(laplaceop.GlobalTrueVSize(), laplaceop.GetComm()); + ErrorIndicators combined_indicators; auto UpdateErrorIndicators = - [this, &estimator, &indicators, &postop](const auto &V, int i, double idx) + [this, &estimator, &combined_indicators, &postop, &laplaceop](const auto &V, int i, double idx) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto estimate = estimator.ComputeIndicators(V, normalized); + auto indicators = estimator.ComputeIndicators(V, normalized); BlockTimer bt1(Timer::POSTPRO); - // Write the indicator for this mode. - postop.SetIndicatorGridFunction(estimate.indicators); + postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators( "i", i, idx, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), + indicators.GetMinErrorIndicator(laplaceop.GetComm()), + indicators.GetMaxErrorIndicator(laplaceop.GetComm()), + indicators.GetMeanErrorIndicator(laplaceop.GetComm()), + indicators.GetNormalization(), normalized); - indicators.AddEstimates(estimate.indicators, estimate.normalization); + combined_indicators.AddIndicators(indicators); }; if (iodata.solver.electrostatic.n_post > 0) { @@ -173,8 +176,13 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, mfem::DenseMatrix Cinv(C); Cinv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(terminal_sources, C, Cinv, Cm); - PostprocessErrorIndicators("Mean", indicators); - return indicators; + PostprocessErrorIndicators("Mean", + combined_indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(laplaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(laplaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(laplaceop.GetComm()), + combined_indicators.GetNormalization()); + return combined_indicators; } void ElectrostaticSolver::PostprocessTerminals( diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index c8000d1a7..ca7c0845b 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -104,21 +104,25 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, Vector Iinc(nstep); // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators indicators(curlcurlop.GlobalTrueVSize(), curlcurlop.GetComm()); + ErrorIndicators combined_indicators; auto UpdateErrorIndicators = - [this, &estimator, &indicators, &postop](const auto &A, int i, double idx) + [this, &estimator, &combined_indicators, &postop, &curlcurlop](const auto &A, int i, double idx) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto estimate = estimator.ComputeIndicators(A, normalized); + auto indicators = estimator.ComputeIndicators(A, normalized); BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. - postop.SetIndicatorGridFunction(estimate.indicators); + postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators( "i", i, idx, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + indicators.GetGlobalErrorIndicator(curlcurlop.GetComm()), + indicators.GetMinErrorIndicator(curlcurlop.GetComm()), + indicators.GetMaxErrorIndicator(curlcurlop.GetComm()), + indicators.GetMeanErrorIndicator(curlcurlop.GetComm()), + indicators.GetNormalization(), normalized); - indicators.AddEstimates(estimate.indicators, estimate.normalization); + combined_indicators.AddIndicators(indicators); }; if (iodata.solver.magnetostatic.n_post > 0) { @@ -141,6 +145,7 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostprocessDomains(postop, "i", i, idx, 0.0, Um, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, 0.0, Um, 0.0, Iinc(i)); PostprocessProbes(postop, "i", i, idx); + Mpi::Print("Computing error estimates for terminal {:d}\n", idx); UpdateErrorIndicators(A[i], i, idx); if (i < iodata.solver.magnetostatic.n_post) { @@ -181,8 +186,13 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, mfem::DenseMatrix Minv(M); Minv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(surf_j_op, M, Minv, Mm); - PostprocessErrorIndicators("Mean", indicators); - return indicators; + PostprocessErrorIndicators("Mean", + combined_indicators.GetGlobalErrorIndicator(curlcurlop.GetComm()), + combined_indicators.GetMinErrorIndicator(curlcurlop.GetComm()), + combined_indicators.GetMaxErrorIndicator(curlcurlop.GetComm()), + combined_indicators.GetMeanErrorIndicator(curlcurlop.GetComm()), + combined_indicators.GetNormalization()); + return combined_indicators; } void MagnetostaticSolver::PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 5f5010455..c8660a89f 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -83,21 +83,25 @@ TransientSolver::Solve(const std::vector> &mesh) BlockTimer bt(Timer::ESTCONSTRUCT); return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); - ErrorIndicators indicators(spaceop.GlobalTrueVSize(), spaceop.GetComm()); + ErrorIndicators combined_indicators; auto UpdateErrorIndicators = - [this, &estimator, &indicators, &postop](const auto &E, int step, double time) + [this, &estimator, &combined_indicators, &postop, &spaceop](const auto &E, int step, double time) { BlockTimer bt0(Timer::ESTSOLVE); // Initial flux of zero would return nan. bool constexpr normalized = false; - auto estimate = estimator.ComputeIndicators(E, normalized); + auto indicators = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); - postop.SetIndicatorGridFunction(estimate.indicators); + postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators( "t (ns)", step, time, - ErrorIndicators{estimate, indicators.GlobalTrueVSize(), indicators.GetComm()}, + indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + indicators.GetMinErrorIndicator(spaceop.GetComm()), + indicators.GetMaxErrorIndicator(spaceop.GetComm()), + indicators.GetMeanErrorIndicator(spaceop.GetComm()), + indicators.GetNormalization(), normalized); - indicators.AddEstimates(estimate.indicators, estimate.normalization); + combined_indicators.AddIndicators(indicators); }; // Main time integration loop. @@ -152,8 +156,13 @@ TransientSolver::Solve(const std::vector> &mesh) step++; } SaveMetadata(timeop.GetLinearSolver()); - PostprocessErrorIndicators("Mean", indicators); - return indicators; + PostprocessErrorIndicators("Mean", + combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), + combined_indicators.GetNormalization()); + return combined_indicators; } std::function TransientSolver::GetTimeExcitation(bool dot) const { diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 90c38a1b8..acc3a515a 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -101,7 +101,7 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( } template <> -IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, +ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, bool normalize) { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); @@ -174,7 +174,7 @@ IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Compl } template <> -IndicatorsAndNormalization CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, +ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); @@ -252,7 +252,7 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( { } -IndicatorsAndNormalization GradFluxErrorEstimator::ComputeIndicators(const Vector &v, +ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const { auto &h1_fespace = h1_fespaces.GetFinestFESpace(); diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 4c0ce7201..567538aab 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -75,7 +75,7 @@ class CurlFluxErrorEstimator // Compute elemental error indicators given a complex vector of true DOF. template - IndicatorsAndNormalization ComputeIndicators(const VectorType &v, bool normalize); + ErrorIndicators ComputeIndicators(const VectorType &v, bool normalize); }; // Class used for computing grad flux error estimate, i.e. || ϵ ∇ ϕₕ - F ||_K where F @@ -95,7 +95,7 @@ class GradFluxErrorEstimator mfem::ParFiniteElementSpaceHierarchy &h1_fespaces); // Compute elemental error indicators given a vector of true DOF. - IndicatorsAndNormalization ComputeIndicators(const Vector &v, bool normalize) const; + ErrorIndicators ComputeIndicators(const Vector &v, bool normalize) const; }; } // namespace palace diff --git a/palace/main.cpp b/palace/main.cpp index 7be814a39..f101ab062 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -189,7 +189,7 @@ int main(int argc, char *argv[]) auto solver_output = solver->Solve(mesh); Mpi::Print(world_comm, "Error Estimate: {:.3e}\n", - solver_output.GetGlobalErrorIndicator()); + solver_output.GetGlobalErrorIndicator(world_comm)); // Print timing summary. BlockTimer::Print(world_comm); diff --git a/palace/utils/errorindicators.cpp b/palace/utils/errorindicators.cpp index d7deb9e72..36bb93bd0 100644 --- a/palace/utils/errorindicators.cpp +++ b/palace/utils/errorindicators.cpp @@ -7,81 +7,44 @@ namespace palace { - -ErrorIndicators::ErrorIndicators(const IndicatorsAndNormalization &indicators, - int global_true_v_size, MPI_Comm comm) - : local_error_indicators(indicators.indicators), global_true_v_size(global_true_v_size), - comm(comm), mean_normalization(indicators.normalization) -{ - // Compute the global indicator across all processors. - constexpr int p = 2; - global_error_indicator = - std::transform_reduce(local_error_indicators.begin(), local_error_indicators.end(), - 0.0, std::plus(), [](auto val) { return std::pow(val, p); }); - Mpi::GlobalSum(1, &global_error_indicator, comm); - global_error_indicator = std::pow(global_error_indicator, 1.0 / p); - min = local_error_indicators.Min(); - max = local_error_indicators.Max(); - int size = local_error_indicators.Size(); - Mpi::GlobalMin(1, &min, comm); - Mpi::GlobalMax(1, &max, comm); - Mpi::GlobalSum(1, &size, comm); - mean = global_error_indicator / size; -} - -void ErrorIndicators::Reset() -{ - n = 0; - local_error_indicators = Vector(); - global_error_indicator = std::numeric_limits::max(); - mean_normalization = 0.0; -} - -void ErrorIndicators::AddEstimates(const Vector &indicators, double normalization) +void ErrorIndicators::AddIndicators(const ErrorIndicators &error_indicators) { - // Compute the global indicator across all processors. - constexpr int p = 2; - double candidate_global_error_indicator = - std::transform_reduce(indicators.begin(), indicators.end(), 0.0, std::plus(), - [](auto val) { return std::pow(val, p); }); - Mpi::GlobalSum(1, &candidate_global_error_indicator, comm); - candidate_global_error_indicator = std::pow(candidate_global_error_indicator, 1.0 / p); - - // Update the global error indicator and local error indicators to the new mean normalized - // values. The average local indicator is used rather than the indicator for the maximum + // The average local indicator is used rather than the indicator for the maximum // error to drive the adaptation, to account for a local error that might be marginally // important to many solves, rather than only large in one solve. if (n > 0) { - global_error_indicator = - (n * global_error_indicator + candidate_global_error_indicator) / (n + 1); - mean_normalization = (n * mean_normalization + normalization) / (n + 1); - auto running_average = [this](const auto &xbar, const auto &x) - { return (xbar * n + x) / (n + 1); }; - MFEM_VERIFY(local_error_indicators.Size() == indicators.Size(), + normalization = (n * normalization + error_indicators.normalization * error_indicators.n) / (n + error_indicators.n); + + // The local indicators must be squared before combining, so that the global error + // calculation is valid: + // E = sqrt(1/N sum_N sum_K eta_{kn}^2) + // from which it follows that: + // E^2 = 1/N sum_N sum_K eta_{kn}^2 + // = 1/N sum_N E_n + // Namely the average of the global error indicators included in the reduction. + // Squaring both sides means the summation can be rearranged, and then the local error + // indicators become: + // e_K = sqrt(1/N sum_N eta_{Kn}^2) + auto running_average = [this, error_indicators](const auto &xbar, const auto &x) + { + return std::sqrt((xbar * xbar * n + x * x * error_indicators.n) / (n + error_indicators.n)); + }; + MFEM_VERIFY(local.Size() == error_indicators.local.Size(), "Local error indicator vectors mismatch."); // Combine these error indicators into the current average. - std::transform(local_error_indicators.begin(), local_error_indicators.end(), - indicators.begin(), local_error_indicators.begin(), running_average); + std::transform(local.begin(), local.end(), + error_indicators.local.begin(), local.begin(), running_average); + + // More samples have been added, update for the running average lambda. + n += error_indicators.n; } else { - global_error_indicator = candidate_global_error_indicator; - local_error_indicators = indicators; - mean_normalization = normalization; + // This indicator was empty, just steal. + (*this) = error_indicators; } - // Assumes that the global error indicator is already reduced across processors. - min = local_error_indicators.Min(); - max = local_error_indicators.Max(); - int size = local_error_indicators.Size(); - Mpi::GlobalMin(1, &min, comm); - Mpi::GlobalMax(1, &max, comm); - Mpi::GlobalSum(1, &size, comm); - mean = global_error_indicator / size; - - // Another sample has been added, increment for the running average lambda. - n++; } } // namespace palace diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index 61457b161..1165685b7 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -9,30 +9,16 @@ namespace palace { -class ErrorReductionOperator; - -// Unnormalized error indicators and the normalization factor. -struct IndicatorsAndNormalization -{ - Vector indicators; - double normalization; -}; - // Storage for error estimation results from the solve. Required in the AMR loop. This is // richer than the IndicatorsAndNormalization because it stores derived quantities, and a // communicator for use in adaptation. class ErrorIndicators { public: - // Construct an Error indicator from an initial set of indicators - explicit ErrorIndicators(const IndicatorsAndNormalization &indicators, - int global_true_v_size, MPI_Comm comm); - // Construct an empty ErrorIndicators. - explicit ErrorIndicators(int global_true_v_size, MPI_Comm comm) - : global_true_v_size(global_true_v_size), comm(comm) - { - } - ErrorIndicators() = delete; + ErrorIndicators(Vector local, double normalization) + : local(std::move(local)), normalization(normalization), n(1) {} + + ErrorIndicators() = default; ErrorIndicators(const ErrorIndicators &) = default; ErrorIndicators(ErrorIndicators &&) = default; ErrorIndicators &operator=(const ErrorIndicators &) = default; @@ -40,43 +26,58 @@ class ErrorIndicators ~ErrorIndicators() = default; // Return the average normalized local error indicators. - const auto &GetLocalErrorIndicators() const { return local_error_indicators; } + const auto &GetLocalErrorIndicators() const { return local; } // Return the global error indicator. - auto GetGlobalErrorIndicator() const { return global_error_indicator; } + inline auto GetGlobalErrorIndicator(MPI_Comm comm) const + { + constexpr int p = 2; + double global_error_indicator = + std::transform_reduce(local.begin(), local.end(), + 0.0, std::plus(), [](auto val) { return std::pow(val, p); }); + Mpi::GlobalSum(1, &global_error_indicator, comm); + return std::pow(global_error_indicator, 1.0 / p); + } // Return the largest normalized local error indicator. - auto GetMaxErrorIndicator() const { return max; } + inline auto GetMaxErrorIndicator(MPI_Comm comm) const + { + double max = local.Max(); + Mpi::GlobalMax(1, &max, comm); + return max; + } // Return the smallest normalized local error indicator. - auto GetMinErrorIndicator() const { return min; } + inline auto GetMinErrorIndicator(MPI_Comm comm) const + { + double min = local.Min(); + Mpi::GlobalMin(1, &min, comm); + return min; + } // Return the mean normalized local error indicator. - auto GetMeanErrorIndicator() const { return mean; } + inline auto GetMeanErrorIndicator(MPI_Comm comm) const + { + int size = local.Size(); + auto global_error_indicator = GetGlobalErrorIndicator(comm); + return global_error_indicator / size; + } // Return the normalization constant for the absolute error. - auto GetNormalization() const { return mean_normalization; } - // The communicator used in any reductions over processors. - const MPI_Comm &GetComm() { return comm; } - // Return the global number of true dofs associated with this set of error indicators. - auto GlobalTrueVSize() const { return global_true_v_size; } + inline auto GetNormalization() const { return normalization; } // Add a set of indicators to the running totals. - void AddEstimates(const Vector &indicators, double normalization); + void AddIndicators(const ErrorIndicators &indicators); // Reset a running total of error indicators ready for computing a new running average. - void Reset(); + inline void Reset() + { + n = 0; + local = Vector(); + normalization = 0; + } protected: // Elemental localized error indicators. Used for marking elements for // refinement and coarsening. - Vector local_error_indicators; - // Global error indicator. Used for driving AMR and diagnostics. - double global_error_indicator = 0; - // Global number of true dof in the finite element solution for which this indicator is - // calculated. - int global_true_v_size; - // Communicator used in calculation of the global error indicator. - MPI_Comm comm; - // Statistics, updated simultaneously with the global error indicator. - double min, max, mean; - // Mean normalization constant. - double mean_normalization; + Vector local; + // Normalization constant. + double normalization; // Number of samples. Mutability required to guarantee operation. - mutable int n = 0; + int n = 0; }; } // namespace palace From d9e23c1a8bb09ce3050b8684ab5459c50942cea7 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 17:12:42 -0400 Subject: [PATCH 16/39] More PR comments and removing more allocations from the estimator calculation methods --- palace/fem/multigrid.hpp | 54 +++++++++++-------------- palace/linalg/errorestimator.cpp | 64 ++++++++++++------------------ palace/linalg/errorestimator.hpp | 8 ++-- palace/main.cpp | 1 - palace/models/curlcurloperator.cpp | 4 +- palace/models/laplaceoperator.cpp | 2 +- palace/models/spaceoperator.cpp | 4 +- 7 files changed, 58 insertions(+), 79 deletions(-) diff --git a/palace/fem/multigrid.hpp b/palace/fem/multigrid.hpp index 71726f97d..0efda94c5 100644 --- a/palace/fem/multigrid.hpp +++ b/palace/fem/multigrid.hpp @@ -15,7 +15,7 @@ namespace palace::fem { // -// Methods for constructing hierarchies of finite element spaces for multigrid. +// Methods for constructing hierarchies of finite element spaces for geometric multigrid. // // Helper function for getting the order of the finite element space underlying a bilinear @@ -131,27 +131,37 @@ std::vector> inline ConstructFECollections( } // Construct a hierarchy of finite element spaces given a sequence of meshes and -// finite element collections. Uses geometric multigrid and p multigrid. +// finite element collections. Dirichlet boundary conditions are additionally +// marked. template inline mfem::ParFiniteElementSpaceHierarchy ConstructFiniteElementSpaceHierarchy( int mg_max_levels, bool mg_legacy_transfer, int pa_order_threshold, const std::vector> &mesh, - const std::vector> &fecs, int dim = 1) + const std::vector> &fecs, + const mfem::Array *dbc_marker = nullptr, + std::vector> *dbc_tdof_lists = nullptr) { - MFEM_VERIFY(!mesh.empty() && !fecs.empty(), + MFEM_VERIFY(!mesh.empty() && !fecs.empty() && + (!dbc_tdof_lists || dbc_tdof_lists->empty()), "Empty mesh or FE collection for FE space construction!"); int coarse_mesh_l = std::max(0, static_cast(mesh.size() + fecs.size()) - 1 - mg_max_levels); - auto *fespace = - new mfem::ParFiniteElementSpace(mesh[coarse_mesh_l].get(), fecs[0].get(), dim); + auto *fespace = new mfem::ParFiniteElementSpace(mesh[coarse_mesh_l].get(), fecs[0].get()); + if (dbc_marker && dbc_tdof_lists) + { + fespace->GetEssentialTrueDofs(*dbc_marker, dbc_tdof_lists->emplace_back()); + } mfem::ParFiniteElementSpaceHierarchy fespaces(mesh[coarse_mesh_l].get(), fespace, false, true); // h-refinement for (std::size_t l = coarse_mesh_l + 1; l < mesh.size(); l++) { - fespace = new mfem::ParFiniteElementSpace(mesh[l].get(), fecs[0].get(), dim); - + fespace = new mfem::ParFiniteElementSpace(mesh[l].get(), fecs[0].get()); + if (dbc_marker && dbc_tdof_lists) + { + fespace->GetEssentialTrueDofs(*dbc_marker, dbc_tdof_lists->emplace_back()); + } auto *P = new ParOperator( std::make_unique(fespaces.GetFinestFESpace(), *fespace), fespaces.GetFinestFESpace(), *fespace, true); @@ -161,8 +171,11 @@ inline mfem::ParFiniteElementSpaceHierarchy ConstructFiniteElementSpaceHierarchy // p-refinement for (std::size_t l = 1; l < fecs.size(); l++) { - fespace = new mfem::ParFiniteElementSpace(mesh.back().get(), fecs[l].get(), dim); - + fespace = new mfem::ParFiniteElementSpace(mesh.back().get(), fecs[l].get()); + if (dbc_marker && dbc_tdof_lists) + { + fespace->GetEssentialTrueDofs(*dbc_marker, dbc_tdof_lists->emplace_back()); + } ParOperator *P; if (!mg_legacy_transfer && mfem::DeviceCanUseCeed()) { @@ -184,27 +197,6 @@ inline mfem::ParFiniteElementSpaceHierarchy ConstructFiniteElementSpaceHierarchy return fespaces; } -// Overload for treatment of Dirichlet boundary conditions, extracts the true -// dof vectors for each level within a finite element space hierarchy. -template typename Container, - typename... MeshT> -mfem::ParFiniteElementSpaceHierarchy ConstructFiniteElementSpaceHierarchy( - int mg_max_levels, bool mg_legacy_transfer, int pa_order_threshold, - const Container &mesh, const std::vector> &fecs, - const mfem::Array &dbc_marker, std::vector> &dbc_tdof_lists, - int dim = 1) -{ - dbc_tdof_lists.clear(); - auto fespaces = ConstructFiniteElementSpaceHierarchy(mg_max_levels, mg_legacy_transfer, - pa_order_threshold, mesh, fecs, dim); - for (int l = 0; l < fespaces.GetNumLevels(); l++) - { - fespaces.GetFESpaceAtLevel(l).GetEssentialTrueDofs(dbc_marker, - dbc_tdof_lists.emplace_back()); - } - return fespaces; -} - } // namespace palace::fem #endif // PALACE_FEM_MULTIGRID_HPP diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index acc3a515a..ca3167ad3 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -96,31 +96,27 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( : mat_op(mat_op), nd_fespaces(nd_fespaces), smooth_projector(nd_fespaces, iodata.solver.linear.tol, 200, 1, iodata.solver.pa_order_threshold), smooth_flux(nd_fespaces.GetFinestFESpace().GetTrueVSize()), - flux_rhs(nd_fespaces.GetFinestFESpace().GetTrueVSize()) + flux_rhs(nd_fespaces.GetFinestFESpace().GetTrueVSize()), + field_gf(&nd_fespaces.GetFinestFESpace()), smooth_flux_gf(&nd_fespaces.GetFinestFESpace()) { } template <> ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, - bool normalize) + bool normalize) const { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); - mfem::ParComplexGridFunction field(&nd_fespace); - field.real().SetFromTrueDofs(v.Real()); - field.imag().SetFromTrueDofs(v.Imag()); const int nelem = nd_fespace.GetNE(); Vector smooth_vec, coarse_vec, estimates(nelem); estimates = 0.0; double normalization = 0.0; - - mfem::ParGridFunction flux_func(&nd_fespace); for (bool real : {true, false}) { - auto &field_component = real ? field.real() : field.imag(); + field_gf.SetFromTrueDofs(real ? v.Real() : v.Imag()); // Coefficients for computing the discontinuous flux component, i.e. (W, μ⁻¹∇ × V). - CurlFluxCoefficient coef(field_component, mat_op); + CurlFluxCoefficient coef(field_gf, mat_op); { // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. @@ -133,8 +129,8 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v nd_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); smooth_projector.Mult(flux_rhs, smooth_flux); } - flux_func.SetFromTrueDofs(smooth_flux); - flux_func.ExchangeFaceNbrData(); + smooth_flux_gf.SetFromTrueDofs(smooth_flux); + smooth_flux_gf.ExchangeFaceNbrData(); // Loop over elements and accumulate the estimates from this component for (int e = 0; e < nd_fespace.GetNE(); e++) @@ -147,7 +143,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v { T.SetIntPoint(&ip); - flux_func.GetVectorValue(e, ip, smooth_vec); + smooth_flux_gf.GetVectorValue(e, ip, smooth_vec); coef.Eval(coarse_vec, T, ip); double w_i = ip.weight * T.Weight(); for (int c = 0; c < 3; c++) @@ -163,7 +159,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v e = std::sqrt(e); } - Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); + Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm()); normalization = std::sqrt(normalization); if (normalize) { @@ -174,16 +170,14 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v } template <> -ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, - bool normalize) +ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); - mfem::ParGridFunction field(&nd_fespace); - field.SetFromTrueDofs(v); + field_gf.SetFromTrueDofs(v); const int nelem = nd_fespace.GetNE(); // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). - CurlFluxCoefficient coef(field, mat_op); + CurlFluxCoefficient coef(field_gf, mat_op); { // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. @@ -199,14 +193,8 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction // for evaluation. - auto build_func = [](const Vector &f, mfem::ParFiniteElementSpace &fespace) - { - mfem::ParGridFunction flux(&fespace); - flux.SetFromTrueDofs(f); - flux.ExchangeFaceNbrData(); - return flux; - }; - auto smooth_flux_func = build_func(smooth_flux, nd_fespace); + smooth_flux_gf.SetFromTrueDofs(smooth_flux); + smooth_flux_gf.ExchangeFaceNbrData(); Vector smooth_vec, coarse_vec, estimates(nelem); estimates = 0.0; @@ -220,7 +208,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, for (const auto &ip : ir) { T.SetIntPoint(&ip); - smooth_flux_func.GetVectorValue(e, ip, smooth_vec); + smooth_flux_gf.GetVectorValue(e, ip, smooth_vec); coef.Eval(coarse_vec, T, ip); const double w_i = ip.weight * T.Weight(); for (int c = 0; c < 3; c++) @@ -232,7 +220,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, estimates[e] = std::sqrt(estimates[e]); } - Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); + Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm()); normalization = std::sqrt(normalization); if (normalize) { @@ -248,7 +236,8 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( : mat_op(mat_op), h1_fespaces(h1_fespaces), smooth_projector(h1_fespaces, iodata.solver.linear.tol, 200, 1, iodata.solver.pa_order_threshold), smooth_flux(h1_fespaces.GetFinestFESpace().GetTrueVSize()), - flux_rhs(h1_fespaces.GetFinestFESpace().GetTrueVSize()) + flux_rhs(h1_fespaces.GetFinestFESpace().GetTrueVSize()), + field_gf(&h1_fespaces.GetFinestFESpace()), smooth_flux_gf(&h1_fespaces.GetFinestFESpace()) { } @@ -257,8 +246,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, { auto &h1_fespace = h1_fespaces.GetFinestFESpace(); const int sdim = h1_fespace.GetMesh()->SpaceDimension(); - mfem::ParGridFunction field(&h1_fespace); - field.SetFromTrueDofs(v); + field_gf.SetFromTrueDofs(v); const int nelem = h1_fespace.GetNE(); Vector estimates(nelem); @@ -266,8 +254,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, double normalization = 0.0; // Coefficient for computing the discontinuous flux., i.e. (Vᵢ, (ϵ ∇ ϕ)ᵢ). - GradFluxCoefficient coef(field, mat_op); - mfem::ParGridFunction smooth_flux_func(&h1_fespace); + GradFluxCoefficient coef(field_gf, mat_op); for (int c = 0; c < 3; c++) { coef.SetComponent(c); @@ -282,8 +269,8 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, h1_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); smooth_projector.Mult(flux_rhs, smooth_flux); } - smooth_flux_func.SetFromTrueDofs(smooth_flux); - smooth_flux_func.ExchangeFaceNbrData(); + smooth_flux_gf.SetFromTrueDofs(smooth_flux); + smooth_flux_gf.ExchangeFaceNbrData(); for (int e = 0; e < h1_fespace.GetNE(); e++) { auto &T = *h1_fespace.GetElementTransformation(e); @@ -293,11 +280,10 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, for (const auto &ip : ir) { T.SetIntPoint(&ip); - double smooth_val = smooth_flux_func.GetValue(e, ip); + double smooth_val = smooth_flux_gf.GetValue(e, ip); double coarse_val = coef.Eval(T, ip); const double w_i = ip.weight * T.Weight(); estimates[e] += w_i * std::pow(smooth_val - coarse_val, 2.0); - // std::cout << "smooth_val " << smooth_val << " coarse_val " << coarse_val << std::endl; normalization += w_i * std::pow(coarse_val, 2.0); } } @@ -306,7 +292,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, // Debugging branch generates some intermediate fields for paraview. mfem::ParaViewDataCollection paraview("debug_coeff" + std::to_string(c), h1_fespace.GetParMesh()); paraview.RegisterCoeffField("Flux", &coef); - paraview.RegisterField("SmoothFlux", &smooth_flux_func); + paraview.RegisterField("SmoothFlux", &smooth_flux_gf); mfem::L2_FECollection est_fec(0, sdim); mfem::ParFiniteElementSpace est_fespace(h1_fespace.GetParMesh(), &est_fec); @@ -322,7 +308,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, e = std::sqrt(e); } - Mpi::GlobalSum(1, &normalization, field.ParFESpace()->GetComm()); + Mpi::GlobalSum(1, &normalization, h1_fespace.GetComm()); normalization = std::sqrt(normalization); if (normalize) diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 567538aab..a195c4182 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -65,8 +65,9 @@ class CurlFluxErrorEstimator const MaterialOperator &mat_op; // The finite element space used to represent V, and F. mfem::ParFiniteElementSpaceHierarchy &nd_fespaces; - mutable FluxProjector smooth_projector; + FluxProjector smooth_projector; mutable Vector smooth_flux, flux_rhs; + mutable mfem::ParGridFunction field_gf, smooth_flux_gf; public: // Constructor for using geometric and p multigrid. @@ -75,7 +76,7 @@ class CurlFluxErrorEstimator // Compute elemental error indicators given a complex vector of true DOF. template - ErrorIndicators ComputeIndicators(const VectorType &v, bool normalize); + ErrorIndicators ComputeIndicators(const VectorType &v, bool normalize) const; }; // Class used for computing grad flux error estimate, i.e. || ϵ ∇ ϕₕ - F ||_K where F @@ -86,8 +87,9 @@ class GradFluxErrorEstimator // The finite element space used to represent ϕ, and components of F mfem::ParFiniteElementSpaceHierarchy &h1_fespaces; - mutable FluxProjector smooth_projector; + FluxProjector smooth_projector; mutable Vector smooth_flux, flux_rhs; + mutable mfem::ParGridFunction field_gf, smooth_flux_gf; public: // Constructor for using geometric and p multigrid. diff --git a/palace/main.cpp b/palace/main.cpp index f101ab062..185c436a9 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -7,7 +7,6 @@ #include #include #include - #include "drivers/drivensolver.hpp" #include "drivers/eigensolver.hpp" #include "drivers/electrostaticsolver.hpp" diff --git a/palace/models/curlcurloperator.cpp b/palace/models/curlcurloperator.cpp index af1019a6e..07d4cc9ab 100644 --- a/palace/models/curlcurloperator.cpp +++ b/palace/models/curlcurloperator.cpp @@ -81,10 +81,10 @@ CurlCurlOperator::CurlCurlOperator(const IoData &iodata, rt_fec(iodata.solver.order - 1, mesh.back()->Dimension()), nd_fespaces(fem::ConstructFiniteElementSpaceHierarchy( iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - pa_order_threshold, mesh, nd_fecs, dbc_marker, dbc_tdof_lists)), + pa_order_threshold, mesh, nd_fecs, &dbc_marker, &dbc_tdof_lists)), h1_fespaces(fem::ConstructFiniteElementSpaceHierarchy( iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - pa_order_threshold, mesh, h1_fecs)), + pa_order_threshold, mesh, h1_fecs, nullptr, nullptr)), rt_fespace(mesh.back().get(), &rt_fec), mat_op(iodata, *mesh.back()), surf_j_op(iodata, GetH1Space()) { diff --git a/palace/models/laplaceoperator.cpp b/palace/models/laplaceoperator.cpp index 289d277fb..b34dadc27 100644 --- a/palace/models/laplaceoperator.cpp +++ b/palace/models/laplaceoperator.cpp @@ -121,7 +121,7 @@ LaplaceOperator::LaplaceOperator(const IoData &iodata, nd_fec(iodata.solver.order, mesh.back()->Dimension()), h1_fespaces(fem::ConstructFiniteElementSpaceHierarchy( iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - pa_order_threshold, mesh, h1_fecs, dbc_marker, dbc_tdof_lists)), + pa_order_threshold, mesh, h1_fecs, &dbc_marker, &dbc_tdof_lists)), nd_fespace(mesh.back().get(), &nd_fec), mat_op(iodata, *mesh.back()), source_attr_lists(ConstructSources(iodata)) { diff --git a/palace/models/spaceoperator.cpp b/palace/models/spaceoperator.cpp index e9d4bb4b3..40c4238f2 100644 --- a/palace/models/spaceoperator.cpp +++ b/palace/models/spaceoperator.cpp @@ -86,10 +86,10 @@ SpaceOperator::SpaceOperator(const IoData &iodata, rt_fec(iodata.solver.order - 1, mesh.back()->Dimension()), nd_fespaces(fem::ConstructFiniteElementSpaceHierarchy( iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - pa_order_threshold, mesh, nd_fecs, dbc_marker, nd_dbc_tdof_lists)), + pa_order_threshold, mesh, nd_fecs, &dbc_marker, &nd_dbc_tdof_lists)), h1_fespaces(fem::ConstructFiniteElementSpaceHierarchy( iodata.solver.linear.mg_max_levels, iodata.solver.linear.mg_legacy_transfer, - pa_order_threshold, mesh, h1_fecs, dbc_marker, h1_dbc_tdof_lists)), + pa_order_threshold, mesh, h1_fecs, &dbc_marker, &h1_dbc_tdof_lists)), rt_fespace(mesh.back().get(), &rt_fec), mat_op(iodata, *mesh.back()), farfield_op(iodata, mat_op, *mesh.back()), surf_sigma_op(iodata, *mesh.back()), surf_z_op(iodata, *mesh.back()), lumped_port_op(iodata, GetH1Space()), From 7cf2dea534574cd47fef695e3b8031236d5fe757 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 17:23:47 -0400 Subject: [PATCH 17/39] style --- palace/drivers/basesolver.cpp | 8 ++-- palace/drivers/basesolver.hpp | 11 +++-- palace/drivers/drivensolver.cpp | 59 ++++++++++++-------------- palace/drivers/eigensolver.cpp | 24 +++++------ palace/drivers/electrostaticsolver.cpp | 33 +++++++------- palace/drivers/magnetostaticsolver.cpp | 30 ++++++------- palace/drivers/transientsolver.cpp | 28 ++++++------ palace/fem/coefficient.hpp | 11 +++-- palace/linalg/errorestimator.cpp | 35 +++++++++------ palace/utils/errorindicators.cpp | 12 +++--- palace/utils/errorindicators.hpp | 8 ++-- 11 files changed, 135 insertions(+), 124 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index a67adf5d2..120910f43 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -567,7 +567,8 @@ void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double } void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, double time, - double global, double min, double max, double mean, double normalization, + double global, double min, double max, + double mean, double normalization, bool normalized) const { if (post_dir.length() == 0) @@ -605,8 +606,9 @@ void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, d } } -void BaseSolver::PostprocessErrorIndicators(const std::string &name, - double global, double min, double max, double mean, double normalization) const +void BaseSolver::PostprocessErrorIndicators(const std::string &name, double global, + double min, double max, double mean, + double normalization) const { if (post_dir.length() == 0) { diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index cfc9fc426..749cef73f 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -72,17 +72,20 @@ class BaseSolver // Postprocess granular error indicator to file. The argument normalized indicates if // supplied indicators have already been normalized. void PostprocessErrorIndicators(const std::string &name, int step, double time, - double global, double min, double max, double mean, double normalization, bool normalized) const; + double global, double min, double max, double mean, + double normalization, bool normalized) const; // Write a string labeled error indicator. Used for writing statistics. - void PostprocessErrorIndicators(const std::string &name, - double global, double min, double max, double mean, double normalization) const; + void PostprocessErrorIndicators(const std::string &name, double global, double min, + double max, double mean, double normalization) const; + public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, const char *git_tag = nullptr); virtual ~BaseSolver() = default; // Performs a solve using the mesh sequence, then reports error indicators. - virtual ErrorIndicators Solve(const std::vector> &mesh) const = 0; + virtual ErrorIndicators + Solve(const std::vector> &mesh) const = 0; // These methods write different simulation metadata to a JSON file in post_dir. void SaveMetadata(const mfem::ParFiniteElementSpaceHierarchy &fespaces) const; diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 067bde5af..1d39b4c2c 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -146,22 +146,20 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = - [this, &estimator, &combined_indicators, &postop, &spaceop](const auto &E, int step, double f) + auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, + &spaceop](const auto &E, int step, double f) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto indicators = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators( - "f (GHz)", step, f, - indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - indicators.GetMinErrorIndicator(spaceop.GetComm()), - indicators.GetMaxErrorIndicator(spaceop.GetComm()), - indicators.GetMeanErrorIndicator(spaceop.GetComm()), - indicators.GetNormalization(), - normalized); + PostprocessErrorIndicators("f (GHz)", step, f, + indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + indicators.GetMinErrorIndicator(spaceop.GetComm()), + indicators.GetMaxErrorIndicator(spaceop.GetComm()), + indicators.GetMeanErrorIndicator(spaceop.GetComm()), + indicators.GetNormalization(), normalized); combined_indicators.AddIndicators(indicators); }; @@ -226,11 +224,11 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator } SaveMetadata(ksp); PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), - combined_indicators.GetNormalization()); + combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), + combined_indicators.GetNormalization()); return combined_indicators; } @@ -290,21 +288,19 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // The error indicators will be calculated for each HDM sample rather than for // the online stage. ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = - [this, &estimator, &combined_indicators, &spaceop](const auto &E, int step, double frequency) + auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, + &spaceop](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto indicators = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); - PostprocessErrorIndicators( - "f (GHz)", step, frequency, - indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - indicators.GetMinErrorIndicator(spaceop.GetComm()), - indicators.GetMaxErrorIndicator(spaceop.GetComm()), - indicators.GetMeanErrorIndicator(spaceop.GetComm()), - indicators.GetNormalization(), - normalized); + PostprocessErrorIndicators("f (GHz)", step, frequency, + indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + indicators.GetMinErrorIndicator(spaceop.GetComm()), + indicators.GetMaxErrorIndicator(spaceop.GetComm()), + indicators.GetMeanErrorIndicator(spaceop.GetComm()), + indicators.GetNormalization(), normalized); combined_indicators.AddIndicators(indicators); }; @@ -318,7 +314,8 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator UpdateErrorIndicators(E, 0, omega0 * f0); prom.AddHDMSample(omega0, E); prom.SolveHDM(omega0 + (nstep - step0 - 1) * delta_omega, E); - Mpi::Print("Computing error estimates for frequency {:d} (GHz)\n", (omega0 + (nstep - step0 - 1) * delta_omega) * f0); + Mpi::Print("Computing error estimates for frequency {:d} (GHz)\n", + (omega0 + (nstep - step0 - 1) * delta_omega) * f0); UpdateErrorIndicators(E, 1, (omega0 + (nstep - step0 - 1) * delta_omega) * f0); prom.AddHDMSample(omega0 + (nstep - step0 - 1) * delta_omega, E); @@ -358,11 +355,11 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // Set the indicator field to the combined field for postprocessing. PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), - combined_indicators.GetNormalization()); + combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), + combined_indicators.GetNormalization()); postop.SetIndicatorGridFunction(combined_indicators.GetLocalErrorIndicators()); // Main fast frequency sweep loop (online phase). diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index c44478e0b..840419363 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -273,14 +273,12 @@ EigenSolver::Solve(const std::vector> &mesh) cons auto indicators = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators( - "m", i, i + 1, - indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - indicators.GetMinErrorIndicator(spaceop.GetComm()), - indicators.GetMaxErrorIndicator(spaceop.GetComm()), - indicators.GetMeanErrorIndicator(spaceop.GetComm()), - indicators.GetNormalization(), - normalized); + PostprocessErrorIndicators("m", i, i + 1, + indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + indicators.GetMinErrorIndicator(spaceop.GetComm()), + indicators.GetMaxErrorIndicator(spaceop.GetComm()), + indicators.GetMeanErrorIndicator(spaceop.GetComm()), + indicators.GetNormalization(), normalized); combined_indicators.AddIndicators(indicators); }; @@ -330,11 +328,11 @@ EigenSolver::Solve(const std::vector> &mesh) cons Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv); } PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), - combined_indicators.GetNormalization()); + combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), + combined_indicators.GetNormalization()); return combined_indicators; } diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 9477760d3..47f124b87 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -73,7 +73,8 @@ ElectrostaticSolver::Solve(const std::vector> &me auto estimator = [&]() { BlockTimer bt(Timer::ESTCONSTRUCT); - return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), laplaceop.GetH1Spaces()); + return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), + laplaceop.GetH1Spaces()); }(); // Postprocess the capacitance matrix from the computed field solutions. @@ -100,22 +101,20 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, mfem::DenseMatrix C(nstep), Cm(nstep); Vector E(Grad->Height()), Vij(Grad->Width()); ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = - [this, &estimator, &combined_indicators, &postop, &laplaceop](const auto &V, int i, double idx) + auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, + &laplaceop](const auto &V, int i, double idx) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto indicators = estimator.ComputeIndicators(V, normalized); BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators( - "i", i, idx, - indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), - indicators.GetMinErrorIndicator(laplaceop.GetComm()), - indicators.GetMaxErrorIndicator(laplaceop.GetComm()), - indicators.GetMeanErrorIndicator(laplaceop.GetComm()), - indicators.GetNormalization(), - normalized); + PostprocessErrorIndicators("i", i, idx, + indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), + indicators.GetMinErrorIndicator(laplaceop.GetComm()), + indicators.GetMaxErrorIndicator(laplaceop.GetComm()), + indicators.GetMeanErrorIndicator(laplaceop.GetComm()), + indicators.GetNormalization(), normalized); combined_indicators.AddIndicators(indicators); }; if (iodata.solver.electrostatic.n_post > 0) @@ -176,12 +175,12 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, mfem::DenseMatrix Cinv(C); Cinv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(terminal_sources, C, Cinv, Cm); - PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(laplaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(laplaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(laplaceop.GetComm()), - combined_indicators.GetNormalization()); + PostprocessErrorIndicators( + "Mean", combined_indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(laplaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(laplaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(laplaceop.GetComm()), + combined_indicators.GetNormalization()); return combined_indicators; } diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index ca7c0845b..3bb3ccd27 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -105,8 +105,8 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, // Initialize structures for storing and reducing the results of error estimation. ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = - [this, &estimator, &combined_indicators, &postop, &curlcurlop](const auto &A, int i, double idx) + auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, + &curlcurlop](const auto &A, int i, double idx) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; @@ -114,14 +114,12 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, BlockTimer bt1(Timer::POSTPRO); // Write the indicator for this mode. postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators( - "i", i, idx, - indicators.GetGlobalErrorIndicator(curlcurlop.GetComm()), - indicators.GetMinErrorIndicator(curlcurlop.GetComm()), - indicators.GetMaxErrorIndicator(curlcurlop.GetComm()), - indicators.GetMeanErrorIndicator(curlcurlop.GetComm()), - indicators.GetNormalization(), - normalized); + PostprocessErrorIndicators("i", i, idx, + indicators.GetGlobalErrorIndicator(curlcurlop.GetComm()), + indicators.GetMinErrorIndicator(curlcurlop.GetComm()), + indicators.GetMaxErrorIndicator(curlcurlop.GetComm()), + indicators.GetMeanErrorIndicator(curlcurlop.GetComm()), + indicators.GetNormalization(), normalized); combined_indicators.AddIndicators(indicators); }; if (iodata.solver.magnetostatic.n_post > 0) @@ -186,12 +184,12 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, mfem::DenseMatrix Minv(M); Minv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(surf_j_op, M, Minv, Mm); - PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(curlcurlop.GetComm()), - combined_indicators.GetMinErrorIndicator(curlcurlop.GetComm()), - combined_indicators.GetMaxErrorIndicator(curlcurlop.GetComm()), - combined_indicators.GetMeanErrorIndicator(curlcurlop.GetComm()), - combined_indicators.GetNormalization()); + PostprocessErrorIndicators( + "Mean", combined_indicators.GetGlobalErrorIndicator(curlcurlop.GetComm()), + combined_indicators.GetMinErrorIndicator(curlcurlop.GetComm()), + combined_indicators.GetMaxErrorIndicator(curlcurlop.GetComm()), + combined_indicators.GetMeanErrorIndicator(curlcurlop.GetComm()), + combined_indicators.GetNormalization()); return combined_indicators; } diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index c8660a89f..90c7209aa 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -84,8 +84,8 @@ TransientSolver::Solve(const std::vector> &mesh) return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = - [this, &estimator, &combined_indicators, &postop, &spaceop](const auto &E, int step, double time) + auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, + &spaceop](const auto &E, int step, double time) { BlockTimer bt0(Timer::ESTSOLVE); // Initial flux of zero would return nan. @@ -93,14 +93,12 @@ TransientSolver::Solve(const std::vector> &mesh) auto indicators = estimator.ComputeIndicators(E, normalized); BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators( - "t (ns)", step, time, - indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - indicators.GetMinErrorIndicator(spaceop.GetComm()), - indicators.GetMaxErrorIndicator(spaceop.GetComm()), - indicators.GetMeanErrorIndicator(spaceop.GetComm()), - indicators.GetNormalization(), - normalized); + PostprocessErrorIndicators("t (ns)", step, time, + indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + indicators.GetMinErrorIndicator(spaceop.GetComm()), + indicators.GetMaxErrorIndicator(spaceop.GetComm()), + indicators.GetMeanErrorIndicator(spaceop.GetComm()), + indicators.GetNormalization(), normalized); combined_indicators.AddIndicators(indicators); }; @@ -157,11 +155,11 @@ TransientSolver::Solve(const std::vector> &mesh) } SaveMetadata(timeop.GetLinearSolver()); PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), - combined_indicators.GetNormalization()); + combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), + combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), + combined_indicators.GetNormalization()); return combined_indicators; } std::function TransientSolver::GetTimeExcitation(bool dot) const diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index 574e4051f..0c1ffc047 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -514,21 +514,24 @@ class GradFluxCoefficient : public mfem::Coefficient public: GradFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) : mfem::Coefficient(), gf(gf), mat_op(mat_op), - grad(gf.ParFESpace()->GetParMesh()->SpaceDimension()), - V(gf.ParFESpace()->GetParMesh()->SpaceDimension()), component(-1) + grad(gf.ParFESpace()->GetParMesh()->SpaceDimension()), + V(gf.ParFESpace()->GetParMesh()->SpaceDimension()), component(-1) { } // Specify the component void SetComponent(int i) { - MFEM_ASSERT(i >= 0 && i < gf.ParFESpace()->GetParMesh()->SpaceDimension(), "Invalid component index!"); + MFEM_ASSERT(i >= 0 && i < gf.ParFESpace()->GetParMesh()->SpaceDimension(), + "Invalid component index!"); component = i; } double Eval(mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip) override { - MFEM_ASSERT(component >= 0 && component < gf.ParFESpace()->GetParMesh()->SpaceDimension(), "Invalid component index, try calling SetComponent(int)!"); + MFEM_ASSERT(component >= 0 && + component < gf.ParFESpace()->GetParMesh()->SpaceDimension(), + "Invalid component index, try calling SetComponent(int)!"); gf.GetGradient(T, grad); mat_op.GetPermittivityReal(T.Attribute).Mult(grad, V); return V(component); diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index ca3167ad3..39e511907 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -94,16 +94,18 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &nd_fespaces) : mat_op(mat_op), nd_fespaces(nd_fespaces), - smooth_projector(nd_fespaces, iodata.solver.linear.tol, 200, 1, iodata.solver.pa_order_threshold), + smooth_projector(nd_fespaces, iodata.solver.linear.tol, 200, 1, + iodata.solver.pa_order_threshold), smooth_flux(nd_fespaces.GetFinestFESpace().GetTrueVSize()), flux_rhs(nd_fespaces.GetFinestFESpace().GetTrueVSize()), - field_gf(&nd_fespaces.GetFinestFESpace()), smooth_flux_gf(&nd_fespaces.GetFinestFESpace()) + field_gf(&nd_fespaces.GetFinestFESpace()), + smooth_flux_gf(&nd_fespaces.GetFinestFESpace()) { } template <> ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, - bool normalize) const + bool normalize) const { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); const int nelem = nd_fespace.GetNE(); @@ -118,9 +120,10 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v // Coefficients for computing the discontinuous flux component, i.e. (W, μ⁻¹∇ × V). CurlFluxCoefficient coef(field_gf, mat_op); { - // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass - // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - Mpi::Print("Computing smooth flux approximation of {} component\n", real ? "real" : "imaginary" ); + // Given the RHS vector of non-smooth flux, construct a flux projector and perform + // mass matrix inversion in the appropriate space, giving f = M⁻¹ f̂. + Mpi::Print("Computing smooth flux approximation of {} component\n", + real ? "real" : "imaginary"); BlockTimer bt(Timer::ESTSOLVE); mfem::LinearForm rhs(&nd_fespace); rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); @@ -146,6 +149,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v smooth_flux_gf.GetVectorValue(e, ip, smooth_vec); coef.Eval(coarse_vec, T, ip); double w_i = ip.weight * T.Weight(); + constexpr int vdim = 3; for (int c = 0; c < 3; c++) { estimates[e] += w_i * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); @@ -170,7 +174,8 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v } template <> -ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool normalize) const +ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, + bool normalize) const { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); field_gf.SetFromTrueDofs(v); @@ -211,7 +216,8 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, bool smooth_flux_gf.GetVectorValue(e, ip, smooth_vec); coef.Eval(coarse_vec, T, ip); const double w_i = ip.weight * T.Weight(); - for (int c = 0; c < 3; c++) + constexpr int vdim = 3; + for (int c = 0; c < vdim; c++) { estimates[e] += w_i * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); normalization += w_i * coarse_vec[c] * coarse_vec[c]; @@ -234,15 +240,17 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &h1_fespaces) : mat_op(mat_op), h1_fespaces(h1_fespaces), - smooth_projector(h1_fespaces, iodata.solver.linear.tol, 200, 1, iodata.solver.pa_order_threshold), + smooth_projector(h1_fespaces, iodata.solver.linear.tol, 200, 1, + iodata.solver.pa_order_threshold), smooth_flux(h1_fespaces.GetFinestFESpace().GetTrueVSize()), flux_rhs(h1_fespaces.GetFinestFESpace().GetTrueVSize()), - field_gf(&h1_fespaces.GetFinestFESpace()), smooth_flux_gf(&h1_fespaces.GetFinestFESpace()) + field_gf(&h1_fespaces.GetFinestFESpace()), + smooth_flux_gf(&h1_fespaces.GetFinestFESpace()) { } ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, - bool normalize) const + bool normalize) const { auto &h1_fespace = h1_fespaces.GetFinestFESpace(); const int sdim = h1_fespace.GetMesh()->SpaceDimension(); @@ -255,7 +263,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, // Coefficient for computing the discontinuous flux., i.e. (Vᵢ, (ϵ ∇ ϕ)ᵢ). GradFluxCoefficient coef(field_gf, mat_op); - for (int c = 0; c < 3; c++) + for (int c = 0; c < sdim; c++) { coef.SetComponent(c); { @@ -290,7 +298,8 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, if constexpr (false) { // Debugging branch generates some intermediate fields for paraview. - mfem::ParaViewDataCollection paraview("debug_coeff" + std::to_string(c), h1_fespace.GetParMesh()); + mfem::ParaViewDataCollection paraview("debug_coeff" + std::to_string(c), + h1_fespace.GetParMesh()); paraview.RegisterCoeffField("Flux", &coef); paraview.RegisterField("SmoothFlux", &smooth_flux_gf); diff --git a/palace/utils/errorindicators.cpp b/palace/utils/errorindicators.cpp index 36bb93bd0..960221489 100644 --- a/palace/utils/errorindicators.cpp +++ b/palace/utils/errorindicators.cpp @@ -14,7 +14,9 @@ void ErrorIndicators::AddIndicators(const ErrorIndicators &error_indicators) // important to many solves, rather than only large in one solve. if (n > 0) { - normalization = (n * normalization + error_indicators.normalization * error_indicators.n) / (n + error_indicators.n); + normalization = + (n * normalization + error_indicators.normalization * error_indicators.n) / + (n + error_indicators.n); // The local indicators must be squared before combining, so that the global error // calculation is valid: @@ -28,13 +30,14 @@ void ErrorIndicators::AddIndicators(const ErrorIndicators &error_indicators) // e_K = sqrt(1/N sum_N eta_{Kn}^2) auto running_average = [this, error_indicators](const auto &xbar, const auto &x) { - return std::sqrt((xbar * xbar * n + x * x * error_indicators.n) / (n + error_indicators.n)); + return std::sqrt((xbar * xbar * n + x * x * error_indicators.n) / + (n + error_indicators.n)); }; MFEM_VERIFY(local.Size() == error_indicators.local.Size(), "Local error indicator vectors mismatch."); // Combine these error indicators into the current average. - std::transform(local.begin(), local.end(), - error_indicators.local.begin(), local.begin(), running_average); + std::transform(local.begin(), local.end(), error_indicators.local.begin(), + local.begin(), running_average); // More samples have been added, update for the running average lambda. n += error_indicators.n; @@ -44,7 +47,6 @@ void ErrorIndicators::AddIndicators(const ErrorIndicators &error_indicators) // This indicator was empty, just steal. (*this) = error_indicators; } - } } // namespace palace diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index 1165685b7..319a291ed 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -16,7 +16,9 @@ class ErrorIndicators { public: ErrorIndicators(Vector local, double normalization) - : local(std::move(local)), normalization(normalization), n(1) {} + : local(std::move(local)), normalization(normalization), n(1) + { + } ErrorIndicators() = default; ErrorIndicators(const ErrorIndicators &) = default; @@ -32,8 +34,8 @@ class ErrorIndicators { constexpr int p = 2; double global_error_indicator = - std::transform_reduce(local.begin(), local.end(), - 0.0, std::plus(), [](auto val) { return std::pow(val, p); }); + std::transform_reduce(local.begin(), local.end(), 0.0, std::plus(), + [](auto val) { return std::pow(val, p); }); Mpi::GlobalSum(1, &global_error_indicator, comm); return std::pow(global_error_indicator, 1.0 / p); } From aed614d73e2e175a0b48c3fa0137eaa391f898a2 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 17:37:01 -0400 Subject: [PATCH 18/39] Change FluxProjector constructor to look more like DivFree --- palace/linalg/errorestimator.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 39e511907..6eb8e8631 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -63,31 +63,36 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie template FluxProjector::FluxProjector( - mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fespace, double tol, int max_it, + mfem::ParFiniteElementSpaceHierarchy &fespaces, double tol, int max_it, int print, int pa_order_threshold) - : M(BuildMassMatrixOperator(smooth_flux_fespace, + : M(BuildMassMatrixOperator(fespaces, pa_order_threshold)) { // The system matrix for the projection is real and SPD. For the coarse-level AMG solve, // we don't use an exact solve on the coarsest level. auto amg = std::make_unique>(std::make_unique(1, 1, 0)); - auto gmg = std::make_unique>( - std::move(amg), smooth_flux_fespace, nullptr, 1, 1, 2, 1.0, 0.0, true, - pa_order_threshold); - - auto pcg = std::make_unique>( - smooth_flux_fespace.GetFinestFESpace().GetComm(), print); + std::unique_ptr> pc; + if (fespaces.GetNumLevels() > 1) + { + pc = std::make_unique>( + std::move(amg), fespaces, nullptr, 1, 1, 2, 1.0, 0.0, true, pa_order_threshold); + } + else + { + pc = std::move(amg); + } + auto pcg = std::make_unique>(fespaces.GetFinestFESpace().GetComm(), print); pcg->SetInitialGuess(false); pcg->SetRelTol(tol); pcg->SetAbsTol(std::numeric_limits::epsilon()); pcg->SetMaxIter(max_it); - ksp = std::make_unique(std::move(pcg), std::move(gmg)); + ksp = std::make_unique(std::move(pcg), std::move(pc)); ksp->SetOperators(*M, *M); - tmp.SetSize(smooth_flux_fespace.GetFinestFESpace().GetTrueVSize()); + tmp.SetSize(fespaces.GetFinestFESpace().GetTrueVSize()); } CurlFluxErrorEstimator::CurlFluxErrorEstimator( From def6d781a5194ec2e6454cea74c8b6616329fba0 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 17:51:21 -0400 Subject: [PATCH 19/39] Move error indicator calculation outside of reporting loop for eigen modes, makes the logfile neater --- palace/drivers/eigensolver.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 840419363..6db81f8f6 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -282,9 +282,17 @@ EigenSolver::Solve(const std::vector> &mesh) cons combined_indicators.AddIndicators(indicators); }; + // Save the eigenvalue estimates. + for (int i = 0; i < iodata.solver.eigenmode.n; i++) + { + eigen->GetEigenvector(i, E); + // Only update the error indicator for targeted modes. + Mpi::Print("\nComputing error estimates for mode {:d}\n", i); + UpdateErrorIndicators(E, i); + } + // Postprocess the results. BlockTimer bt2(Timer::POSTPRO); - for (int i = 0; i < num_conv; i++) { // Get the eigenvalue and relative error. @@ -313,12 +321,6 @@ EigenSolver::Solve(const std::vector> &mesh) cons eigen->GetEigenvector(i, E); Curl->Mult(E, B); B *= -1.0 / (1i * omega); - if (i < iodata.solver.eigenmode.n) - { - // Only update the error indicator for targeted modes. - Mpi::Print("Computing error estimates for mode {:d}\n", i); - UpdateErrorIndicators(E, i); - } postop.SetEGridFunction(E); postop.SetBGridFunction(B); From e1251add14a9bc1154f471f248d8d4f9d7be87bf Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 18:30:33 -0400 Subject: [PATCH 20/39] Fixes for some log printing --- palace/drivers/drivensolver.cpp | 5 ----- palace/drivers/eigensolver.cpp | 1 - palace/drivers/electrostaticsolver.cpp | 1 - palace/drivers/transientsolver.cpp | 3 +-- palace/linalg/errorestimator.cpp | 2 +- palace/utils/errorindicators.hpp | 8 ++++---- palace/utils/timer.hpp | 24 +++++++++++++++++------- 7 files changed, 23 insertions(+), 21 deletions(-) diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 1d39b4c2c..73cd1ce40 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -152,7 +152,6 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto indicators = estimator.ComputeIndicators(E, normalized); - BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("f (GHz)", step, f, indicators.GetGlobalErrorIndicator(spaceop.GetComm()), @@ -294,7 +293,6 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto indicators = estimator.ComputeIndicators(E, normalized); - BlockTimer bt1(Timer::POSTPRO); PostprocessErrorIndicators("f (GHz)", step, frequency, indicators.GetGlobalErrorIndicator(spaceop.GetComm()), indicators.GetMinErrorIndicator(spaceop.GetComm()), @@ -310,12 +308,9 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // of the RomOperator. BlockTimer bt1(Timer::CONSTRUCTPROM); prom.SolveHDM(omega0, E); // Print matrix stats at first HDM solve - Mpi::Print("Computing error estimates for frequency {:d} (GHz)\n", omega0 * f0); UpdateErrorIndicators(E, 0, omega0 * f0); prom.AddHDMSample(omega0, E); prom.SolveHDM(omega0 + (nstep - step0 - 1) * delta_omega, E); - Mpi::Print("Computing error estimates for frequency {:d} (GHz)\n", - (omega0 + (nstep - step0 - 1) * delta_omega) * f0); UpdateErrorIndicators(E, 1, (omega0 + (nstep - step0 - 1) * delta_omega) * f0); prom.AddHDMSample(omega0 + (nstep - step0 - 1) * delta_omega, E); diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 6db81f8f6..9f254d581 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -271,7 +271,6 @@ EigenSolver::Solve(const std::vector> &mesh) cons BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto indicators = estimator.ComputeIndicators(E, normalized); - BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("m", i, i + 1, indicators.GetGlobalErrorIndicator(spaceop.GetComm()), diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 47f124b87..d79ad1ab0 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -107,7 +107,6 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; auto indicators = estimator.ComputeIndicators(V, normalized); - BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("i", i, idx, indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 90c7209aa..ec930c9bb 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -87,11 +87,10 @@ TransientSolver::Solve(const std::vector> &mesh) auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, &spaceop](const auto &E, int step, double time) { - BlockTimer bt0(Timer::ESTSOLVE); + BlockTimer bt0(Timer::ESTIMATION); // Initial flux of zero would return nan. bool constexpr normalized = false; auto indicators = estimator.ComputeIndicators(E, normalized); - BlockTimer bt1(Timer::POSTPRO); postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("t (ns)", step, time, indicators.GetGlobalErrorIndicator(spaceop.GetComm()), diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 6eb8e8631..438326002 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -155,7 +155,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v coef.Eval(coarse_vec, T, ip); double w_i = ip.weight * T.Weight(); constexpr int vdim = 3; - for (int c = 0; c < 3; c++) + for (int c = 0; c < vdim; c++) { estimates[e] += w_i * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); normalization += w_i * coarse_vec[c] * coarse_vec[c]; diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index 319a291ed..9bc70bc5a 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -27,7 +27,7 @@ class ErrorIndicators ErrorIndicators &operator=(ErrorIndicators &&) = default; ~ErrorIndicators() = default; - // Return the average normalized local error indicators. + // Return the local error indicators. const auto &GetLocalErrorIndicators() const { return local; } // Return the global error indicator. inline auto GetGlobalErrorIndicator(MPI_Comm comm) const @@ -39,21 +39,21 @@ class ErrorIndicators Mpi::GlobalSum(1, &global_error_indicator, comm); return std::pow(global_error_indicator, 1.0 / p); } - // Return the largest normalized local error indicator. + // Return the largest local error indicator. inline auto GetMaxErrorIndicator(MPI_Comm comm) const { double max = local.Max(); Mpi::GlobalMax(1, &max, comm); return max; } - // Return the smallest normalized local error indicator. + // Return the smallest local error indicator. inline auto GetMinErrorIndicator(MPI_Comm comm) const { double min = local.Min(); Mpi::GlobalMin(1, &min, comm); return min; } - // Return the mean normalized local error indicator. + // Return the mean local error indicator. inline auto GetMeanErrorIndicator(MPI_Comm comm) const { int size = local.Size(); diff --git a/palace/utils/timer.hpp b/palace/utils/timer.hpp index 30f686ee3..89c6da817 100644 --- a/palace/utils/timer.hpp +++ b/palace/utils/timer.hpp @@ -43,14 +43,24 @@ class Timer NUMTIMINGS }; + + // clang-format off inline static const std::vector descriptions{ - "Initialization", "Operator Construction", - " Wave Ports", "Solve", - " Preconditioner", " Coarse Solve", - "Estimation", " Construction", - " Evaluation", "PROM Construction", - "PROM Solve", "Postprocessing", - "Disk IO", "Total"}; + "Initialization", + "Operator Construction", + " Wave Ports", + "Solve", + " Preconditioner", + " Coarse Solve", + "Estimation", + " Construction", + " Solve", + "PROM Construction", + "PROM Solve", + "Postprocessing", + "Disk IO", + "Total"}; + // clang-format on private: const TimePoint start_time; From a7d9f69ad33c281ed23bce335f8183a1044c9348 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 18:35:54 -0400 Subject: [PATCH 21/39] Regenerate reference error indicator data. Reflects changes to mean calculations and to the matrix free calculation of the elemental norms --- palace/utils/errorindicators.hpp | 6 +- .../ref/cavity/impedance/error-indicators.csv | 32 +- test/ref/cavity/pec/error-indicators.csv | 32 +- test/ref/coaxial/matched/error-indicators.csv | 402 +++++++++--------- test/ref/coaxial/open/error-indicators.csv | 402 +++++++++--------- .../cpw/lumped_adaptive/error-indicators.csv | 20 +- .../cpw/lumped_uniform/error-indicators.csv | 32 +- .../cpw/wave_adaptive/error-indicators.csv | 20 +- .../ref/cpw/wave_uniform/error-indicators.csv | 32 +- test/ref/rings/error-indicators.csv | 6 +- test/ref/spheres/error-indicators.csv | 6 +- 11 files changed, 496 insertions(+), 494 deletions(-) diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index 9bc70bc5a..83ffbf799 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -57,8 +57,10 @@ class ErrorIndicators inline auto GetMeanErrorIndicator(MPI_Comm comm) const { int size = local.Size(); - auto global_error_indicator = GetGlobalErrorIndicator(comm); - return global_error_indicator / size; + Mpi::GlobalSum(1, &size, comm); + auto sum = std::accumulate(local.begin(), local.end(), 0.0); + Mpi::GlobalSum(1, &sum, comm); + return sum / size; } // Return the normalization constant for the absolute error. inline auto GetNormalization() const { return normalization; } diff --git a/test/ref/cavity/impedance/error-indicators.csv b/test/ref/cavity/impedance/error-indicators.csv index 1a34e270a..c88a44205 100644 --- a/test/ref/cavity/impedance/error-indicators.csv +++ b/test/ref/cavity/impedance/error-indicators.csv @@ -1,17 +1,17 @@ m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +8.211772854e-03, +9.214906491e-05, +6.517143501e-04, +1.425655009e-05, +6.964432246e-01 - 2.000000e+00, +7.376205362e-03, +1.120203634e-04, +5.122692155e-04, +1.280591209e-05, +4.545414510e-01 - 3.000000e+00, +7.965439217e-03, +1.093565748e-04, +6.153863198e-04, +1.382888753e-05, +5.168938583e-01 - 4.000000e+00, +8.289455761e-03, +6.591008815e-05, +8.975703459e-04, +1.439141625e-05, +7.254717341e-01 - 5.000000e+00, +1.680165062e-02, +2.686897967e-04, +1.173118394e-03, +2.916953233e-05, +7.021942152e-01 - 6.000000e+00, +1.724222117e-02, +2.679646989e-04, +1.285588040e-03, +2.993441175e-05, +6.917661151e-01 - 7.000000e+00, +1.541389064e-02, +2.982023357e-04, +1.088582974e-03, +2.676022681e-05, +6.836991197e-01 - 8.000000e+00, +1.589306497e-02, +2.709931996e-04, +1.182754190e-03, +2.759212668e-05, +7.774857333e-01 - 9.000000e+00, +2.161744473e-02, +3.939248464e-04, +1.590949752e-03, +3.753028600e-05, +1.082707043e+00 - 1.000000e+01, +2.568013712e-02, +3.749227031e-04, +2.345062909e-03, +4.458357139e-05, +1.143427854e+00 - 1.100000e+01, +1.171936065e-02, +2.548287325e-04, +8.771786122e-04, +2.034611225e-05, +8.880851779e-01 - 1.200000e+01, +2.424099577e-02, +3.311786201e-04, +2.230405154e-03, +4.208506210e-05, +8.563665976e-01 - 1.300000e+01, +2.164706045e-02, +1.439794677e-04, +2.192103020e-03, +3.758170218e-05, +1.123784317e+00 - 1.400000e+01, +2.570506964e-02, +1.191879269e-04, +3.229783266e-03, +4.462685701e-05, +1.125048654e+00 - 1.500000e+01, +2.690499163e-02, +4.630246596e-04, +1.771494907e-03, +4.671005492e-05, +8.954809561e-01 - Mean, +1.698058404e-02, +3.754059600e-04, +1.180751362e-03, +2.948018062e-05, +8.242264034e-01 + 1.000000e+00, +8.196374239e-03, +9.214842342e-05, +6.510932304e-04, +3.254431675e-04, +6.964480813e-01 + 2.000000e+00, +7.387353181e-03, +1.120183148e-04, +5.134253635e-04, +2.965305671e-04, +4.545494361e-01 + 3.000000e+00, +7.977895730e-03, +1.062690571e-04, +6.123658757e-04, +3.123956087e-04, +5.169059866e-01 + 4.000000e+00, +8.274201489e-03, +6.590964834e-05, +8.966948182e-04, +3.045023806e-04, +7.254767932e-01 + 5.000000e+00, +1.685960941e-02, +2.820942105e-04, +1.153640116e-03, +6.780447656e-04, +7.021998742e-01 + 6.000000e+00, +1.730399293e-02, +2.679568469e-04, +1.285571611e-03, +6.941394542e-04, +6.917741537e-01 + 7.000000e+00, +1.541951446e-02, +3.015087299e-04, +1.088775657e-03, +6.184944349e-04, +6.837241462e-01 + 8.000000e+00, +1.589962692e-02, +2.715177297e-04, +1.182648337e-03, +6.283845490e-04, +7.775219321e-01 + 9.000000e+00, +2.159859358e-02, +3.943793250e-04, +1.605016900e-03, +8.505673899e-04, +1.082720492e+00 + 1.000000e+01, +2.566043946e-02, +3.654560100e-04, +2.348156651e-03, +9.735827558e-04, +1.143440851e+00 + 1.100000e+01, +1.170857739e-02, +2.548269260e-04, +8.762332197e-04, +4.764991796e-04, +8.880913705e-01 + 1.200000e+01, +2.430340822e-02, +3.361537088e-04, +2.235911969e-03, +9.591480064e-04, +8.564002177e-01 + 1.300000e+01, +2.162823528e-02, +1.439773671e-04, +2.211508889e-03, +7.807885965e-04, +1.123798284e+00 + 1.400000e+01, +2.568539055e-02, +1.169360347e-04, +3.234035290e-03, +8.927006761e-04, +1.125061440e+00 + 1.500000e+01, +2.696836169e-02, +4.629929123e-04, +1.772177016e-03, +1.080937100e-03, +8.955419511e-01 + Mean, +1.832232363e-02, +4.343505522e-04, +1.399356727e-03, +7.453986847e-04, +8.242436674e-01 diff --git a/test/ref/cavity/pec/error-indicators.csv b/test/ref/cavity/pec/error-indicators.csv index e1f56438e..2323bb511 100644 --- a/test/ref/cavity/pec/error-indicators.csv +++ b/test/ref/cavity/pec/error-indicators.csv @@ -1,17 +1,17 @@ m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +8.211772523e-03, +9.214886535e-05, +6.517144226e-04, +1.425654952e-05, +6.964432122e-01 - 2.000000e+00, +7.376205452e-03, +1.120202197e-04, +5.122680096e-04, +1.280591224e-05, +4.545414691e-01 - 3.000000e+00, +7.965439333e-03, +1.093560248e-04, +6.153853635e-04, +1.382888773e-05, +5.168938091e-01 - 4.000000e+00, +8.289455219e-03, +6.591005173e-05, +8.975704252e-04, +1.439141531e-05, +7.254717195e-01 - 5.000000e+00, +1.680164766e-02, +2.686717116e-04, +1.173092213e-03, +2.916952718e-05, +7.021936684e-01 - 6.000000e+00, +1.724222477e-02, +2.679503077e-04, +1.285582570e-03, +2.993441800e-05, +6.917667336e-01 - 7.000000e+00, +1.541389133e-02, +2.982045184e-04, +1.088587389e-03, +2.676022801e-05, +6.836990559e-01 - 8.000000e+00, +1.589306779e-02, +2.709696912e-04, +1.182729517e-03, +2.759213158e-05, +7.774838203e-01 - 9.000000e+00, +2.161744463e-02, +3.939247378e-04, +1.590950569e-03, +3.753028582e-05, +1.082707024e+00 - 1.000000e+01, +2.568013828e-02, +3.749227076e-04, +2.345063736e-03, +4.458357340e-05, +1.143427857e+00 - 1.100000e+01, +1.171936185e-02, +2.548286854e-04, +8.771787530e-04, +2.034611433e-05, +8.880851654e-01 - 1.200000e+01, +2.424099407e-02, +3.311788996e-04, +2.230405073e-03, +4.208505915e-05, +8.563665731e-01 - 1.300000e+01, +2.164706072e-02, +1.439800054e-04, +2.192106108e-03, +3.758170264e-05, +1.123784347e+00 - 1.400000e+01, +2.570507035e-02, +1.191878736e-04, +3.229783780e-03, +4.462685825e-05, +1.125048642e+00 - 1.500000e+01, +2.690494192e-02, +4.630260577e-04, +1.771483008e-03, +4.670996860e-05, +8.954790792e-01 - Mean, +1.698058106e-02, +3.754074357e-04, +1.180748376e-03, +2.948017545e-05, +8.242261450e-01 + 1.000000e+00, +8.196373525e-03, +9.214821861e-05, +6.510933257e-04, +3.254431264e-04, +6.964480689e-01 + 2.000000e+00, +7.387353222e-03, +1.120182421e-04, +5.134245793e-04, +2.965305493e-04, +4.545494456e-01 + 3.000000e+00, +7.977895796e-03, +1.062685103e-04, +6.123649464e-04, +3.123955986e-04, +5.169059367e-01 + 4.000000e+00, +8.274201169e-03, +6.590958800e-05, +8.966948963e-04, +3.045023647e-04, +7.254767786e-01 + 5.000000e+00, +1.685960472e-02, +2.820836869e-04, +1.153598318e-03, +6.780453428e-04, +7.021989802e-01 + 6.000000e+00, +1.730399491e-02, +2.679493494e-04, +1.285568826e-03, +6.941391330e-04, +6.917744665e-01 + 7.000000e+00, +1.541950856e-02, +3.013866066e-04, +1.088523264e-03, +6.184933404e-04, +6.837273782e-01 + 8.000000e+00, +1.589962884e-02, +2.714996993e-04, +1.182628298e-03, +6.283841145e-04, +7.775204071e-01 + 9.000000e+00, +2.159859353e-02, +3.943791856e-04, +1.605017990e-03, +8.505674052e-04, +1.082720471e+00 + 1.000000e+01, +2.566044031e-02, +3.654560313e-04, +2.348157389e-03, +9.735827810e-04, +1.143440852e+00 + 1.100000e+01, +1.170857794e-02, +2.548269622e-04, +8.762333072e-04, +4.764992002e-04, +8.880913583e-01 + 1.200000e+01, +2.430340709e-02, +3.361540626e-04, +2.235911950e-03, +9.591479625e-04, +8.564001863e-01 + 1.300000e+01, +2.162823566e-02, +1.439781616e-04, +2.211512532e-03, +7.807886467e-04, +1.123798302e+00 + 1.400000e+01, +2.568539163e-02, +1.169360360e-04, +3.234036921e-03, +8.927007077e-04, +1.125061428e+00 + 1.500000e+01, +2.696828680e-02, +4.629949994e-04, +1.772157010e-03, +1.080939278e-03, +8.955391704e-01 + Mean, +1.832231600e-02, +4.343440209e-04, +1.399356089e-03, +7.453990959e-04, +8.242435486e-01 diff --git a/test/ref/coaxial/matched/error-indicators.csv b/test/ref/coaxial/matched/error-indicators.csv index c3f574b19..6569d6a7a 100644 --- a/test/ref/coaxial/matched/error-indicators.csv +++ b/test/ref/coaxial/matched/error-indicators.csv @@ -1,203 +1,203 @@ t (ns), Sum Abs., Min Elem Abs., Max Elem Abs., Mean Elem Abs., Normalization 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 - 3.747406e-02, +5.656919329e-06, +4.877275943e-19, +1.429146542e-06, +4.419468225e-08, +1.015026012e-04 - 7.494811e-02, +1.331394786e-05, +1.262176118e-18, +3.354187386e-06, +1.040152177e-07, +1.286467416e-04 - 1.124222e-01, +4.533166152e-06, +8.797629984e-19, +1.144058873e-06, +3.541536056e-08, +1.363432445e-04 - 1.498962e-01, +5.316539312e-06, +5.339102739e-19, +1.352378049e-06, +4.153546337e-08, +2.237453773e-04 - 1.873703e-01, +2.287184397e-05, +2.999088821e-18, +5.789590609e-06, +1.786862810e-07, +5.988251413e-04 - 2.248443e-01, +1.598438319e-05, +2.232109810e-18, +4.071906508e-06, +1.248779937e-07, +1.231318974e-03 - 2.623184e-01, +1.150013517e-05, +5.196882665e-18, +3.089062447e-06, +8.984480601e-08, +2.183514588e-03 - 2.997925e-01, +2.092441721e-05, +1.274335458e-17, +5.172537104e-06, +1.634720094e-07, +3.443881315e-03 - 3.372665e-01, +3.023129660e-05, +2.000023090e-17, +8.595200606e-06, +2.361820047e-07, +4.896471117e-03 - 3.747406e-01, +2.966149896e-05, +5.645231840e-17, +8.160478469e-06, +2.317304606e-07, +6.288238192e-03 - 4.122146e-01, +3.934747639e-05, +1.414197098e-17, +1.043816137e-05, +3.074021593e-07, +7.236958436e-03 - 4.496887e-01, +5.496375443e-05, +1.412135240e-16, +1.376602992e-05, +4.294043315e-07, +7.438657288e-03 - 4.871627e-01, +5.317715171e-05, +1.765052700e-16, +1.345278687e-05, +4.154464978e-07, +7.655736081e-03 - 5.246368e-01, +6.345987510e-05, +3.578882920e-16, +1.616235667e-05, +4.957802742e-07, +1.144507368e-02 - 5.621109e-01, +1.000188544e-04, +6.683863674e-16, +2.890344790e-05, +7.813973001e-07, +2.129749051e-02 - 5.995849e-01, +1.569853883e-04, +6.894686202e-16, +5.054827882e-05, +1.226448346e-06, +3.669225071e-02 - 6.370590e-01, +2.371414640e-04, +2.988956019e-15, +7.811657109e-05, +1.852667687e-06, +5.641515864e-02 - 6.745330e-01, +3.436988525e-04, +2.171452447e-15, +1.084030298e-04, +2.685147285e-06, +7.828737440e-02 - 7.120071e-01, +4.503449432e-04, +8.422790023e-15, +1.334532772e-04, +3.518319868e-06, +9.885303849e-02 - 7.494811e-01, +5.387201999e-04, +2.207064524e-14, +1.460192415e-04, +4.208751561e-06, +1.137546488e-01 - 7.869552e-01, +5.854634666e-04, +3.678749831e-14, +1.426211774e-04, +4.573933333e-06, +1.195362049e-01 - 8.244293e-01, +5.945252636e-04, +1.530621531e-13, +1.553115755e-04, +4.644728622e-06, +1.188356113e-01 - 8.619033e-01, +6.277794930e-04, +8.375528408e-13, +1.446548827e-04, +4.904527289e-06, +1.309570623e-01 - 8.993774e-01, +8.196023618e-04, +3.513126401e-12, +2.064570778e-04, +6.403143451e-06, +1.857784636e-01 - 9.368514e-01, +1.245063026e-03, +1.222913601e-11, +3.796113147e-04, +9.727054889e-06, +2.867343617e-01 - 9.743255e-01, +1.824265444e-03, +3.836946290e-11, +5.734398444e-04, +1.425207378e-05, +4.167726757e-01 - 1.011800e+00, +2.450890631e-03, +1.109091742e-10, +7.521115026e-04, +1.914758306e-05, +5.549120054e-01 - 1.049274e+00, +3.008970324e-03, +2.905630779e-10, +8.708173545e-04, +2.350758066e-05, +6.782193157e-01 - 1.086748e+00, +3.391209599e-03, +6.708721725e-10, +8.861180142e-04, +2.649382500e-05, +7.649055092e-01 - 1.124222e+00, +3.537834075e-03, +1.316366440e-09, +8.504123104e-04, +2.763932871e-05, +8.022591780e-01 - 1.161696e+00, +3.511779316e-03, +2.035246014e-09, +8.840349986e-04, +2.743577591e-05, +8.009806200e-01 - 1.199170e+00, +3.584287867e-03, +1.923006182e-09, +8.159752264e-04, +2.800224896e-05, +8.143332817e-01 - 1.236644e+00, +4.160738346e-03, +3.636930649e-09, +8.694210875e-04, +3.250576833e-05, +9.294499013e-01 - 1.274118e+00, +5.352965483e-03, +1.872213299e-08, +1.361582714e-03, +4.182004284e-05, +1.185780064e+00 - 1.311592e+00, +6.884036964e-03, +5.579983321e-08, +1.934136736e-03, +5.378153878e-05, +1.534661136e+00 - 1.349066e+00, +8.393574770e-03, +1.270371506e-07, +2.361243466e-03, +6.557480289e-05, +1.896361173e+00 - 1.386540e+00, +9.587144888e-03, +2.368634115e-07, +2.543459610e-03, +7.489956944e-05, +2.199964362e+00 - 1.424014e+00, +1.030847846e-02, +3.669557621e-07, +2.421466309e-03, +8.053498795e-05, +2.397709121e+00 - 1.461488e+00, +1.058479828e-02, +4.699423030e-07, +2.459542440e-03, +8.269373653e-05, +2.477248853e+00 - 1.498962e+00, +1.065114462e-02, +5.188095796e-07, +2.478084232e-03, +8.321206732e-05, +2.474045346e+00 - 1.536436e+00, +1.088707745e-02, +7.069806391e-07, +2.347785514e-03, +8.505529260e-05, +2.473655996e+00 - 1.573910e+00, +1.161350500e-02, +1.366657274e-06, +2.453010484e-03, +9.073050782e-05, +2.579211451e+00 - 1.611384e+00, +1.284893909e-02, +2.472582291e-06, +2.368957236e-03, +1.003823366e-04, +2.837696929e+00 - 1.648859e+00, +1.431859013e-02, +3.857804985e-06, +3.068473826e-03, +1.118639854e-04, +3.200566296e+00 - 1.686333e+00, +1.568466897e-02, +5.309872634e-06, +3.469083243e-03, +1.225364764e-04, +3.569988955e+00 - 1.723807e+00, +1.672073194e-02, +6.527746720e-06, +3.499417909e-03, +1.306307183e-04, +3.860977237e+00 - 1.761281e+00, +1.735917505e-02, +7.046758211e-06, +3.289526062e-03, +1.356185551e-04, +4.031108681e+00 - 1.798755e+00, +1.767118126e-02, +6.301190978e-06, +3.511818671e-03, +1.380561036e-04, +4.086235159e+00 - 1.836229e+00, +1.781514385e-02, +5.066555573e-06, +3.414133251e-03, +1.391808113e-04, +4.072115006e+00 - 1.873703e+00, +1.795734364e-02, +9.785530080e-06, +3.394013652e-03, +1.402917472e-04, +4.053080663e+00 - 1.911177e+00, +1.820297960e-02, +2.153285729e-05, +3.420849841e-03, +1.422107781e-04, +4.080884955e+00 - 1.948651e+00, +1.856531984e-02, +3.797782051e-05, +3.234892450e-03, +1.450415613e-04, +4.169137462e+00 - 1.986125e+00, +1.899015399e-02, +5.721899660e-05, +3.396745120e-03, +1.483605781e-04, +4.292812394e+00 - 2.023599e+00, +1.939524079e-02, +6.688604613e-05, +3.250581869e-03, +1.515253187e-04, +4.411264745e+00 - 2.061073e+00, +1.970534621e-02, +8.499884299e-05, +3.312961563e-03, +1.539480172e-04, +4.493132813e+00 - 2.098547e+00, +1.988283047e-02, +6.241843847e-05, +3.301605834e-03, +1.553346131e-04, +4.527959658e+00 - 2.136021e+00, +1.993607358e-02, +7.807678218e-05, +3.172906433e-03, +1.557505749e-04, +4.524294614e+00 - 2.173495e+00, +1.989960316e-02, +4.892650876e-05, +3.293718876e-03, +1.554656497e-04, +4.500332857e+00 - 2.210969e+00, +1.981531470e-02, +6.512328567e-05, +3.115928915e-03, +1.548071461e-04, +4.473391974e+00 - 2.248443e+00, +1.972658286e-02, +1.542799595e-04, +3.228336502e-03, +1.541139286e-04, +4.453059505e+00 - 2.285917e+00, +1.965740159e-02, +2.721087405e-04, +3.180446523e-03, +1.535734500e-04, +4.440114615e+00 - 2.323392e+00, +1.959904705e-02, +3.980665951e-04, +3.107500308e-03, +1.531175551e-04, +4.429876188e+00 - 2.360866e+00, +1.953282420e-02, +4.003191631e-04, +3.189015088e-03, +1.526001891e-04, +4.416676540e+00 - 2.398340e+00, +1.944869644e-02, +5.464091330e-04, +2.981980375e-03, +1.519429410e-04, +4.396869067e+00 - 2.435814e+00, +1.933806044e-02, +4.155885738e-04, +3.140904615e-03, +1.510785972e-04, +4.369773349e+00 - 2.473288e+00, +1.920073398e-02, +4.446796367e-04, +3.059026759e-03, +1.500057342e-04, +4.337298959e+00 - 2.510762e+00, +1.905522218e-02, +3.406503422e-04, +3.038745874e-03, +1.488689233e-04, +4.303041277e+00 - 2.548236e+00, +1.892141281e-02, +1.821448184e-04, +3.082892831e-03, +1.478235376e-04, +4.271004959e+00 - 2.585710e+00, +1.879970376e-02, +8.506633659e-05, +2.887937012e-03, +1.468726856e-04, +4.243717289e+00 - 2.623184e+00, +1.867373515e-02, +1.004256191e-04, +3.051640505e-03, +1.458885559e-04, +4.219927993e+00 - 2.660658e+00, +1.852070404e-02, +1.196737672e-04, +2.938772388e-03, +1.446930003e-04, +4.193080930e+00 - 2.698132e+00, +1.830775800e-02, +1.071572532e-04, +2.968150277e-03, +1.430293593e-04, +4.152255197e+00 - 2.735606e+00, +1.800218945e-02, +1.320213036e-04, +2.975736631e-03, +1.406421051e-04, +4.086458524e+00 - 2.773080e+00, +1.760560892e-02, +9.506618993e-05, +2.829530709e-03, +1.375438197e-04, +3.991233526e+00 - 2.810554e+00, +1.716131434e-02, +7.317906244e-05, +2.954310998e-03, +1.340727683e-04, +3.874560228e+00 - 2.848028e+00, +1.672503705e-02, +4.353865043e-05, +2.810145815e-03, +1.306643519e-04, +3.757798136e+00 - 2.885502e+00, +1.634861092e-02, +2.010917331e-05, +2.884632857e-03, +1.277235228e-04, +3.667738464e+00 - 2.922976e+00, +1.607034328e-02, +7.273872482e-06, +2.862475827e-03, +1.255495569e-04, +3.620214926e+00 - 2.960451e+00, +1.586218229e-02, +9.724124820e-06, +2.809723238e-03, +1.239232991e-04, +3.604829965e+00 - 2.997925e+00, +1.562061457e-02, +1.277801777e-05, +2.899257020e-03, +1.220360513e-04, +3.584446446e+00 - 3.035399e+00, +1.522782617e-02, +1.233271869e-05, +2.721416525e-03, +1.189673920e-04, +3.511909116e+00 - 3.072873e+00, +1.458718439e-02, +1.012356765e-05, +2.782997075e-03, +1.139623780e-04, +3.352720251e+00 - 3.110347e+00, +1.366320273e-02, +7.294721417e-06, +2.762079543e-03, +1.067437714e-04, +3.102486911e+00 - 3.147821e+00, +1.251768481e-02, +4.420404970e-06, +2.455956523e-03, +9.779441258e-05, +2.794887923e+00 - 3.185295e+00, +1.133578876e-02, +2.426619315e-06, +2.045393409e-03, +8.856084967e-05, +2.497506493e+00 - 3.222769e+00, +1.036779336e-02, +1.089662333e-06, +2.156643882e-03, +8.099838562e-05, +2.287032135e+00 - 3.260243e+00, +9.775726216e-03, +7.775481631e-07, +2.063789978e-03, +7.637286106e-05, +2.199308734e+00 - 3.297717e+00, +9.532734129e-03, +6.108364501e-07, +2.153749277e-03, +7.447448538e-05, +2.194284318e+00 - 3.335191e+00, +9.398661297e-03, +8.216045826e-07, +2.135688544e-03, +7.342704138e-05, +2.187377027e+00 - 3.372665e+00, +9.084908041e-03, +4.990304566e-07, +2.040467603e-03, +7.097584407e-05, +2.107636508e+00 - 3.410139e+00, +8.406765284e-03, +2.134421476e-07, +2.145048814e-03, +6.567785378e-05, +1.925405968e+00 - 3.447613e+00, +7.345670750e-03, +2.218387414e-07, +1.993262691e-03, +5.738805273e-05, +1.652352466e+00 - 3.485087e+00, +6.047746911e-03, +3.358003217e-07, +1.633531203e-03, +4.724802274e-05, +1.332422196e+00 - 3.522561e+00, +4.746403671e-03, +5.213325765e-07, +1.144017234e-03, +3.708127868e-05, +1.031986493e+00 - 3.560035e+00, +3.769640679e-03, +7.310929636e-07, +8.020516541e-04, +2.945031781e-05, +8.248254263e-01 - 3.597509e+00, +3.333383976e-03, +7.229544209e-07, +7.732301786e-04, +2.604206231e-05, +7.467641311e-01 - 3.634984e+00, +3.288196133e-03, +5.789536836e-07, +8.209433331e-04, +2.568903229e-05, +7.476238539e-01 - 3.672458e+00, +3.293284166e-03, +8.017460625e-07, +7.778578434e-04, +2.572878255e-05, +7.473541982e-01 - 3.709932e+00, +3.117292919e-03, +4.714346928e-07, +8.140977771e-04, +2.435385093e-05, +7.038828045e-01 - 3.747406e+00, +2.729593487e-03, +5.298796474e-07, +7.857588558e-04, +2.132494912e-05, +6.126782673e-01 - 3.784880e+00, +2.180764441e-03, +1.216180682e-06, +6.625316699e-04, +1.703722219e-05, +4.890491612e-01 - 3.822354e+00, +1.586088429e-03, +2.384923708e-06, +4.875486122e-04, +1.239131585e-05, +3.558862685e-01 - 3.859828e+00, +1.066809698e-03, +3.511481400e-06, +3.033193280e-04, +8.334450763e-06, +2.372274680e-01 - 3.897302e+00, +7.233506887e-04, +4.708544179e-06, +1.463310009e-04, +5.651177256e-06, +1.561842423e-01 - 3.934776e+00, +6.203759634e-04, +3.573109786e-06, +1.545835346e-04, +4.846687214e-06, +1.265111525e-01 - 3.972250e+00, +6.226640658e-04, +4.851805755e-06, +1.561429812e-04, +4.864563014e-06, +1.273690025e-01 - 4.009724e+00, +6.087857640e-04, +4.093612810e-06, +1.485704092e-04, +4.756138781e-06, +1.276030578e-01 - 4.047198e+00, +5.400191464e-04, +3.742942575e-06, +1.478591034e-04, +4.218899581e-06, +1.171779781e-01 - 4.084672e+00, +4.401182623e-04, +4.379635266e-06, +1.273842800e-04, +3.438423924e-06, +9.815150341e-02 - 4.122146e+00, +3.324057871e-04, +3.390264825e-06, +9.627154161e-05, +2.596920212e-06, +7.584800868e-02 - 4.159620e+00, +2.370075780e-04, +4.831246141e-06, +6.338579234e-05, +1.851621704e-06, +5.553016806e-02 - 4.197094e+00, +1.811418738e-04, +5.641649803e-06, +3.631936953e-05, +1.415170889e-06, +4.116690846e-02 - 4.234568e+00, +1.557249175e-04, +2.668727426e-06, +2.221487393e-05, +1.216600918e-06, +3.415513459e-02 - 4.272043e+00, +1.465187123e-04, +1.674662891e-06, +2.170675953e-05, +1.144677440e-06, +3.215710958e-02 - 4.309517e+00, +1.405423711e-04, +1.924531308e-06, +2.149418176e-05, +1.097987274e-06, +3.160248470e-02 - 4.346991e+00, +1.357847028e-04, +1.605850099e-06, +2.192148957e-05, +1.060817991e-06, +3.065472344e-02 - 4.384465e+00, +1.258550769e-04, +1.284603555e-06, +2.046648335e-05, +9.832427884e-07, +2.904522959e-02 - 4.421939e+00, +1.169869494e-04, +1.278083034e-06, +2.159284921e-05, +9.139605420e-07, +2.713926010e-02 - 4.459413e+00, +1.110703419e-04, +1.509709848e-06, +2.079338937e-05, +8.677370462e-07, +2.542558185e-02 - 4.496887e+00, +1.057701132e-04, +5.790175045e-07, +2.100616009e-05, +8.263290094e-07, +2.427050930e-02 - 4.534361e+00, +1.026251684e-04, +7.810732683e-07, +2.116271305e-05, +8.017591281e-07, +2.376362919e-02 - 4.571835e+00, +1.011917515e-04, +5.963700164e-07, +2.046230847e-05, +7.905605584e-07, +2.366920631e-02 - 4.609309e+00, +1.001634828e-04, +8.947496930e-07, +2.134477231e-05, +7.825272092e-07, +2.354631490e-02 - 4.646783e+00, +9.707341240e-05, +3.485090968e-07, +2.031439678e-05, +7.583860344e-07, +2.295738668e-02 - 4.684257e+00, +9.167802975e-05, +3.892734021e-07, +2.066985033e-05, +7.162346074e-07, +2.162696749e-02 - 4.721731e+00, +8.383810233e-05, +2.180439299e-07, +2.059725780e-05, +6.549851744e-07, +1.951430734e-02 - 4.759205e+00, +7.380265088e-05, +2.824979897e-07, +1.856655807e-05, +5.765832100e-07, +1.682504287e-02 - 4.796679e+00, +6.231475173e-05, +3.941094597e-07, +1.489212553e-05, +4.868339979e-07, +1.398756018e-02 - 4.834153e+00, +5.205405290e-05, +9.286779095e-08, +1.031777326e-05, +4.066722883e-07, +1.158631725e-02 - 4.871627e+00, +4.563461790e-05, +2.947284984e-07, +9.944042898e-06, +3.565204523e-07, +1.014939239e-02 - 4.909101e+00, +4.281885319e-05, +2.428192628e-07, +9.489779107e-06, +3.345222905e-07, +9.727415693e-03 - 4.946576e+00, +4.221795820e-05, +3.153020264e-07, +1.010970700e-05, +3.298277985e-07, +9.783286724e-03 - 4.984050e+00, +4.193062269e-05, +1.303311765e-07, +9.664688198e-06, +3.275829898e-07, +9.697488727e-03 - 5.021524e+00, +3.983627123e-05, +2.205995585e-07, +9.929565065e-06, +3.112208689e-07, +9.137274998e-03 - 5.058998e+00, +3.557555053e-05, +4.469302524e-07, +9.812803061e-06, +2.779339885e-07, +8.056891307e-03 - 5.096472e+00, +2.974014513e-05, +3.707369784e-07, +8.638593246e-06, +2.323448838e-07, +6.593883084e-03 - 5.133946e+00, +2.283060209e-05, +1.698980880e-07, +6.683001259e-06, +1.783640788e-07, +4.979337600e-03 - 5.171420e+00, +1.666241488e-05, +3.152850598e-07, +4.477294414e-06, +1.301751163e-07, +3.482333541e-03 - 5.208894e+00, +1.250577530e-05, +1.567322049e-07, +2.385718952e-06, +9.770136956e-08, +2.384295551e-03 - 5.246368e+00, +1.120766256e-05, +2.127376472e-07, +2.042794895e-06, +8.755986377e-08, +1.893867546e-03 - 5.283842e+00, +1.074669282e-05, +1.995524573e-07, +2.278776422e-06, +8.395853768e-08, +1.857462365e-03 - 5.321316e+00, +1.005005248e-05, +3.020729087e-07, +2.048453694e-06, +7.851603503e-08, +1.884026962e-03 - 5.358790e+00, +1.027674791e-05, +1.968048130e-07, +2.146663943e-06, +8.028709303e-08, +1.781458878e-03 - 5.396264e+00, +9.992556393e-06, +2.897864871e-07, +2.019372838e-06, +7.806684682e-08, +1.539662271e-03 - 5.433738e+00, +8.130178520e-06, +9.113637532e-08, +1.644339013e-06, +6.351701969e-08, +1.215813603e-03 - 5.471212e+00, +6.073720860e-06, +1.478698592e-07, +1.146645185e-06, +4.745094422e-08, +8.794652133e-04 - 5.508686e+00, +6.698145957e-06, +9.692050058e-08, +9.740123784e-07, +5.232926529e-08, +5.925767028e-04 - 5.546160e+00, +7.402377781e-06, +3.073583532e-07, +1.094274187e-06, +5.783107641e-08, +4.025875390e-04 - 5.583635e+00, +6.171371328e-06, +1.350665052e-07, +9.761632436e-07, +4.821383850e-08, +3.250012544e-04 - 5.621109e+00, +5.253378796e-06, +9.249799012e-08, +7.532469469e-07, +4.104202184e-08, +3.144552650e-04 - 5.658583e+00, +6.034991375e-06, +1.524639589e-07, +8.379380711e-07, +4.714837011e-08, +3.143233654e-04 - 5.696057e+00, +6.954997063e-06, +1.738911611e-07, +1.032799808e-06, +5.433591456e-08, +3.053781279e-04 - 5.733531e+00, +6.618310969e-06, +1.393653114e-07, +9.706904912e-07, +5.170555445e-08, +2.905274734e-04 - 5.771005e+00, +4.983683461e-06, +7.755124598e-08, +7.959423977e-07, +3.893502704e-08, +2.767846518e-04 - 5.808479e+00, +5.498600911e-06, +5.420209390e-08, +8.069513404e-07, +4.295781962e-08, +2.672840112e-04 - 5.845953e+00, +6.952197529e-06, +1.517748078e-07, +1.004761064e-06, +5.431404319e-08, +2.606043537e-04 - 5.883427e+00, +6.329200324e-06, +9.133362427e-08, +8.427275928e-07, +4.944687753e-08, +2.537151853e-04 - 5.920901e+00, +5.215814043e-06, +2.025757389e-08, +8.189526702e-07, +4.074854721e-08, +2.439748930e-04 - 5.958375e+00, +5.305473522e-06, +1.698308984e-07, +6.730686945e-07, +4.144901189e-08, +2.288006244e-04 - 5.995849e+00, +6.176556538e-06, +1.308471106e-07, +9.587655854e-07, +4.825434796e-08, +2.091970502e-04 - 6.033323e+00, +6.645687902e-06, +8.608255961e-09, +8.996641328e-07, +5.191943673e-08, +1.868959007e-04 - 6.070797e+00, +5.398006420e-06, +1.508346841e-07, +8.539653084e-07, +4.217192516e-08, +1.639107998e-04 - 6.108271e+00, +4.639737004e-06, +1.461880318e-07, +5.846045576e-07, +3.624794534e-08, +1.453359455e-04 - 6.145745e+00, +6.145503127e-06, +8.241347046e-08, +9.145260134e-07, +4.801174318e-08, +1.340081738e-04 - 6.183219e+00, +6.485408075e-06, +6.256052170e-08, +9.129623215e-07, +5.066725059e-08, +1.300350502e-04 - 6.220694e+00, +5.542152354e-06, +8.574382827e-08, +7.821938023e-07, +4.329806526e-08, +1.300871660e-04 - 6.258168e+00, +4.985299793e-06, +1.082258410e-07, +7.100280797e-07, +3.894765463e-08, +1.297254971e-04 - 6.295642e+00, +5.334224443e-06, +4.704382370e-08, +7.980106490e-07, +4.167362846e-08, +1.253586088e-04 - 6.333116e+00, +6.421229966e-06, +2.899044795e-07, +9.204385162e-07, +5.016585911e-08, +1.158965803e-04 - 6.370590e+00, +6.073729617e-06, +8.837766608e-08, +7.781931780e-07, +4.745101264e-08, +1.015407485e-04 - 6.408064e+00, +4.266462607e-06, +7.837010701e-08, +6.328355156e-07, +3.333173912e-08, +8.295426639e-05 - 6.445538e+00, +5.024334961e-06, +9.562404129e-08, +6.546439061e-07, +3.925261688e-08, +6.413712390e-05 - 6.483012e+00, +6.379472633e-06, +2.230152814e-07, +9.421354668e-07, +4.983962995e-08, +4.719626387e-05 - 6.520486e+00, +5.693647492e-06, +1.453002947e-07, +7.169230581e-07, +4.448162103e-08, +3.428721761e-05 - 6.557960e+00, +4.451825136e-06, +6.643439179e-08, +7.111721174e-07, +3.477988388e-08, +2.871636911e-05 - 6.595434e+00, +4.899532173e-06, +1.455459486e-07, +7.337733172e-07, +3.827759510e-08, +2.830713222e-05 - 6.632908e+00, +5.715246554e-06, +1.362644650e-07, +8.893272823e-07, +4.465036370e-08, +2.868930373e-05 - 6.670382e+00, +5.627211642e-06, +1.506604533e-07, +7.797610449e-07, +4.396259095e-08, +2.757669232e-05 - 6.707856e+00, +4.948318403e-06, +1.411311683e-07, +7.615920776e-07, +3.865873753e-08, +2.478379040e-05 - 6.745330e+00, +4.504617494e-06, +8.895024618e-08, +7.798763415e-07, +3.519232417e-08, +2.028553515e-05 - 6.782804e+00, +4.993989569e-06, +8.728616039e-08, +7.967544777e-07, +3.901554351e-08, +1.596593919e-05 - 6.820278e+00, +5.747899974e-06, +1.409305399e-07, +8.337878046e-07, +4.490546855e-08, +1.262078777e-05 - 6.857752e+00, +5.247855022e-06, +1.813322600e-07, +7.726407085e-07, +4.099886736e-08, +9.582348020e-06 - 6.895227e+00, +3.985176119e-06, +3.643501908e-09, +6.312561916e-07, +3.113418843e-08, +7.451330020e-06 - 6.932701e+00, +4.586520602e-06, +8.433082430e-08, +6.862483344e-07, +3.583219221e-08, +7.714032597e-06 - 6.970175e+00, +5.649888792e-06, +9.980674145e-08, +8.434611994e-07, +4.413975619e-08, +8.153444926e-06 - 7.007649e+00, +5.253207318e-06, +6.315408219e-08, +6.957074664e-07, +4.104068217e-08, +8.013247188e-06 - 7.045123e+00, +3.957427885e-06, +3.892856681e-08, +6.417153927e-07, +3.091740535e-08, +7.103148867e-06 - 7.082597e+00, +4.067553643e-06, +1.030158487e-07, +5.483071957e-07, +3.177776283e-08, +6.251987237e-06 - 7.120071e+00, +5.339245594e-06, +3.932283532e-08, +8.433147156e-07, +4.171285620e-08, +7.632282159e-06 - 7.157545e+00, +5.327917860e-06, +1.177531273e-07, +6.938631171e-07, +4.162435828e-08, +7.329495938e-06 - 7.195019e+00, +3.969772302e-06, +7.950854911e-08, +6.036690244e-07, +3.101384611e-08, +6.045781632e-06 - 7.232493e+00, +3.715633257e-06, +1.404288927e-07, +5.594073805e-07, +2.902838482e-08, +5.792377053e-06 - 7.269967e+00, +5.014965578e-06, +1.612842163e-07, +7.753567255e-07, +3.917941858e-08, +6.622437127e-06 - 7.307441e+00, +5.352770855e-06, +1.184136808e-07, +7.375166156e-07, +4.181852231e-08, +7.298828655e-06 - 7.344915e+00, +4.153471107e-06, +4.480572672e-08, +6.123685349e-07, +3.244899303e-08, +5.538450355e-06 - 7.382389e+00, +3.599982642e-06, +3.650950617e-08, +5.446719495e-07, +2.812486439e-08, +5.496705485e-06 - 7.419863e+00, +4.830764950e-06, +5.884614929e-08, +7.174872599e-07, +3.774035117e-08, +6.712350142e-06 - 7.457337e+00, +5.146869399e-06, +1.163178472e-07, +7.555272227e-07, +4.020991718e-08, +6.428643692e-06 - 7.494811e+00, +4.397401248e-06, +1.080430387e-07, +5.798179600e-07, +3.435469725e-08, +6.238567531e-06 - Mean, +4.855456550e-03, +2.700723318e-04, +3.584548722e-04, +3.793325429e-05, +1.099258140e+00 + 3.747406e-02, +5.656659454e-06, +4.839334474e-19, +1.429016906e-06, +1.884177417e-07, +1.015013075e-04 + 7.494811e-02, +1.331364706e-05, +1.250394489e-18, +3.354073322e-06, +4.532531474e-07, +1.286451027e-04 + 1.124222e-01, +4.532668493e-06, +8.678982156e-19, +1.143753276e-06, +1.587978169e-07, +1.363415075e-04 + 1.498962e-01, +5.315482992e-06, +5.365268297e-19, +1.352478573e-06, +1.844891507e-07, +2.237425263e-04 + 1.873703e-01, +2.286991326e-05, +3.008666113e-18, +5.789494315e-06, +7.781958289e-07, +5.988175109e-04 + 2.248443e-01, +1.597419759e-05, +2.193338141e-18, +4.067429237e-06, +5.797052699e-07, +1.231303285e-03 + 2.623184e-01, +1.145612024e-05, +5.204731821e-18, +3.069144677e-06, +4.523653992e-07, +2.183486767e-03 + 2.997925e-01, +2.086398938e-05, +1.270987622e-17, +5.143395002e-06, +8.183715070e-07, +3.443837437e-03 + 3.372665e-01, +3.014646152e-05, +1.995241492e-17, +8.564706304e-06, +1.096508508e-06, +4.896408733e-03 + 3.747406e-01, +2.951847940e-05, +5.642532431e-17, +8.109678263e-06, +1.146518725e-06, +6.288158078e-03 + 4.122146e-01, +3.920398129e-05, +1.410851956e-17, +1.039662099e-05, +1.506290838e-06, +7.236866239e-03 + 4.496887e-01, +5.485446652e-05, +1.411258486e-16, +1.373626813e-05, +2.138216877e-06, +7.438562524e-03 + 4.871627e-01, +5.305798814e-05, +1.745521413e-16, +1.341910982e-05, +2.244500004e-06, +7.655638548e-03 + 5.246368e-01, +6.324063047e-05, +3.538137933e-16, +1.609936398e-05, +2.567937221e-06, +1.144492786e-02 + 5.621109e-01, +9.953861113e-05, +6.647586246e-16, +2.871846797e-05, +3.979070330e-06, +2.129721915e-02 + 5.995849e-01, +1.560743589e-04, +6.830591915e-16, +5.021692981e-05, +5.826054663e-06, +3.669178320e-02 + 6.370590e-01, +2.357106590e-04, +2.975830288e-15, +7.762490221e-05, +8.212787663e-06, +5.641443987e-02 + 6.745330e-01, +3.417912838e-04, +2.135164474e-15, +1.077684524e-04, +1.198621562e-05, +7.828637698e-02 + 7.120071e-01, +4.480155462e-04, +8.422598055e-15, +1.327195791e-04, +1.653183123e-05, +9.885177909e-02 + 7.494811e-01, +5.361316672e-04, +2.205573753e-14, +1.452888272e-04, +2.046928227e-05, +1.137531996e-01 + 7.869552e-01, +5.828228357e-04, +3.676720758e-14, +1.419796915e-04, +2.316840106e-05, +1.195346821e-01 + 8.244293e-01, +5.919511494e-04, +1.531242976e-13, +1.546158786e-04, +2.529399186e-05, +1.188340974e-01 + 8.619033e-01, +6.248414803e-04, +8.377203739e-13, +1.440128726e-04, +2.829825492e-05, +1.309553939e-01 + 8.993774e-01, +8.151146601e-04, +3.513353292e-12, +2.051054800e-04, +3.659201856e-05, +1.857760966e-01 + 9.368514e-01, +1.238030606e-03, +1.222959768e-11, +3.772512881e-04, +5.099828692e-05, +2.867307084e-01 + 9.743255e-01, +1.814098313e-03, +3.837194374e-11, +5.700103264e-04, +7.220437043e-05, +4.167673658e-01 + 1.011800e+00, +2.437431108e-03, +1.109193524e-10, +7.477328124e-04, +9.448610394e-05, +5.549049357e-01 + 1.049274e+00, +2.992538657e-03, +2.905930910e-10, +8.658103031e-04, +1.186157958e-04, +6.782106751e-01 + 1.086748e+00, +3.372602378e-03, +6.709425516e-10, +8.810228476e-04, +1.408439490e-04, +7.648957645e-01 + 1.124222e+00, +3.518151571e-03, +1.316486442e-09, +8.457004166e-04, +1.549572611e-04, +8.022489576e-01 + 1.161696e+00, +3.491978300e-03, +2.035218488e-09, +8.789616841e-04, +1.613984637e-04, +8.009704160e-01 + 1.199170e+00, +3.564279227e-03, +1.920368097e-09, +8.112931293e-04, +1.690581835e-04, +8.143229074e-01 + 1.236644e+00, +4.138418534e-03, +3.626219426e-09, +8.645553519e-04, +2.078914976e-04, +9.294380600e-01 + 1.274118e+00, +5.324827324e-03, +1.871304041e-08, +1.353962850e-03, +2.601141620e-04, +1.185764957e+00 + 1.311592e+00, +6.847371238e-03, +5.579282980e-08, +1.923072314e-03, +3.232892188e-04, +1.534641583e+00 + 1.349066e+00, +8.347554914e-03, +1.270395048e-07, +2.347347422e-03, +3.889148474e-04, +1.896337013e+00 + 1.386540e+00, +9.532772095e-03, +2.368876072e-07, +2.527956003e-03, +4.514086852e-04, +2.199936335e+00 + 1.424014e+00, +1.024824705e-02, +3.669957628e-07, +2.406072494e-03, +5.056009733e-04, +2.397678575e+00 + 1.461488e+00, +1.052203770e-02, +4.698294987e-07, +2.444667123e-03, +5.393140228e-04, +2.477217295e+00 + 1.498962e+00, +1.058885932e-02, +5.175642757e-07, +2.462910797e-03, +5.619191776e-04, +2.474013829e+00 + 1.536436e+00, +1.082620304e-02, +7.023524853e-07, +2.333885666e-03, +5.875106486e-04, +2.473624483e+00 + 1.573910e+00, +1.155161542e-02, +1.358321114e-06, +2.437891160e-03, +6.540547261e-04, +2.579178593e+00 + 1.611384e+00, +1.278137828e-02, +2.460277494e-06, +2.356865899e-03, +7.374532022e-04, +2.837660777e+00 + 1.648859e+00, +1.424151084e-02, +3.841030656e-06, +3.050855223e-03, +8.230119162e-04, +3.200525521e+00 + 1.686333e+00, +1.559704403e-02, +5.289170111e-06, +3.447606735e-03, +9.115234740e-04, +3.569943474e+00 + 1.723807e+00, +1.662444075e-02, +6.505705770e-06, +3.476585555e-03, +9.874082185e-04, +3.860928050e+00 + 1.761281e+00, +1.725790653e-02, +7.028429069e-06, +3.269676383e-03, +1.054718212e-03, +4.031057327e+00 + 1.798755e+00, +1.756883176e-02, +6.292185134e-06, +3.490071839e-03, +1.100129526e-03, +4.086183103e+00 + 1.836229e+00, +1.771426204e-02, +5.053355061e-06, +3.393376632e-03, +1.130205560e-03, +4.072063131e+00 + 1.873703e+00, +1.785821205e-02, +9.718791907e-06, +3.372883096e-03, +1.161533370e-03, +4.053029029e+00 + 1.911177e+00, +1.810390712e-02, +2.140071884e-05, +3.399366407e-03, +1.205529210e-03, +4.080832967e+00 + 1.948651e+00, +1.846400001e-02, +3.776416235e-05, +3.215132108e-03, +1.262075112e-03, +4.169084349e+00 + 1.986125e+00, +1.888516033e-02, +5.691108071e-05, +3.375389793e-03, +1.312380663e-03, +4.292757706e+00 + 2.023599e+00, +1.928666314e-02, +6.662337504e-05, +3.230487063e-03, +1.367886609e-03, +4.411208547e+00 + 2.061073e+00, +1.959441958e-02, +8.468956787e-05, +3.292283141e-03, +1.415407565e-03, +4.493075573e+00 + 2.098547e+00, +1.977112668e-02, +6.223830173e-05, +3.280926680e-03, +1.450258368e-03, +4.527901974e+00 + 2.136021e+00, +1.982480843e-02, +7.762739708e-05, +3.153403839e-03, +1.477408436e-03, +4.524236977e+00 + 2.173495e+00, +1.978929475e-02, +4.860199538e-05, +3.273005134e-03, +1.485745556e-03, +4.500275525e+00 + 2.210969e+00, +1.970586078e-02, +6.477189253e-05, +3.096774043e-03, +1.493075419e-03, +4.473334986e+00 + 2.248443e+00, +1.961764306e-02, +1.534757406e-04, +3.208131565e-03, +1.497128893e-03, +4.453002776e+00 + 2.285917e+00, +1.954872068e-02, +2.706281300e-04, +3.160574331e-03, +1.507315970e-03, +4.440058051e+00 + 2.323392e+00, +1.949054619e-02, +3.957934621e-04, +3.088297962e-03, +1.515726763e-03, +4.429819755e+00 + 2.360866e+00, +1.942459939e-02, +3.988303302e-04, +3.168967366e-03, +1.518408645e-03, +4.416620275e+00 + 2.398340e+00, +1.934097007e-02, +5.433495872e-04, +2.963763007e-03, +1.520978987e-03, +4.396813054e+00 + 2.435814e+00, +1.923104349e-02, +4.136980080e-04, +3.121203146e-03, +1.509413450e-03, +4.369717681e+00 + 2.473288e+00, +1.909454427e-02, +4.433117467e-04, +3.039974758e-03, +1.497084750e-03, +4.337243705e+00 + 2.510762e+00, +1.894990382e-02, +3.389526031e-04, +3.019884470e-03, +1.477309988e-03, +4.302986459e+00 + 2.548236e+00, +1.881692297e-02, +1.813552221e-04, +3.063530458e-03, +1.457192590e-03, +4.270950549e+00 + 2.585710e+00, +1.869587728e-02, +8.465299821e-05, +2.870368490e-03, +1.442518900e-03, +4.243663227e+00 + 2.623184e+00, +1.857037699e-02, +9.976407527e-05, +3.032463236e-03, +1.425813526e-03, +4.219874234e+00 + 2.660658e+00, +1.841781342e-02, +1.192521509e-04, +2.920544574e-03, +1.407031587e-03, +4.193027513e+00 + 2.698132e+00, +1.820568732e-02, +1.067729875e-04, +2.949664425e-03, +1.373492487e-03, +4.152202300e+00 + 2.735606e+00, +1.790164964e-02, +1.313201035e-04, +2.957071910e-03, +1.333220921e-03, +4.086406465e+00 + 2.773080e+00, +1.750753979e-02, +9.473131355e-05, +2.812202072e-03, +1.282149903e-03, +3.991182680e+00 + 2.810554e+00, +1.706650266e-02, +7.279758402e-05, +2.935739859e-03, +1.226809624e-03, +3.874510868e+00 + 2.848028e+00, +1.663352768e-02, +4.330250216e-05, +2.792808427e-03, +1.175650443e-03, +3.757750265e+00 + 2.885502e+00, +1.625942813e-02, +1.998524832e-05, +2.866590860e-03, +1.122929499e-03, +3.667691740e+00 + 2.922976e+00, +1.598195181e-02, +7.232592982e-06, +2.844532135e-03, +1.081737190e-03, +3.620168807e+00 + 2.960451e+00, +1.577338960e-02, +9.700374697e-06, +2.792731012e-03, +1.049770950e-03, +3.604784042e+00 + 2.997925e+00, +1.553146450e-02, +1.273990493e-05, +2.881285129e-03, +1.016515754e-03, +3.584400783e+00 + 3.035399e+00, +1.514003980e-02, +1.228964683e-05, +2.704823364e-03, +9.688133854e-04, +3.511864377e+00 + 3.072873e+00, +1.450366264e-02, +1.008435101e-05, +2.764476254e-03, +9.052991708e-04, +3.352677539e+00 + 3.110347e+00, +1.358684772e-02, +7.263619326e-06, +2.744507612e-03, +8.326497080e-04, +3.102447387e+00 + 3.147821e+00, +1.245005031e-02, +4.397706483e-06, +2.441482369e-03, +7.515377896e-04, +2.794852318e+00 + 3.185295e+00, +1.127615095e-02, +2.412352806e-06, +2.032806362e-03, +6.752724457e-04, +2.497474676e+00 + 3.222769e+00, +1.031311461e-02, +1.082456494e-06, +2.143285167e-03, +6.040960991e-04, +2.287003000e+00 + 3.260243e+00, +9.722098447e-03, +7.766219178e-07, +2.051504040e-03, +5.487466088e-04, +2.199280716e+00 + 3.297717e+00, +9.477988980e-03, +6.106994583e-07, +2.140591601e-03, +5.231365919e-04, +2.194256365e+00 + 3.335191e+00, +9.343483349e-03, +8.212455600e-07, +2.122709661e-03, +5.020052066e-04, +2.187349161e+00 + 3.372665e+00, +9.031910827e-03, +4.985683249e-07, +2.027202802e-03, +4.683100142e-04, +2.107609658e+00 + 3.410139e+00, +8.358969454e-03, +2.133098618e-07, +2.131712733e-03, +4.161829301e-04, +1.925381439e+00 + 3.447613e+00, +7.305386060e-03, +2.216151869e-07, +1.981383473e-03, +3.560405545e-04, +1.652331416e+00 + 3.485087e+00, +6.015930589e-03, +3.339547581e-07, +1.624217000e-03, +2.931416733e-04, +1.332405222e+00 + 3.522561e+00, +4.722084916e-03, +5.182195532e-07, +1.137772700e-03, +2.357286023e-04, +1.031973346e+00 + 3.560035e+00, +3.750080035e-03, +7.270196567e-07, +7.975887764e-04, +1.902983164e-04, +8.248149186e-01 + 3.597509e+00, +3.315251942e-03, +7.201736437e-07, +7.688036370e-04, +1.618009553e-04, +7.467546178e-01 + 3.634984e+00, +3.269772328e-03, +5.774292198e-07, +8.162085931e-04, +1.582682324e-04, +7.476143297e-01 + 3.672458e+00, +3.274902133e-03, +7.976089495e-07, +7.735399332e-04, +1.531263374e-04, +7.473446775e-01 + 3.709932e+00, +3.100066670e-03, +4.682881045e-07, +8.093187229e-04, +1.390009277e-04, +7.038738376e-01 + 3.747406e+00, +2.714688495e-03, +5.266096132e-07, +7.812101182e-04, +1.164492725e-04, +6.126704622e-01 + 3.784880e+00, +2.168877746e-03, +1.208501048e-06, +6.586765463e-04, +9.254620437e-05, +4.890429311e-01 + 3.822354e+00, +1.577433577e-03, +2.371185309e-06, +4.846493602e-04, +7.036170965e-05, +3.558817348e-01 + 3.859828e+00, +1.061092217e-03, +3.489844866e-06, +3.014387380e-04, +5.200299178e-05, +2.372244458e-01 + 3.897302e+00, +7.196957159e-04, +4.678915832e-06, +1.453501800e-04, +3.991080599e-05, +1.561822526e-01 + 3.934776e+00, +6.175797222e-04, +3.556228444e-06, +1.538658023e-04, +3.448591183e-05, +1.265095408e-01 + 3.972250e+00, +6.198402098e-04, +4.834120749e-06, +1.554232146e-04, +3.396382816e-05, +1.273673799e-01 + 4.009724e+00, +6.058869618e-04, +4.073170316e-06, +1.478534484e-04, +3.240172211e-05, +1.276014323e-01 + 4.047198e+00, +5.372633895e-04, +3.729312485e-06, +1.470744798e-04, +2.867360832e-05, +1.171764854e-01 + 4.084672e+00, +4.377458730e-04, +4.352552351e-06, +1.266662599e-04, +2.420054957e-05, +9.815025303e-02 + 4.122146e+00, +3.305299603e-04, +3.376120114e-06, +9.569403430e-05, +2.028618803e-05, +7.584704242e-02 + 4.159620e+00, +2.355973660e-04, +4.810808574e-06, +6.297385767e-05, +1.629290325e-05, +5.552946065e-02 + 4.197094e+00, +1.801278525e-04, +5.620603917e-06, +3.607504563e-05, +1.410172159e-05, +4.116638402e-02 + 4.234568e+00, +1.549129975e-04, +2.661303486e-06, +2.207187955e-05, +1.253413700e-05, +3.415469948e-02 + 4.272043e+00, +1.457537798e-04, +1.670028107e-06, +2.156859992e-05, +1.145563068e-05, +3.215669992e-02 + 4.309517e+00, +1.397721813e-04, +1.917059150e-06, +2.135580367e-05, +1.088938993e-05, +3.160208211e-02 + 4.346991e+00, +1.350346130e-04, +1.599818122e-06, +2.178265361e-05, +1.022898783e-05, +3.065433292e-02 + 4.384465e+00, +1.251285395e-04, +1.280776212e-06, +2.033729556e-05, +9.244076055e-06, +2.904485957e-02 + 4.421939e+00, +1.163045493e-04, +1.277606632e-06, +2.145481482e-05, +8.237540618e-06, +2.713891437e-02 + 4.459413e+00, +1.104395022e-04, +1.506864487e-06, +2.066125793e-05, +7.780941814e-06, +2.542525795e-02 + 4.496887e+00, +1.051664861e-04, +5.785581780e-07, +2.087104344e-05, +7.010699791e-06, +2.427020011e-02 + 4.534361e+00, +1.020287435e-04, +7.808767188e-07, +2.102781511e-05, +6.486364065e-06, +2.376332646e-02 + 4.571835e+00, +1.005916786e-04, +5.963162817e-07, +2.033419762e-05, +6.226742951e-06, +2.366890478e-02 + 4.609309e+00, +9.956350360e-05, +8.948174405e-07, +2.120950949e-05, +6.203908051e-06, +2.354601494e-02 + 4.646783e+00, +9.648493091e-05, +3.461590621e-07, +2.018627070e-05, +5.734260658e-06, +2.295709422e-02 + 4.684257e+00, +9.112502939e-05, +3.883062686e-07, +2.053310382e-05, +5.164905501e-06, +2.162669198e-02 + 4.721731e+00, +8.334578085e-05, +2.177835208e-07, +2.046520648e-05, +4.687682932e-06, +1.951405874e-02 + 4.759205e+00, +7.338692516e-05, +2.824948814e-07, +1.845233238e-05, +4.257504749e-06, +1.682482853e-02 + 4.796679e+00, +6.197444759e-05, +3.940908030e-07, +1.480488406e-05, +3.596556545e-06, +1.398738199e-02 + 4.834153e+00, +5.177454751e-05, +9.287846008e-08, +1.026217963e-05, +2.976275229e-06, +1.158616965e-02 + 4.871627e+00, +4.538996392e-05, +2.947300616e-07, +9.884692522e-06, +2.721672544e-06, +1.014926309e-02 + 4.909101e+00, +4.257934145e-05, +2.417527055e-07, +9.432253934e-06, +2.508730323e-06, +9.727291773e-03 + 4.946576e+00, +4.197222756e-05, +3.145511971e-07, +1.004826459e-05, +2.415124360e-06, +9.783162092e-03 + 4.984050e+00, +4.168753775e-05, +1.303284528e-07, +9.606901820e-06, +2.260500356e-06, +9.697365188e-03 + 5.021524e+00, +3.960910741e-05, +2.205747653e-07, +9.867715713e-06, +2.091158785e-06, +9.137158595e-03 + 5.058998e+00, +3.537778784e-05, +4.468634674e-07, +9.753153858e-06, +1.922101446e-06, +8.056788668e-03 + 5.096472e+00, +2.958169142e-05, +3.707332509e-07, +8.587240753e-06, +1.633811265e-06, +6.593799082e-03 + 5.133946e+00, +2.271289986e-05, +1.698979251e-07, +6.643049350e-06, +1.279078028e-06, +4.979274167e-03 + 5.171420e+00, +1.658353716e-05, +3.152510878e-07, +4.450288142e-06, +1.046492112e-06, +3.482289178e-03 + 5.208894e+00, +1.245649774e-05, +1.564776138e-07, +2.370501380e-06, +9.150443492e-07, +2.384265177e-03 + 5.246368e+00, +1.117295932e-05, +2.121257863e-07, +2.031983450e-06, +8.607076636e-07, +1.893843419e-03 + 5.283842e+00, +1.071187897e-05, +1.993146525e-07, +2.267768381e-06, +7.809792359e-07, +1.857438703e-03 + 5.321316e+00, +1.001176624e-05, +3.013669505e-07, +2.037861830e-06, +7.219680786e-07, +1.884002961e-03 + 5.358790e+00, +1.024326393e-05, +1.955515665e-07, +2.134760475e-06, +7.278118421e-07, +1.781436184e-03 + 5.396264e+00, +9.966811528e-06, +2.886789209e-07, +2.008113485e-06, +7.744211989e-07, +1.539642657e-03 + 5.433738e+00, +8.110450112e-06, +9.102295863e-08, +1.634829761e-06, +6.330492611e-07, +1.215798114e-03 + 5.471212e+00, +6.059893755e-06, +1.477892133e-07, +1.139279007e-06, +4.637514657e-07, +8.794540096e-04 + 5.508686e+00, +6.692405771e-06, +9.674164163e-08, +9.740456409e-07, +5.245857386e-07, +5.925691539e-04 + 5.546160e+00, +7.399923072e-06, +3.073116713e-07, +1.094159758e-06, +6.025615222e-07, +4.025824102e-04 + 5.583635e+00, +6.169448434e-06, +1.344622774e-07, +9.761330755e-07, +4.978628371e-07, +3.249971140e-04 + 5.621109e+00, +5.251281156e-06, +9.219144089e-08, +7.529512542e-07, +4.044328657e-07, +3.144512592e-04 + 5.658583e+00, +6.033152707e-06, +1.517798126e-07, +8.376661419e-07, +4.900050921e-07, +3.143193611e-04 + 5.696057e+00, +6.953461481e-06, +1.731350010e-07, +1.032658432e-06, +5.408160264e-07, +3.053742378e-04 + 5.733531e+00, +6.616852919e-06, +1.393709780e-07, +9.707101787e-07, +5.299127849e-07, +2.905237722e-04 + 5.771005e+00, +4.981964770e-06, +7.734669672e-08, +7.959476411e-07, +3.910720171e-07, +2.767811258e-04 + 5.808479e+00, +5.497129429e-06, +5.415541090e-08, +8.069138412e-07, +4.159162126e-07, +2.672806062e-04 + 5.845953e+00, +6.951056971e-06, +1.517623716e-07, +1.004289950e-06, +5.465321079e-07, +2.606010338e-04 + 5.883427e+00, +6.328022933e-06, +9.130260613e-08, +8.427665708e-07, +4.999347176e-07, +2.537119531e-04 + 5.920901e+00, +5.214519405e-06, +2.023362231e-08, +8.188184714e-07, +3.956866551e-07, +2.439717849e-04 + 5.958375e+00, +5.304342109e-06, +1.698335842e-07, +6.730635294e-07, +4.418913029e-07, +2.287977096e-04 + 5.995849e+00, +6.175716100e-06, +1.307996671e-07, +9.587490709e-07, +4.892736786e-07, +2.091943852e-04 + 6.033323e+00, +6.645036012e-06, +8.548111301e-09, +8.996693798e-07, +5.325461238e-07, +1.868935198e-04 + 6.070797e+00, +5.397402497e-06, +1.508119283e-07, +8.539238540e-07, +4.257453984e-07, +1.639087117e-04 + 6.108271e+00, +4.639189681e-06, +1.459404705e-07, +5.846006585e-07, +3.914171158e-07, +1.453340940e-04 + 6.145745e+00, +6.145110009e-06, +8.240901159e-08, +9.145533702e-07, +4.947475022e-07, +1.340064667e-04 + 6.183219e+00, +6.485045280e-06, +6.256111494e-08, +9.130002723e-07, +5.250157101e-07, +1.300333935e-04 + 6.220694e+00, +5.541752537e-06, +8.574644437e-08, +7.822126843e-07, +4.586351499e-07, +1.300855088e-04 + 6.258168e+00, +4.984874654e-06, +1.077156320e-07, +7.100207062e-07, +3.962645038e-07, +1.297238444e-04 + 6.295642e+00, +5.333840078e-06, +4.701696829e-08, +7.978862286e-07, +4.319465292e-07, +1.253570119e-04 + 6.333116e+00, +6.420923110e-06, +2.898989892e-07, +9.204602489e-07, +5.400349919e-07, +1.158951038e-04 + 6.370590e+00, +6.073469406e-06, +8.837274556e-08, +7.781719401e-07, +4.990416886e-07, +1.015394549e-04 + 6.408064e+00, +4.266235556e-06, +7.837221964e-08, +6.328306312e-07, +3.203590672e-07, +8.295320962e-05 + 6.445538e+00, +5.024184876e-06, +9.562255997e-08, +6.546089918e-07, +4.170534811e-07, +6.413630695e-05 + 6.483012e+00, +6.379354444e-06, +2.230168975e-07, +9.421503967e-07, +5.297344304e-07, +4.719566269e-05 + 6.520486e+00, +5.693552615e-06, +1.452961270e-07, +7.169097015e-07, +4.702344561e-07, +3.428678049e-05 + 6.557960e+00, +4.451748648e-06, +6.643351337e-08, +7.111598228e-07, +3.392180044e-07, +2.871600307e-05 + 6.595434e+00, +4.899454217e-06, +1.455408984e-07, +7.337662947e-07, +4.063277411e-07, +2.830677177e-05 + 6.632908e+00, +5.715159740e-06, +1.362595137e-07, +8.892945808e-07, +4.551871839e-07, +2.868893844e-05 + 6.670382e+00, +5.627125656e-06, +1.506598940e-07, +7.797586350e-07, +4.640234721e-07, +2.757634084e-05 + 6.707856e+00, +4.948242219e-06, +1.410741165e-07, +7.615815071e-07, +3.976485709e-07, +2.478347455e-05 + 6.745330e+00, +4.504551740e-06, +8.895032984e-08, +7.798697482e-07, +3.526128990e-07, +2.028527677e-05 + 6.782804e+00, +4.993922067e-06, +8.728064318e-08, +7.967415424e-07, +3.890334694e-07, +1.596573591e-05 + 6.820278e+00, +5.747823872e-06, +1.409228846e-07, +8.337814478e-07, +4.659644325e-07, +1.262062648e-05 + 6.857752e+00, +5.247785988e-06, +1.813323881e-07, +7.726341183e-07, +4.312019165e-07, +9.582225142e-06 + 6.895227e+00, +3.985125339e-06, +3.623797444e-09, +6.312485712e-07, +3.104924942e-07, +7.451235411e-06 + 6.932701e+00, +4.586463251e-06, +8.432940932e-08, +6.862440602e-07, +3.657252939e-07, +7.713935361e-06 + 6.970175e+00, +5.649816560e-06, +9.980611945e-08, +8.434497191e-07, +4.545842056e-07, +8.153341350e-06 + 7.007649e+00, +5.253138935e-06, +6.315394513e-08, +6.956947871e-07, +4.363129718e-07, +8.013144415e-06 + 7.045123e+00, +3.957376831e-06, +3.892659965e-08, +6.417031076e-07, +2.828830275e-07, +7.103058561e-06 + 7.082597e+00, +4.067503119e-06, +1.030145131e-07, +5.482994277e-07, +3.324391553e-07, +6.251908497e-06 + 7.120071e+00, +5.339177720e-06, +3.932289647e-08, +8.433000917e-07, +4.337002769e-07, +7.632185039e-06 + 7.157545e+00, +5.327848789e-06, +1.177484112e-07, +6.938530724e-07, +4.217932669e-07, +7.329401859e-06 + 7.195019e+00, +3.969720665e-06, +7.950795856e-08, +6.036635361e-07, +2.959133757e-07, +6.045703988e-06 + 7.232493e+00, +3.715587160e-06, +1.404244897e-07, +5.594021142e-07, +3.068399705e-07, +5.792304398e-06 + 7.269967e+00, +5.014902828e-06, +1.612835846e-07, +7.753467579e-07, +3.960848066e-07, +6.622353708e-06 + 7.307441e+00, +5.352702037e-06, +1.184137760e-07, +7.375087236e-07, +4.240320490e-07, +7.298735179e-06 + 7.344915e+00, +4.153416797e-06, +4.480648236e-08, +6.123600421e-07, +3.202807598e-07, +5.538378559e-06 + 7.382389e+00, +3.599936856e-06, +3.650725801e-08, +5.446679750e-07, +2.773792997e-07, +5.496635574e-06 + 7.419863e+00, +4.830704479e-06, +5.884631298e-08, +7.174825239e-07, +3.835192893e-07, +6.712265589e-06 + 7.457337e+00, +5.146804022e-06, +1.163190193e-07, +7.555170452e-07, +4.169979523e-07, +6.428561978e-06 + 7.494811e+00, +4.397343693e-06, +1.080439747e-07, +5.798133525e-07, +3.671442609e-07, +6.238486822e-06 + Mean, +8.612739103e-03, +6.513527808e-04, +8.444567972e-04, +7.594154458e-04, +1.099244137e+00 diff --git a/test/ref/coaxial/open/error-indicators.csv b/test/ref/coaxial/open/error-indicators.csv index ab63216db..57fa73ea4 100644 --- a/test/ref/coaxial/open/error-indicators.csv +++ b/test/ref/coaxial/open/error-indicators.csv @@ -1,203 +1,203 @@ t (ns), Sum Abs., Min Elem Abs., Max Elem Abs., Mean Elem Abs., Normalization 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 - 3.747406e-02, +5.656919329e-06, +4.859397461e-19, +1.429146542e-06, +4.419468226e-08, +1.015026012e-04 - 7.494811e-02, +1.331394786e-05, +1.261528861e-18, +3.354187386e-06, +1.040152177e-07, +1.286467416e-04 - 1.124222e-01, +4.533166150e-06, +8.827124714e-19, +1.144058873e-06, +3.541536055e-08, +1.363432444e-04 - 1.498962e-01, +5.316539313e-06, +5.373308021e-19, +1.352378049e-06, +4.153546339e-08, +2.237453773e-04 - 1.873703e-01, +2.287184397e-05, +3.000946784e-18, +5.789590609e-06, +1.786862810e-07, +5.988251413e-04 - 2.248443e-01, +1.598438319e-05, +2.230531364e-18, +4.071906507e-06, +1.248779937e-07, +1.231318974e-03 - 2.623184e-01, +1.150013517e-05, +5.191338644e-18, +3.089062447e-06, +8.984480600e-08, +2.183514588e-03 - 2.997925e-01, +2.092441721e-05, +1.273595463e-17, +5.172537104e-06, +1.634720094e-07, +3.443881315e-03 - 3.372665e-01, +3.023129660e-05, +1.998560108e-17, +8.595200606e-06, +2.361820047e-07, +4.896471117e-03 - 3.747406e-01, +2.966149896e-05, +5.643586447e-17, +8.160478469e-06, +2.317304606e-07, +6.288238192e-03 - 4.122146e-01, +3.934747639e-05, +1.415540740e-17, +1.043816137e-05, +3.074021593e-07, +7.236958436e-03 - 4.496887e-01, +5.496375443e-05, +1.412705025e-16, +1.376602992e-05, +4.294043315e-07, +7.438657288e-03 - 4.871627e-01, +5.317715172e-05, +1.765597600e-16, +1.345278688e-05, +4.154464978e-07, +7.655736081e-03 - 5.246368e-01, +6.345987510e-05, +3.578921520e-16, +1.616235667e-05, +4.957802742e-07, +1.144507368e-02 - 5.621109e-01, +1.000188544e-04, +6.683968190e-16, +2.890344790e-05, +7.813973001e-07, +2.129749051e-02 - 5.995849e-01, +1.569853883e-04, +6.893744780e-16, +5.054827882e-05, +1.226448346e-06, +3.669225071e-02 - 6.370590e-01, +2.371414640e-04, +2.988883429e-15, +7.811657109e-05, +1.852667687e-06, +5.641515864e-02 - 6.745330e-01, +3.436988525e-04, +2.170694938e-15, +1.084030298e-04, +2.685147285e-06, +7.828737440e-02 - 7.120071e-01, +4.503449432e-04, +8.427839505e-15, +1.334532772e-04, +3.518319869e-06, +9.885303849e-02 - 7.494811e-01, +5.387201999e-04, +2.210536808e-14, +1.460192415e-04, +4.208751561e-06, +1.137546488e-01 - 7.869552e-01, +5.854634666e-04, +3.700525228e-14, +1.426211774e-04, +4.573933333e-06, +1.195362049e-01 - 8.244293e-01, +5.945252636e-04, +1.542935864e-13, +1.553115756e-04, +4.644728622e-06, +1.188356113e-01 - 8.619033e-01, +6.277794930e-04, +8.438596361e-13, +1.446548827e-04, +4.904527289e-06, +1.309570623e-01 - 8.993774e-01, +8.196023618e-04, +3.542527947e-12, +2.064570778e-04, +6.403143451e-06, +1.857784636e-01 - 9.368514e-01, +1.245063026e-03, +1.235451610e-11, +3.796113148e-04, +9.727054891e-06, +2.867343617e-01 - 9.743255e-01, +1.824265444e-03, +3.885931987e-11, +5.734398443e-04, +1.425207378e-05, +4.167726757e-01 - 1.011800e+00, +2.450890631e-03, +1.126666778e-10, +7.521115025e-04, +1.914758306e-05, +5.549120054e-01 - 1.049274e+00, +3.008970325e-03, +2.963474220e-10, +8.708173546e-04, +2.350758066e-05, +6.782193157e-01 - 1.086748e+00, +3.391209599e-03, +6.882824319e-10, +8.861180141e-04, +2.649382499e-05, +7.649055092e-01 - 1.124222e+00, +3.537834075e-03, +1.363904131e-09, +8.504123105e-04, +2.763932871e-05, +8.022591780e-01 - 1.161696e+00, +3.511779317e-03, +2.150129927e-09, +8.840349987e-04, +2.743577591e-05, +8.009806200e-01 - 1.199170e+00, +3.584287866e-03, +2.125158607e-09, +8.159752263e-04, +2.800224896e-05, +8.143332817e-01 - 1.236644e+00, +4.160738345e-03, +3.088718318e-09, +8.694210871e-04, +3.250576832e-05, +9.294499013e-01 - 1.274118e+00, +5.352965483e-03, +1.778270100e-08, +1.361582714e-03, +4.182004283e-05, +1.185780064e+00 - 1.311592e+00, +6.884036964e-03, +5.482403929e-08, +1.934136735e-03, +5.378153878e-05, +1.534661136e+00 - 1.349066e+00, +8.393574770e-03, +1.274739058e-07, +2.361243467e-03, +6.557480289e-05, +1.896361173e+00 - 1.386540e+00, +9.587144890e-03, +2.427922136e-07, +2.543459610e-03, +7.489956945e-05, +2.199964363e+00 - 1.424014e+00, +1.030847847e-02, +3.870792896e-07, +2.421466309e-03, +8.053498806e-05, +2.397709121e+00 - 1.461488e+00, +1.058479832e-02, +5.172682213e-07, +2.459542440e-03, +8.269373687e-05, +2.477248854e+00 - 1.498962e+00, +1.065114467e-02, +5.900127780e-07, +2.478084232e-03, +8.321206773e-05, +2.474045348e+00 - 1.536436e+00, +1.088707735e-02, +6.933527190e-07, +2.347785514e-03, +8.505529183e-05, +2.473655990e+00 - 1.573910e+00, +1.161350427e-02, +1.128992731e-06, +2.453010483e-03, +9.073050211e-05, +2.579211406e+00 - 1.611384e+00, +1.284893699e-02, +1.982499253e-06, +2.368957236e-03, +1.003823202e-04, +2.837696762e+00 - 1.648859e+00, +1.431858621e-02, +3.112369697e-06, +3.068473826e-03, +1.118639547e-04, +3.200565871e+00 - 1.686333e+00, +1.568466438e-02, +4.376325047e-06, +3.469083243e-03, +1.225364404e-04, +3.569988173e+00 - 1.723807e+00, +1.672073186e-02, +5.660135676e-06, +3.499417909e-03, +1.306307177e-04, +3.860976369e+00 - 1.761281e+00, +1.735919126e-02, +6.836616389e-06, +3.289526062e-03, +1.356186817e-04, +4.031109202e+00 - 1.798755e+00, +1.767123056e-02, +7.821965102e-06, +3.511818671e-03, +1.380564888e-04, +4.086241163e+00 - 1.836229e+00, +1.781523617e-02, +9.246722069e-06, +3.414133250e-03, +1.391815326e-04, +4.072134164e+00 - 1.873703e+00, +1.795745680e-02, +1.328997588e-05, +3.394013651e-03, +1.402926312e-04, +4.053121061e+00 - 1.911177e+00, +1.820302843e-02, +2.161284504e-05, +3.420849841e-03, +1.422111596e-04, +4.080942379e+00 - 1.948651e+00, +1.856514782e-02, +3.356781622e-05, +3.234892449e-03, +1.450402173e-04, +4.169173455e+00 - 1.986125e+00, +1.898957175e-02, +4.723138491e-05, +3.396745121e-03, +1.483560293e-04, +4.292737520e+00 - 2.023599e+00, +1.939414576e-02, +5.977349433e-05, +3.250581870e-03, +1.515167637e-04, +4.410955044e+00 - 2.061073e+00, +1.970394332e-02, +6.737232249e-05, +3.312961591e-03, +1.539370572e-04, +4.492517857e+00 - 2.098547e+00, +1.988190624e-02, +6.610345079e-05, +3.301605870e-03, +1.553273925e-04, +4.527215199e+00 - 2.136021e+00, +1.993731501e-02, +5.671442401e-05, +3.172906485e-03, +1.557602735e-04, +4.524104599e+00 - 2.173495e+00, +1.990562099e-02, +6.101944972e-05, +3.293718918e-03, +1.555126640e-04, +4.502015291e+00 - 2.210969e+00, +1.982893776e-02, +1.088878828e-04, +3.115929763e-03, +1.549135762e-04, +4.478491192e+00 - 2.248443e+00, +1.974876097e-02, +1.873801567e-04, +3.228307111e-03, +1.542871951e-04, +4.462065326e+00 - 2.285917e+00, +1.968329015e-02, +2.772981887e-04, +3.180410440e-03, +1.537757043e-04, +4.450543317e+00 - 2.323392e+00, +1.961439719e-02, +3.585051976e-04, +3.107072907e-03, +1.532374780e-04, +4.434922062e+00 - 2.360866e+00, +1.951363987e-02, +4.075955778e-04, +3.188334861e-03, +1.524503115e-04, +4.406444465e+00 - 2.398340e+00, +1.937089614e-02, +4.015209202e-04, +2.980831106e-03, +1.513351261e-04, +4.364029184e+00 - 2.435814e+00, +1.920186060e-02, +3.225496007e-04, +3.138425993e-03, +1.500145360e-04, +4.319424711e+00 - 2.473288e+00, +1.906306151e-02, +1.840331470e-04, +3.055918212e-03, +1.489301681e-04, +4.296324704e+00 - 2.510762e+00, +1.905052604e-02, +2.217725636e-04, +3.046208004e-03, +1.488322347e-04, +4.319068012e+00 - 2.548236e+00, +1.922732384e-02, +1.821453542e-04, +3.098865945e-03, +1.502134675e-04, +4.391888403e+00 - 2.585710e+00, +1.953172619e-02, +8.506759637e-05, +2.933659318e-03, +1.525916109e-04, +4.481531511e+00 - 2.623184e+00, +1.974732686e-02, +1.004249255e-04, +3.113361469e-03, +1.542759911e-04, +4.519891310e+00 - 2.660658e+00, +1.956384336e-02, +1.196983105e-04, +3.005984908e-03, +1.528425263e-04, +4.429739436e+00 - 2.698132e+00, +1.871919390e-02, +1.071559579e-04, +2.901948705e-03, +1.462437023e-04, +4.164358438e+00 - 2.735606e+00, +1.718960117e-02, +1.320053755e-04, +2.803797671e-03, +1.342937591e-04, +3.755791737e+00 - 2.773080e+00, +1.546491643e-02, +9.509285091e-05, +2.408371021e-03, +1.208196596e-04, +3.369663722e+00 - 2.810554e+00, +1.468011525e-02, +7.315500821e-05, +2.441928107e-03, +1.146884004e-04, +3.300120354e+00 - 2.848028e+00, +1.586922847e-02, +4.353556598e-05, +2.665555836e-03, +1.239783474e-04, +3.715710458e+00 - 2.885502e+00, +1.862615408e-02, +2.018520823e-05, +3.373944166e-03, +1.455168288e-04, +4.412752860e+00 - 2.922976e+00, +2.144870961e-02, +7.479911328e-06, +4.038673871e-03, +1.675680439e-04, +5.035126169e+00 - 2.960451e+00, +2.292343193e-02, +9.559526669e-06, +4.563613681e-03, +1.790893120e-04, +5.296819140e+00 - 2.997925e+00, +2.211029681e-02, +1.216757195e-05, +4.513888960e-03, +1.727366938e-04, +5.025358059e+00 - 3.035399e+00, +1.863331031e-02, +1.063158688e-05, +3.843556356e-03, +1.455727368e-04, +4.165761053e+00 - 3.072873e+00, +1.272017111e-02, +8.184202773e-06, +2.614012422e-03, +9.937633682e-05, +2.782770985e+00 - 3.110347e+00, +5.289924617e-03, +4.602990989e-06, +9.892105005e-04, +4.132753607e-05, +1.091922648e+00 - 3.147821e+00, +4.602799913e-03, +1.691676596e-06, +8.722525324e-04, +3.595937432e-05, +1.087911650e+00 - 3.185295e+00, +1.215215563e-02, +2.265588759e-06, +2.536343693e-03, +9.493871585e-05, +2.795375974e+00 - 3.222769e+00, +1.838959293e-02, +3.425650080e-06, +3.876930233e-03, +1.436686948e-04, +4.198258824e+00 - 3.260243e+00, +2.210912954e-02, +3.716661787e-06, +4.665403620e-03, +1.727275745e-04, +5.054847159e+00 - 3.297717e+00, +2.287279666e-02, +5.465969699e-06, +4.800014042e-03, +1.786937239e-04, +5.271021749e+00 - 3.335191e+00, +2.087582241e-02, +1.094747916e-05, +4.296501227e-03, +1.630923626e-04, +4.875726472e+00 - 3.372665e+00, +1.693740099e-02, +1.957205640e-05, +3.274370495e-03, +1.323234452e-04, +4.026175767e+00 - 3.410139e+00, +1.265391042e-02, +3.033725212e-05, +2.351759896e-03, +9.885867518e-05, +3.029504715e+00 - 3.447613e+00, +1.056944488e-02, +3.699160287e-05, +1.868764054e-03, +8.257378814e-05, +2.402834715e+00 - 3.485087e+00, +1.203465460e-02, +5.157331508e-05, +1.987519538e-03, +9.402073902e-05, +2.582889975e+00 - 3.522561e+00, +1.483191162e-02, +4.264808044e-05, +2.431130279e-03, +1.158743095e-04, +3.207270944e+00 - 3.560035e+00, +1.694364856e-02, +4.348712938e-05, +2.651516726e-03, +1.323722544e-04, +3.758945722e+00 - 3.597509e+00, +1.777812513e-02, +3.816772972e-05, +2.830632832e-03, +1.388916026e-04, +4.038955569e+00 - 3.634984e+00, +1.749152618e-02, +2.610873465e-05, +2.685137944e-03, +1.366525483e-04, +4.041150977e+00 - 3.672458e+00, +1.659354033e-02, +6.478739420e-05, +2.504827606e-03, +1.296370338e-04, +3.857101060e+00 - 3.709932e+00, +1.567966850e-02, +1.324957253e-04, +2.523973389e-03, +1.224974102e-04, +3.618583613e+00 - 3.747406e+00, +1.515242979e-02, +1.837750152e-04, +2.479688081e-03, +1.183783577e-04, +3.441317861e+00 - 3.784880e+00, +1.505241220e-02, +2.589866907e-04, +2.413883219e-03, +1.175969703e-04, +3.374168903e+00 - 3.822354e+00, +1.517594324e-02, +2.648845550e-04, +2.482655253e-03, +1.185620565e-04, +3.390825944e+00 - 3.859828e+00, +1.530534707e-02, +3.460441858e-04, +2.325030107e-03, +1.195730240e-04, +3.433304933e+00 - 3.897302e+00, +1.533180643e-02, +2.591782740e-04, +2.448645600e-03, +1.197797377e-04, +3.458360447e+00 - 3.934776e+00, +1.525209324e-02, +3.263088666e-04, +2.386365345e-03, +1.191569785e-04, +3.452521374e+00 - 3.972250e+00, +1.511733830e-02, +2.656776913e-04, +2.367449970e-03, +1.181042055e-04, +3.423956689e+00 - 4.009724e+00, +1.498169883e-02, +1.893347752e-04, +2.402930160e-03, +1.170445221e-04, +3.388114585e+00 - 4.047198e+00, +1.486760911e-02, +8.599168463e-05, +2.247589218e-03, +1.161531962e-04, +3.356524519e+00 - 4.084672e+00, +1.476249591e-02, +7.080242635e-05, +2.377243440e-03, +1.153319993e-04, +3.331853567e+00 - 4.122146e+00, +1.464115774e-02, +1.002760332e-04, +2.290380531e-03, +1.143840448e-04, +3.308603435e+00 - 4.159620e+00, +1.448100039e-02, +1.127755160e-04, +2.309758242e-03, +1.131328156e-04, +3.277119137e+00 - 4.197094e+00, +1.425845180e-02, +1.046536919e-04, +2.317095442e-03, +1.113941547e-04, +3.228713172e+00 - 4.234568e+00, +1.397280754e-02, +8.171048574e-05, +2.200893910e-03, +1.091625589e-04, +3.160502439e+00 - 4.272043e+00, +1.365800498e-02, +5.594417538e-05, +2.299295461e-03, +1.067031639e-04, +3.078660427e+00 - 4.309517e+00, +1.335595257e-02, +3.262293522e-05, +2.188198722e-03, +1.043433795e-04, +2.998121297e+00 - 4.346991e+00, +1.310147746e-02, +1.743680856e-05, +2.244566034e-03, +1.023552927e-04, +2.936669456e+00 - 4.384465e+00, +1.291316404e-02, +1.318336764e-05, +2.228431201e-03, +1.008840941e-04, +2.903997885e+00 - 4.421939e+00, +1.276376145e-02, +1.435774037e-05, +2.182482625e-03, +9.971688630e-05, +2.891883014e+00 - 4.459413e+00, +1.257886363e-02, +1.385785895e-05, +2.254463017e-03, +9.827237213e-05, +2.873950817e+00 - 4.496887e+00, +1.227354455e-02, +1.139016156e-05, +2.117954940e-03, +9.588706676e-05, +2.816794882e+00 - 4.534361e+00, +1.178139292e-02, +8.224354336e-06, +2.159095858e-03, +9.204213215e-05, +2.695716179e+00 - 4.571835e+00, +1.108386625e-02, +4.702489471e-06, +2.144208192e-03, +8.659270510e-05, +2.507677639e+00 - 4.609309e+00, +1.023691014e-02, +2.918640342e-06, +1.911635765e-03, +7.997586047e-05, +2.277833009e+00 - 4.646783e+00, +9.371323031e-03, +1.545409171e-06, +1.700487237e-03, +7.321346118e-05, +2.056659349e+00 - 4.684257e+00, +8.657719189e-03, +2.177147837e-06, +1.788518719e-03, +6.763843116e-05, +1.900934905e+00 - 4.721731e+00, +8.217440495e-03, +1.601037153e-07, +1.709076276e-03, +6.419875387e-05, +1.836493022e+00 - 4.759205e+00, +8.019652879e-03, +1.376983597e-06, +1.786480796e-03, +6.265353811e-05, +1.832790721e+00 - 4.796679e+00, +7.890364257e-03, +1.486445721e-06, +1.768891622e-03, +6.164347076e-05, +1.825328220e+00 - 4.834153e+00, +7.621359223e-03, +9.019622716e-07, +1.685597957e-03, +5.954186893e-05, +1.758585885e+00 - 4.871627e+00, +7.068989784e-03, +9.009948019e-07, +1.773969888e-03, +5.522648269e-05, +1.608502805e+00 - 4.909101e+00, +6.213005292e-03, +2.594278515e-07, +1.651811893e-03, +4.853910384e-05, +1.384459441e+00 - 4.946576e+00, +5.162372595e-03, +1.085336870e-06, +1.352451286e-03, +4.033103590e-05, +1.123478548e+00 - 4.984050e+00, +4.122759929e-03, +6.012850877e-07, +9.429225286e-04, +3.220906194e-05, +8.826005714e-01 - 5.021524e+00, +3.350707601e-03, +6.078344522e-07, +7.200088076e-04, +2.617740313e-05, +7.236678618e-01 - 5.058998e+00, +3.013498510e-03, +6.067704731e-07, +7.103653051e-04, +2.354295711e-05, +6.701157353e-01 - 5.096472e+00, +2.982577268e-03, +6.275275747e-08, +7.400639948e-04, +2.330138491e-05, +6.737270462e-01 - 5.133946e+00, +2.967454278e-03, +7.366752407e-07, +6.898839516e-04, +2.318323655e-05, +6.688583217e-01 - 5.171420e+00, +2.790092474e-03, +5.208324035e-07, +7.384504272e-04, +2.179759745e-05, +6.231475505e-01 - 5.208894e+00, +2.418548988e-03, +9.276752457e-07, +7.000383276e-04, +1.889491397e-05, +5.351074185e-01 - 5.246368e+00, +1.911062361e-03, +1.344568805e-06, +5.773793295e-04, +1.493017470e-05, +4.200268783e-01 - 5.283842e+00, +1.372440606e-03, +2.056363962e-06, +4.096268533e-04, +1.072219224e-05, +3.000708009e-01 - 5.321316e+00, +9.213288711e-04, +2.114823211e-06, +2.372578999e-04, +7.197881805e-06, +1.991713402e-01 - 5.358790e+00, +6.722993615e-04, +2.927023810e-06, +1.457815447e-04, +5.252338761e-06, +1.401144692e-01 - 5.396264e+00, +6.283225291e-04, +2.643605712e-06, +1.625307574e-04, +4.908769758e-06, +1.268544987e-01 - 5.433738e+00, +6.348530854e-04, +1.837387106e-06, +1.550414836e-04, +4.959789730e-06, +1.297308749e-01 - 5.471212e+00, +6.001491653e-04, +2.469214990e-06, +1.570842275e-04, +4.688665354e-06, +1.254275991e-01 - 5.508686e+00, +5.139588852e-04, +6.781779592e-07, +1.472964293e-04, +4.015303791e-06, +1.099729019e-01 - 5.546160e+00, +3.974099564e-04, +2.282771424e-06, +1.184808925e-04, +3.104765285e-06, +8.751673075e-02 - 5.583635e+00, +2.814766597e-04, +4.142124481e-06, +8.237874526e-05, +2.199036404e-06, +6.404830704e-02 - 5.621109e+00, +1.971418372e-04, +5.379288482e-06, +4.879963190e-05, +1.540170604e-06, +4.499940303e-02 - 5.658583e+00, +1.566562903e-04, +5.397792871e-06, +2.429803265e-05, +1.223877268e-06, +3.378394289e-02 - 5.696057e+00, +1.405994051e-04, +2.383392345e-06, +2.103885737e-05, +1.098432853e-06, +2.942426165e-02 - 5.733531e+00, +1.313588361e-04, +1.776515077e-06, +1.911272236e-05, +1.026240907e-06, +2.730267726e-02 - 5.771005e+00, +1.162490257e-04, +2.192974526e-06, +1.780103944e-05, +9.081955133e-07, +2.433964679e-02 - 5.808479e+00, +9.660978209e-05, +1.832208312e-06, +1.546361775e-05, +7.547639226e-07, +2.048189674e-02 - 5.845953e+00, +7.972386355e-05, +2.032041548e-06, +1.277530080e-05, +6.228426840e-07, +1.785059556e-02 - 5.883427e+00, +8.075440741e-05, +9.629804944e-07, +1.405092356e-05, +6.308938079e-07, +1.905071629e-02 - 5.920901e+00, +9.847125181e-05, +1.617458010e-06, +1.897478279e-05, +7.693066548e-07, +2.355183009e-02 - 5.958375e+00, +1.201247757e-04, +1.250282725e-06, +2.394888269e-05, +9.384748101e-07, +2.860407565e-02 - 5.995849e+00, +1.353459675e-04, +4.093061374e-07, +2.700896494e-05, +1.057390371e-06, +3.211115798e-02 - 6.033323e+00, +1.393064156e-04, +1.148348275e-06, +2.893048894e-05, +1.088331372e-06, +3.289020287e-02 - 6.070797e+00, +1.306427217e-04, +8.982553633e-07, +2.770021651e-05, +1.020646263e-06, +3.044147732e-02 - 6.108271e+00, +1.061818821e-04, +7.339142826e-07, +2.278162355e-05, +8.295459542e-07, +2.486915720e-02 - 6.145745e+00, +7.209552695e-05, +3.375269493e-07, +1.566264693e-05, +5.632463043e-07, +1.688222621e-02 - 6.183219e+00, +3.514370025e-05, +6.910549004e-07, +6.938825859e-06, +2.745601582e-07, +8.222569600e-03 - 6.220694e+00, +3.332434404e-05, +8.008708289e-07, +5.858394718e-06, +2.603464378e-07, +7.112443417e-03 - 6.258168e+00, +6.596901561e-05, +4.918351346e-07, +1.173531422e-05, +5.153829345e-07, +1.487473373e-02 - 6.295642e+00, +9.667915064e-05, +3.027019372e-07, +1.881777929e-05, +7.553058644e-07, +2.195995150e-02 - 6.333116e+00, +1.151311728e-04, +1.042992568e-06, +2.293042378e-05, +8.994622877e-07, +2.658857944e-02 - 6.370590e+00, +1.207366960e-04, +7.528801062e-07, +2.380623776e-05, +9.432554375e-07, +2.836724125e-02 - 6.408064e+00, +1.164009914e-04, +2.748897183e-07, +2.213552786e-05, +9.093827456e-07, +2.754771847e-02 - 6.445538e+00, +1.035095295e-04, +1.131867308e-06, +1.908048890e-05, +8.086681991e-07, +2.492432935e-02 - 6.483012e+00, +9.055022966e-05, +1.313210738e-06, +1.572048715e-05, +7.074236692e-07, +2.173397508e-02 - 6.520486e+00, +8.224274468e-05, +7.030108963e-07, +1.364569358e-05, +6.425214429e-07, +1.939641242e-02 - 6.557960e+00, +8.240698886e-05, +1.148314156e-06, +1.353813700e-05, +6.438046004e-07, +1.881287963e-02 - 6.595434e+00, +8.663895944e-05, +1.631803893e-06, +1.483972908e-05, +6.768668706e-07, +1.965705398e-02 - 6.632908e+00, +9.150938696e-05, +8.959336730e-07, +1.500001344e-05, +7.149170856e-07, +2.089087372e-02 - 6.670382e+00, +9.411555999e-05, +7.071728276e-07, +1.471136650e-05, +7.352778124e-07, +2.174361592e-02 - 6.707856e+00, +9.427881810e-05, +2.232689138e-06, +1.527592348e-05, +7.365532664e-07, +2.197690465e-02 - 6.745330e+00, +9.283419611e-05, +1.674328273e-06, +1.475470634e-05, +7.252671571e-07, +2.172213826e-02 - 6.782804e+00, +9.091313126e-05, +2.172434071e-06, +1.494130972e-05, +7.102588379e-07, +2.125742296e-02 - 6.820278e+00, +8.947559322e-05, +1.208738908e-06, +1.467196182e-05, +6.990280721e-07, +2.082523460e-02 - 6.857752e+00, +8.849046137e-05, +2.183296734e-07, +1.422391044e-05, +6.913317295e-07, +2.053535812e-02 - 6.895227e+00, +8.869124720e-05, +1.872610390e-06, +1.475311687e-05, +6.929003687e-07, +2.037002683e-02 - 6.932701e+00, +8.703583659e-05, +1.303861073e-06, +1.403557469e-05, +6.799674734e-07, +2.025706959e-02 - 6.970175e+00, +8.705236101e-05, +1.310170334e-06, +1.440134753e-05, +6.800965704e-07, +2.014118630e-02 - 7.007649e+00, +8.649188868e-05, +1.642909372e-06, +1.413717956e-05, +6.757178803e-07, +2.000795493e-02 - 7.045123e+00, +8.557114199e-05, +8.070387273e-07, +1.402625353e-05, +6.685245468e-07, +1.987188346e-02 - 7.082597e+00, +8.556574398e-05, +6.303919740e-07, +1.437910359e-05, +6.684823748e-07, +1.974815441e-02 - 7.120071e+00, +8.395754835e-05, +2.492708841e-07, +1.333948625e-05, +6.559183464e-07, +1.962815933e-02 - 7.157545e+00, +8.469831735e-05, +4.394354772e-07, +1.399445249e-05, +6.617056043e-07, +1.947236585e-02 - 7.195019e+00, +8.219518954e-05, +2.016472612e-07, +1.374633414e-05, +6.421499183e-07, +1.922019946e-02 - 7.232493e+00, +8.060990055e-05, +7.126280650e-07, +1.375634861e-05, +6.297648481e-07, +1.881533865e-02 - 7.269967e+00, +7.975417976e-05, +2.119912937e-07, +1.379736945e-05, +6.230795294e-07, +1.823277535e-02 - 7.307441e+00, +7.470325725e-05, +1.169669241e-07, +1.285632322e-05, +5.836191972e-07, +1.750653819e-02 - 7.344915e+00, +7.330184956e-05, +2.570909088e-07, +1.365659262e-05, +5.726706997e-07, +1.673934977e-02 - 7.382389e+00, +7.020343991e-05, +4.248388020e-07, +1.310218265e-05, +5.484643743e-07, +1.607601725e-02 - 7.419863e+00, +6.896813444e-05, +4.112239141e-07, +1.338802699e-05, +5.388135503e-07, +1.564435535e-02 - 7.457337e+00, +6.715833136e-05, +1.872803050e-07, +1.323199471e-05, +5.246744638e-07, +1.547011399e-02 - 7.494811e+00, +6.573000791e-05, +1.923954388e-07, +1.301561943e-05, +5.135156868e-07, +1.543460452e-02 - Mean, +7.685673982e-03, +2.479807792e-04, +5.882818519e-04, +6.004432799e-05, +1.742443487e+00 + 3.747406e-02, +5.656659455e-06, +4.851768538e-19, +1.429016907e-06, +1.884177417e-07, +1.015013075e-04 + 7.494811e-02, +1.331364706e-05, +1.252047160e-18, +3.354073322e-06, +4.532531473e-07, +1.286451027e-04 + 1.124222e-01, +4.532668491e-06, +8.692216473e-19, +1.143753275e-06, +1.587978168e-07, +1.363415075e-04 + 1.498962e-01, +5.315482993e-06, +5.382843762e-19, +1.352478574e-06, +1.844891507e-07, +2.237425263e-04 + 1.873703e-01, +2.286991326e-05, +3.007030970e-18, +5.789494315e-06, +7.781958288e-07, +5.988175109e-04 + 2.248443e-01, +1.597419758e-05, +2.187676401e-18, +4.067429237e-06, +5.797052698e-07, +1.231303285e-03 + 2.623184e-01, +1.145612024e-05, +5.207848925e-18, +3.069144677e-06, +4.523653991e-07, +2.183486767e-03 + 2.997925e-01, +2.086398938e-05, +1.270112296e-17, +5.143395002e-06, +8.183715069e-07, +3.443837437e-03 + 3.372665e-01, +3.014646152e-05, +1.995375749e-17, +8.564706304e-06, +1.096508508e-06, +4.896408733e-03 + 3.747406e-01, +2.951847940e-05, +5.642253167e-17, +8.109678263e-06, +1.146518725e-06, +6.288158078e-03 + 4.122146e-01, +3.920398129e-05, +1.409960031e-17, +1.039662099e-05, +1.506290838e-06, +7.236866239e-03 + 4.496887e-01, +5.485446653e-05, +1.411154384e-16, +1.373626813e-05, +2.138216877e-06, +7.438562524e-03 + 4.871627e-01, +5.305798815e-05, +1.745355459e-16, +1.341910982e-05, +2.244500004e-06, +7.655638548e-03 + 5.246368e-01, +6.324063047e-05, +3.538112111e-16, +1.609936398e-05, +2.567937221e-06, +1.144492786e-02 + 5.621109e-01, +9.953861113e-05, +6.647608180e-16, +2.871846797e-05, +3.979070330e-06, +2.129721915e-02 + 5.995849e-01, +1.560743589e-04, +6.830393652e-16, +5.021692981e-05, +5.826054663e-06, +3.669178320e-02 + 6.370590e-01, +2.357106590e-04, +2.975840305e-15, +7.762490221e-05, +8.212787663e-06, +5.641443987e-02 + 6.745330e-01, +3.417912838e-04, +2.134700483e-15, +1.077684524e-04, +1.198621562e-05, +7.828637698e-02 + 7.120071e-01, +4.480155462e-04, +8.427337322e-15, +1.327195791e-04, +1.653183123e-05, +9.885177909e-02 + 7.494811e-01, +5.361316672e-04, +2.209015576e-14, +1.452888272e-04, +2.046928227e-05, +1.137531996e-01 + 7.869552e-01, +5.828228357e-04, +3.698456781e-14, +1.419796915e-04, +2.316840106e-05, +1.195346821e-01 + 8.244293e-01, +5.919511494e-04, +1.543555294e-13, +1.546158786e-04, +2.529399186e-05, +1.188340974e-01 + 8.619033e-01, +6.248414803e-04, +8.440272402e-13, +1.440128726e-04, +2.829825492e-05, +1.309553939e-01 + 8.993774e-01, +8.151146601e-04, +3.542755319e-12, +2.051054800e-04, +3.659201856e-05, +1.857760966e-01 + 9.368514e-01, +1.238030606e-03, +1.235495048e-11, +3.772512882e-04, +5.099828694e-05, +2.867307084e-01 + 9.743255e-01, +1.814098313e-03, +3.886184200e-11, +5.700103263e-04, +7.220437046e-05, +4.167673658e-01 + 1.011800e+00, +2.437431108e-03, +1.126769718e-10, +7.477328123e-04, +9.448610408e-05, +5.549049357e-01 + 1.049274e+00, +2.992538657e-03, +2.963780279e-10, +8.658103032e-04, +1.186157963e-04, +6.782106751e-01 + 1.086748e+00, +3.372602378e-03, +6.883561366e-10, +8.810228475e-04, +1.408439505e-04, +7.648957645e-01 + 1.124222e+00, +3.518151571e-03, +1.364041638e-09, +8.457004167e-04, +1.549572659e-04, +8.022489576e-01 + 1.161696e+00, +3.491978301e-03, +2.150204961e-09, +8.789616842e-04, +1.613984718e-04, +8.009704160e-01 + 1.199170e+00, +3.564279227e-03, +2.123409813e-09, +8.112931293e-04, +1.690581924e-04, +8.143229074e-01 + 1.236644e+00, +4.138418533e-03, +3.078381559e-09, +8.645553516e-04, +2.078914509e-04, +9.294380600e-01 + 1.274118e+00, +5.324827324e-03, +1.777478124e-08, +1.353962850e-03, +2.601140741e-04, +1.185764957e+00 + 1.311592e+00, +6.847371239e-03, +5.481861739e-08, +1.923072314e-03, +3.232891049e-04, +1.534641584e+00 + 1.349066e+00, +8.347554914e-03, +1.274777982e-07, +2.347347422e-03, +3.889148264e-04, +1.896337013e+00 + 1.386540e+00, +9.532772096e-03, +2.428165880e-07, +2.527956003e-03, +4.514092009e-04, +2.199936335e+00 + 1.424014e+00, +1.024824707e-02, +3.871187955e-07, +2.406072494e-03, +5.056030501e-04, +2.397678575e+00 + 1.461488e+00, +1.052203775e-02, +5.171912012e-07, +2.444667124e-03, +5.393185100e-04, +2.477217296e+00 + 1.498962e+00, +1.058885937e-02, +5.891194816e-07, +2.462910797e-03, +5.619245208e-04, +2.474013830e+00 + 1.536436e+00, +1.082620294e-02, +6.896210906e-07, +2.333885666e-03, +5.875077909e-04, +2.473624477e+00 + 1.573910e+00, +1.155161469e-02, +1.121475533e-06, +2.437891160e-03, +6.540312672e-04, +2.579178548e+00 + 1.611384e+00, +1.278137618e-02, +1.972066894e-06, +2.356865899e-03, +7.374026260e-04, +2.837660610e+00 + 1.648859e+00, +1.424150692e-02, +3.100040281e-06, +3.050855223e-03, +8.229301974e-04, +3.200525096e+00 + 1.686333e+00, +1.559703945e-02, +4.364327287e-06, +3.447606735e-03, +9.114278074e-04, +3.569942692e+00 + 1.723807e+00, +1.662444071e-02, +5.652063847e-06, +3.476585555e-03, +9.873524406e-04, +3.860927182e+00 + 1.761281e+00, +1.725792281e-02, +6.835154067e-06, +3.269676383e-03, +1.054802446e-03, +4.031057848e+00 + 1.798755e+00, +1.756888105e-02, +7.822221900e-06, +3.490071839e-03, +1.100407720e-03, +4.086189106e+00 + 1.836229e+00, +1.771435393e-02, +9.224119034e-06, +3.393376631e-03, +1.130683221e-03, +4.072082288e+00 + 1.873703e+00, +1.785832386e-02, +1.320969740e-05, +3.372883096e-03, +1.161958398e-03, +4.053069427e+00 + 1.911177e+00, +1.810395343e-02, +2.146664968e-05, +3.399366408e-03, +1.205675107e-03, +4.080890391e+00 + 1.948651e+00, +1.846382530e-02, +3.336302682e-05, +3.215132108e-03, +1.261773627e-03, +4.169120342e+00 + 1.986125e+00, +1.888457853e-02, +4.698430886e-05, +3.375389794e-03, +1.311524450e-03, +4.292682833e+00 + 2.023599e+00, +1.928557725e-02, +5.951672357e-05, +3.230487064e-03, +1.366405838e-03, +4.410898850e+00 + 2.061073e+00, +1.959303920e-02, +6.715821177e-05, +3.292283170e-03, +1.413551116e-03, +4.492460625e+00 + 2.098547e+00, +1.977023401e-02, +6.599675127e-05, +3.280926715e-03, +1.448890774e-03, +4.527157525e+00 + 2.136021e+00, +1.982606615e-02, +5.672179952e-05, +3.153403886e-03, +1.477660705e-03, +4.524046965e+00 + 2.173495e+00, +1.979526345e-02, +6.082263690e-05, +3.273005166e-03, +1.489008848e-03, +4.501957938e+00 + 2.210969e+00, +1.971930948e-02, +1.081631263e-04, +3.096774883e-03, +1.499471252e-03, +4.478434139e+00 + 2.248443e+00, +1.963950279e-02, +1.861344323e-04, +3.208102115e-03, +1.504260282e-03, +4.462008482e+00 + 2.285917e+00, +1.957424152e-02, +2.755866023e-04, +3.160538427e-03, +1.513687739e-03, +4.450486620e+00 + 2.323392e+00, +1.950573403e-02, +3.564622283e-04, +3.087873225e-03, +1.519283140e-03, +4.434865564e+00 + 2.360866e+00, +1.940581024e-02, +4.054684373e-04, +3.168292697e-03, +1.515796398e-03, +4.406388329e+00 + 2.398340e+00, +1.926434723e-02, +3.996770705e-04, +2.962622826e-03, +1.506568723e-03, +4.363973589e+00 + 2.435814e+00, +1.909655442e-02, +3.214407182e-04, +3.118749615e-03, +1.482647495e-03, +4.319369685e+00 + 2.473288e+00, +1.895811647e-02, +1.839689235e-04, +3.036885498e-03, +1.468540491e-03, +4.296269972e+00 + 2.510762e+00, +1.894439512e-02, +2.204911120e-04, +3.027284537e-03, +1.470201519e-03, +4.319012990e+00 + 2.548236e+00, +1.911859202e-02, +1.813557604e-04, +3.079371068e-03, +1.490279423e-03, +4.391832454e+00 + 2.585710e+00, +1.942027452e-02, +8.465426247e-05, +2.915624197e-03, +1.509126434e-03, +4.481474420e+00 + 2.623184e+00, +1.963519733e-02, +9.976338258e-05, +3.093706609e-03, +1.511211369e-03, +4.519833729e+00 + 2.660658e+00, +1.945513343e-02, +1.192767917e-04, +2.987367916e-03, +1.485119786e-03, +4.429683004e+00 + 2.698132e+00, +1.861878549e-02, +1.067715430e-04, +2.884183524e-03, +1.407976497e-03, +4.164305386e+00 + 2.735606e+00, +1.710066160e-02, +1.313040409e-04, +2.786532587e-03, +1.279638252e-03, +3.755743891e+00 + 2.773080e+00, +1.538534017e-02, +9.475720213e-05, +2.394757280e-03, +1.120242820e-03, +3.369620795e+00 + 2.810554e+00, +1.459970790e-02, +7.277303229e-05, +2.426897616e-03, +1.022789237e-03, +3.300078313e+00 + 2.848028e+00, +1.577492744e-02, +4.329855808e-05, +2.648390438e-03, +1.105397512e-03, +3.715663123e+00 + 2.885502e+00, +1.851283738e-02, +2.006000941e-05, +3.351903461e-03, +1.249233074e-03, +4.412696644e+00 + 2.922976e+00, +2.132059162e-02, +7.437820796e-06, +4.013069564e-03, +1.384748240e-03, +5.035062025e+00 + 2.960451e+00, +2.279077487e-02, +9.538190125e-06, +4.535289462e-03, +1.438249671e-03, +5.296751662e+00 + 2.997925e+00, +2.198649980e-02, +1.213423224e-05, +4.486425727e-03, +1.365182572e-03, +5.025294040e+00 + 3.035399e+00, +1.853237043e-02, +1.062292095e-05, +3.820619063e-03, +1.141992427e-03, +4.165707984e+00 + 3.072873e+00, +1.265418979e-02, +8.153948486e-06, +2.598816283e-03, +7.812765378e-04, +2.782735535e+00 + 3.110347e+00, +5.265495931e-03, +4.582142878e-06, +9.839733384e-04, +3.411259218e-04, +1.091908737e+00 + 3.147821e+00, +4.574928367e-03, +1.680469188e-06, +8.666484916e-04, +2.912393939e-04, +1.087897791e+00 + 3.185295e+00, +1.208246023e-02, +2.263397939e-06, +2.520770130e-03, +7.276559379e-04, +2.795340363e+00 + 3.222769e+00, +1.828571121e-02, +3.423453186e-06, +3.853354933e-03, +1.093030170e-03, +4.198205341e+00 + 3.260243e+00, +2.198386792e-02, +3.709070615e-06, +4.637021302e-03, +1.312583527e-03, +5.054782764e+00 + 3.297717e+00, +2.274113830e-02, +5.430777524e-06, +4.770573156e-03, +1.364056751e-03, +5.270954600e+00 + 3.335191e+00, +2.075239208e-02, +1.087448501e-05, +4.269770291e-03, +1.264037484e-03, +4.875664359e+00 + 3.372665e+00, +1.683366259e-02, +1.945262936e-05, +3.253540776e-03, +1.058387811e-03, +4.026124476e+00 + 3.410139e+00, +1.257529174e-02, +3.016371246e-05, +2.336277033e-03, +8.425637502e-04, +3.029466121e+00 + 3.447613e+00, +1.051023882e-02, +3.687698139e-05, +1.857166545e-03, +7.127467496e-04, +2.402804105e+00 + 3.485087e+00, +1.197457373e-02, +5.129570561e-05, +1.975832266e-03, +8.549372312e-04, +2.582857071e+00 + 3.522561e+00, +1.475674376e-02, +4.252621015e-05, +2.416287574e-03, +1.061933682e-03, +3.207230086e+00 + 3.560035e+00, +1.685326549e-02, +4.330328377e-05, +2.636066290e-03, +1.211007114e-03, +3.758897835e+00 + 3.597509e+00, +1.767867104e-02, +3.794721935e-05, +2.813350409e-03, +1.280542243e-03, +4.038904116e+00 + 3.634984e+00, +1.739033047e-02, +2.593251981e-05, +2.668158619e-03, +1.277447128e-03, +4.041099495e+00 + 3.672458e+00, +1.649636267e-02, +6.441647584e-05, +2.489142507e-03, +1.239911039e-03, +3.857051923e+00 + 3.709932e+00, +1.558915378e-02, +1.317476718e-04, +2.508135890e-03, +1.183332706e-03, +3.618537515e+00 + 3.747406e+00, +1.506771922e-02, +1.827996373e-04, +2.464165400e-03, +1.139564839e-03, +3.441274021e+00 + 3.784880e+00, +1.497043510e-02, +2.583602594e-04, +2.398976769e-03, +1.138677928e-03, +3.374125919e+00 + 3.822354e+00, +1.509382884e-02, +2.639451574e-04, +2.467045163e-03, +1.167383022e-03, +3.390782747e+00 + 3.859828e+00, +1.522187392e-02, +3.441247598e-04, +2.310794739e-03, +1.195800440e-03, +3.433261195e+00 + 3.897302e+00, +1.524725623e-02, +2.580789787e-04, +2.433286221e-03, +1.202259383e-03, +3.458316390e+00 + 3.934776e+00, +1.516738767e-02, +3.251563904e-04, +2.371467061e-03, +1.197977832e-03, +3.452477391e+00 + 3.972250e+00, +1.503328590e-02, +2.641111457e-04, +2.352762597e-03, +1.181160532e-03, +3.423913070e+00 + 4.009724e+00, +1.489865194e-02, +1.880624459e-04, +2.387811628e-03, +1.164102973e-03, +3.388071423e+00 + 4.047198e+00, +1.478547828e-02, +8.550478354e-05, +2.233912073e-03, +1.149898734e-03, +3.356481759e+00 + 4.084672e+00, +1.468099176e-02, +7.081310759e-05, +2.362301588e-03, +1.134568267e-03, +3.331811122e+00 + 4.122146e+00, +1.456012095e-02, +1.000589609e-04, +2.276136299e-03, +1.120143566e-03, +3.308561286e+00 + 4.159620e+00, +1.440061914e-02, +1.123758522e-04, +2.295370230e-03, +1.096086058e-03, +3.277077389e+00 + 4.197094e+00, +1.417920979e-02, +1.041989270e-04, +2.302534672e-03, +1.066757394e-03, +3.228672040e+00 + 4.234568e+00, +1.389532622e-02, +8.128856513e-05, +2.187418699e-03, +1.029125720e-03, +3.160462176e+00 + 4.272043e+00, +1.358279022e-02, +5.561281064e-05, +2.284829119e-03, +9.884369661e-04, +3.078621207e+00 + 4.309517e+00, +1.328300870e-02, +3.240403157e-05, +2.174652578e-03, +9.511227346e-04, +2.998083103e+00 + 4.346991e+00, +1.303013392e-02, +1.734442280e-05, +2.230526466e-03, +9.130664661e-04, +2.936632045e+00 + 4.384465e+00, +1.284238151e-02, +1.317936068e-05, +2.214435654e-03, +8.833845738e-04, +2.903960890e+00 + 4.421939e+00, +1.269274612e-02, +1.435346878e-05, +2.169259921e-03, +8.602679546e-04, +2.891846173e+00 + 4.459413e+00, +1.250769470e-02, +1.383626335e-05, +2.240466421e-03, +8.338864541e-04, +2.873914205e+00 + 4.496887e+00, +1.220347711e-02, +1.135949579e-05, +2.105010524e-03, +7.956943879e-04, +2.816758998e+00 + 4.534361e+00, +1.171453920e-02, +8.193555952e-06, +2.144672476e-03, +7.462399314e-04, +2.695681837e+00 + 4.571835e+00, +1.102237397e-02, +4.675370098e-06, +2.130475113e-03, +6.893655959e-04, +2.507645693e+00 + 4.609309e+00, +1.018197664e-02, +2.900858519e-06, +1.900309735e-03, +6.269664928e-04, +2.277803991e+00 + 4.646783e+00, +9.322403390e-03, +1.537370332e-06, +1.690064076e-03, +5.675653591e-04, +2.056633148e+00 + 4.684257e+00, +8.612482540e-03, +2.177024166e-06, +1.777449120e-03, +5.126679272e-04, +1.900910688e+00 + 4.721731e+00, +8.172956269e-03, +1.597348476e-07, +1.698925799e-03, +4.697649227e-04, +1.836469626e+00 + 4.759205e+00, +7.974254164e-03, +1.376759447e-06, +1.775601935e-03, +4.478961087e-04, +1.832767372e+00 + 4.796679e+00, +7.844595933e-03, +1.486059100e-06, +1.758174419e-03, +4.293124594e-04, +1.825304967e+00 + 4.834153e+00, +7.577377359e-03, +9.019852725e-07, +1.674581099e-03, +4.004209283e-04, +1.758563482e+00 + 4.871627e+00, +7.029320252e-03, +9.009408167e-07, +1.762912950e-03, +3.576671923e-04, +1.608482314e+00 + 4.909101e+00, +6.179568725e-03, +2.593617651e-07, +1.642024118e-03, +3.075982349e-04, +1.384441804e+00 + 4.946576e+00, +5.135873059e-03, +1.084927890e-06, +1.344862366e-03, +2.560184412e-04, +1.123464235e+00 + 4.984050e+00, +4.102281426e-03, +6.009974120e-07, +9.379570169e-04, +2.098103295e-04, +8.825893277e-01 + 5.021524e+00, +3.333768130e-03, +6.069854336e-07, +7.160525289e-04, +1.710763679e-04, +7.236586428e-01 + 5.058998e+00, +2.997347813e-03, +6.062333039e-07, +7.063083875e-04, +1.480832886e-04, +6.701071985e-01 + 5.096472e+00, +2.966082582e-03, +6.251374827e-08, +7.358345835e-04, +1.447742713e-04, +6.737184634e-01 + 5.133946e+00, +2.951114323e-03, +7.366551806e-07, +6.861277482e-04, +1.388125984e-04, +6.688498009e-01 + 5.171420e+00, +2.775008111e-03, +5.174370339e-07, +7.341489445e-04, +1.241847277e-04, +6.231396120e-01 + 5.208894e+00, +2.405717198e-03, +9.223463232e-07, +6.960389026e-04, +1.031301521e-04, +5.351006016e-01 + 5.246368e+00, +1.901056861e-03, +1.335454083e-06, +5.740866334e-04, +8.197917225e-05, +4.200215275e-01 + 5.283842e+00, +1.365329913e-03, +2.044962887e-06, +4.072460566e-04, +6.179777067e-05, +3.000669782e-01 + 5.321316e+00, +9.166623131e-04, +2.100458062e-06, +2.358018741e-04, +4.553893307e-05, +1.991688029e-01 + 5.358790e+00, +6.691344192e-04, +2.913257371e-06, +1.451168458e-04, +3.589872006e-05, +1.401126843e-01 + 5.396264e+00, +6.255466124e-04, +2.630196304e-06, +1.617746153e-04, +3.315845096e-05, +1.268528827e-01 + 5.433738e+00, +6.319797684e-04, +1.826400061e-06, +1.543450588e-04, +3.227005096e-05, +1.297292222e-01 + 5.471212e+00, +5.973080640e-04, +2.467362248e-06, +1.563021036e-04, +2.989323370e-05, +1.254260012e-01 + 5.508686e+00, +5.114085168e-04, +6.749695340e-07, +1.465183244e-04, +2.551092103e-05, +1.099715010e-01 + 5.546160e+00, +3.953211408e-04, +2.267292894e-06, +1.178128914e-04, +2.135373909e-05, +8.751561585e-02 + 5.583635e+00, +2.798970741e-04, +4.114592451e-06, +8.187619566e-05, +1.730379474e-05, +6.404749111e-02 + 5.621109e+00, +1.960285336e-04, +5.362207249e-06, +4.847757778e-05, +1.414317116e-05, +4.499882977e-02 + 5.658583e+00, +1.558666654e-04, +5.386080396e-06, +2.414455282e-05, +1.262666437e-05, +3.378351251e-02 + 5.696057e+00, +1.399319971e-04, +2.380848476e-06, +2.098415500e-05, +1.112674671e-05, +2.942388680e-02 + 5.733531e+00, +1.307437706e-04, +1.766280363e-06, +1.905675918e-05, +1.024969719e-05, +2.730232945e-02 + 5.771005e+00, +1.156967022e-04, +2.179098966e-06, +1.773792113e-05, +9.011591185e-06, +2.433933672e-02 + 5.808479e+00, +9.613916015e-05, +1.829222517e-06, +1.539190377e-05, +7.488412599e-06, +2.048163582e-02 + 5.845953e+00, +7.929064937e-05, +2.024222810e-06, +1.269795076e-05, +5.983825506e-06, +1.785036816e-02 + 5.883427e+00, +8.026728622e-05, +9.612655320e-07, +1.395848839e-05, +5.767006684e-06, +1.905047360e-02 + 5.920901e+00, +9.786066890e-05, +1.616263390e-06, +1.884901042e-05, +6.916660805e-06, +2.355153006e-02 + 5.958375e+00, +1.193864783e-04, +1.249289289e-06, +2.379396756e-05, +7.728439405e-06, +2.860371125e-02 + 5.995849e+00, +1.345201914e-04, +4.088511303e-07, +2.683752440e-05, +8.277988716e-06, +3.211074890e-02 + 6.033323e+00, +1.384647182e-04, +1.148339916e-06, +2.874751155e-05, +8.447988514e-06, +3.288978387e-02 + 6.070797e+00, +1.298738806e-04, +8.929752626e-07, +2.752913259e-05, +7.815986825e-06, +3.044108952e-02 + 6.108271e+00, +1.055505567e-04, +7.319142990e-07, +2.263722024e-05, +6.213020788e-06, +2.486884038e-02 + 6.145745e+00, +7.166703123e-05, +3.366496572e-07, +1.556595803e-05, +4.216763327e-06, +1.688201114e-02 + 6.183219e+00, +3.493517869e-05, +6.909111035e-07, +6.894011008e-06, +2.378664019e-06, +8.222464850e-03 + 6.220694e+00, +3.315980737e-05, +7.955861069e-07, +5.819273465e-06, +2.558211822e-06, +7.112352809e-03 + 6.258168e+00, +6.560551270e-05, +4.917076246e-07, +1.166288699e-05, +4.572165491e-06, +1.487454424e-02 + 6.295642e+00, +9.613850820e-05, +3.025743178e-07, +1.870489049e-05, +6.344660049e-06, +2.195967174e-02 + 6.333116e+00, +1.144656347e-04, +1.042577705e-06, +2.279021770e-05, +7.500785919e-06, +2.658824072e-02 + 6.370590e+00, +1.200142843e-04, +7.525397926e-07, +2.365592156e-05, +7.845707667e-06, +2.836687987e-02 + 6.408064e+00, +1.156943204e-04, +2.734515578e-07, +2.199714989e-05, +7.613771837e-06, +2.754736754e-02 + 6.445538e+00, +1.028589941e-04, +1.131230488e-06, +1.895496169e-05, +7.067578951e-06, +2.492401183e-02 + 6.483012e+00, +8.998476195e-05, +1.312943251e-06, +1.561633829e-05, +6.506193902e-06, +2.173369821e-02 + 6.520486e+00, +8.174691766e-05, +7.018622596e-07, +1.355662912e-05, +5.905477292e-06, +1.939616532e-02 + 6.557960e+00, +8.194147886e-05, +1.147425739e-06, +1.345797658e-05, +5.929276191e-06, +1.881263997e-02 + 6.595434e+00, +8.615558632e-05, +1.631522428e-06, +1.474983448e-05, +6.453523240e-06, +1.965680357e-02 + 6.632908e+00, +9.099245733e-05, +8.959004766e-07, +1.490351472e-05, +6.947049143e-06, +2.089060758e-02 + 6.670382e+00, +9.357109056e-05, +7.047876828e-07, +1.461727660e-05, +7.176279953e-06, +2.174333892e-02 + 6.707856e+00, +9.372354548e-05, +2.228534400e-06, +1.517600551e-05, +7.306445383e-06, +2.197662468e-02 + 6.745330e+00, +9.228327205e-05, +1.665893489e-06, +1.466430309e-05, +7.167443892e-06, +2.172186153e-02 + 6.782804e+00, +9.037440491e-05, +2.158474481e-06, +1.484760547e-05, +7.053754296e-06, +2.125715216e-02 + 6.820278e+00, +8.895020888e-05, +1.201712459e-06, +1.457706151e-05, +6.904806409e-06, +2.082496930e-02 + 6.857752e+00, +8.797395582e-05, +2.175566004e-07, +1.413235007e-05, +6.696578320e-06, +2.053509651e-02 + 6.895227e+00, +8.818415187e-05, +1.870305267e-06, +1.465944635e-05, +6.823121947e-06, +2.036976733e-02 + 6.932701e+00, +8.652482398e-05, +1.297190621e-06, +1.394898637e-05, +6.590002143e-06, +2.025681153e-02 + 6.970175e+00, +8.654726097e-05, +1.302455999e-06, +1.430750605e-05, +6.647337445e-06, +2.014092972e-02 + 7.007649e+00, +8.599023697e-05, +1.632653526e-06, +1.404656976e-05, +6.560173926e-06, +2.000770004e-02 + 7.045123e+00, +8.507094701e-05, +8.018955482e-07, +1.393717929e-05, +6.460613964e-06, +1.987163031e-02 + 7.082597e+00, +8.507173670e-05, +6.272024716e-07, +1.428921333e-05, +6.385954443e-06, +1.974790284e-02 + 7.120071e+00, +8.346018328e-05, +2.475724018e-07, +1.325648636e-05, +6.243644764e-06, +1.962790928e-02 + 7.157545e+00, +8.421307523e-05, +4.394951594e-07, +1.390228436e-05, +6.285879059e-06, +1.947211778e-02 + 7.195019e+00, +8.170806833e-05, +2.014984360e-07, +1.366106331e-05, +5.956409962e-06, +1.921995461e-02 + 7.232493e+00, +8.013386787e-05, +7.122899917e-07, +1.367002400e-05, +5.816296736e-06, +1.881509895e-02 + 7.269967e+00, +7.930240825e-05, +2.115088068e-07, +1.370943668e-05, +5.595438478e-06, +1.823254307e-02 + 7.307441e+00, +7.425857303e-05, +1.161750331e-07, +1.277195401e-05, +5.156725539e-06, +1.750631517e-02 + 7.344915e+00, +7.288753011e-05, +2.559503828e-07, +1.356922915e-05, +5.000841773e-06, +1.673913653e-02 + 7.382389e+00, +6.980444151e-05, +4.240898323e-07, +1.302044783e-05, +4.767250185e-06, +1.607581245e-02 + 7.419863e+00, +6.858350498e-05, +4.111819579e-07, +1.330329130e-05, +4.515920254e-06, +1.564415606e-02 + 7.457337e+00, +6.677209617e-05, +1.872692108e-07, +1.314575231e-05, +4.251436669e-06, +1.546991691e-02 + 7.494811e+00, +6.533714996e-05, +1.918803774e-07, +1.293346135e-05, +4.171945397e-06, +1.543440789e-02 + Mean, +1.092334635e-02, +6.102304230e-04, +1.136639064e-03, +9.589290912e-04, +1.742421290e+00 diff --git a/test/ref/cpw/lumped_adaptive/error-indicators.csv b/test/ref/cpw/lumped_adaptive/error-indicators.csv index 9a627cf4a..1dd1777bc 100644 --- a/test/ref/cpw/lumped_adaptive/error-indicators.csv +++ b/test/ref/cpw/lumped_adaptive/error-indicators.csv @@ -1,11 +1,11 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +7.050139872e-01, +1.089715623e-06, +1.591585952e-02, +6.373526318e-06, +2.186423623e-01 - 3.000000e+01, +7.048573159e-01, +1.801749017e-06, +1.911288427e-02, +6.372109965e-06, +4.552939305e+00 - 1.780000e+01, +7.085941898e-01, +1.239011768e-06, +1.967802015e-02, +6.405892364e-06, +2.505814500e+00 - 1.280000e+01, +7.050022016e-01, +1.023153730e-06, +1.779303491e-02, +6.373419773e-06, +1.992681983e+00 - 2.510000e+01, +7.064238116e-01, +9.104840972e-07, +1.919993236e-02, +6.386271531e-06, +3.534944010e+00 - 5.200000e+00, +7.032239220e-01, +7.481839787e-07, +1.760116584e-02, +6.357343621e-06, +6.351597186e-01 - 2.790000e+01, +7.052578369e-01, +1.050803997e-06, +1.902258442e-02, +6.375730789e-06, +4.240374356e+00 - 1.540000e+01, +7.069908612e-01, +1.449849369e-06, +1.912291719e-02, +6.391397819e-06, +2.357765408e+00 - 8.300000e+00, +7.029555592e-01, +6.862523461e-07, +1.801637153e-02, +6.354917545e-06, +1.133642571e+00 - Mean, +7.053688539e-01, +1.196501131e-06, +1.658438367e-02, +6.376734414e-06, +2.352440468e+00 + 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585962e-02, +8.806573475e-04, +2.186423623e-01 + 3.000000e+01, +7.048573140e-01, +1.801733890e-06, +1.911288392e-02, +8.604087499e-04, +4.552939315e+00 + 1.780000e+01, +7.085941902e-01, +1.239015712e-06, +1.967801959e-02, +8.584871537e-04, +2.505814499e+00 + 1.270000e+01, +7.049313558e-01, +1.005197299e-06, +1.776072999e-02, +8.634324526e-04, +1.974457243e+00 + 2.510000e+01, +7.064238109e-01, +1.100529477e-06, +1.919993195e-02, +8.642147215e-04, +3.534944010e+00 + 5.100000e+00, +7.032588007e-01, +7.538436764e-07, +1.758484364e-02, +8.696505248e-04, +6.206815574e-01 + 2.830000e+01, +7.051385177e-01, +9.644838144e-07, +1.903597935e-02, +8.599617820e-04, +4.323348710e+00 + 1.540000e+01, +7.069908609e-01, +1.449833545e-06, +1.912291735e-02, +8.545513705e-04, +2.357765408e+00 + 9.000000e+00, +7.030971609e-01, +7.011305477e-07, +1.801203000e-02, +8.639941115e-04, +1.260202551e+00 + Mean, +7.053692541e-01, +1.260046792e-06, +1.675928718e-02, +8.746827136e-04, +2.372088406e+00 diff --git a/test/ref/cpw/lumped_uniform/error-indicators.csv b/test/ref/cpw/lumped_uniform/error-indicators.csv index 2cd6296ed..f40e72b5b 100644 --- a/test/ref/cpw/lumped_uniform/error-indicators.csv +++ b/test/ref/cpw/lumped_uniform/error-indicators.csv @@ -1,17 +1,17 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +7.050139872e-01, +1.089715622e-06, +1.591585952e-02, +6.373526318e-06, +2.186423623e-01 - 4.000000e+00, +7.037534941e-01, +8.348646390e-07, +1.716164813e-02, +6.362131103e-06, +4.676837690e-01 - 6.000000e+00, +7.030062925e-01, +7.118692553e-07, +1.767838746e-02, +6.355376189e-06, +7.544551064e-01 - 8.000000e+00, +7.029165341e-01, +6.832340564e-07, +1.800232389e-02, +6.354564748e-06, +1.081001046e+00 - 1.000000e+01, +7.034192143e-01, +7.417840765e-07, +1.790794998e-02, +6.359109119e-06, +1.449355687e+00 - 1.200000e+01, +7.044599266e-01, +9.086861419e-07, +1.751260195e-02, +6.368517453e-06, +1.842149229e+00 - 1.400000e+01, +7.058985483e-01, +1.260916409e-06, +1.833900406e-02, +6.381523001e-06, +2.190892578e+00 - 1.600000e+01, +7.074564926e-01, +1.093990747e-06, +1.938386316e-02, +6.395607259e-06, +2.406792037e+00 - 1.800000e+01, +7.086746629e-01, +1.209021510e-06, +1.965872720e-02, +6.406619864e-06, +2.515530177e+00 - 2.000000e+01, +7.088448208e-01, +8.917049253e-07, +1.963851901e-02, +6.408158140e-06, +2.646279233e+00 - 2.200000e+01, +7.081043249e-01, +7.960618462e-07, +1.946623450e-02, +6.401463847e-06, +2.889633320e+00 - 2.400000e+01, +7.070103252e-01, +8.888321469e-07, +1.928974719e-02, +6.391573780e-06, +3.271855293e+00 - 2.600000e+01, +7.059908737e-01, +1.025100239e-06, +1.913395101e-02, +6.382357649e-06, +3.767453334e+00 - 2.800000e+01, +7.052267089e-01, +9.840962024e-07, +1.902600862e-02, +6.375449382e-06, +4.261952652e+00 - 3.000000e+01, +7.048573116e-01, +1.801773182e-06, +1.911288411e-02, +6.372109927e-06, +4.552939148e+00 - Mean, +7.056422345e-01, +1.101172703e-06, +1.672465773e-02, +6.379205852e-06, +2.287774331e+00 + 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +8.806573475e-04, +2.186423623e-01 + 4.000000e+00, +7.037534929e-01, +8.348638678e-07, +1.716164791e-02, +8.740225193e-04, +4.676837686e-01 + 6.000000e+00, +7.030062911e-01, +7.118689649e-07, +1.767838766e-02, +8.667794453e-04, +7.544551055e-01 + 8.000000e+00, +7.029165326e-01, +6.832337447e-07, +1.800232414e-02, +8.638803686e-04, +1.081001045e+00 + 1.000000e+01, +7.034192125e-01, +7.418014192e-07, +1.790795027e-02, +8.645952292e-04, +1.449355685e+00 + 1.200000e+01, +7.044599247e-01, +9.086471535e-07, +1.751260139e-02, +8.646185630e-04, +1.842149227e+00 + 1.400000e+01, +7.058985472e-01, +1.260914522e-06, +1.833900466e-02, +8.591277673e-04, +2.190892577e+00 + 1.600000e+01, +7.074564924e-01, +1.093986277e-06, +1.938386319e-02, +8.542665420e-04, +2.406792037e+00 + 1.800000e+01, +7.086746633e-01, +1.209024855e-06, +1.965872658e-02, +8.590913096e-04, +2.515530177e+00 + 2.000000e+01, +7.088448209e-01, +8.917186070e-07, +1.963851961e-02, +8.636924028e-04, +2.646279233e+00 + 2.200000e+01, +7.081043245e-01, +7.960606263e-07, +1.946623509e-02, +8.649306217e-04, +2.889633318e+00 + 2.400000e+01, +7.070103246e-01, +9.980955322e-07, +1.928974677e-02, +8.646218376e-04, +3.271855291e+00 + 2.600000e+01, +7.059908728e-01, +1.195377481e-06, +1.913395059e-02, +8.635749385e-04, +3.767453335e+00 + 2.800000e+01, +7.052267075e-01, +9.841023722e-07, +1.902600821e-02, +8.604645102e-04, +4.261952660e+00 + 3.000000e+01, +7.048573097e-01, +1.801758125e-06, +1.911288376e-02, +8.604087621e-04, +4.552939157e+00 + Mean, +7.056448946e-01, +1.148429619e-06, +1.689680225e-02, +8.751153438e-04, +2.287774332e+00 diff --git a/test/ref/cpw/wave_adaptive/error-indicators.csv b/test/ref/cpw/wave_adaptive/error-indicators.csv index 19728fc81..f4164b5e9 100644 --- a/test/ref/cpw/wave_adaptive/error-indicators.csv +++ b/test/ref/cpw/wave_adaptive/error-indicators.csv @@ -1,11 +1,11 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +6.993092200e-01, +1.566265933e-06, +1.685105942e-02, +6.731376290e-06, +2.614559004e-01 - 3.000000e+01, +6.802918578e-01, +5.681633843e-06, +1.845184430e-02, +6.548319901e-06, +4.266998699e+00 - 1.740000e+01, +6.984728390e-01, +3.015199052e-06, +1.743761926e-02, +6.723325495e-06, +2.320170390e+00 - 2.670000e+01, +6.990163126e-01, +3.326374904e-06, +1.698618907e-02, +6.728556836e-06, +3.579808172e+00 - 2.030000e+01, +6.990616766e-01, +2.913715164e-06, +1.755160089e-02, +6.728993498e-06, +2.698250801e+00 - 1.640000e+01, +6.962139900e-01, +4.324939119e-06, +1.717774037e-02, +6.701582378e-06, +2.200470547e+00 - 2.960000e+01, +6.281346087e-01, +8.180986750e-06, +1.580175763e-02, +6.046267217e-06, +5.361105316e+00 - 1.530000e+01, +6.988269093e-01, +4.708649999e-06, +1.727962179e-02, +6.726733687e-06, +2.051893344e+00 - 8.200000e+00, +6.993384974e-01, +2.436893373e-06, +1.675967485e-02, +6.731658107e-06, +1.094416733e+00 - Mean, +6.887406568e-01, +5.788786385e-06, +1.631021899e-02, +6.629645934e-06, +2.648285545e+00 + 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +9.263986181e-04, +2.614559004e-01 + 3.000000e+01, +6.802918578e-01, +5.681633835e-06, +1.845184430e-02, +1.069612630e-03, +4.266998699e+00 + 1.760000e+01, +6.986524402e-01, +2.412989991e-06, +1.745780169e-02, +9.343913445e-04, +2.345607124e+00 + 2.740000e+01, +6.989786511e-01, +3.347415932e-06, +1.699249148e-02, +9.402676176e-04, +3.678349350e+00 + 2.210000e+01, +6.990013886e-01, +3.195355480e-06, +1.749122001e-02, +9.296641221e-04, +2.940510061e+00 + 1.650000e+01, +6.962225150e-01, +5.006960530e-06, +1.720841009e-02, +9.616412940e-04, +2.212464633e+00 + 1.450000e+01, +6.993481111e-01, +1.362223224e-06, +1.708077753e-02, +9.386436767e-04, +1.944614579e+00 + 1.510000e+01, +6.964183126e-01, +4.767143173e-06, +1.723792019e-02, +9.938502198e-04, +2.042324804e+00 + 2.950000e+01, +6.692857361e-01, +5.896183956e-06, +1.725486449e-02, +1.115952350e-03, +4.433714589e+00 + Mean, +6.931309928e-01, +5.968865405e-06, +1.679209924e-02, +1.019613084e-03, +2.680671082e+00 diff --git a/test/ref/cpw/wave_uniform/error-indicators.csv b/test/ref/cpw/wave_uniform/error-indicators.csv index 7347e6188..b177ec561 100644 --- a/test/ref/cpw/wave_uniform/error-indicators.csv +++ b/test/ref/cpw/wave_uniform/error-indicators.csv @@ -1,17 +1,17 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +6.993092200e-01, +1.566265933e-06, +1.685105942e-02, +6.731376290e-06, +2.614559004e-01 - 4.000000e+00, +6.992890992e-01, +1.736035553e-06, +1.680419113e-02, +6.731182612e-06, +5.258892594e-01 - 6.000000e+00, +6.992881535e-01, +2.021152765e-06, +1.676125294e-02, +6.731173509e-06, +7.947256627e-01 - 8.000000e+00, +6.993312910e-01, +2.448759168e-06, +1.675682757e-02, +6.731588740e-06, +1.067084152e+00 - 1.000000e+01, +6.994243806e-01, +2.399252545e-06, +1.681663205e-02, +6.732484797e-06, +1.340172770e+00 - 1.200000e+01, +6.995448511e-01, +2.366910744e-06, +1.694287832e-02, +6.733644416e-06, +1.610652070e+00 - 1.400000e+01, +6.995942415e-01, +1.874364476e-06, +1.709174983e-02, +6.734119836e-06, +1.876886099e+00 - 1.600000e+01, +6.980880025e-01, +5.469569269e-06, +1.721279606e-02, +6.719621155e-06, +2.144504886e+00 - 1.800000e+01, +6.988641967e-01, +1.805639364e-06, +1.748858870e-02, +6.727092606e-06, +2.396980543e+00 - 2.000000e+01, +6.990669364e-01, +2.883025512e-06, +1.755245634e-02, +6.729044128e-06, +2.658455702e+00 - 2.200000e+01, +6.990046675e-01, +3.191459322e-06, +1.749703902e-02, +6.728444743e-06, +2.926893770e+00 - 2.400000e+01, +6.989710885e-01, +2.953254976e-06, +1.732841600e-02, +6.728121520e-06, +3.202248951e+00 - 2.600000e+01, +6.990090439e-01, +3.218910288e-06, +1.709437140e-02, +6.728486869e-06, +3.481668528e+00 - 2.800000e+01, +6.988244472e-01, +3.387084530e-06, +1.706546999e-02, +6.726709988e-06, +3.764267665e+00 - 3.000000e+01, +6.802918601e-01, +5.681637487e-06, +1.845184332e-02, +6.548319923e-06, +4.266998672e+00 - Mean, +6.978600986e-01, +4.485174041e-06, +1.714127920e-02, +6.717427409e-06, +2.154592309e+00 + 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +9.263986181e-04, +2.614559004e-01 + 4.000000e+00, +6.992890992e-01, +1.736035539e-06, +1.680419113e-02, +9.263152213e-04, +5.258892594e-01 + 6.000000e+00, +6.992881535e-01, +2.021152752e-06, +1.676125294e-02, +9.261540058e-04, +7.947256627e-01 + 8.000000e+00, +6.993312910e-01, +2.448759169e-06, +1.675682757e-02, +9.260015170e-04, +1.067084152e+00 + 1.000000e+01, +6.994243806e-01, +2.399252549e-06, +1.681663205e-02, +9.259792773e-04, +1.340172770e+00 + 1.200000e+01, +6.995448511e-01, +2.366910745e-06, +1.694287832e-02, +9.262810134e-04, +1.610652070e+00 + 1.400000e+01, +6.995942415e-01, +1.874364476e-06, +1.709174983e-02, +9.296637533e-04, +1.876886099e+00 + 1.600000e+01, +6.980880025e-01, +5.469569257e-06, +1.721279606e-02, +9.594361262e-04, +2.144504886e+00 + 1.800000e+01, +6.988641967e-01, +1.805639359e-06, +1.748858870e-02, +9.321067031e-04, +2.396980543e+00 + 2.000000e+01, +6.990669364e-01, +2.883025515e-06, +1.755245634e-02, +9.293916043e-04, +2.658455702e+00 + 2.200000e+01, +6.990046675e-01, +3.191459315e-06, +1.749703902e-02, +9.296276956e-04, +2.926893770e+00 + 2.400000e+01, +6.989710885e-01, +2.953254947e-06, +1.732841600e-02, +9.308023476e-04, +3.202248951e+00 + 2.600000e+01, +6.990090439e-01, +3.218910262e-06, +1.709437140e-02, +9.338867734e-04, +3.481668528e+00 + 2.800000e+01, +6.988244472e-01, +3.387084517e-06, +1.706546999e-02, +9.468574632e-04, +3.764267665e+00 + 3.000000e+01, +6.802918601e-01, +5.681637479e-06, +1.845184332e-02, +1.069612628e-03, +4.266998672e+00 + Mean, +6.978759848e-01, +4.868939341e-06, +1.714752106e-02, +9.637101249e-04, +2.154592309e+00 diff --git a/test/ref/rings/error-indicators.csv b/test/ref/rings/error-indicators.csv index 189aa400c..a42cbf899 100644 --- a/test/ref/rings/error-indicators.csv +++ b/test/ref/rings/error-indicators.csv @@ -1,4 +1,4 @@ i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +3.546095276e-01, +4.103305750e-09, +2.980197521e-02, +6.790554136e-06, +1.115530397e-01 - 2.000000e+00, +2.764870097e-01, +1.023265200e-07, +7.245396381e-03, +5.294556016e-06, +4.557585167e-01 - Mean, +3.155482686e-01, +5.321491286e-08, +1.495441012e-02, +6.042555076e-06, +2.836557782e-01 + 1.000000e+00, +3.544895565e-01, +4.103268389e-09, +2.982074859e-02, +2.217169423e-04, +1.115529612e-01 + 2.000000e+00, +2.764894272e-01, +1.023265139e-07, +7.245458667e-03, +5.519522597e-04, +4.557585527e-01 + Mean, +3.178909003e-01, +7.241392224e-08, +2.108658880e-02, +5.448306923e-04, +2.836557570e-01 diff --git a/test/ref/spheres/error-indicators.csv b/test/ref/spheres/error-indicators.csv index 29704f4a0..74b941f6f 100644 --- a/test/ref/spheres/error-indicators.csv +++ b/test/ref/spheres/error-indicators.csv @@ -1,4 +1,4 @@ i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +7.827776488e-03, +7.729127800e-07, +5.589600332e-04, +7.474244713e-07, +3.052392581e-01 - 2.000000e+00, +7.529026549e-03, +8.147247392e-07, +7.245667936e-04, +7.188987443e-07, +4.319876077e-01 - Mean, +7.678401519e-03, +1.617075494e-06, +4.783729070e-04, +7.331616078e-07, +3.686134329e-01 + 1.000000e+00, +7.835018957e-03, +7.728228292e-07, +5.659925890e-04, +4.310697912e-05, +3.052442134e-01 + 2.000000e+00, +7.529893122e-03, +8.146906715e-07, +7.245606637e-04, +4.195362244e-05, +4.319918577e-01 + Mean, +7.683970734e-03, +1.578279084e-06, +5.153241491e-04, +4.815649458e-05, +3.686180355e-01 From d275f02dbb62546b2b3658cea19ed3cf59cbec4d Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Wed, 4 Oct 2023 18:36:55 -0400 Subject: [PATCH 22/39] style --- palace/linalg/errorestimator.cpp | 7 ++++--- palace/utils/timer.hpp | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 438326002..d2b9f474f 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -63,8 +63,8 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie template FluxProjector::FluxProjector( - mfem::ParFiniteElementSpaceHierarchy &fespaces, double tol, int max_it, - int print, int pa_order_threshold) + mfem::ParFiniteElementSpaceHierarchy &fespaces, double tol, int max_it, int print, + int pa_order_threshold) : M(BuildMassMatrixOperator(fespaces, pa_order_threshold)) { @@ -83,7 +83,8 @@ FluxProjector::FluxProjector( pc = std::move(amg); } - auto pcg = std::make_unique>(fespaces.GetFinestFESpace().GetComm(), print); + auto pcg = + std::make_unique>(fespaces.GetFinestFESpace().GetComm(), print); pcg->SetInitialGuess(false); pcg->SetRelTol(tol); pcg->SetAbsTol(std::numeric_limits::epsilon()); diff --git a/palace/utils/timer.hpp b/palace/utils/timer.hpp index 89c6da817..4c74325c7 100644 --- a/palace/utils/timer.hpp +++ b/palace/utils/timer.hpp @@ -43,7 +43,6 @@ class Timer NUMTIMINGS }; - // clang-format off inline static const std::vector descriptions{ "Initialization", From fc5000601a6bd4ede4cfe331884a67c37bcb7054 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Thu, 5 Oct 2023 11:08:44 -0400 Subject: [PATCH 23/39] Removing printing the mean row --- palace/drivers/basesolver.cpp | 26 ------------- palace/drivers/basesolver.hpp | 3 -- palace/drivers/drivensolver.cpp | 12 ------ palace/drivers/eigensolver.cpp | 6 --- palace/drivers/electrostaticsolver.cpp | 6 --- palace/drivers/magnetostaticsolver.cpp | 6 --- palace/drivers/transientsolver.cpp | 6 --- palace/main.cpp | 3 -- .../ref/cavity/impedance/error-indicators.csv | 3 +- test/ref/cavity/pec/error-indicators.csv | 5 +-- test/ref/coaxial/matched/error-indicators.csv | 1 - test/ref/coaxial/open/error-indicators.csv | 1 - .../cpw/lumped_adaptive/error-indicators.csv | 20 +++++----- .../cpw/lumped_uniform/error-indicators.csv | 11 +++--- .../cpw/wave_adaptive/error-indicators.csv | 12 +++--- .../ref/cpw/wave_uniform/error-indicators.csv | 1 - test/ref/rings/error-indicators.csv | 1 - test/ref/spheres/error-indicators.csv | 1 - test/testcase.jl | 39 +++---------------- 19 files changed, 30 insertions(+), 133 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index 120910f43..9a4f8fa93 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -606,32 +606,6 @@ void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, d } } -void BaseSolver::PostprocessErrorIndicators(const std::string &name, double global, - double min, double max, double mean, - double normalization) const -{ - if (post_dir.length() == 0) - { - return; - } - - // Write the indicator statistics - if (root) - { - std::string path = post_dir + "error-indicators.csv"; - auto output = OutputFile(path, true); - // clang-format off - output.print("{:>{}s},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", - name, table.w1, - global, table.w, table.p, - min, table.w, table.p, - max, table.w, table.p, - mean, table.w, table.p, - normalization, table.w, table.p); - // clang-format on - } -} - template void BaseSolver::SaveMetadata(const KspSolver &) const; template void BaseSolver::SaveMetadata(const ComplexKspSolver &) const; diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index 749cef73f..d92cb3e7b 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -74,9 +74,6 @@ class BaseSolver void PostprocessErrorIndicators(const std::string &name, int step, double time, double global, double min, double max, double mean, double normalization, bool normalized) const; - // Write a string labeled error indicator. Used for writing statistics. - void PostprocessErrorIndicators(const std::string &name, double global, double min, - double max, double mean, double normalization) const; public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 73cd1ce40..3b85bfc1c 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -222,12 +222,6 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator omega += delta_omega; } SaveMetadata(ksp); - PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), - combined_indicators.GetNormalization()); return combined_indicators; } @@ -349,12 +343,6 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator SaveMetadata(prom.GetLinearSolver()); // Set the indicator field to the combined field for postprocessing. - PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), - combined_indicators.GetNormalization()); postop.SetIndicatorGridFunction(combined_indicators.GetLocalErrorIndicators()); // Main fast frequency sweep loop (online phase). diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 9f254d581..a8d95bb55 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -328,12 +328,6 @@ EigenSolver::Solve(const std::vector> &mesh) cons // Postprocess the mode. Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv); } - PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), - combined_indicators.GetNormalization()); return combined_indicators; } diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index d79ad1ab0..fa2c93ab2 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -174,12 +174,6 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, mfem::DenseMatrix Cinv(C); Cinv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(terminal_sources, C, Cinv, Cm); - PostprocessErrorIndicators( - "Mean", combined_indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(laplaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(laplaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(laplaceop.GetComm()), - combined_indicators.GetNormalization()); return combined_indicators; } diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index 3bb3ccd27..9a37c58d0 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -184,12 +184,6 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, mfem::DenseMatrix Minv(M); Minv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(surf_j_op, M, Minv, Mm); - PostprocessErrorIndicators( - "Mean", combined_indicators.GetGlobalErrorIndicator(curlcurlop.GetComm()), - combined_indicators.GetMinErrorIndicator(curlcurlop.GetComm()), - combined_indicators.GetMaxErrorIndicator(curlcurlop.GetComm()), - combined_indicators.GetMeanErrorIndicator(curlcurlop.GetComm()), - combined_indicators.GetNormalization()); return combined_indicators; } diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index ec930c9bb..69dcea092 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -153,12 +153,6 @@ TransientSolver::Solve(const std::vector> &mesh) step++; } SaveMetadata(timeop.GetLinearSolver()); - PostprocessErrorIndicators("Mean", - combined_indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMinErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMaxErrorIndicator(spaceop.GetComm()), - combined_indicators.GetMeanErrorIndicator(spaceop.GetComm()), - combined_indicators.GetNormalization()); return combined_indicators; } std::function TransientSolver::GetTimeExcitation(bool dot) const diff --git a/palace/main.cpp b/palace/main.cpp index 185c436a9..4609f977a 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -187,9 +187,6 @@ int main(int argc, char *argv[]) // Run the problem driver. auto solver_output = solver->Solve(mesh); - Mpi::Print(world_comm, "Error Estimate: {:.3e}\n", - solver_output.GetGlobalErrorIndicator(world_comm)); - // Print timing summary. BlockTimer::Print(world_comm); solver->SaveMetadata(BlockTimer::GlobalTimer()); diff --git a/test/ref/cavity/impedance/error-indicators.csv b/test/ref/cavity/impedance/error-indicators.csv index c88a44205..a81f09c9f 100644 --- a/test/ref/cavity/impedance/error-indicators.csv +++ b/test/ref/cavity/impedance/error-indicators.csv @@ -3,7 +3,7 @@ 2.000000e+00, +7.387353181e-03, +1.120183148e-04, +5.134253635e-04, +2.965305671e-04, +4.545494361e-01 3.000000e+00, +7.977895730e-03, +1.062690571e-04, +6.123658757e-04, +3.123956087e-04, +5.169059866e-01 4.000000e+00, +8.274201489e-03, +6.590964834e-05, +8.966948182e-04, +3.045023806e-04, +7.254767932e-01 - 5.000000e+00, +1.685960941e-02, +2.820942105e-04, +1.153640116e-03, +6.780447656e-04, +7.021998742e-01 + 5.000000e+00, +1.685960941e-02, +2.820942104e-04, +1.153640116e-03, +6.780447656e-04, +7.021998742e-01 6.000000e+00, +1.730399293e-02, +2.679568469e-04, +1.285571611e-03, +6.941394542e-04, +6.917741537e-01 7.000000e+00, +1.541951446e-02, +3.015087299e-04, +1.088775657e-03, +6.184944349e-04, +6.837241462e-01 8.000000e+00, +1.589962692e-02, +2.715177297e-04, +1.182648337e-03, +6.283845490e-04, +7.775219321e-01 @@ -14,4 +14,3 @@ 1.300000e+01, +2.162823528e-02, +1.439773671e-04, +2.211508889e-03, +7.807885965e-04, +1.123798284e+00 1.400000e+01, +2.568539055e-02, +1.169360347e-04, +3.234035290e-03, +8.927006761e-04, +1.125061440e+00 1.500000e+01, +2.696836169e-02, +4.629929123e-04, +1.772177016e-03, +1.080937100e-03, +8.955419511e-01 - Mean, +1.832232363e-02, +4.343505522e-04, +1.399356727e-03, +7.453986847e-04, +8.242436674e-01 diff --git a/test/ref/cavity/pec/error-indicators.csv b/test/ref/cavity/pec/error-indicators.csv index 2323bb511..afe734cbc 100644 --- a/test/ref/cavity/pec/error-indicators.csv +++ b/test/ref/cavity/pec/error-indicators.csv @@ -1,7 +1,7 @@ m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization 1.000000e+00, +8.196373525e-03, +9.214821861e-05, +6.510933257e-04, +3.254431264e-04, +6.964480689e-01 - 2.000000e+00, +7.387353222e-03, +1.120182421e-04, +5.134245793e-04, +2.965305493e-04, +4.545494456e-01 - 3.000000e+00, +7.977895796e-03, +1.062685103e-04, +6.123649464e-04, +3.123955986e-04, +5.169059367e-01 + 2.000000e+00, +7.387353222e-03, +1.120182421e-04, +5.134245794e-04, +2.965305493e-04, +4.545494456e-01 + 3.000000e+00, +7.977895796e-03, +1.062685104e-04, +6.123649465e-04, +3.123955986e-04, +5.169059367e-01 4.000000e+00, +8.274201169e-03, +6.590958800e-05, +8.966948963e-04, +3.045023647e-04, +7.254767786e-01 5.000000e+00, +1.685960472e-02, +2.820836869e-04, +1.153598318e-03, +6.780453428e-04, +7.021989802e-01 6.000000e+00, +1.730399491e-02, +2.679493494e-04, +1.285568826e-03, +6.941391330e-04, +6.917744665e-01 @@ -14,4 +14,3 @@ 1.300000e+01, +2.162823566e-02, +1.439781616e-04, +2.211512532e-03, +7.807886467e-04, +1.123798302e+00 1.400000e+01, +2.568539163e-02, +1.169360360e-04, +3.234036921e-03, +8.927007077e-04, +1.125061428e+00 1.500000e+01, +2.696828680e-02, +4.629949994e-04, +1.772157010e-03, +1.080939278e-03, +8.955391704e-01 - Mean, +1.832231600e-02, +4.343440209e-04, +1.399356089e-03, +7.453990959e-04, +8.242435486e-01 diff --git a/test/ref/coaxial/matched/error-indicators.csv b/test/ref/coaxial/matched/error-indicators.csv index 6569d6a7a..ce7cdd968 100644 --- a/test/ref/coaxial/matched/error-indicators.csv +++ b/test/ref/coaxial/matched/error-indicators.csv @@ -200,4 +200,3 @@ 7.419863e+00, +4.830704479e-06, +5.884631298e-08, +7.174825239e-07, +3.835192893e-07, +6.712265589e-06 7.457337e+00, +5.146804022e-06, +1.163190193e-07, +7.555170452e-07, +4.169979523e-07, +6.428561978e-06 7.494811e+00, +4.397343693e-06, +1.080439747e-07, +5.798133525e-07, +3.671442609e-07, +6.238486822e-06 - Mean, +8.612739103e-03, +6.513527808e-04, +8.444567972e-04, +7.594154458e-04, +1.099244137e+00 diff --git a/test/ref/coaxial/open/error-indicators.csv b/test/ref/coaxial/open/error-indicators.csv index 57fa73ea4..7e1a5bacf 100644 --- a/test/ref/coaxial/open/error-indicators.csv +++ b/test/ref/coaxial/open/error-indicators.csv @@ -200,4 +200,3 @@ 7.419863e+00, +6.858350498e-05, +4.111819579e-07, +1.330329130e-05, +4.515920254e-06, +1.564415606e-02 7.457337e+00, +6.677209617e-05, +1.872692108e-07, +1.314575231e-05, +4.251436669e-06, +1.546991691e-02 7.494811e+00, +6.533714996e-05, +1.918803774e-07, +1.293346135e-05, +4.171945397e-06, +1.543440789e-02 - Mean, +1.092334635e-02, +6.102304230e-04, +1.136639064e-03, +9.589290912e-04, +1.742421290e+00 diff --git a/test/ref/cpw/lumped_adaptive/error-indicators.csv b/test/ref/cpw/lumped_adaptive/error-indicators.csv index 1dd1777bc..9739c51b9 100644 --- a/test/ref/cpw/lumped_adaptive/error-indicators.csv +++ b/test/ref/cpw/lumped_adaptive/error-indicators.csv @@ -1,11 +1,11 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585962e-02, +8.806573475e-04, +2.186423623e-01 - 3.000000e+01, +7.048573140e-01, +1.801733890e-06, +1.911288392e-02, +8.604087499e-04, +4.552939315e+00 - 1.780000e+01, +7.085941902e-01, +1.239015712e-06, +1.967801959e-02, +8.584871537e-04, +2.505814499e+00 - 1.270000e+01, +7.049313558e-01, +1.005197299e-06, +1.776072999e-02, +8.634324526e-04, +1.974457243e+00 - 2.510000e+01, +7.064238109e-01, +1.100529477e-06, +1.919993195e-02, +8.642147215e-04, +3.534944010e+00 - 5.100000e+00, +7.032588007e-01, +7.538436764e-07, +1.758484364e-02, +8.696505248e-04, +6.206815574e-01 - 2.830000e+01, +7.051385177e-01, +9.644838144e-07, +1.903597935e-02, +8.599617820e-04, +4.323348710e+00 - 1.540000e+01, +7.069908609e-01, +1.449833545e-06, +1.912291735e-02, +8.545513705e-04, +2.357765408e+00 - 9.000000e+00, +7.030971609e-01, +7.011305477e-07, +1.801203000e-02, +8.639941115e-04, +1.260202551e+00 - Mean, +7.053692541e-01, +1.260046792e-06, +1.675928718e-02, +8.746827136e-04, +2.372088406e+00 + 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +8.806573475e-04, +2.186423623e-01 + 3.000000e+01, +7.048573140e-01, +1.801733889e-06, +1.911288392e-02, +8.604087499e-04, +4.552939315e+00 + 1.760000e+01, +7.085030395e-01, +1.269973795e-06, +1.968683336e-02, +8.578765318e-04, +2.496240545e+00 + 1.280000e+01, +7.050021999e-01, +1.023120281e-06, +1.779303441e-02, +8.631933312e-04, +1.992681982e+00 + 2.530000e+01, +7.063233638e-01, +1.119274219e-06, +1.918472307e-02, +8.641051749e-04, +3.585707764e+00 + 5.700000e+00, +7.030751823e-01, +7.236778828e-07, +1.762962944e-02, +8.676377055e-04, +7.089908259e-01 + 2.880000e+01, +7.050114123e-01, +1.148903905e-06, +1.905258155e-02, +8.593880474e-04, +4.413027198e+00 + 2.090000e+01, +7.085921616e-01, +7.822419313e-07, +1.961007783e-02, +8.645942925e-04, +2.739108293e+00 + 3.100000e+00, +7.042983087e-01, +9.303255016e-07, +1.650025839e-02, +8.775457687e-04, +3.510194976e-01 + 3.500000e+00, +7.040400654e-01, +8.844716811e-07, +1.682088474e-02, +8.760540565e-04, +4.019232247e-01 diff --git a/test/ref/cpw/lumped_uniform/error-indicators.csv b/test/ref/cpw/lumped_uniform/error-indicators.csv index f40e72b5b..78281b387 100644 --- a/test/ref/cpw/lumped_uniform/error-indicators.csv +++ b/test/ref/cpw/lumped_uniform/error-indicators.csv @@ -2,16 +2,15 @@ 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +8.806573475e-04, +2.186423623e-01 4.000000e+00, +7.037534929e-01, +8.348638678e-07, +1.716164791e-02, +8.740225193e-04, +4.676837686e-01 6.000000e+00, +7.030062911e-01, +7.118689649e-07, +1.767838766e-02, +8.667794453e-04, +7.544551055e-01 - 8.000000e+00, +7.029165326e-01, +6.832337447e-07, +1.800232414e-02, +8.638803686e-04, +1.081001045e+00 - 1.000000e+01, +7.034192125e-01, +7.418014192e-07, +1.790795027e-02, +8.645952292e-04, +1.449355685e+00 + 8.000000e+00, +7.029165326e-01, +6.832337446e-07, +1.800232414e-02, +8.638803686e-04, +1.081001045e+00 + 1.000000e+01, +7.034192125e-01, +7.418014191e-07, +1.790795027e-02, +8.645952292e-04, +1.449355685e+00 1.200000e+01, +7.044599247e-01, +9.086471535e-07, +1.751260139e-02, +8.646185630e-04, +1.842149227e+00 1.400000e+01, +7.058985472e-01, +1.260914522e-06, +1.833900466e-02, +8.591277673e-04, +2.190892577e+00 1.600000e+01, +7.074564924e-01, +1.093986277e-06, +1.938386319e-02, +8.542665420e-04, +2.406792037e+00 - 1.800000e+01, +7.086746633e-01, +1.209024855e-06, +1.965872658e-02, +8.590913096e-04, +2.515530177e+00 - 2.000000e+01, +7.088448209e-01, +8.917186070e-07, +1.963851961e-02, +8.636924028e-04, +2.646279233e+00 - 2.200000e+01, +7.081043245e-01, +7.960606263e-07, +1.946623509e-02, +8.649306217e-04, +2.889633318e+00 + 1.800000e+01, +7.086746633e-01, +1.209024854e-06, +1.965872658e-02, +8.590913096e-04, +2.515530177e+00 + 2.000000e+01, +7.088448209e-01, +8.917186071e-07, +1.963851961e-02, +8.636924028e-04, +2.646279233e+00 + 2.200000e+01, +7.081043245e-01, +7.960606264e-07, +1.946623509e-02, +8.649306217e-04, +2.889633318e+00 2.400000e+01, +7.070103246e-01, +9.980955322e-07, +1.928974677e-02, +8.646218376e-04, +3.271855291e+00 2.600000e+01, +7.059908728e-01, +1.195377481e-06, +1.913395059e-02, +8.635749385e-04, +3.767453335e+00 2.800000e+01, +7.052267075e-01, +9.841023722e-07, +1.902600821e-02, +8.604645102e-04, +4.261952660e+00 3.000000e+01, +7.048573097e-01, +1.801758125e-06, +1.911288376e-02, +8.604087621e-04, +4.552939157e+00 - Mean, +7.056448946e-01, +1.148429619e-06, +1.689680225e-02, +8.751153438e-04, +2.287774332e+00 diff --git a/test/ref/cpw/wave_adaptive/error-indicators.csv b/test/ref/cpw/wave_adaptive/error-indicators.csv index f4164b5e9..f896ced91 100644 --- a/test/ref/cpw/wave_adaptive/error-indicators.csv +++ b/test/ref/cpw/wave_adaptive/error-indicators.csv @@ -1,11 +1,11 @@ f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +9.263986181e-04, +2.614559004e-01 3.000000e+01, +6.802918578e-01, +5.681633835e-06, +1.845184430e-02, +1.069612630e-03, +4.266998699e+00 - 1.760000e+01, +6.986524402e-01, +2.412989991e-06, +1.745780169e-02, +9.343913445e-04, +2.345607124e+00 - 2.740000e+01, +6.989786511e-01, +3.347415932e-06, +1.699249148e-02, +9.402676176e-04, +3.678349350e+00 - 2.210000e+01, +6.990013886e-01, +3.195355480e-06, +1.749122001e-02, +9.296641221e-04, +2.940510061e+00 + 1.800000e+01, +6.988641990e-01, +1.805646533e-06, +1.748858742e-02, +9.321067007e-04, +2.396980554e+00 + 2.750000e+01, +6.989649048e-01, +3.352201085e-06, +1.700252668e-02, +9.410894391e-04, +3.692517431e+00 + 2.960000e+01, +6.281346087e-01, +8.180986748e-06, +1.580175763e-02, +1.177590100e-03, +5.361105316e+00 + 2.430000e+01, +6.989731036e-01, +2.948013356e-06, +1.729382892e-02, +9.310877420e-04, +3.243999699e+00 1.650000e+01, +6.962225150e-01, +5.006960530e-06, +1.720841009e-02, +9.616412940e-04, +2.212464633e+00 - 1.450000e+01, +6.993481111e-01, +1.362223224e-06, +1.708077753e-02, +9.386436767e-04, +1.944614579e+00 + 1.420000e+01, +6.995569363e-01, +1.848544401e-06, +1.709761753e-02, +9.315059097e-04, +1.903555185e+00 + 8.500000e+00, +6.993502755e-01, +2.435632799e-06, +1.676522029e-02, +9.259791653e-04, +1.135422506e+00 1.510000e+01, +6.964183126e-01, +4.767143173e-06, +1.723792019e-02, +9.938502198e-04, +2.042324804e+00 - 2.950000e+01, +6.692857361e-01, +5.896183956e-06, +1.725486449e-02, +1.115952350e-03, +4.433714589e+00 - Mean, +6.931309928e-01, +5.968865405e-06, +1.679209924e-02, +1.019613084e-03, +2.680671082e+00 diff --git a/test/ref/cpw/wave_uniform/error-indicators.csv b/test/ref/cpw/wave_uniform/error-indicators.csv index b177ec561..a317b58a0 100644 --- a/test/ref/cpw/wave_uniform/error-indicators.csv +++ b/test/ref/cpw/wave_uniform/error-indicators.csv @@ -14,4 +14,3 @@ 2.600000e+01, +6.990090439e-01, +3.218910262e-06, +1.709437140e-02, +9.338867734e-04, +3.481668528e+00 2.800000e+01, +6.988244472e-01, +3.387084517e-06, +1.706546999e-02, +9.468574632e-04, +3.764267665e+00 3.000000e+01, +6.802918601e-01, +5.681637479e-06, +1.845184332e-02, +1.069612628e-03, +4.266998672e+00 - Mean, +6.978759848e-01, +4.868939341e-06, +1.714752106e-02, +9.637101249e-04, +2.154592309e+00 diff --git a/test/ref/rings/error-indicators.csv b/test/ref/rings/error-indicators.csv index a42cbf899..855775f8c 100644 --- a/test/ref/rings/error-indicators.csv +++ b/test/ref/rings/error-indicators.csv @@ -1,4 +1,3 @@ i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization 1.000000e+00, +3.544895565e-01, +4.103268389e-09, +2.982074859e-02, +2.217169423e-04, +1.115529612e-01 2.000000e+00, +2.764894272e-01, +1.023265139e-07, +7.245458667e-03, +5.519522597e-04, +4.557585527e-01 - Mean, +3.178909003e-01, +7.241392224e-08, +2.108658880e-02, +5.448306923e-04, +2.836557570e-01 diff --git a/test/ref/spheres/error-indicators.csv b/test/ref/spheres/error-indicators.csv index 74b941f6f..2586439ba 100644 --- a/test/ref/spheres/error-indicators.csv +++ b/test/ref/spheres/error-indicators.csv @@ -1,4 +1,3 @@ i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization 1.000000e+00, +7.835018957e-03, +7.728228292e-07, +5.659925890e-04, +4.310697912e-05, +3.052442134e-01 2.000000e+00, +7.529893122e-03, +8.146906715e-07, +7.245606637e-04, +4.195362244e-05, +4.319918577e-01 - Mean, +7.683970734e-03, +1.578279084e-06, +5.153241491e-04, +4.815649458e-05, +3.686180355e-01 diff --git a/test/testcase.jl b/test/testcase.jl index f74bc25db..d26af19aa 100644 --- a/test/testcase.jl +++ b/test/testcase.jl @@ -53,29 +53,6 @@ function testcase( @test proc.exitcode == 0 end - # Convert variables to Float64, or replace with zeros. - function asfloat(x::Float64) - return x - end - function asfloat(str)::Float64 - try - return parse(Float64, str) - catch - return 0.0 - end - end - # Convert variables to String15, or replace with empty. - function asstring(x::String15) - return x - end - function asstring(str)::String15 - try - return parse(String15, str) - catch - return "" - end - end - @testset "Results" begin # Test that directories were created @test isdir(postprodir) @@ -90,7 +67,11 @@ function testcase( for file in filesref dataref = CSV.File(joinpath(refpostprodir, file); header=1) |> DataFrame data = CSV.File(joinpath(postprodir, file); header=1) |> DataFrame - data = data[1:size(dataref, 1), :] + if (size(data, 1) < size(dataref, 1)) + # pad the data with duplicates of final row. + append!(data, [data[end, :] for _ ∈ 1:(size(dataref, 1) - size(data, 1))]) + end + data = data[1:size(dataref, 1), :] # Check the number of columns matches, before removing any excluded_columns @test ncol(data) == ncol(dataref) @@ -99,15 +80,7 @@ function testcase( select!(dataref, Not(Cols(contains(col)))) end - fdata = asfloat.(data) - fdataref = asfloat.(dataref) - sdata = asstring.(data) - sdataref = asstring.(dataref) - - test = isapprox.(fdata, fdataref; rtol=rtol, atol=atol) - if (atol < Inf && rtol < Inf) - test .&= (sdata .== sdataref) - end + test = isapprox.(data, dataref; rtol=rtol, atol=atol) for (row, rowdataref, rowdata) in zip(eachrow(test), eachrow(dataref), eachrow(data)) for (rowcol, rowcoldataref, rowcoldata) in From cf6a2b00ed6df0ac777f21726950eec3fd07ad69 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Thu, 5 Oct 2023 15:01:08 -0400 Subject: [PATCH 24/39] Reduce LOC in drivers --- palace/drivers/basesolver.cpp | 13 ++++---- palace/drivers/basesolver.hpp | 3 +- palace/drivers/drivensolver.cpp | 44 +++++++++++--------------- palace/drivers/eigensolver.cpp | 20 +++++------- palace/drivers/electrostaticsolver.cpp | 23 ++++++-------- palace/drivers/magnetostaticsolver.cpp | 23 ++++++-------- palace/drivers/transientsolver.cpp | 19 +++++------ palace/utils/errorindicators.hpp | 6 ++++ 8 files changed, 67 insertions(+), 84 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index 9a4f8fa93..e19ec8d46 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -567,8 +567,7 @@ void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double } void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, double time, - double global, double min, double max, - double mean, double normalization, + const std::array &data, bool normalized) const { if (post_dir.length() == 0) @@ -597,11 +596,11 @@ void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, d // clang-format off output.print("{:{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", time, table.w1, table.p1, - global, table.w, table.p, - min, table.w, table.p, - max, table.w, table.p, - mean, table.w, table.p, - normalization, table.w, table.p); + data[0], table.w, table.p, + data[1], table.w, table.p, + data[2], table.w, table.p, + data[3], table.w, table.p, + data[4], table.w, table.p); // clang-format on } } diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index d92cb3e7b..1228ef567 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -72,8 +72,7 @@ class BaseSolver // Postprocess granular error indicator to file. The argument normalized indicates if // supplied indicators have already been normalized. void PostprocessErrorIndicators(const std::string &name, int step, double time, - double global, double min, double max, double mean, - double normalization, bool normalized) const; + const std::array &data, bool normalized) const; public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 3b85bfc1c..de6277c43 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -145,21 +145,18 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator B = 0.0; // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, - &spaceop](const auto &E, int step, double f) + ErrorIndicators indicators; + auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop, + &spaceop](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto indicators = estimator.ComputeIndicators(E, normalized); - postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators("f (GHz)", step, f, - indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - indicators.GetMinErrorIndicator(spaceop.GetComm()), - indicators.GetMaxErrorIndicator(spaceop.GetComm()), - indicators.GetMeanErrorIndicator(spaceop.GetComm()), - indicators.GetNormalization(), normalized); - combined_indicators.AddIndicators(indicators); + auto sample_indicators = estimator.ComputeIndicators(E, normalized); + postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); + PostprocessErrorIndicators("f (GHz)", step, frequency, + sample_indicators.GetPostprocessData(spaceop.GetComm()), + normalized); + indicators.AddIndicators(sample_indicators); }; // Main frequency sweep loop. @@ -222,7 +219,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator omega += delta_omega; } SaveMetadata(ksp); - return combined_indicators; + return indicators; } ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, @@ -280,20 +277,17 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // The error indicators will be calculated for each HDM sample rather than for // the online stage. - ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, - &spaceop](const auto &E, int step, double frequency) + ErrorIndicators indicators; + auto UpdateErrorIndicators = + [this, &estimator, &indicators, &spaceop](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto indicators = estimator.ComputeIndicators(E, normalized); + auto sample_indicators = estimator.ComputeIndicators(E, normalized); PostprocessErrorIndicators("f (GHz)", step, frequency, - indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - indicators.GetMinErrorIndicator(spaceop.GetComm()), - indicators.GetMaxErrorIndicator(spaceop.GetComm()), - indicators.GetMeanErrorIndicator(spaceop.GetComm()), - indicators.GetNormalization(), normalized); - combined_indicators.AddIndicators(indicators); + sample_indicators.GetPostprocessData(spaceop.GetComm()), + normalized); + indicators.AddIndicators(sample_indicators); }; // Initialize the basis with samples from the top and bottom of the frequency @@ -343,7 +337,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator SaveMetadata(prom.GetLinearSolver()); // Set the indicator field to the combined field for postprocessing. - postop.SetIndicatorGridFunction(combined_indicators.GetLocalErrorIndicators()); + postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); // Main fast frequency sweep loop (online phase). BlockTimer bt2(Timer::CONSTRUCT); @@ -392,7 +386,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator step++; omega += delta_omega; } - return combined_indicators; + return indicators; } int DrivenSolver::GetNumSteps(double start, double end, double delta) const diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index a8d95bb55..6100ddbe5 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -264,21 +264,17 @@ EigenSolver::Solve(const std::vector> &mesh) cons SaveMetadata(*ksp); // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators combined_indicators; + ErrorIndicators indicators; auto UpdateErrorIndicators = - [this, &estimator, &combined_indicators, &postop, &spaceop](const auto &E, int i) + [this, &estimator, &indicators, &postop, &spaceop](const auto &E, int i) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto indicators = estimator.ComputeIndicators(E, normalized); - postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators("m", i, i + 1, - indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - indicators.GetMinErrorIndicator(spaceop.GetComm()), - indicators.GetMaxErrorIndicator(spaceop.GetComm()), - indicators.GetMeanErrorIndicator(spaceop.GetComm()), - indicators.GetNormalization(), normalized); - combined_indicators.AddIndicators(indicators); + auto sample_indicators = estimator.ComputeIndicators(E, normalized); + postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); + PostprocessErrorIndicators( + "m", i, i + 1, sample_indicators.GetPostprocessData(spaceop.GetComm()), normalized); + indicators.AddIndicators(sample_indicators); }; // Save the eigenvalue estimates. @@ -328,7 +324,7 @@ EigenSolver::Solve(const std::vector> &mesh) cons // Postprocess the mode. Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv); } - return combined_indicators; + return indicators; } void EigenSolver::Postprocess(const PostOperator &postop, diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index fa2c93ab2..51f05d1ab 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -100,21 +100,18 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, int nstep = static_cast(terminal_sources.size()); mfem::DenseMatrix C(nstep), Cm(nstep); Vector E(Grad->Height()), Vij(Grad->Width()); - ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, - &laplaceop](const auto &V, int i, double idx) + ErrorIndicators indicators; + auto UpdateErrorIndicators = + [this, &estimator, &indicators, &postop, &laplaceop](const auto &V, int i, double idx) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto indicators = estimator.ComputeIndicators(V, normalized); - postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators("i", i, idx, - indicators.GetGlobalErrorIndicator(laplaceop.GetComm()), - indicators.GetMinErrorIndicator(laplaceop.GetComm()), - indicators.GetMaxErrorIndicator(laplaceop.GetComm()), - indicators.GetMeanErrorIndicator(laplaceop.GetComm()), - indicators.GetNormalization(), normalized); - combined_indicators.AddIndicators(indicators); + auto sample_indicators = estimator.ComputeIndicators(V, normalized); + postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); + PostprocessErrorIndicators("i", i, i + 1, + sample_indicators.GetPostprocessData(laplaceop.GetComm()), + normalized); + indicators.AddIndicators(sample_indicators); }; if (iodata.solver.electrostatic.n_post > 0) { @@ -174,7 +171,7 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, mfem::DenseMatrix Cinv(C); Cinv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(terminal_sources, C, Cinv, Cm); - return combined_indicators; + return indicators; } void ElectrostaticSolver::PostprocessTerminals( diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index 9a37c58d0..922f0e1d4 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -104,23 +104,18 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, Vector Iinc(nstep); // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, + ErrorIndicators indicators; + auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop, &curlcurlop](const auto &A, int i, double idx) { BlockTimer bt0(Timer::ESTIMATION); constexpr bool normalized = true; - auto indicators = estimator.ComputeIndicators(A, normalized); - BlockTimer bt1(Timer::POSTPRO); - // Write the indicator for this mode. - postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators("i", i, idx, - indicators.GetGlobalErrorIndicator(curlcurlop.GetComm()), - indicators.GetMinErrorIndicator(curlcurlop.GetComm()), - indicators.GetMaxErrorIndicator(curlcurlop.GetComm()), - indicators.GetMeanErrorIndicator(curlcurlop.GetComm()), - indicators.GetNormalization(), normalized); - combined_indicators.AddIndicators(indicators); + auto sample_indicators = estimator.ComputeIndicators(A, normalized); + postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); + PostprocessErrorIndicators("i", i, i + 1, + sample_indicators.GetPostprocessData(curlcurlop.GetComm()), + normalized); + indicators.AddIndicators(sample_indicators); }; if (iodata.solver.magnetostatic.n_post > 0) { @@ -184,7 +179,7 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, mfem::DenseMatrix Minv(M); Minv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(surf_j_op, M, Minv, Mm); - return combined_indicators; + return indicators; } void MagnetostaticSolver::PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 69dcea092..838d396fd 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -83,22 +83,19 @@ TransientSolver::Solve(const std::vector> &mesh) BlockTimer bt(Timer::ESTCONSTRUCT); return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); - ErrorIndicators combined_indicators; - auto UpdateErrorIndicators = [this, &estimator, &combined_indicators, &postop, + ErrorIndicators indicators; + auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop, &spaceop](const auto &E, int step, double time) { BlockTimer bt0(Timer::ESTIMATION); // Initial flux of zero would return nan. bool constexpr normalized = false; - auto indicators = estimator.ComputeIndicators(E, normalized); - postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); + auto sample_indicators = estimator.ComputeIndicators(E, normalized); + postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("t (ns)", step, time, - indicators.GetGlobalErrorIndicator(spaceop.GetComm()), - indicators.GetMinErrorIndicator(spaceop.GetComm()), - indicators.GetMaxErrorIndicator(spaceop.GetComm()), - indicators.GetMeanErrorIndicator(spaceop.GetComm()), - indicators.GetNormalization(), normalized); - combined_indicators.AddIndicators(indicators); + sample_indicators.GetPostprocessData(spaceop.GetComm()), + normalized); + indicators.AddIndicators(sample_indicators); }; // Main time integration loop. @@ -153,7 +150,7 @@ TransientSolver::Solve(const std::vector> &mesh) step++; } SaveMetadata(timeop.GetLinearSolver()); - return combined_indicators; + return indicators; } std::function TransientSolver::GetTimeExcitation(bool dot) const { diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index 83ffbf799..a6b0e3c21 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -73,6 +73,12 @@ class ErrorIndicators local = Vector(); normalization = 0; } + // Return a vector of postprocess data. + std::array GetPostprocessData(MPI_Comm comm) const + { + return {GetGlobalErrorIndicator(comm), GetMinErrorIndicator(comm), + GetMaxErrorIndicator(comm), GetMeanErrorIndicator(comm), GetNormalization()}; + } protected: // Elemental localized error indicators. Used for marking elements for From 5dfadba7de7234f4821dbc3f69c951745c4fef89 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Thu, 5 Oct 2023 16:49:27 -0400 Subject: [PATCH 25/39] Remove normalization option from compute indicators. Treat the transient problem special case by only normalizing if non-zero --- palace/drivers/basesolver.cpp | 11 +- palace/drivers/basesolver.hpp | 2 +- palace/drivers/drivensolver.cpp | 12 +- palace/drivers/eigensolver.cpp | 5 +- palace/drivers/electrostaticsolver.cpp | 6 +- palace/drivers/magnetostaticsolver.cpp | 6 +- palace/drivers/transientsolver.cpp | 7 +- palace/linalg/errorestimator.cpp | 15 +- palace/linalg/errorestimator.hpp | 4 +- test/ref/coaxial/matched/error-indicators.csv | 402 +++++++++--------- test/ref/coaxial/open/error-indicators.csv | 402 +++++++++--------- 11 files changed, 428 insertions(+), 444 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index e19ec8d46..966d5e7d4 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -567,8 +567,7 @@ void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double } void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, double time, - const std::array &data, - bool normalized) const + const std::array &data) const { if (post_dir.length() == 0) { @@ -585,10 +584,10 @@ void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, d // clang-format off output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", name, table.w1, - normalized ? "Sum Rel." : "Sum Abs.", table.w, - normalized ? "Min Elem Rel." : "Min Elem Abs.", table.w, - normalized ? "Max Elem Rel." : "Max Elem Abs.", table.w, - normalized ? "Mean Elem Rel." : "Mean Elem Abs.", table.w, + "Sum Rel.", table.w, + "Min Elem Rel.", table.w, + "Max Elem Rel.", table.w, + "Mean Elem Rel.", table.w, "Normalization", table.w); // clang-format on } diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index 1228ef567..6bb39d565 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -72,7 +72,7 @@ class BaseSolver // Postprocess granular error indicator to file. The argument normalized indicates if // supplied indicators have already been normalized. void PostprocessErrorIndicators(const std::string &name, int step, double time, - const std::array &data, bool normalized) const; + const std::array &data) const; public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index de6277c43..12322f3a3 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -150,12 +150,10 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &spaceop](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTIMATION); - constexpr bool normalized = true; - auto sample_indicators = estimator.ComputeIndicators(E, normalized); + auto sample_indicators = estimator.ComputeIndicators(E); postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("f (GHz)", step, frequency, - sample_indicators.GetPostprocessData(spaceop.GetComm()), - normalized); + sample_indicators.GetPostprocessData(spaceop.GetComm())); indicators.AddIndicators(sample_indicators); }; @@ -282,11 +280,9 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator [this, &estimator, &indicators, &spaceop](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTIMATION); - constexpr bool normalized = true; - auto sample_indicators = estimator.ComputeIndicators(E, normalized); + auto sample_indicators = estimator.ComputeIndicators(E); PostprocessErrorIndicators("f (GHz)", step, frequency, - sample_indicators.GetPostprocessData(spaceop.GetComm()), - normalized); + sample_indicators.GetPostprocessData(spaceop.GetComm())); indicators.AddIndicators(sample_indicators); }; diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 6100ddbe5..ee4e54677 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -269,11 +269,10 @@ EigenSolver::Solve(const std::vector> &mesh) cons [this, &estimator, &indicators, &postop, &spaceop](const auto &E, int i) { BlockTimer bt0(Timer::ESTIMATION); - constexpr bool normalized = true; - auto sample_indicators = estimator.ComputeIndicators(E, normalized); + auto sample_indicators = estimator.ComputeIndicators(E); postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators( - "m", i, i + 1, sample_indicators.GetPostprocessData(spaceop.GetComm()), normalized); + "m", i, i + 1, sample_indicators.GetPostprocessData(spaceop.GetComm())); indicators.AddIndicators(sample_indicators); }; diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 51f05d1ab..f3cee50cf 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -105,12 +105,10 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, [this, &estimator, &indicators, &postop, &laplaceop](const auto &V, int i, double idx) { BlockTimer bt0(Timer::ESTIMATION); - constexpr bool normalized = true; - auto sample_indicators = estimator.ComputeIndicators(V, normalized); + auto sample_indicators = estimator.ComputeIndicators(V); postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("i", i, i + 1, - sample_indicators.GetPostprocessData(laplaceop.GetComm()), - normalized); + sample_indicators.GetPostprocessData(laplaceop.GetComm())); indicators.AddIndicators(sample_indicators); }; if (iodata.solver.electrostatic.n_post > 0) diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index 922f0e1d4..b86835963 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -109,12 +109,10 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, &curlcurlop](const auto &A, int i, double idx) { BlockTimer bt0(Timer::ESTIMATION); - constexpr bool normalized = true; - auto sample_indicators = estimator.ComputeIndicators(A, normalized); + auto sample_indicators = estimator.ComputeIndicators(A); postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("i", i, i + 1, - sample_indicators.GetPostprocessData(curlcurlop.GetComm()), - normalized); + sample_indicators.GetPostprocessData(curlcurlop.GetComm())); indicators.AddIndicators(sample_indicators); }; if (iodata.solver.magnetostatic.n_post > 0) diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 838d396fd..d96efc4fc 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -88,13 +88,10 @@ TransientSolver::Solve(const std::vector> &mesh) &spaceop](const auto &E, int step, double time) { BlockTimer bt0(Timer::ESTIMATION); - // Initial flux of zero would return nan. - bool constexpr normalized = false; - auto sample_indicators = estimator.ComputeIndicators(E, normalized); + auto sample_indicators = estimator.ComputeIndicators(E); postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); PostprocessErrorIndicators("t (ns)", step, time, - sample_indicators.GetPostprocessData(spaceop.GetComm()), - normalized); + sample_indicators.GetPostprocessData(spaceop.GetComm())); indicators.AddIndicators(sample_indicators); }; diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index d2b9f474f..42b0aedfc 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -110,8 +110,7 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( } template <> -ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v, - bool normalize) const +ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v) const { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); const int nelem = nd_fespace.GetNE(); @@ -171,7 +170,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm()); normalization = std::sqrt(normalization); - if (normalize) + if (normalization > 0) { std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); @@ -180,8 +179,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v } template <> -ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, - bool normalize) const +ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); field_gf.SetFromTrueDofs(v); @@ -234,7 +232,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v, Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm()); normalization = std::sqrt(normalization); - if (normalize) + if (normalization > 0) { std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); @@ -255,8 +253,7 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( { } -ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, - bool normalize) const +ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const { auto &h1_fespace = h1_fespaces.GetFinestFESpace(); const int sdim = h1_fespace.GetMesh()->SpaceDimension(); @@ -326,7 +323,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v, Mpi::GlobalSum(1, &normalization, h1_fespace.GetComm()); normalization = std::sqrt(normalization); - if (normalize) + if (normalization > 0) { std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index a195c4182..1eb613f0b 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -76,7 +76,7 @@ class CurlFluxErrorEstimator // Compute elemental error indicators given a complex vector of true DOF. template - ErrorIndicators ComputeIndicators(const VectorType &v, bool normalize) const; + ErrorIndicators ComputeIndicators(const VectorType &v) const; }; // Class used for computing grad flux error estimate, i.e. || ϵ ∇ ϕₕ - F ||_K where F @@ -97,7 +97,7 @@ class GradFluxErrorEstimator mfem::ParFiniteElementSpaceHierarchy &h1_fespaces); // Compute elemental error indicators given a vector of true DOF. - ErrorIndicators ComputeIndicators(const Vector &v, bool normalize) const; + ErrorIndicators ComputeIndicators(const Vector &v) const; }; } // namespace palace diff --git a/test/ref/coaxial/matched/error-indicators.csv b/test/ref/coaxial/matched/error-indicators.csv index ce7cdd968..eedd72de9 100644 --- a/test/ref/coaxial/matched/error-indicators.csv +++ b/test/ref/coaxial/matched/error-indicators.csv @@ -1,202 +1,202 @@ - t (ns), Sum Abs., Min Elem Abs., Max Elem Abs., Mean Elem Abs., Normalization + t (ns), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 - 3.747406e-02, +5.656659454e-06, +4.839334474e-19, +1.429016906e-06, +1.884177417e-07, +1.015013075e-04 - 7.494811e-02, +1.331364706e-05, +1.250394489e-18, +3.354073322e-06, +4.532531474e-07, +1.286451027e-04 - 1.124222e-01, +4.532668493e-06, +8.678982156e-19, +1.143753276e-06, +1.587978169e-07, +1.363415075e-04 - 1.498962e-01, +5.315482992e-06, +5.365268297e-19, +1.352478573e-06, +1.844891507e-07, +2.237425263e-04 - 1.873703e-01, +2.286991326e-05, +3.008666113e-18, +5.789494315e-06, +7.781958289e-07, +5.988175109e-04 - 2.248443e-01, +1.597419759e-05, +2.193338141e-18, +4.067429237e-06, +5.797052699e-07, +1.231303285e-03 - 2.623184e-01, +1.145612024e-05, +5.204731821e-18, +3.069144677e-06, +4.523653992e-07, +2.183486767e-03 - 2.997925e-01, +2.086398938e-05, +1.270987622e-17, +5.143395002e-06, +8.183715070e-07, +3.443837437e-03 - 3.372665e-01, +3.014646152e-05, +1.995241492e-17, +8.564706304e-06, +1.096508508e-06, +4.896408733e-03 - 3.747406e-01, +2.951847940e-05, +5.642532431e-17, +8.109678263e-06, +1.146518725e-06, +6.288158078e-03 - 4.122146e-01, +3.920398129e-05, +1.410851956e-17, +1.039662099e-05, +1.506290838e-06, +7.236866239e-03 - 4.496887e-01, +5.485446652e-05, +1.411258486e-16, +1.373626813e-05, +2.138216877e-06, +7.438562524e-03 - 4.871627e-01, +5.305798814e-05, +1.745521413e-16, +1.341910982e-05, +2.244500004e-06, +7.655638548e-03 - 5.246368e-01, +6.324063047e-05, +3.538137933e-16, +1.609936398e-05, +2.567937221e-06, +1.144492786e-02 - 5.621109e-01, +9.953861113e-05, +6.647586246e-16, +2.871846797e-05, +3.979070330e-06, +2.129721915e-02 - 5.995849e-01, +1.560743589e-04, +6.830591915e-16, +5.021692981e-05, +5.826054663e-06, +3.669178320e-02 - 6.370590e-01, +2.357106590e-04, +2.975830288e-15, +7.762490221e-05, +8.212787663e-06, +5.641443987e-02 - 6.745330e-01, +3.417912838e-04, +2.135164474e-15, +1.077684524e-04, +1.198621562e-05, +7.828637698e-02 - 7.120071e-01, +4.480155462e-04, +8.422598055e-15, +1.327195791e-04, +1.653183123e-05, +9.885177909e-02 - 7.494811e-01, +5.361316672e-04, +2.205573753e-14, +1.452888272e-04, +2.046928227e-05, +1.137531996e-01 - 7.869552e-01, +5.828228357e-04, +3.676720758e-14, +1.419796915e-04, +2.316840106e-05, +1.195346821e-01 - 8.244293e-01, +5.919511494e-04, +1.531242976e-13, +1.546158786e-04, +2.529399186e-05, +1.188340974e-01 - 8.619033e-01, +6.248414803e-04, +8.377203739e-13, +1.440128726e-04, +2.829825492e-05, +1.309553939e-01 - 8.993774e-01, +8.151146601e-04, +3.513353292e-12, +2.051054800e-04, +3.659201856e-05, +1.857760966e-01 - 9.368514e-01, +1.238030606e-03, +1.222959768e-11, +3.772512881e-04, +5.099828692e-05, +2.867307084e-01 - 9.743255e-01, +1.814098313e-03, +3.837194374e-11, +5.700103264e-04, +7.220437043e-05, +4.167673658e-01 - 1.011800e+00, +2.437431108e-03, +1.109193524e-10, +7.477328124e-04, +9.448610394e-05, +5.549049357e-01 - 1.049274e+00, +2.992538657e-03, +2.905930910e-10, +8.658103031e-04, +1.186157958e-04, +6.782106751e-01 - 1.086748e+00, +3.372602378e-03, +6.709425516e-10, +8.810228476e-04, +1.408439490e-04, +7.648957645e-01 - 1.124222e+00, +3.518151571e-03, +1.316486442e-09, +8.457004166e-04, +1.549572611e-04, +8.022489576e-01 - 1.161696e+00, +3.491978300e-03, +2.035218488e-09, +8.789616841e-04, +1.613984637e-04, +8.009704160e-01 - 1.199170e+00, +3.564279227e-03, +1.920368097e-09, +8.112931293e-04, +1.690581835e-04, +8.143229074e-01 - 1.236644e+00, +4.138418534e-03, +3.626219426e-09, +8.645553519e-04, +2.078914976e-04, +9.294380600e-01 - 1.274118e+00, +5.324827324e-03, +1.871304041e-08, +1.353962850e-03, +2.601141620e-04, +1.185764957e+00 - 1.311592e+00, +6.847371238e-03, +5.579282980e-08, +1.923072314e-03, +3.232892188e-04, +1.534641583e+00 - 1.349066e+00, +8.347554914e-03, +1.270395048e-07, +2.347347422e-03, +3.889148474e-04, +1.896337013e+00 - 1.386540e+00, +9.532772095e-03, +2.368876072e-07, +2.527956003e-03, +4.514086852e-04, +2.199936335e+00 - 1.424014e+00, +1.024824705e-02, +3.669957628e-07, +2.406072494e-03, +5.056009733e-04, +2.397678575e+00 - 1.461488e+00, +1.052203770e-02, +4.698294987e-07, +2.444667123e-03, +5.393140228e-04, +2.477217295e+00 - 1.498962e+00, +1.058885932e-02, +5.175642757e-07, +2.462910797e-03, +5.619191776e-04, +2.474013829e+00 - 1.536436e+00, +1.082620304e-02, +7.023524853e-07, +2.333885666e-03, +5.875106486e-04, +2.473624483e+00 - 1.573910e+00, +1.155161542e-02, +1.358321114e-06, +2.437891160e-03, +6.540547261e-04, +2.579178593e+00 - 1.611384e+00, +1.278137828e-02, +2.460277494e-06, +2.356865899e-03, +7.374532022e-04, +2.837660777e+00 - 1.648859e+00, +1.424151084e-02, +3.841030656e-06, +3.050855223e-03, +8.230119162e-04, +3.200525521e+00 - 1.686333e+00, +1.559704403e-02, +5.289170111e-06, +3.447606735e-03, +9.115234740e-04, +3.569943474e+00 - 1.723807e+00, +1.662444075e-02, +6.505705770e-06, +3.476585555e-03, +9.874082185e-04, +3.860928050e+00 - 1.761281e+00, +1.725790653e-02, +7.028429069e-06, +3.269676383e-03, +1.054718212e-03, +4.031057327e+00 - 1.798755e+00, +1.756883176e-02, +6.292185134e-06, +3.490071839e-03, +1.100129526e-03, +4.086183103e+00 - 1.836229e+00, +1.771426204e-02, +5.053355061e-06, +3.393376632e-03, +1.130205560e-03, +4.072063131e+00 - 1.873703e+00, +1.785821205e-02, +9.718791907e-06, +3.372883096e-03, +1.161533370e-03, +4.053029029e+00 - 1.911177e+00, +1.810390712e-02, +2.140071884e-05, +3.399366407e-03, +1.205529210e-03, +4.080832967e+00 - 1.948651e+00, +1.846400001e-02, +3.776416235e-05, +3.215132108e-03, +1.262075112e-03, +4.169084349e+00 - 1.986125e+00, +1.888516033e-02, +5.691108071e-05, +3.375389793e-03, +1.312380663e-03, +4.292757706e+00 - 2.023599e+00, +1.928666314e-02, +6.662337504e-05, +3.230487063e-03, +1.367886609e-03, +4.411208547e+00 - 2.061073e+00, +1.959441958e-02, +8.468956787e-05, +3.292283141e-03, +1.415407565e-03, +4.493075573e+00 - 2.098547e+00, +1.977112668e-02, +6.223830173e-05, +3.280926680e-03, +1.450258368e-03, +4.527901974e+00 - 2.136021e+00, +1.982480843e-02, +7.762739708e-05, +3.153403839e-03, +1.477408436e-03, +4.524236977e+00 - 2.173495e+00, +1.978929475e-02, +4.860199538e-05, +3.273005134e-03, +1.485745556e-03, +4.500275525e+00 - 2.210969e+00, +1.970586078e-02, +6.477189253e-05, +3.096774043e-03, +1.493075419e-03, +4.473334986e+00 - 2.248443e+00, +1.961764306e-02, +1.534757406e-04, +3.208131565e-03, +1.497128893e-03, +4.453002776e+00 - 2.285917e+00, +1.954872068e-02, +2.706281300e-04, +3.160574331e-03, +1.507315970e-03, +4.440058051e+00 - 2.323392e+00, +1.949054619e-02, +3.957934621e-04, +3.088297962e-03, +1.515726763e-03, +4.429819755e+00 - 2.360866e+00, +1.942459939e-02, +3.988303302e-04, +3.168967366e-03, +1.518408645e-03, +4.416620275e+00 - 2.398340e+00, +1.934097007e-02, +5.433495872e-04, +2.963763007e-03, +1.520978987e-03, +4.396813054e+00 - 2.435814e+00, +1.923104349e-02, +4.136980080e-04, +3.121203146e-03, +1.509413450e-03, +4.369717681e+00 - 2.473288e+00, +1.909454427e-02, +4.433117467e-04, +3.039974758e-03, +1.497084750e-03, +4.337243705e+00 - 2.510762e+00, +1.894990382e-02, +3.389526031e-04, +3.019884470e-03, +1.477309988e-03, +4.302986459e+00 - 2.548236e+00, +1.881692297e-02, +1.813552221e-04, +3.063530458e-03, +1.457192590e-03, +4.270950549e+00 - 2.585710e+00, +1.869587728e-02, +8.465299821e-05, +2.870368490e-03, +1.442518900e-03, +4.243663227e+00 - 2.623184e+00, +1.857037699e-02, +9.976407527e-05, +3.032463236e-03, +1.425813526e-03, +4.219874234e+00 - 2.660658e+00, +1.841781342e-02, +1.192521509e-04, +2.920544574e-03, +1.407031587e-03, +4.193027513e+00 - 2.698132e+00, +1.820568732e-02, +1.067729875e-04, +2.949664425e-03, +1.373492487e-03, +4.152202300e+00 - 2.735606e+00, +1.790164964e-02, +1.313201035e-04, +2.957071910e-03, +1.333220921e-03, +4.086406465e+00 - 2.773080e+00, +1.750753979e-02, +9.473131355e-05, +2.812202072e-03, +1.282149903e-03, +3.991182680e+00 - 2.810554e+00, +1.706650266e-02, +7.279758402e-05, +2.935739859e-03, +1.226809624e-03, +3.874510868e+00 - 2.848028e+00, +1.663352768e-02, +4.330250216e-05, +2.792808427e-03, +1.175650443e-03, +3.757750265e+00 - 2.885502e+00, +1.625942813e-02, +1.998524832e-05, +2.866590860e-03, +1.122929499e-03, +3.667691740e+00 - 2.922976e+00, +1.598195181e-02, +7.232592982e-06, +2.844532135e-03, +1.081737190e-03, +3.620168807e+00 - 2.960451e+00, +1.577338960e-02, +9.700374697e-06, +2.792731012e-03, +1.049770950e-03, +3.604784042e+00 - 2.997925e+00, +1.553146450e-02, +1.273990493e-05, +2.881285129e-03, +1.016515754e-03, +3.584400783e+00 - 3.035399e+00, +1.514003980e-02, +1.228964683e-05, +2.704823364e-03, +9.688133854e-04, +3.511864377e+00 - 3.072873e+00, +1.450366264e-02, +1.008435101e-05, +2.764476254e-03, +9.052991708e-04, +3.352677539e+00 - 3.110347e+00, +1.358684772e-02, +7.263619326e-06, +2.744507612e-03, +8.326497080e-04, +3.102447387e+00 - 3.147821e+00, +1.245005031e-02, +4.397706483e-06, +2.441482369e-03, +7.515377896e-04, +2.794852318e+00 - 3.185295e+00, +1.127615095e-02, +2.412352806e-06, +2.032806362e-03, +6.752724457e-04, +2.497474676e+00 - 3.222769e+00, +1.031311461e-02, +1.082456494e-06, +2.143285167e-03, +6.040960991e-04, +2.287003000e+00 - 3.260243e+00, +9.722098447e-03, +7.766219178e-07, +2.051504040e-03, +5.487466088e-04, +2.199280716e+00 - 3.297717e+00, +9.477988980e-03, +6.106994583e-07, +2.140591601e-03, +5.231365919e-04, +2.194256365e+00 - 3.335191e+00, +9.343483349e-03, +8.212455600e-07, +2.122709661e-03, +5.020052066e-04, +2.187349161e+00 - 3.372665e+00, +9.031910827e-03, +4.985683249e-07, +2.027202802e-03, +4.683100142e-04, +2.107609658e+00 - 3.410139e+00, +8.358969454e-03, +2.133098618e-07, +2.131712733e-03, +4.161829301e-04, +1.925381439e+00 - 3.447613e+00, +7.305386060e-03, +2.216151869e-07, +1.981383473e-03, +3.560405545e-04, +1.652331416e+00 - 3.485087e+00, +6.015930589e-03, +3.339547581e-07, +1.624217000e-03, +2.931416733e-04, +1.332405222e+00 - 3.522561e+00, +4.722084916e-03, +5.182195532e-07, +1.137772700e-03, +2.357286023e-04, +1.031973346e+00 - 3.560035e+00, +3.750080035e-03, +7.270196567e-07, +7.975887764e-04, +1.902983164e-04, +8.248149186e-01 - 3.597509e+00, +3.315251942e-03, +7.201736437e-07, +7.688036370e-04, +1.618009553e-04, +7.467546178e-01 - 3.634984e+00, +3.269772328e-03, +5.774292198e-07, +8.162085931e-04, +1.582682324e-04, +7.476143297e-01 - 3.672458e+00, +3.274902133e-03, +7.976089495e-07, +7.735399332e-04, +1.531263374e-04, +7.473446775e-01 - 3.709932e+00, +3.100066670e-03, +4.682881045e-07, +8.093187229e-04, +1.390009277e-04, +7.038738376e-01 - 3.747406e+00, +2.714688495e-03, +5.266096132e-07, +7.812101182e-04, +1.164492725e-04, +6.126704622e-01 - 3.784880e+00, +2.168877746e-03, +1.208501048e-06, +6.586765463e-04, +9.254620437e-05, +4.890429311e-01 - 3.822354e+00, +1.577433577e-03, +2.371185309e-06, +4.846493602e-04, +7.036170965e-05, +3.558817348e-01 - 3.859828e+00, +1.061092217e-03, +3.489844866e-06, +3.014387380e-04, +5.200299178e-05, +2.372244458e-01 - 3.897302e+00, +7.196957159e-04, +4.678915832e-06, +1.453501800e-04, +3.991080599e-05, +1.561822526e-01 - 3.934776e+00, +6.175797222e-04, +3.556228444e-06, +1.538658023e-04, +3.448591183e-05, +1.265095408e-01 - 3.972250e+00, +6.198402098e-04, +4.834120749e-06, +1.554232146e-04, +3.396382816e-05, +1.273673799e-01 - 4.009724e+00, +6.058869618e-04, +4.073170316e-06, +1.478534484e-04, +3.240172211e-05, +1.276014323e-01 - 4.047198e+00, +5.372633895e-04, +3.729312485e-06, +1.470744798e-04, +2.867360832e-05, +1.171764854e-01 - 4.084672e+00, +4.377458730e-04, +4.352552351e-06, +1.266662599e-04, +2.420054957e-05, +9.815025303e-02 - 4.122146e+00, +3.305299603e-04, +3.376120114e-06, +9.569403430e-05, +2.028618803e-05, +7.584704242e-02 - 4.159620e+00, +2.355973660e-04, +4.810808574e-06, +6.297385767e-05, +1.629290325e-05, +5.552946065e-02 - 4.197094e+00, +1.801278525e-04, +5.620603917e-06, +3.607504563e-05, +1.410172159e-05, +4.116638402e-02 - 4.234568e+00, +1.549129975e-04, +2.661303486e-06, +2.207187955e-05, +1.253413700e-05, +3.415469948e-02 - 4.272043e+00, +1.457537798e-04, +1.670028107e-06, +2.156859992e-05, +1.145563068e-05, +3.215669992e-02 - 4.309517e+00, +1.397721813e-04, +1.917059150e-06, +2.135580367e-05, +1.088938993e-05, +3.160208211e-02 - 4.346991e+00, +1.350346130e-04, +1.599818122e-06, +2.178265361e-05, +1.022898783e-05, +3.065433292e-02 - 4.384465e+00, +1.251285395e-04, +1.280776212e-06, +2.033729556e-05, +9.244076055e-06, +2.904485957e-02 - 4.421939e+00, +1.163045493e-04, +1.277606632e-06, +2.145481482e-05, +8.237540618e-06, +2.713891437e-02 - 4.459413e+00, +1.104395022e-04, +1.506864487e-06, +2.066125793e-05, +7.780941814e-06, +2.542525795e-02 - 4.496887e+00, +1.051664861e-04, +5.785581780e-07, +2.087104344e-05, +7.010699791e-06, +2.427020011e-02 - 4.534361e+00, +1.020287435e-04, +7.808767188e-07, +2.102781511e-05, +6.486364065e-06, +2.376332646e-02 - 4.571835e+00, +1.005916786e-04, +5.963162817e-07, +2.033419762e-05, +6.226742951e-06, +2.366890478e-02 - 4.609309e+00, +9.956350360e-05, +8.948174405e-07, +2.120950949e-05, +6.203908051e-06, +2.354601494e-02 - 4.646783e+00, +9.648493091e-05, +3.461590621e-07, +2.018627070e-05, +5.734260658e-06, +2.295709422e-02 - 4.684257e+00, +9.112502939e-05, +3.883062686e-07, +2.053310382e-05, +5.164905501e-06, +2.162669198e-02 - 4.721731e+00, +8.334578085e-05, +2.177835208e-07, +2.046520648e-05, +4.687682932e-06, +1.951405874e-02 - 4.759205e+00, +7.338692516e-05, +2.824948814e-07, +1.845233238e-05, +4.257504749e-06, +1.682482853e-02 - 4.796679e+00, +6.197444759e-05, +3.940908030e-07, +1.480488406e-05, +3.596556545e-06, +1.398738199e-02 - 4.834153e+00, +5.177454751e-05, +9.287846008e-08, +1.026217963e-05, +2.976275229e-06, +1.158616965e-02 - 4.871627e+00, +4.538996392e-05, +2.947300616e-07, +9.884692522e-06, +2.721672544e-06, +1.014926309e-02 - 4.909101e+00, +4.257934145e-05, +2.417527055e-07, +9.432253934e-06, +2.508730323e-06, +9.727291773e-03 - 4.946576e+00, +4.197222756e-05, +3.145511971e-07, +1.004826459e-05, +2.415124360e-06, +9.783162092e-03 - 4.984050e+00, +4.168753775e-05, +1.303284528e-07, +9.606901820e-06, +2.260500356e-06, +9.697365188e-03 - 5.021524e+00, +3.960910741e-05, +2.205747653e-07, +9.867715713e-06, +2.091158785e-06, +9.137158595e-03 - 5.058998e+00, +3.537778784e-05, +4.468634674e-07, +9.753153858e-06, +1.922101446e-06, +8.056788668e-03 - 5.096472e+00, +2.958169142e-05, +3.707332509e-07, +8.587240753e-06, +1.633811265e-06, +6.593799082e-03 - 5.133946e+00, +2.271289986e-05, +1.698979251e-07, +6.643049350e-06, +1.279078028e-06, +4.979274167e-03 - 5.171420e+00, +1.658353716e-05, +3.152510878e-07, +4.450288142e-06, +1.046492112e-06, +3.482289178e-03 - 5.208894e+00, +1.245649774e-05, +1.564776138e-07, +2.370501380e-06, +9.150443492e-07, +2.384265177e-03 - 5.246368e+00, +1.117295932e-05, +2.121257863e-07, +2.031983450e-06, +8.607076636e-07, +1.893843419e-03 - 5.283842e+00, +1.071187897e-05, +1.993146525e-07, +2.267768381e-06, +7.809792359e-07, +1.857438703e-03 - 5.321316e+00, +1.001176624e-05, +3.013669505e-07, +2.037861830e-06, +7.219680786e-07, +1.884002961e-03 - 5.358790e+00, +1.024326393e-05, +1.955515665e-07, +2.134760475e-06, +7.278118421e-07, +1.781436184e-03 - 5.396264e+00, +9.966811528e-06, +2.886789209e-07, +2.008113485e-06, +7.744211989e-07, +1.539642657e-03 - 5.433738e+00, +8.110450112e-06, +9.102295863e-08, +1.634829761e-06, +6.330492611e-07, +1.215798114e-03 - 5.471212e+00, +6.059893755e-06, +1.477892133e-07, +1.139279007e-06, +4.637514657e-07, +8.794540096e-04 - 5.508686e+00, +6.692405771e-06, +9.674164163e-08, +9.740456409e-07, +5.245857386e-07, +5.925691539e-04 - 5.546160e+00, +7.399923072e-06, +3.073116713e-07, +1.094159758e-06, +6.025615222e-07, +4.025824102e-04 - 5.583635e+00, +6.169448434e-06, +1.344622774e-07, +9.761330755e-07, +4.978628371e-07, +3.249971140e-04 - 5.621109e+00, +5.251281156e-06, +9.219144089e-08, +7.529512542e-07, +4.044328657e-07, +3.144512592e-04 - 5.658583e+00, +6.033152707e-06, +1.517798126e-07, +8.376661419e-07, +4.900050921e-07, +3.143193611e-04 - 5.696057e+00, +6.953461481e-06, +1.731350010e-07, +1.032658432e-06, +5.408160264e-07, +3.053742378e-04 - 5.733531e+00, +6.616852919e-06, +1.393709780e-07, +9.707101787e-07, +5.299127849e-07, +2.905237722e-04 - 5.771005e+00, +4.981964770e-06, +7.734669672e-08, +7.959476411e-07, +3.910720171e-07, +2.767811258e-04 - 5.808479e+00, +5.497129429e-06, +5.415541090e-08, +8.069138412e-07, +4.159162126e-07, +2.672806062e-04 - 5.845953e+00, +6.951056971e-06, +1.517623716e-07, +1.004289950e-06, +5.465321079e-07, +2.606010338e-04 - 5.883427e+00, +6.328022933e-06, +9.130260613e-08, +8.427665708e-07, +4.999347176e-07, +2.537119531e-04 - 5.920901e+00, +5.214519405e-06, +2.023362231e-08, +8.188184714e-07, +3.956866551e-07, +2.439717849e-04 - 5.958375e+00, +5.304342109e-06, +1.698335842e-07, +6.730635294e-07, +4.418913029e-07, +2.287977096e-04 - 5.995849e+00, +6.175716100e-06, +1.307996671e-07, +9.587490709e-07, +4.892736786e-07, +2.091943852e-04 - 6.033323e+00, +6.645036012e-06, +8.548111301e-09, +8.996693798e-07, +5.325461238e-07, +1.868935198e-04 - 6.070797e+00, +5.397402497e-06, +1.508119283e-07, +8.539238540e-07, +4.257453984e-07, +1.639087117e-04 - 6.108271e+00, +4.639189681e-06, +1.459404705e-07, +5.846006585e-07, +3.914171158e-07, +1.453340940e-04 - 6.145745e+00, +6.145110009e-06, +8.240901159e-08, +9.145533702e-07, +4.947475022e-07, +1.340064667e-04 - 6.183219e+00, +6.485045280e-06, +6.256111494e-08, +9.130002723e-07, +5.250157101e-07, +1.300333935e-04 - 6.220694e+00, +5.541752537e-06, +8.574644437e-08, +7.822126843e-07, +4.586351499e-07, +1.300855088e-04 - 6.258168e+00, +4.984874654e-06, +1.077156320e-07, +7.100207062e-07, +3.962645038e-07, +1.297238444e-04 - 6.295642e+00, +5.333840078e-06, +4.701696829e-08, +7.978862286e-07, +4.319465292e-07, +1.253570119e-04 - 6.333116e+00, +6.420923110e-06, +2.898989892e-07, +9.204602489e-07, +5.400349919e-07, +1.158951038e-04 - 6.370590e+00, +6.073469406e-06, +8.837274556e-08, +7.781719401e-07, +4.990416886e-07, +1.015394549e-04 - 6.408064e+00, +4.266235556e-06, +7.837221964e-08, +6.328306312e-07, +3.203590672e-07, +8.295320962e-05 - 6.445538e+00, +5.024184876e-06, +9.562255997e-08, +6.546089918e-07, +4.170534811e-07, +6.413630695e-05 - 6.483012e+00, +6.379354444e-06, +2.230168975e-07, +9.421503967e-07, +5.297344304e-07, +4.719566269e-05 - 6.520486e+00, +5.693552615e-06, +1.452961270e-07, +7.169097015e-07, +4.702344561e-07, +3.428678049e-05 - 6.557960e+00, +4.451748648e-06, +6.643351337e-08, +7.111598228e-07, +3.392180044e-07, +2.871600307e-05 - 6.595434e+00, +4.899454217e-06, +1.455408984e-07, +7.337662947e-07, +4.063277411e-07, +2.830677177e-05 - 6.632908e+00, +5.715159740e-06, +1.362595137e-07, +8.892945808e-07, +4.551871839e-07, +2.868893844e-05 - 6.670382e+00, +5.627125656e-06, +1.506598940e-07, +7.797586350e-07, +4.640234721e-07, +2.757634084e-05 - 6.707856e+00, +4.948242219e-06, +1.410741165e-07, +7.615815071e-07, +3.976485709e-07, +2.478347455e-05 - 6.745330e+00, +4.504551740e-06, +8.895032984e-08, +7.798697482e-07, +3.526128990e-07, +2.028527677e-05 - 6.782804e+00, +4.993922067e-06, +8.728064318e-08, +7.967415424e-07, +3.890334694e-07, +1.596573591e-05 - 6.820278e+00, +5.747823872e-06, +1.409228846e-07, +8.337814478e-07, +4.659644325e-07, +1.262062648e-05 - 6.857752e+00, +5.247785988e-06, +1.813323881e-07, +7.726341183e-07, +4.312019165e-07, +9.582225142e-06 - 6.895227e+00, +3.985125339e-06, +3.623797444e-09, +6.312485712e-07, +3.104924942e-07, +7.451235411e-06 - 6.932701e+00, +4.586463251e-06, +8.432940932e-08, +6.862440602e-07, +3.657252939e-07, +7.713935361e-06 - 6.970175e+00, +5.649816560e-06, +9.980611945e-08, +8.434497191e-07, +4.545842056e-07, +8.153341350e-06 - 7.007649e+00, +5.253138935e-06, +6.315394513e-08, +6.956947871e-07, +4.363129718e-07, +8.013144415e-06 - 7.045123e+00, +3.957376831e-06, +3.892659965e-08, +6.417031076e-07, +2.828830275e-07, +7.103058561e-06 - 7.082597e+00, +4.067503119e-06, +1.030145131e-07, +5.482994277e-07, +3.324391553e-07, +6.251908497e-06 - 7.120071e+00, +5.339177720e-06, +3.932289647e-08, +8.433000917e-07, +4.337002769e-07, +7.632185039e-06 - 7.157545e+00, +5.327848789e-06, +1.177484112e-07, +6.938530724e-07, +4.217932669e-07, +7.329401859e-06 - 7.195019e+00, +3.969720665e-06, +7.950795856e-08, +6.036635361e-07, +2.959133757e-07, +6.045703988e-06 - 7.232493e+00, +3.715587160e-06, +1.404244897e-07, +5.594021142e-07, +3.068399705e-07, +5.792304398e-06 - 7.269967e+00, +5.014902828e-06, +1.612835846e-07, +7.753467579e-07, +3.960848066e-07, +6.622353708e-06 - 7.307441e+00, +5.352702037e-06, +1.184137760e-07, +7.375087236e-07, +4.240320490e-07, +7.298735179e-06 - 7.344915e+00, +4.153416797e-06, +4.480648236e-08, +6.123600421e-07, +3.202807598e-07, +5.538378559e-06 - 7.382389e+00, +3.599936856e-06, +3.650725801e-08, +5.446679750e-07, +2.773792997e-07, +5.496635574e-06 - 7.419863e+00, +4.830704479e-06, +5.884631298e-08, +7.174825239e-07, +3.835192893e-07, +6.712265589e-06 - 7.457337e+00, +5.146804022e-06, +1.163190193e-07, +7.555170452e-07, +4.169979523e-07, +6.428561978e-06 - 7.494811e+00, +4.397343693e-06, +1.080439747e-07, +5.798133525e-07, +3.671442609e-07, +6.238486822e-06 + 3.747406e-02, +5.572991710e-02, +4.767755798e-15, +1.407880294e-02, +1.856308518e-03, +1.015013075e-04 + 7.494811e-02, +1.034912856e-01, +9.719720872e-15, +2.607229698e-02, +3.523283344e-03, +1.286451027e-04 + 1.124222e-01, +3.324496389e-02, +6.365619917e-15, +8.388885356e-03, +1.164706330e-03, +1.363415075e-04 + 1.498962e-01, +2.375714211e-02, +2.397965370e-15, +6.044798885e-03, +8.245600591e-04, +2.237425263e-04 + 1.873703e-01, +3.819179106e-02, +5.024345578e-15, +9.668211450e-03, +1.299554229e-03, +5.988175109e-04 + 2.248443e-01, +1.297340613e-02, +1.781314293e-15, +3.303352868e-03, +4.708062401e-04, +1.231303285e-03 + 2.623184e-01, +5.246709261e-03, +2.383679123e-15, +1.405616340e-03, +2.071756999e-04, +2.183486767e-03 + 2.997925e-01, +6.058354891e-03, +3.690614455e-15, +1.493506908e-03, +2.376336055e-04, +3.443837437e-03 + 3.372665e-01, +6.156851514e-03, +4.074907959e-15, +1.749181241e-03, +2.239413758e-04, +4.896408733e-03 + 3.747406e-01, +4.694296649e-03, +8.973267467e-15, +1.289674681e-03, +1.823298192e-04, +6.288158078e-03 + 4.122146e-01, +5.417259350e-03, +1.949534383e-15, +1.436619200e-03, +2.081413126e-04, +7.236866239e-03 + 4.496887e-01, +7.374336957e-03, +1.897219364e-14, +1.846629384e-03, +2.874502795e-04, +7.438562524e-03 + 4.871627e-01, +6.930576439e-03, +2.280046795e-14, +1.752840045e-03, +2.931825986e-04, +7.655638548e-03 + 5.246368e-01, +5.525646929e-03, +3.091446252e-14, +1.406681124e-03, +2.243733864e-04, +1.144492786e-02 + 5.621109e-01, +4.673784424e-03, +3.121340021e-14, +1.348460932e-03, +1.868352062e-04, +2.129721915e-02 + 5.995849e-01, +4.253659682e-03, +1.861613505e-14, +1.368615134e-03, +1.587836337e-04, +3.669178320e-02 + 6.370590e-01, +4.178197277e-03, +5.274944314e-14, +1.375975768e-03, +1.455795304e-04, +5.641443987e-02 + 6.745330e-01, +4.365910098e-03, +2.727376788e-14, +1.376592666e-03, +1.531072976e-04, +7.828637698e-02 + 7.120071e-01, +4.532195073e-03, +8.520431431e-14, +1.342611942e-03, +1.672385806e-04, +9.885177909e-02 + 7.494811e-01, +4.713112853e-03, +1.938911398e-13, +1.277228489e-03, +1.799446727e-04, +1.137531996e-01 + 7.869552e-01, +4.875763465e-03, +3.075861075e-13, +1.187769851e-03, +1.938215810e-04, +1.195346821e-01 + 8.244293e-01, +4.981324068e-03, +1.288555229e-12, +1.301107022e-03, +2.128512979e-04, +1.188340974e-01 + 8.619033e-01, +4.771406979e-03, +6.396990220e-12, +1.099709362e-03, +2.160907931e-04, +1.309553939e-01 + 8.993774e-01, +4.387618618e-03, +1.891176183e-11, +1.104046666e-03, +1.969683895e-04, +1.857760966e-01 + 9.368514e-01, +4.317746826e-03, +4.265185878e-11, +1.315698936e-03, +1.778612664e-04, +2.867307084e-01 + 9.743255e-01, +4.352783980e-03, +9.207041359e-11, +1.367694242e-03, +1.732486187e-04, +4.167673658e-01 + 1.011800e+00, +4.392520145e-03, +1.998889274e-10, +1.347497137e-03, +1.702743981e-04, +5.549049357e-01 + 1.049274e+00, +4.412402763e-03, +4.284702403e-10, +1.276609665e-03, +1.748952061e-04, +6.782106751e-01 + 1.086748e+00, +4.409231342e-03, +8.771686060e-10, +1.151820795e-03, +1.841348266e-04, +7.648957645e-01 + 1.124222e+00, +4.385361349e-03, +1.640994893e-09, +1.054162063e-03, +1.931535836e-04, +8.022489576e-01 + 1.161696e+00, +4.359684491e-03, +2.540940898e-09, +1.097370972e-03, +2.015036517e-04, +8.009704160e-01 + 1.199170e+00, +4.376985093e-03, +2.358239072e-09, +9.962793898e-04, +2.076058305e-04, +8.143229074e-01 + 1.236644e+00, +4.452602827e-03, +3.901518113e-09, +9.301914663e-04, +2.236743970e-04, +9.294380600e-01 + 1.274118e+00, +4.490626319e-03, +1.578140786e-08, +1.141847583e-03, +2.193640151e-04, +1.185764957e+00 + 1.311592e+00, +4.461869998e-03, +3.635560929e-08, +1.253108436e-03, +2.106610575e-04, +1.534641583e+00 + 1.349066e+00, +4.401936395e-03, +6.699205042e-08, +1.237832414e-03, +2.050874105e-04, +1.896337013e+00 + 1.386540e+00, +4.333203621e-03, +1.076793012e-07, +1.149104164e-03, +2.051917039e-04, +2.199936335e+00 + 1.424014e+00, +4.274237239e-03, +1.530629529e-07, +1.003500853e-03, +2.108710395e-04, +2.397678575e+00 + 1.461488e+00, +4.247523108e-03, +1.896601883e-07, +9.868601873e-04, +2.177096147e-04, +2.477217295e+00 + 1.498962e+00, +4.280032390e-03, +2.092002356e-07, +9.955121384e-04, +2.271285516e-04, +2.474013829e+00 + 1.536436e+00, +4.376655840e-03, +2.839365838e-07, +9.435084758e-04, +2.375100394e-04, +2.473624483e+00 + 1.573910e+00, +4.478796254e-03, +5.266487239e-07, +9.452199888e-04, +2.535903206e-04, +2.579178593e+00 + 1.611384e+00, +4.504195280e-03, +8.670090215e-07, +8.305664715e-04, +2.598806764e-04, +2.837660777e+00 + 1.648859e+00, +4.449741379e-03, +1.200124989e-06, +9.532357119e-04, +2.571489935e-04, +3.200525521e+00 + 1.686333e+00, +4.368989074e-03, +1.481583714e-06, +9.657314632e-04, +2.553327470e-04, +3.569943474e+00 + 1.723807e+00, +4.305814700e-03, +1.685010880e-06, +9.004533391e-04, +2.557437501e-04, +3.860928050e+00 + 1.761281e+00, +4.281235699e-03, +1.743569614e-06, +8.111212810e-04, +2.616480310e-04, +4.031057327e+00 + 1.798755e+00, +4.299570361e-03, +1.539868619e-06, +8.541153812e-04, +2.692315782e-04, +4.086183103e+00 + 1.836229e+00, +4.350193371e-03, +1.240981512e-06, +8.333310469e-04, +2.775510898e-04, +4.072063131e+00 + 1.873703e+00, +4.406139686e-03, +2.397908290e-06, +8.321882404e-04, +2.865840244e-04, +4.053029029e+00 + 1.911177e+00, +4.436326424e-03, +5.244203576e-06, +8.330079752e-04, +2.954125346e-04, +4.080832967e+00 + 1.948651e+00, +4.428790225e-03, +9.058143034e-06, +7.711842311e-04, +3.027223741e-04, +4.169084349e+00 + 1.986125e+00, +4.399307303e-03, +1.325746399e-05, +7.862986976e-04, +3.057197152e-04, +4.292757706e+00 + 2.023599e+00, +4.372194815e-03, +1.510320229e-05, +7.323360545e-04, +3.100933893e-04, +4.411208547e+00 + 2.061073e+00, +4.361026041e-03, +1.884890795e-05, +7.327459971e-04, +3.150197546e-04, +4.493075573e+00 + 2.098547e+00, +4.366509432e-03, +1.374550555e-05, +7.246019676e-04, +3.202936759e-04, +4.527901974e+00 + 2.136021e+00, +4.381912029e-03, +1.715811914e-05, +6.970023574e-04, +3.265541668e-04, +4.524236977e+00 + 2.173495e+00, +4.397351815e-03, +1.079978217e-05, +7.272899437e-04, +3.301454651e-04, +4.500275525e+00 + 2.210969e+00, +4.405183345e-03, +1.447955334e-05, +6.922741206e-04, +3.337723250e-04, +4.473334986e+00 + 2.248443e+00, +4.405486375e-03, +3.446567370e-05, +7.204423006e-04, +3.362065933e-04, +4.453002776e+00 + 2.285917e+00, +4.402807453e-03, +6.095148461e-05, +7.118317587e-04, +3.394811402e-04, +4.440058051e+00 + 2.323392e+00, +4.399850845e-03, +8.934753195e-05, +6.971610885e-04, +3.421644326e-04, +4.429819755e+00 + 2.360866e+00, +4.398068699e-03, +9.030215537e-05, +7.175095818e-04, +3.437942479e-04, +4.416620275e+00 + 2.398340e+00, +4.398861137e-03, +1.235780509e-04, +6.740707350e-04, +3.459275998e-04, +4.396813054e+00 + 2.435814e+00, +4.400980771e-03, +9.467385267e-05, +7.142802747e-04, +3.454258513e-04, +4.369717681e+00 + 2.473288e+00, +4.402460543e-03, +1.022104767e-04, +7.009001488e-04, +3.451696172e-04, +4.337243705e+00 + 2.510762e+00, +4.403895760e-03, +7.877147797e-05, +7.018112881e-04, +3.433220166e-04, +4.302986459e+00 + 2.548236e+00, +4.405792751e-03, +4.246249635e-05, +7.172947620e-04, +3.411869495e-04, +4.270950549e+00 + 2.585710e+00, +4.405598720e-03, +1.994809524e-05, +6.763893213e-04, +3.399230388e-04, +4.243663227e+00 + 2.623184e+00, +4.400694419e-03, +2.364148071e-05, +7.186146004e-04, +3.378805735e-04, +4.219874234e+00 + 2.660658e+00, +4.392485708e-03, +2.844058393e-05, +6.965240664e-04, +3.355645968e-04, +4.193027513e+00 + 2.698132e+00, +4.384585819e-03, +2.571478454e-05, +7.103855285e-04, +3.307865051e-04, +4.152202300e+00 + 2.735606e+00, +4.380780471e-03, +3.213583979e-05, +7.236362645e-04, +3.262575401e-04, +4.086406465e+00 + 2.773080e+00, +4.386554361e-03, +2.373514849e-05, +7.046036970e-04, +3.212456071e-04, +3.991182680e+00 + 2.810554e+00, +4.404814760e-03, +1.878884496e-05, +7.577059295e-04, +3.166360002e-04, +3.874510868e+00 + 2.848028e+00, +4.426459055e-03, +1.152351783e-05, +7.432128881e-04, +3.128601851e-04, +3.757750265e+00 + 2.885502e+00, +4.433150133e-03, +5.448998919e-06, +7.815790047e-04, +3.061679058e-04, +3.667691740e+00 + 2.922976e+00, +4.414697950e-03, +1.997860698e-06, +7.857457169e-04, +2.988084941e-04, +3.620168807e+00 + 2.960451e+00, +4.375682264e-03, +2.690972492e-06, +7.747290766e-04, +2.912160446e-04, +3.604784042e+00 + 2.997925e+00, +4.333071394e-03, +3.554263518e-06, +8.038401126e-04, +2.835943343e-04, +3.584400783e+00 + 3.035399e+00, +4.311111757e-03, +3.499465101e-06, +7.701958486e-04, +2.758686787e-04, +3.511864377e+00 + 3.072873e+00, +4.325993916e-03, +3.007849963e-06, +8.245577518e-04, +2.700227386e-04, +3.352677539e+00 + 3.110347e+00, +4.379396657e-03, +2.341254635e-06, +8.846266414e-04, +2.683847956e-04, +3.102447387e+00 + 3.147821e+00, +4.454636199e-03, +1.573502276e-06, +8.735639996e-04, +2.689007160e-04, +2.794852318e+00 + 3.185295e+00, +4.515021137e-03, +9.659168234e-07, +8.139447345e-04, +2.703820992e-04, +2.497474676e+00 + 3.222769e+00, +4.509445159e-03, +4.733078592e-07, +9.371588790e-04, +2.641431162e-04, +2.287003000e+00 + 3.260243e+00, +4.420580954e-03, +3.531254160e-07, +9.328068147e-04, +2.495118539e-04, +2.199280716e+00 + 3.297717e+00, +4.319453795e-03, +2.783172778e-07, +9.755430751e-04, +2.384117920e-04, +2.194256365e+00 + 3.335191e+00, +4.271601222e-03, +3.754524310e-07, +9.704484767e-04, +2.295039199e-04, +2.187349161e+00 + 3.372665e+00, +4.285381211e-03, +2.365562917e-07, +9.618492656e-04, +2.221995959e-04, +2.107609658e+00 + 3.410139e+00, +4.341461532e-03, +1.107883651e-07, +1.107163853e-03, +2.161560933e-04, +1.925381439e+00 + 3.447613e+00, +4.421259555e-03, +1.341227219e-07, +1.199144103e-03, +2.154776888e-04, +1.652331416e+00 + 3.485087e+00, +4.515090823e-03, +2.506405354e-07, +1.219011284e-03, +2.200093999e-04, +1.332405222e+00 + 3.522561e+00, +4.575781860e-03, +5.021636995e-07, +1.102521402e-03, +2.284250880e-04, +1.031973346e+00 + 3.560035e+00, +4.546571541e-03, +8.814336894e-07, +9.669912103e-04, +2.307163851e-04, +8.248149186e-01 + 3.597509e+00, +4.439546624e-03, +9.644046739e-07, +1.029526458e-03, +2.166721858e-04, +7.467546178e-01 + 3.634984e+00, +4.373608421e-03, +7.723624291e-07, +1.091750868e-03, +2.116976978e-04, +7.476143297e-01 + 3.672458e+00, +4.382050521e-03, +1.067257149e-06, +1.035051104e-03, +2.048938622e-04, +7.473446775e-01 + 3.709932e+00, +4.404293078e-03, +6.653011939e-07, +1.149806513e-03, +1.974798896e-04, +7.038738376e-01 + 3.747406e+00, +4.430911334e-03, +8.595315846e-07, +1.275090226e-03, +1.900683641e-04, +6.126704622e-01 + 3.784880e+00, +4.434943455e-03, +2.471155334e-06, +1.346868556e-03, +1.892394276e-04, +4.890429311e-01 + 3.822354e+00, +4.432465684e-03, +6.662846327e-06, +1.361827014e-03, +1.977109325e-04, +3.558817348e-01 + 3.859828e+00, +4.472946341e-03, +1.471115194e-05, +1.270690029e-03, +2.192143040e-04, +2.372244458e-01 + 3.897302e+00, +4.608050556e-03, +2.995805064e-05, +9.306446641e-04, +2.555399562e-04, +1.561822526e-01 + 3.934776e+00, +4.881684955e-03, +2.811035769e-05, +1.216238723e-03, +2.725953442e-04, +1.265095408e-01 + 3.972250e+00, +4.866553825e-03, +3.795415085e-05, +1.220274883e-03, +2.666603348e-04, +1.273673799e-01 + 4.009724e+00, +4.748277124e-03, +3.192103916e-05, +1.158713078e-03, +2.539291412e-04, +1.276014323e-01 + 4.047198e+00, +4.585078550e-03, +3.182645796e-05, +1.255153535e-03, +2.447044579e-04, +1.171764854e-01 + 4.084672e+00, +4.459956643e-03, +4.434580877e-05, +1.290534216e-03, +2.465663493e-04, +9.815025303e-02 + 4.122146e+00, +4.357849031e-03, +4.451221836e-05, +1.261671269e-03, +2.674618203e-04, +7.584704242e-02 + 4.159620e+00, +4.242745441e-03, +8.663524762e-05, +1.134062116e-03, +2.934100756e-04, +5.552946065e-02 + 4.197094e+00, +4.375605406e-03, +1.365338261e-04, +8.763229146e-04, +3.425542935e-04, +4.116638402e-02 + 4.234568e+00, +4.535627597e-03, +7.791910123e-05, +6.462325798e-04, +3.669813290e-04, +3.415469948e-02 + 4.272043e+00, +4.532610005e-03, +5.193406384e-05, +6.707342473e-04, +3.562439774e-04, +3.215669992e-02 + 4.309517e+00, +4.422878873e-03, +6.066243178e-05, +6.757720455e-04, +3.445782429e-04, +3.160208211e-02 + 4.346991e+00, +4.405074263e-03, +5.218897198e-05, +7.105897121e-04, +3.336881563e-04, +3.065433292e-02 + 4.384465e+00, +4.308113084e-03, +4.409648491e-05, +7.002029226e-04, +3.182689189e-04, +2.904485957e-02 + 4.421939e+00, +4.285526963e-03, +4.707655638e-05, +7.905553822e-04, +3.035324297e-04, +2.713891437e-02 + 4.459413e+00, +4.343692498e-03, +5.926643851e-05, +8.126272690e-04, +3.060319714e-04, +2.542525795e-02 + 4.496887e+00, +4.333152821e-03, +2.383821210e-05, +8.599452557e-04, +2.888604032e-04, +2.427020011e-02 + 4.534361e+00, +4.293537928e-03, +3.286058120e-05, +8.848851674e-04, +2.729569059e-04, +2.376332646e-02 + 4.571835e+00, +4.249950706e-03, +2.519408005e-05, +8.591102042e-04, +2.630769361e-04, +2.366890478e-02 + 4.609309e+00, +4.228465151e-03, +3.800292504e-05, +9.007685395e-04, +2.634801714e-04, +2.354601494e-02 + 4.646783e+00, +4.202837257e-03, +1.507852251e-05, +8.793042581e-04, +2.497816406e-04, +2.295709422e-02 + 4.684257e+00, +4.213544516e-03, +1.795495441e-05, +9.494334056e-04, +2.388208750e-04, +2.162669198e-02 + 4.721731e+00, +4.271063337e-03, +1.116033951e-05, +1.048741666e-03, +2.402208066e-04, +1.951405874e-02 + 4.759205e+00, +4.361823065e-03, +1.679035723e-05, +1.096732269e-03, +2.530489235e-04, +1.682482853e-02 + 4.796679e+00, +4.430739622e-03, +2.817473658e-05, +1.058445681e-03, +2.571286426e-04, +1.398738199e-02 + 4.834153e+00, +4.468650907e-03, +8.016321433e-06, +8.857266847e-04, +2.568817235e-04, +1.158616965e-02 + 4.871627e+00, +4.472242319e-03, +2.903955281e-05, +9.739320412e-04, +2.681645474e-04, +1.014926309e-02 + 4.909101e+00, +4.377306905e-03, +2.485303321e-05, +9.696690666e-04, +2.579063507e-04, +9.727291773e-03 + 4.946576e+00, +4.290251676e-03, +3.215230353e-05, +1.027097834e-03, +2.468654140e-04, +9.783162092e-03 + 4.984050e+00, +4.298852002e-03, +1.343957356e-05, +9.906713457e-04, +2.331045920e-04, +9.697365188e-03 + 5.021524e+00, +4.334948003e-03, +2.414041116e-05, +1.079954519e-03, +2.288631376e-04, +9.137158595e-03 + 5.058998e+00, +4.391053222e-03, +5.546421606e-05, +1.210551035e-03, +2.385691775e-04, +8.056788668e-03 + 5.096472e+00, +4.486289474e-03, +5.622452948e-05, +1.302320657e-03, +2.477799587e-04, +6.593799082e-03 + 5.133946e+00, +4.561488101e-03, +3.412102235e-05, +1.334140103e-03, +2.568804178e-04, +4.979274167e-03 + 5.171420e+00, +4.762251584e-03, +9.052984162e-05, +1.277977765e-03, +3.005184401e-04, +3.482289178e-03 + 5.208894e+00, +5.224459873e-03, +6.562928293e-05, +9.942272374e-04, +3.837846386e-04, +2.384265177e-03 + 5.246368e+00, +5.899621480e-03, +1.120080911e-04, +1.072941632e-03, +4.544766768e-04, +1.893843419e-03 + 5.283842e+00, +5.767016137e-03, +1.073061804e-04, +1.220911558e-03, +4.204603009e-04, +1.857438703e-03 + 5.321316e+00, +5.314092626e-03, +1.599609750e-04, +1.081665938e-03, +3.832096305e-04, +1.884002961e-03 + 5.358790e+00, +5.750003298e-03, +1.097718618e-04, +1.198336766e-03, +4.085534182e-04, +1.781436184e-03 + 5.396264e+00, +6.473457644e-03, +1.874973518e-04, +1.304272440e-03, +5.029876221e-04, +1.539642657e-03 + 5.433738e+00, +6.670885584e-03, +7.486683651e-05, +1.344655616e-03, +5.206861680e-04, +1.215798114e-03 + 5.471212e+00, +6.890518082e-03, +1.680465513e-04, +1.295438982e-03, +5.273174727e-04, +8.794540096e-04 + 5.508686e+00, +1.129388144e-02, +1.632579776e-04, +1.643767035e-03, +8.852734490e-04, +5.925691539e-04 + 5.546160e+00, +1.838113858e-02, +7.633509650e-04, +2.717852868e-03, +1.496740809e-03, +4.025824102e-04 + 5.583635e+00, +1.898308683e-02, +4.137337581e-04, +3.003513057e-03, +1.531899256e-03, +3.249971140e-04 + 5.621109e+00, +1.669982550e-02, +2.931819740e-04, +2.394492731e-03, +1.286154385e-03, +3.144512592e-04 + 5.658583e+00, +1.919434007e-02, +4.828840707e-04, +2.665016049e-03, +1.558940214e-03, +3.143193611e-04 + 5.696057e+00, +2.277029501e-02, +5.669600759e-04, +3.381616077e-03, +1.770994274e-03, +3.053742378e-04 + 5.733531e+00, +2.277559895e-02, +4.797231463e-04, +3.341241825e-03, +1.823991135e-03, +2.905237722e-04 + 5.771005e+00, +1.799965499e-02, +2.794507628e-04, +2.875729474e-03, +1.412928776e-03, +2.767811258e-04 + 5.808479e+00, +2.056688477e-02, +2.026163128e-04, +3.018976396e-03, +1.556103223e-03, +2.672806062e-04 + 5.845953e+00, +2.667317497e-02, +5.823552172e-04, +3.853745073e-03, +2.097198542e-03, +2.606010338e-04 + 5.883427e+00, +2.494176114e-02, +3.598671841e-04, +3.321745627e-03, +1.970481530e-03, +2.537119531e-04 + 5.920901e+00, +2.137345270e-02, +8.293427173e-05, +3.356201503e-03, +1.621854163e-03, +2.439717849e-04 + 5.958375e+00, +2.318354549e-02, +7.422870817e-04, +2.941740678e-03, +1.931362441e-03, +2.287977096e-04 + 5.995849e+00, +2.952142379e-02, +6.252541959e-04, +4.583053556e-03, +2.338847088e-03, +2.091943852e-04 + 6.033323e+00, +3.555519752e-02, +4.573786887e-05, +4.813807246e-03, +2.849462755e-03, +1.868935198e-04 + 6.070797e+00, +3.292932048e-02, +9.200970879e-04, +5.209752704e-03, +2.597454364e-03, +1.639087117e-04 + 6.108271e+00, +3.192086283e-02, +1.004172293e-03, +4.022460541e-03, +2.693222938e-03, +1.453340940e-04 + 6.145745e+00, +4.585681692e-02, +6.149629464e-04, +6.824695798e-03, +3.691967369e-03, +1.340064667e-04 + 6.183219e+00, +4.987215286e-02, +4.811157598e-04, +7.021275439e-03, +4.037545247e-03, +1.300333935e-04 + 6.220694e+00, +4.260084454e-02, +6.591544682e-04, +6.013065495e-03, +3.525643664e-03, +1.300855088e-04 + 6.258168e+00, +3.842681872e-02, +8.303456662e-04, +5.473324578e-03, +3.054677461e-03, +1.297238444e-04 + 6.295642e+00, +4.254919607e-02, +3.750645263e-04, +6.364911037e-03, +3.445730899e-03, +1.253570119e-04 + 6.333116e+00, +5.540288504e-02, +2.501391170e-03, +7.942184088e-03, +4.659687721e-03, +1.158951038e-04 + 6.370590e+00, +5.981388626e-02, +8.703291313e-04, +7.663739587e-03, +4.914756428e-03, +1.015394549e-04 + 6.408064e+00, +5.142942118e-02, +9.447762178e-04, +7.628766072e-03, +3.861924917e-03, +8.295320962e-05 + 6.445538e+00, +7.833604887e-02, +1.490927129e-03, +1.020652767e-02, +6.502611406e-03, +6.413630695e-05 + 6.483012e+00, +1.351682354e-01, +4.725368493e-03, +1.996264790e-02, +1.122421850e-02, +4.719566269e-05 + 6.520486e+00, +1.660567873e-01, +4.237671924e-03, +2.090921607e-02, +1.371474514e-02, +3.428678049e-05 + 6.557960e+00, +1.550267507e-01, +2.313466579e-03, +2.476527883e-02, +1.181285584e-02, +2.871600307e-05 + 6.595434e+00, +1.730841742e-01, +5.141557630e-03, +2.592193488e-02, +1.435443590e-02, +2.830677177e-05 + 6.632908e+00, +1.992112658e-01, +4.749548819e-03, +3.099782108e-02, +1.586629581e-02, +2.868893844e-05 + 6.670382e+00, +2.040562846e-01, +5.463375103e-03, +2.827636340e-02, +1.682686890e-02, +2.757634084e-05 + 6.707856e+00, +1.996589384e-01, +5.692265472e-03, +3.072940824e-02, +1.604490807e-02, +2.478347455e-05 + 6.745330e+00, +2.220601568e-01, +4.384969989e-03, +3.844511254e-02, +1.738270091e-02, +2.028527677e-05 + 6.782804e+00, +3.127899706e-01, +5.466747267e-03, +4.990321440e-02, +2.436677342e-02, +1.596573591e-05 + 6.820278e+00, +4.554309472e-01, +1.116607680e-02, +6.606498094e-02, +3.692086390e-02, +1.262062648e-05 + 6.857752e+00, +5.476583894e-01, +1.892382880e-02, +8.063201468e-02, +4.500018629e-02, +9.582225142e-06 + 6.895227e+00, +5.348274640e-01, +4.863351168e-04, +8.471730342e-02, +4.166993486e-02, +7.451235411e-06 + 6.932701e+00, +5.945685356e-01, +1.093208659e-02, +8.896160366e-02, +4.741098762e-02, +7.713935361e-06 + 6.970175e+00, +6.929449310e-01, +1.224113099e-02, +1.034483512e-01, +5.575434489e-02, +8.153341350e-06 + 7.007649e+00, +6.555652393e-01, +7.881293768e-03, +8.681919994e-02, +5.444965786e-02, +8.013144415e-06 + 7.045123e+00, +5.571370132e-01, +5.480258866e-03, +9.034180164e-02, +3.982552376e-02, +7.103058561e-06 + 7.082597e+00, +6.506018316e-01, +1.647729060e-02, +8.770112806e-02, +5.317402765e-02, +6.251908497e-06 + 7.120071e+00, +6.995608325e-01, +5.152246213e-03, +1.104926161e-01, +5.682517846e-02, +7.632185039e-06 + 7.157545e+00, +7.269145411e-01, +1.606521425e-02, +9.466708003e-02, +5.754811580e-02, +7.329401859e-06 + 7.195019e+00, +6.566184307e-01, +1.315114976e-02, +9.984999884e-02, +4.894605762e-02, +6.045703988e-06 + 7.232493e+00, +6.414695957e-01, +2.424328558e-02, +9.657678114e-02, +5.297373021e-02, +5.792304398e-06 + 7.269967e+00, +7.572689483e-01, +2.435442014e-02, +1.170802395e-01, +5.981027653e-02, +6.622353708e-06 + 7.307441e+00, +7.333739211e-01, +1.622387621e-02, +1.010460998e-01, +5.809664807e-02, +7.298735179e-06 + 7.344915e+00, +7.499337130e-01, +8.090180525e-03, +1.105666642e-01, +5.782933693e-02, +5.538378559e-06 + 7.382389e+00, +6.549346065e-01, +6.641746123e-03, +9.909115634e-02, +5.046346915e-02, +5.496635574e-06 + 7.419863e+00, +7.196831554e-01, +8.766982205e-03, +1.068912596e-01, +5.713708497e-02, +6.712265589e-06 + 7.457337e+00, +8.006151360e-01, +1.809409626e-02, +1.175250465e-01, +6.486644349e-02, +6.428561978e-06 + 7.494811e+00, +7.048734441e-01, +1.731893931e-02, +9.294134442e-02, +5.885149257e-02, +6.238486822e-06 diff --git a/test/ref/coaxial/open/error-indicators.csv b/test/ref/coaxial/open/error-indicators.csv index 7e1a5bacf..741fd9506 100644 --- a/test/ref/coaxial/open/error-indicators.csv +++ b/test/ref/coaxial/open/error-indicators.csv @@ -1,202 +1,202 @@ - t (ns), Sum Abs., Min Elem Abs., Max Elem Abs., Mean Elem Abs., Normalization + t (ns), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 - 3.747406e-02, +5.656659455e-06, +4.851768538e-19, +1.429016907e-06, +1.884177417e-07, +1.015013075e-04 - 7.494811e-02, +1.331364706e-05, +1.252047160e-18, +3.354073322e-06, +4.532531473e-07, +1.286451027e-04 - 1.124222e-01, +4.532668491e-06, +8.692216473e-19, +1.143753275e-06, +1.587978168e-07, +1.363415075e-04 - 1.498962e-01, +5.315482993e-06, +5.382843762e-19, +1.352478574e-06, +1.844891507e-07, +2.237425263e-04 - 1.873703e-01, +2.286991326e-05, +3.007030970e-18, +5.789494315e-06, +7.781958288e-07, +5.988175109e-04 - 2.248443e-01, +1.597419758e-05, +2.187676401e-18, +4.067429237e-06, +5.797052698e-07, +1.231303285e-03 - 2.623184e-01, +1.145612024e-05, +5.207848925e-18, +3.069144677e-06, +4.523653991e-07, +2.183486767e-03 - 2.997925e-01, +2.086398938e-05, +1.270112296e-17, +5.143395002e-06, +8.183715069e-07, +3.443837437e-03 - 3.372665e-01, +3.014646152e-05, +1.995375749e-17, +8.564706304e-06, +1.096508508e-06, +4.896408733e-03 - 3.747406e-01, +2.951847940e-05, +5.642253167e-17, +8.109678263e-06, +1.146518725e-06, +6.288158078e-03 - 4.122146e-01, +3.920398129e-05, +1.409960031e-17, +1.039662099e-05, +1.506290838e-06, +7.236866239e-03 - 4.496887e-01, +5.485446653e-05, +1.411154384e-16, +1.373626813e-05, +2.138216877e-06, +7.438562524e-03 - 4.871627e-01, +5.305798815e-05, +1.745355459e-16, +1.341910982e-05, +2.244500004e-06, +7.655638548e-03 - 5.246368e-01, +6.324063047e-05, +3.538112111e-16, +1.609936398e-05, +2.567937221e-06, +1.144492786e-02 - 5.621109e-01, +9.953861113e-05, +6.647608180e-16, +2.871846797e-05, +3.979070330e-06, +2.129721915e-02 - 5.995849e-01, +1.560743589e-04, +6.830393652e-16, +5.021692981e-05, +5.826054663e-06, +3.669178320e-02 - 6.370590e-01, +2.357106590e-04, +2.975840305e-15, +7.762490221e-05, +8.212787663e-06, +5.641443987e-02 - 6.745330e-01, +3.417912838e-04, +2.134700483e-15, +1.077684524e-04, +1.198621562e-05, +7.828637698e-02 - 7.120071e-01, +4.480155462e-04, +8.427337322e-15, +1.327195791e-04, +1.653183123e-05, +9.885177909e-02 - 7.494811e-01, +5.361316672e-04, +2.209015576e-14, +1.452888272e-04, +2.046928227e-05, +1.137531996e-01 - 7.869552e-01, +5.828228357e-04, +3.698456781e-14, +1.419796915e-04, +2.316840106e-05, +1.195346821e-01 - 8.244293e-01, +5.919511494e-04, +1.543555294e-13, +1.546158786e-04, +2.529399186e-05, +1.188340974e-01 - 8.619033e-01, +6.248414803e-04, +8.440272402e-13, +1.440128726e-04, +2.829825492e-05, +1.309553939e-01 - 8.993774e-01, +8.151146601e-04, +3.542755319e-12, +2.051054800e-04, +3.659201856e-05, +1.857760966e-01 - 9.368514e-01, +1.238030606e-03, +1.235495048e-11, +3.772512882e-04, +5.099828694e-05, +2.867307084e-01 - 9.743255e-01, +1.814098313e-03, +3.886184200e-11, +5.700103263e-04, +7.220437046e-05, +4.167673658e-01 - 1.011800e+00, +2.437431108e-03, +1.126769718e-10, +7.477328123e-04, +9.448610408e-05, +5.549049357e-01 - 1.049274e+00, +2.992538657e-03, +2.963780279e-10, +8.658103032e-04, +1.186157963e-04, +6.782106751e-01 - 1.086748e+00, +3.372602378e-03, +6.883561366e-10, +8.810228475e-04, +1.408439505e-04, +7.648957645e-01 - 1.124222e+00, +3.518151571e-03, +1.364041638e-09, +8.457004167e-04, +1.549572659e-04, +8.022489576e-01 - 1.161696e+00, +3.491978301e-03, +2.150204961e-09, +8.789616842e-04, +1.613984718e-04, +8.009704160e-01 - 1.199170e+00, +3.564279227e-03, +2.123409813e-09, +8.112931293e-04, +1.690581924e-04, +8.143229074e-01 - 1.236644e+00, +4.138418533e-03, +3.078381559e-09, +8.645553516e-04, +2.078914509e-04, +9.294380600e-01 - 1.274118e+00, +5.324827324e-03, +1.777478124e-08, +1.353962850e-03, +2.601140741e-04, +1.185764957e+00 - 1.311592e+00, +6.847371239e-03, +5.481861739e-08, +1.923072314e-03, +3.232891049e-04, +1.534641584e+00 - 1.349066e+00, +8.347554914e-03, +1.274777982e-07, +2.347347422e-03, +3.889148264e-04, +1.896337013e+00 - 1.386540e+00, +9.532772096e-03, +2.428165880e-07, +2.527956003e-03, +4.514092009e-04, +2.199936335e+00 - 1.424014e+00, +1.024824707e-02, +3.871187955e-07, +2.406072494e-03, +5.056030501e-04, +2.397678575e+00 - 1.461488e+00, +1.052203775e-02, +5.171912012e-07, +2.444667124e-03, +5.393185100e-04, +2.477217296e+00 - 1.498962e+00, +1.058885937e-02, +5.891194816e-07, +2.462910797e-03, +5.619245208e-04, +2.474013830e+00 - 1.536436e+00, +1.082620294e-02, +6.896210906e-07, +2.333885666e-03, +5.875077909e-04, +2.473624477e+00 - 1.573910e+00, +1.155161469e-02, +1.121475533e-06, +2.437891160e-03, +6.540312672e-04, +2.579178548e+00 - 1.611384e+00, +1.278137618e-02, +1.972066894e-06, +2.356865899e-03, +7.374026260e-04, +2.837660610e+00 - 1.648859e+00, +1.424150692e-02, +3.100040281e-06, +3.050855223e-03, +8.229301974e-04, +3.200525096e+00 - 1.686333e+00, +1.559703945e-02, +4.364327287e-06, +3.447606735e-03, +9.114278074e-04, +3.569942692e+00 - 1.723807e+00, +1.662444071e-02, +5.652063847e-06, +3.476585555e-03, +9.873524406e-04, +3.860927182e+00 - 1.761281e+00, +1.725792281e-02, +6.835154067e-06, +3.269676383e-03, +1.054802446e-03, +4.031057848e+00 - 1.798755e+00, +1.756888105e-02, +7.822221900e-06, +3.490071839e-03, +1.100407720e-03, +4.086189106e+00 - 1.836229e+00, +1.771435393e-02, +9.224119034e-06, +3.393376631e-03, +1.130683221e-03, +4.072082288e+00 - 1.873703e+00, +1.785832386e-02, +1.320969740e-05, +3.372883096e-03, +1.161958398e-03, +4.053069427e+00 - 1.911177e+00, +1.810395343e-02, +2.146664968e-05, +3.399366408e-03, +1.205675107e-03, +4.080890391e+00 - 1.948651e+00, +1.846382530e-02, +3.336302682e-05, +3.215132108e-03, +1.261773627e-03, +4.169120342e+00 - 1.986125e+00, +1.888457853e-02, +4.698430886e-05, +3.375389794e-03, +1.311524450e-03, +4.292682833e+00 - 2.023599e+00, +1.928557725e-02, +5.951672357e-05, +3.230487064e-03, +1.366405838e-03, +4.410898850e+00 - 2.061073e+00, +1.959303920e-02, +6.715821177e-05, +3.292283170e-03, +1.413551116e-03, +4.492460625e+00 - 2.098547e+00, +1.977023401e-02, +6.599675127e-05, +3.280926715e-03, +1.448890774e-03, +4.527157525e+00 - 2.136021e+00, +1.982606615e-02, +5.672179952e-05, +3.153403886e-03, +1.477660705e-03, +4.524046965e+00 - 2.173495e+00, +1.979526345e-02, +6.082263690e-05, +3.273005166e-03, +1.489008848e-03, +4.501957938e+00 - 2.210969e+00, +1.971930948e-02, +1.081631263e-04, +3.096774883e-03, +1.499471252e-03, +4.478434139e+00 - 2.248443e+00, +1.963950279e-02, +1.861344323e-04, +3.208102115e-03, +1.504260282e-03, +4.462008482e+00 - 2.285917e+00, +1.957424152e-02, +2.755866023e-04, +3.160538427e-03, +1.513687739e-03, +4.450486620e+00 - 2.323392e+00, +1.950573403e-02, +3.564622283e-04, +3.087873225e-03, +1.519283140e-03, +4.434865564e+00 - 2.360866e+00, +1.940581024e-02, +4.054684373e-04, +3.168292697e-03, +1.515796398e-03, +4.406388329e+00 - 2.398340e+00, +1.926434723e-02, +3.996770705e-04, +2.962622826e-03, +1.506568723e-03, +4.363973589e+00 - 2.435814e+00, +1.909655442e-02, +3.214407182e-04, +3.118749615e-03, +1.482647495e-03, +4.319369685e+00 - 2.473288e+00, +1.895811647e-02, +1.839689235e-04, +3.036885498e-03, +1.468540491e-03, +4.296269972e+00 - 2.510762e+00, +1.894439512e-02, +2.204911120e-04, +3.027284537e-03, +1.470201519e-03, +4.319012990e+00 - 2.548236e+00, +1.911859202e-02, +1.813557604e-04, +3.079371068e-03, +1.490279423e-03, +4.391832454e+00 - 2.585710e+00, +1.942027452e-02, +8.465426247e-05, +2.915624197e-03, +1.509126434e-03, +4.481474420e+00 - 2.623184e+00, +1.963519733e-02, +9.976338258e-05, +3.093706609e-03, +1.511211369e-03, +4.519833729e+00 - 2.660658e+00, +1.945513343e-02, +1.192767917e-04, +2.987367916e-03, +1.485119786e-03, +4.429683004e+00 - 2.698132e+00, +1.861878549e-02, +1.067715430e-04, +2.884183524e-03, +1.407976497e-03, +4.164305386e+00 - 2.735606e+00, +1.710066160e-02, +1.313040409e-04, +2.786532587e-03, +1.279638252e-03, +3.755743891e+00 - 2.773080e+00, +1.538534017e-02, +9.475720213e-05, +2.394757280e-03, +1.120242820e-03, +3.369620795e+00 - 2.810554e+00, +1.459970790e-02, +7.277303229e-05, +2.426897616e-03, +1.022789237e-03, +3.300078313e+00 - 2.848028e+00, +1.577492744e-02, +4.329855808e-05, +2.648390438e-03, +1.105397512e-03, +3.715663123e+00 - 2.885502e+00, +1.851283738e-02, +2.006000941e-05, +3.351903461e-03, +1.249233074e-03, +4.412696644e+00 - 2.922976e+00, +2.132059162e-02, +7.437820796e-06, +4.013069564e-03, +1.384748240e-03, +5.035062025e+00 - 2.960451e+00, +2.279077487e-02, +9.538190125e-06, +4.535289462e-03, +1.438249671e-03, +5.296751662e+00 - 2.997925e+00, +2.198649980e-02, +1.213423224e-05, +4.486425727e-03, +1.365182572e-03, +5.025294040e+00 - 3.035399e+00, +1.853237043e-02, +1.062292095e-05, +3.820619063e-03, +1.141992427e-03, +4.165707984e+00 - 3.072873e+00, +1.265418979e-02, +8.153948486e-06, +2.598816283e-03, +7.812765378e-04, +2.782735535e+00 - 3.110347e+00, +5.265495931e-03, +4.582142878e-06, +9.839733384e-04, +3.411259218e-04, +1.091908737e+00 - 3.147821e+00, +4.574928367e-03, +1.680469188e-06, +8.666484916e-04, +2.912393939e-04, +1.087897791e+00 - 3.185295e+00, +1.208246023e-02, +2.263397939e-06, +2.520770130e-03, +7.276559379e-04, +2.795340363e+00 - 3.222769e+00, +1.828571121e-02, +3.423453186e-06, +3.853354933e-03, +1.093030170e-03, +4.198205341e+00 - 3.260243e+00, +2.198386792e-02, +3.709070615e-06, +4.637021302e-03, +1.312583527e-03, +5.054782764e+00 - 3.297717e+00, +2.274113830e-02, +5.430777524e-06, +4.770573156e-03, +1.364056751e-03, +5.270954600e+00 - 3.335191e+00, +2.075239208e-02, +1.087448501e-05, +4.269770291e-03, +1.264037484e-03, +4.875664359e+00 - 3.372665e+00, +1.683366259e-02, +1.945262936e-05, +3.253540776e-03, +1.058387811e-03, +4.026124476e+00 - 3.410139e+00, +1.257529174e-02, +3.016371246e-05, +2.336277033e-03, +8.425637502e-04, +3.029466121e+00 - 3.447613e+00, +1.051023882e-02, +3.687698139e-05, +1.857166545e-03, +7.127467496e-04, +2.402804105e+00 - 3.485087e+00, +1.197457373e-02, +5.129570561e-05, +1.975832266e-03, +8.549372312e-04, +2.582857071e+00 - 3.522561e+00, +1.475674376e-02, +4.252621015e-05, +2.416287574e-03, +1.061933682e-03, +3.207230086e+00 - 3.560035e+00, +1.685326549e-02, +4.330328377e-05, +2.636066290e-03, +1.211007114e-03, +3.758897835e+00 - 3.597509e+00, +1.767867104e-02, +3.794721935e-05, +2.813350409e-03, +1.280542243e-03, +4.038904116e+00 - 3.634984e+00, +1.739033047e-02, +2.593251981e-05, +2.668158619e-03, +1.277447128e-03, +4.041099495e+00 - 3.672458e+00, +1.649636267e-02, +6.441647584e-05, +2.489142507e-03, +1.239911039e-03, +3.857051923e+00 - 3.709932e+00, +1.558915378e-02, +1.317476718e-04, +2.508135890e-03, +1.183332706e-03, +3.618537515e+00 - 3.747406e+00, +1.506771922e-02, +1.827996373e-04, +2.464165400e-03, +1.139564839e-03, +3.441274021e+00 - 3.784880e+00, +1.497043510e-02, +2.583602594e-04, +2.398976769e-03, +1.138677928e-03, +3.374125919e+00 - 3.822354e+00, +1.509382884e-02, +2.639451574e-04, +2.467045163e-03, +1.167383022e-03, +3.390782747e+00 - 3.859828e+00, +1.522187392e-02, +3.441247598e-04, +2.310794739e-03, +1.195800440e-03, +3.433261195e+00 - 3.897302e+00, +1.524725623e-02, +2.580789787e-04, +2.433286221e-03, +1.202259383e-03, +3.458316390e+00 - 3.934776e+00, +1.516738767e-02, +3.251563904e-04, +2.371467061e-03, +1.197977832e-03, +3.452477391e+00 - 3.972250e+00, +1.503328590e-02, +2.641111457e-04, +2.352762597e-03, +1.181160532e-03, +3.423913070e+00 - 4.009724e+00, +1.489865194e-02, +1.880624459e-04, +2.387811628e-03, +1.164102973e-03, +3.388071423e+00 - 4.047198e+00, +1.478547828e-02, +8.550478354e-05, +2.233912073e-03, +1.149898734e-03, +3.356481759e+00 - 4.084672e+00, +1.468099176e-02, +7.081310759e-05, +2.362301588e-03, +1.134568267e-03, +3.331811122e+00 - 4.122146e+00, +1.456012095e-02, +1.000589609e-04, +2.276136299e-03, +1.120143566e-03, +3.308561286e+00 - 4.159620e+00, +1.440061914e-02, +1.123758522e-04, +2.295370230e-03, +1.096086058e-03, +3.277077389e+00 - 4.197094e+00, +1.417920979e-02, +1.041989270e-04, +2.302534672e-03, +1.066757394e-03, +3.228672040e+00 - 4.234568e+00, +1.389532622e-02, +8.128856513e-05, +2.187418699e-03, +1.029125720e-03, +3.160462176e+00 - 4.272043e+00, +1.358279022e-02, +5.561281064e-05, +2.284829119e-03, +9.884369661e-04, +3.078621207e+00 - 4.309517e+00, +1.328300870e-02, +3.240403157e-05, +2.174652578e-03, +9.511227346e-04, +2.998083103e+00 - 4.346991e+00, +1.303013392e-02, +1.734442280e-05, +2.230526466e-03, +9.130664661e-04, +2.936632045e+00 - 4.384465e+00, +1.284238151e-02, +1.317936068e-05, +2.214435654e-03, +8.833845738e-04, +2.903960890e+00 - 4.421939e+00, +1.269274612e-02, +1.435346878e-05, +2.169259921e-03, +8.602679546e-04, +2.891846173e+00 - 4.459413e+00, +1.250769470e-02, +1.383626335e-05, +2.240466421e-03, +8.338864541e-04, +2.873914205e+00 - 4.496887e+00, +1.220347711e-02, +1.135949579e-05, +2.105010524e-03, +7.956943879e-04, +2.816758998e+00 - 4.534361e+00, +1.171453920e-02, +8.193555952e-06, +2.144672476e-03, +7.462399314e-04, +2.695681837e+00 - 4.571835e+00, +1.102237397e-02, +4.675370098e-06, +2.130475113e-03, +6.893655959e-04, +2.507645693e+00 - 4.609309e+00, +1.018197664e-02, +2.900858519e-06, +1.900309735e-03, +6.269664928e-04, +2.277803991e+00 - 4.646783e+00, +9.322403390e-03, +1.537370332e-06, +1.690064076e-03, +5.675653591e-04, +2.056633148e+00 - 4.684257e+00, +8.612482540e-03, +2.177024166e-06, +1.777449120e-03, +5.126679272e-04, +1.900910688e+00 - 4.721731e+00, +8.172956269e-03, +1.597348476e-07, +1.698925799e-03, +4.697649227e-04, +1.836469626e+00 - 4.759205e+00, +7.974254164e-03, +1.376759447e-06, +1.775601935e-03, +4.478961087e-04, +1.832767372e+00 - 4.796679e+00, +7.844595933e-03, +1.486059100e-06, +1.758174419e-03, +4.293124594e-04, +1.825304967e+00 - 4.834153e+00, +7.577377359e-03, +9.019852725e-07, +1.674581099e-03, +4.004209283e-04, +1.758563482e+00 - 4.871627e+00, +7.029320252e-03, +9.009408167e-07, +1.762912950e-03, +3.576671923e-04, +1.608482314e+00 - 4.909101e+00, +6.179568725e-03, +2.593617651e-07, +1.642024118e-03, +3.075982349e-04, +1.384441804e+00 - 4.946576e+00, +5.135873059e-03, +1.084927890e-06, +1.344862366e-03, +2.560184412e-04, +1.123464235e+00 - 4.984050e+00, +4.102281426e-03, +6.009974120e-07, +9.379570169e-04, +2.098103295e-04, +8.825893277e-01 - 5.021524e+00, +3.333768130e-03, +6.069854336e-07, +7.160525289e-04, +1.710763679e-04, +7.236586428e-01 - 5.058998e+00, +2.997347813e-03, +6.062333039e-07, +7.063083875e-04, +1.480832886e-04, +6.701071985e-01 - 5.096472e+00, +2.966082582e-03, +6.251374827e-08, +7.358345835e-04, +1.447742713e-04, +6.737184634e-01 - 5.133946e+00, +2.951114323e-03, +7.366551806e-07, +6.861277482e-04, +1.388125984e-04, +6.688498009e-01 - 5.171420e+00, +2.775008111e-03, +5.174370339e-07, +7.341489445e-04, +1.241847277e-04, +6.231396120e-01 - 5.208894e+00, +2.405717198e-03, +9.223463232e-07, +6.960389026e-04, +1.031301521e-04, +5.351006016e-01 - 5.246368e+00, +1.901056861e-03, +1.335454083e-06, +5.740866334e-04, +8.197917225e-05, +4.200215275e-01 - 5.283842e+00, +1.365329913e-03, +2.044962887e-06, +4.072460566e-04, +6.179777067e-05, +3.000669782e-01 - 5.321316e+00, +9.166623131e-04, +2.100458062e-06, +2.358018741e-04, +4.553893307e-05, +1.991688029e-01 - 5.358790e+00, +6.691344192e-04, +2.913257371e-06, +1.451168458e-04, +3.589872006e-05, +1.401126843e-01 - 5.396264e+00, +6.255466124e-04, +2.630196304e-06, +1.617746153e-04, +3.315845096e-05, +1.268528827e-01 - 5.433738e+00, +6.319797684e-04, +1.826400061e-06, +1.543450588e-04, +3.227005096e-05, +1.297292222e-01 - 5.471212e+00, +5.973080640e-04, +2.467362248e-06, +1.563021036e-04, +2.989323370e-05, +1.254260012e-01 - 5.508686e+00, +5.114085168e-04, +6.749695340e-07, +1.465183244e-04, +2.551092103e-05, +1.099715010e-01 - 5.546160e+00, +3.953211408e-04, +2.267292894e-06, +1.178128914e-04, +2.135373909e-05, +8.751561585e-02 - 5.583635e+00, +2.798970741e-04, +4.114592451e-06, +8.187619566e-05, +1.730379474e-05, +6.404749111e-02 - 5.621109e+00, +1.960285336e-04, +5.362207249e-06, +4.847757778e-05, +1.414317116e-05, +4.499882977e-02 - 5.658583e+00, +1.558666654e-04, +5.386080396e-06, +2.414455282e-05, +1.262666437e-05, +3.378351251e-02 - 5.696057e+00, +1.399319971e-04, +2.380848476e-06, +2.098415500e-05, +1.112674671e-05, +2.942388680e-02 - 5.733531e+00, +1.307437706e-04, +1.766280363e-06, +1.905675918e-05, +1.024969719e-05, +2.730232945e-02 - 5.771005e+00, +1.156967022e-04, +2.179098966e-06, +1.773792113e-05, +9.011591185e-06, +2.433933672e-02 - 5.808479e+00, +9.613916015e-05, +1.829222517e-06, +1.539190377e-05, +7.488412599e-06, +2.048163582e-02 - 5.845953e+00, +7.929064937e-05, +2.024222810e-06, +1.269795076e-05, +5.983825506e-06, +1.785036816e-02 - 5.883427e+00, +8.026728622e-05, +9.612655320e-07, +1.395848839e-05, +5.767006684e-06, +1.905047360e-02 - 5.920901e+00, +9.786066890e-05, +1.616263390e-06, +1.884901042e-05, +6.916660805e-06, +2.355153006e-02 - 5.958375e+00, +1.193864783e-04, +1.249289289e-06, +2.379396756e-05, +7.728439405e-06, +2.860371125e-02 - 5.995849e+00, +1.345201914e-04, +4.088511303e-07, +2.683752440e-05, +8.277988716e-06, +3.211074890e-02 - 6.033323e+00, +1.384647182e-04, +1.148339916e-06, +2.874751155e-05, +8.447988514e-06, +3.288978387e-02 - 6.070797e+00, +1.298738806e-04, +8.929752626e-07, +2.752913259e-05, +7.815986825e-06, +3.044108952e-02 - 6.108271e+00, +1.055505567e-04, +7.319142990e-07, +2.263722024e-05, +6.213020788e-06, +2.486884038e-02 - 6.145745e+00, +7.166703123e-05, +3.366496572e-07, +1.556595803e-05, +4.216763327e-06, +1.688201114e-02 - 6.183219e+00, +3.493517869e-05, +6.909111035e-07, +6.894011008e-06, +2.378664019e-06, +8.222464850e-03 - 6.220694e+00, +3.315980737e-05, +7.955861069e-07, +5.819273465e-06, +2.558211822e-06, +7.112352809e-03 - 6.258168e+00, +6.560551270e-05, +4.917076246e-07, +1.166288699e-05, +4.572165491e-06, +1.487454424e-02 - 6.295642e+00, +9.613850820e-05, +3.025743178e-07, +1.870489049e-05, +6.344660049e-06, +2.195967174e-02 - 6.333116e+00, +1.144656347e-04, +1.042577705e-06, +2.279021770e-05, +7.500785919e-06, +2.658824072e-02 - 6.370590e+00, +1.200142843e-04, +7.525397926e-07, +2.365592156e-05, +7.845707667e-06, +2.836687987e-02 - 6.408064e+00, +1.156943204e-04, +2.734515578e-07, +2.199714989e-05, +7.613771837e-06, +2.754736754e-02 - 6.445538e+00, +1.028589941e-04, +1.131230488e-06, +1.895496169e-05, +7.067578951e-06, +2.492401183e-02 - 6.483012e+00, +8.998476195e-05, +1.312943251e-06, +1.561633829e-05, +6.506193902e-06, +2.173369821e-02 - 6.520486e+00, +8.174691766e-05, +7.018622596e-07, +1.355662912e-05, +5.905477292e-06, +1.939616532e-02 - 6.557960e+00, +8.194147886e-05, +1.147425739e-06, +1.345797658e-05, +5.929276191e-06, +1.881263997e-02 - 6.595434e+00, +8.615558632e-05, +1.631522428e-06, +1.474983448e-05, +6.453523240e-06, +1.965680357e-02 - 6.632908e+00, +9.099245733e-05, +8.959004766e-07, +1.490351472e-05, +6.947049143e-06, +2.089060758e-02 - 6.670382e+00, +9.357109056e-05, +7.047876828e-07, +1.461727660e-05, +7.176279953e-06, +2.174333892e-02 - 6.707856e+00, +9.372354548e-05, +2.228534400e-06, +1.517600551e-05, +7.306445383e-06, +2.197662468e-02 - 6.745330e+00, +9.228327205e-05, +1.665893489e-06, +1.466430309e-05, +7.167443892e-06, +2.172186153e-02 - 6.782804e+00, +9.037440491e-05, +2.158474481e-06, +1.484760547e-05, +7.053754296e-06, +2.125715216e-02 - 6.820278e+00, +8.895020888e-05, +1.201712459e-06, +1.457706151e-05, +6.904806409e-06, +2.082496930e-02 - 6.857752e+00, +8.797395582e-05, +2.175566004e-07, +1.413235007e-05, +6.696578320e-06, +2.053509651e-02 - 6.895227e+00, +8.818415187e-05, +1.870305267e-06, +1.465944635e-05, +6.823121947e-06, +2.036976733e-02 - 6.932701e+00, +8.652482398e-05, +1.297190621e-06, +1.394898637e-05, +6.590002143e-06, +2.025681153e-02 - 6.970175e+00, +8.654726097e-05, +1.302455999e-06, +1.430750605e-05, +6.647337445e-06, +2.014092972e-02 - 7.007649e+00, +8.599023697e-05, +1.632653526e-06, +1.404656976e-05, +6.560173926e-06, +2.000770004e-02 - 7.045123e+00, +8.507094701e-05, +8.018955482e-07, +1.393717929e-05, +6.460613964e-06, +1.987163031e-02 - 7.082597e+00, +8.507173670e-05, +6.272024716e-07, +1.428921333e-05, +6.385954443e-06, +1.974790284e-02 - 7.120071e+00, +8.346018328e-05, +2.475724018e-07, +1.325648636e-05, +6.243644764e-06, +1.962790928e-02 - 7.157545e+00, +8.421307523e-05, +4.394951594e-07, +1.390228436e-05, +6.285879059e-06, +1.947211778e-02 - 7.195019e+00, +8.170806833e-05, +2.014984360e-07, +1.366106331e-05, +5.956409962e-06, +1.921995461e-02 - 7.232493e+00, +8.013386787e-05, +7.122899917e-07, +1.367002400e-05, +5.816296736e-06, +1.881509895e-02 - 7.269967e+00, +7.930240825e-05, +2.115088068e-07, +1.370943668e-05, +5.595438478e-06, +1.823254307e-02 - 7.307441e+00, +7.425857303e-05, +1.161750331e-07, +1.277195401e-05, +5.156725539e-06, +1.750631517e-02 - 7.344915e+00, +7.288753011e-05, +2.559503828e-07, +1.356922915e-05, +5.000841773e-06, +1.673913653e-02 - 7.382389e+00, +6.980444151e-05, +4.240898323e-07, +1.302044783e-05, +4.767250185e-06, +1.607581245e-02 - 7.419863e+00, +6.858350498e-05, +4.111819579e-07, +1.330329130e-05, +4.515920254e-06, +1.564415606e-02 - 7.457337e+00, +6.677209617e-05, +1.872692108e-07, +1.314575231e-05, +4.251436669e-06, +1.546991691e-02 - 7.494811e+00, +6.533714996e-05, +1.918803774e-07, +1.293346135e-05, +4.171945397e-06, +1.543440789e-02 + 3.747406e-02, +5.572991711e-02, +4.780005949e-15, +1.407880294e-02, +1.856308518e-03, +1.015013075e-04 + 7.494811e-02, +1.034912856e-01, +9.732567615e-15, +2.607229698e-02, +3.523283343e-03, +1.286451027e-04 + 1.124222e-01, +3.324496388e-02, +6.375326658e-15, +8.388885354e-03, +1.164706330e-03, +1.363415075e-04 + 1.498962e-01, +2.375714211e-02, +2.405820589e-15, +6.044798887e-03, +8.245600593e-04, +2.237425263e-04 + 1.873703e-01, +3.819179106e-02, +5.021614958e-15, +9.668211449e-03, +1.299554229e-03, +5.988175109e-04 + 2.248443e-01, +1.297340613e-02, +1.776716125e-15, +3.303352868e-03, +4.708062400e-04, +1.231303285e-03 + 2.623184e-01, +5.246709260e-03, +2.385106704e-15, +1.405616340e-03, +2.071756999e-04, +2.183486767e-03 + 2.997925e-01, +6.058354891e-03, +3.688072737e-15, +1.493506908e-03, +2.376336055e-04, +3.443837437e-03 + 3.372665e-01, +6.156851514e-03, +4.075182156e-15, +1.749181241e-03, +2.239413758e-04, +4.896408733e-03 + 3.747406e-01, +4.694296650e-03, +8.972823355e-15, +1.289674681e-03, +1.823298192e-04, +6.288158078e-03 + 4.122146e-01, +5.417259349e-03, +1.948301910e-15, +1.436619200e-03, +2.081413126e-04, +7.236866239e-03 + 4.496887e-01, +7.374336957e-03, +1.897079415e-14, +1.846629384e-03, +2.874502795e-04, +7.438562524e-03 + 4.871627e-01, +6.930576439e-03, +2.279830021e-14, +1.752840045e-03, +2.931825986e-04, +7.655638548e-03 + 5.246368e-01, +5.525646929e-03, +3.091423690e-14, +1.406681124e-03, +2.243733865e-04, +1.144492786e-02 + 5.621109e-01, +4.673784425e-03, +3.121350320e-14, +1.348460932e-03, +1.868352062e-04, +2.129721915e-02 + 5.995849e-01, +4.253659682e-03, +1.861559471e-14, +1.368615134e-03, +1.587836337e-04, +3.669178320e-02 + 6.370590e-01, +4.178197277e-03, +5.274962070e-14, +1.375975768e-03, +1.455795304e-04, +5.641443987e-02 + 6.745330e-01, +4.365910098e-03, +2.726784104e-14, +1.376592666e-03, +1.531072976e-04, +7.828637698e-02 + 7.120071e-01, +4.532195073e-03, +8.525225747e-14, +1.342611942e-03, +1.672385806e-04, +9.885177909e-02 + 7.494811e-01, +4.713112853e-03, +1.941937091e-13, +1.277228489e-03, +1.799446727e-04, +1.137531996e-01 + 7.869552e-01, +4.875763465e-03, +3.094044938e-13, +1.187769851e-03, +1.938215810e-04, +1.195346821e-01 + 8.244293e-01, +4.981324068e-03, +1.298916160e-12, +1.301107022e-03, +2.128512979e-04, +1.188340974e-01 + 8.619033e-01, +4.771406979e-03, +6.445150636e-12, +1.099709362e-03, +2.160907931e-04, +1.309553939e-01 + 8.993774e-01, +4.387618618e-03, +1.907002776e-11, +1.104046666e-03, +1.969683896e-04, +1.857760966e-01 + 9.368514e-01, +4.317746827e-03, +4.308903833e-11, +1.315698937e-03, +1.778612665e-04, +2.867307084e-01 + 9.743255e-01, +4.352783979e-03, +9.324588533e-11, +1.367694242e-03, +1.732486187e-04, +4.167673658e-01 + 1.011800e+00, +4.392520145e-03, +2.030563517e-10, +1.347497137e-03, +1.702743984e-04, +5.549049357e-01 + 1.049274e+00, +4.412402764e-03, +4.369999452e-10, +1.276609666e-03, +1.748952068e-04, +6.782106751e-01 + 1.086748e+00, +4.409231341e-03, +8.999345643e-10, +1.151820795e-03, +1.841348285e-04, +7.648957645e-01 + 1.124222e+00, +4.385361349e-03, +1.700272247e-09, +1.054162064e-03, +1.931535896e-04, +8.022489576e-01 + 1.161696e+00, +4.359684491e-03, +2.684499850e-09, +1.097370972e-03, +2.015036618e-04, +8.009704160e-01 + 1.199170e+00, +4.376985093e-03, +2.607577158e-09, +9.962793897e-04, +2.076058414e-04, +8.143229074e-01 + 1.236644e+00, +4.452602826e-03, +3.312088983e-09, +9.301914660e-04, +2.236743467e-04, +9.294380600e-01 + 1.274118e+00, +4.490626318e-03, +1.499013876e-08, +1.141847583e-03, +2.193639410e-04, +1.185764957e+00 + 1.311592e+00, +4.461869998e-03, +3.572079499e-08, +1.253108436e-03, +2.106609832e-04, +1.534641584e+00 + 1.349066e+00, +4.401936395e-03, +6.722317675e-08, +1.237832414e-03, +2.050873994e-04, +1.896337013e+00 + 1.386540e+00, +4.333203622e-03, +1.103743704e-07, +1.149104164e-03, +2.051919384e-04, +2.199936335e+00 + 1.424014e+00, +4.274237244e-03, +1.614556678e-07, +1.003500852e-03, +2.108719056e-04, +2.397678575e+00 + 1.461488e+00, +4.247523124e-03, +2.087791015e-07, +9.868601870e-04, +2.177114260e-04, +2.477217296e+00 + 1.498962e+00, +4.280032410e-03, +2.381229541e-07, +9.955121378e-04, +2.271307112e-04, +2.474013830e+00 + 1.536436e+00, +4.376655811e-03, +2.787897261e-07, +9.435084780e-04, +2.375088848e-04, +2.473624477e+00 + 1.573910e+00, +4.478796048e-03, +4.348188822e-07, +9.452200051e-04, +2.535812295e-04, +2.579178548e+00 + 1.611384e+00, +4.504194805e-03, +6.949622119e-07, +8.305665204e-04, +2.598628685e-04, +2.837660610e+00 + 1.648859e+00, +4.449740742e-03, +9.686036472e-07, +9.532358385e-04, +2.571234946e-04, +3.200525096e+00 + 1.686333e+00, +4.368988747e-03, +1.222520265e-06, +9.657316748e-04, +2.553060052e-04, +3.569942692e+00 + 1.723807e+00, +4.305815658e-03, +1.463913610e-06, +9.004535416e-04, +2.557293609e-04, +3.860927182e+00 + 1.761281e+00, +4.281239183e-03, +1.695622917e-06, +8.111211761e-04, +2.616688934e-04, +4.031057848e+00 + 1.798755e+00, +4.299576106e-03, +1.914307365e-06, +8.541141264e-04, +2.692992642e-04, +4.086189106e+00 + 1.836229e+00, +4.350195471e-03, +2.265209390e-06, +8.333271263e-04, +2.776670855e-04, +4.072082288e+00 + 1.873703e+00, +4.406123355e-03, +3.259183597e-06, +8.321799456e-04, +2.866860335e-04, +4.053069427e+00 + 1.911177e+00, +4.436275347e-03, +5.260285777e-06, +8.329962539e-04, +2.954441289e-04, +4.080890391e+00 + 1.948651e+00, +4.428710083e-03, +8.002413959e-06, +7.711775732e-04, +3.026474470e-04, +4.169120342e+00 + 1.986125e+00, +4.399248505e-03, +1.094520855e-05, +7.863124125e-04, +3.055255888e-04, +4.292682833e+00 + 2.023599e+00, +4.372255611e-03, +1.349310551e-05, +7.323874733e-04, +3.097794542e-04, +4.410898850e+00 + 2.061073e+00, +4.361315731e-03, +1.494909302e-05, +7.328463050e-04, +3.146496395e-04, +4.492460625e+00 + 2.098547e+00, +4.367030285e-03, +1.457796662e-05, +7.247211296e-04, +3.200442587e-04, +4.527157525e+00 + 2.136021e+00, +4.382374080e-03, +1.253784498e-05, +6.970316424e-04, +3.266236440e-04, +4.524046965e+00 + 2.173495e+00, +4.397034295e-03, +1.351026325e-05, +7.270181577e-04, +3.307469479e-04, +4.501957938e+00 + 2.210969e+00, +4.403170587e-03, +2.415199665e-05, +6.914860835e-04, +3.348204316e-04, +4.478434139e+00 + 2.248443e+00, +4.401493826e-03, +4.171539184e-05, +7.189816263e-04, +3.371262712e-04, +4.462008482e+00 + 2.285917e+00, +4.398225001e-03, +6.192280212e-05, +7.101556969e-04, +3.401173553e-04, +4.450486620e+00 + 2.323392e+00, +4.398269519e-03, +8.037723425e-05, +6.962721148e-04, +3.425770451e-04, +4.434865564e+00 + 2.360866e+00, +4.404017255e-03, +9.201831682e-05, +7.190225781e-04, +3.439997305e-04, +4.406388329e+00 + 2.398340e+00, +4.414405092e-03, +9.158558418e-05, +6.788819331e-04, +3.452286529e-04, +4.363973589e+00 + 2.435814e+00, +4.421143782e-03, +7.441843177e-05, +7.220381311e-04, +3.432555218e-04, +4.319369685e+00 + 2.473288e+00, +4.412692077e-03, +4.282061526e-05, +7.068656107e-04, +3.418175535e-04, +4.296269972e+00 + 2.510762e+00, +4.386278801e-03, +5.105127318e-05, +7.009204519e-04, +3.404021990e-04, +4.319012990e+00 + 2.548236e+00, +4.353215251e-03, +4.129387046e-05, +7.011585940e-04, +3.393297532e-04, +4.391832454e+00 + 2.585710e+00, +4.333456514e-03, +1.888982387e-05, +6.505948543e-04, +3.367477514e-04, +4.481474420e+00 + 2.623184e+00, +4.344230010e-03, +2.207235676e-05, +6.844735434e-04, +3.343510978e-04, +4.519833729e+00 + 2.660658e+00, +4.391992250e-03, +2.692671046e-05, +6.743976743e-04, +3.352654771e-04, +4.429683004e+00 + 2.698132e+00, +4.471042289e-03, +2.563970053e-05, +6.925965452e-04, +3.381059664e-04, +4.164305386e+00 + 2.735606e+00, +4.553202267e-03, +3.496086120e-05, +7.419389256e-04, +3.407149926e-04, +3.755743891e+00 + 2.773080e+00, +4.565896612e-03, +2.812102842e-05, +7.106904383e-04, +3.324536761e-04, +3.369620795e+00 + 2.810554e+00, +4.424048922e-03, +2.205191071e-05, +7.354060681e-04, +3.099287774e-04, +3.300078313e+00 + 2.848028e+00, +4.245521437e-03, +1.165298270e-05, +7.127638729e-04, +2.974966986e-04, +3.715663123e+00 + 2.885502e+00, +4.195356915e-03, +4.545975177e-06, +7.596043263e-04, +2.830996949e-04, +4.412696644e+00 + 2.922976e+00, +4.234424822e-03, +1.477205397e-06, +7.970248517e-04, +2.750210888e-04, +5.035062025e+00 + 2.960451e+00, +4.302783352e-03, +1.800762190e-06, +8.562397769e-04, +2.715342841e-04, +5.296751662e+00 + 2.997925e+00, +4.375166831e-03, +2.414631292e-06, +8.927687994e-04, +2.716622273e-04, +5.025294040e+00 + 3.035399e+00, +4.448792499e-03, +2.550087762e-06, +9.171595987e-04, +2.741412579e-04, +4.165707984e+00 + 3.072873e+00, +4.547392172e-03, +2.930191671e-06, +9.339070315e-04, +2.807584580e-04, +2.782735535e+00 + 3.110347e+00, +4.822285737e-03, +4.196452250e-06, +9.011497982e-04, +3.124124847e-04, +1.091908737e+00 + 3.147821e+00, +4.205292450e-03, +1.544693998e-06, +7.966267592e-04, +2.677084155e-04, +1.087897791e+00 + 3.185295e+00, +4.322357444e-03, +8.097038803e-07, +9.017757419e-04, +2.603103176e-04, +2.795340363e+00 + 3.222769e+00, +4.355601912e-03, +8.154563458e-07, +9.178576606e-04, +2.603565288e-04, +4.198205341e+00 + 3.260243e+00, +4.349122198e-03, +7.337744842e-07, +9.173532313e-04, +2.596715998e-04, +5.054782764e+00 + 3.297717e+00, +4.314425000e-03, +1.030321438e-06, +9.050681552e-04, +2.587874217e-04, +5.270954600e+00 + 3.335191e+00, +4.256320894e-03, +2.230359642e-06, +8.757309727e-04, +2.592544095e-04, +4.875664359e+00 + 3.372665e+00, +4.181108331e-03, +4.831601574e-06, +8.081073485e-04, +2.628800519e-04, +4.026124476e+00 + 3.410139e+00, +4.150992695e-03, +9.956774975e-06, +7.711844066e-04, +2.781228495e-04, +3.029466121e+00 + 3.447613e+00, +4.374155513e-03, +1.534747727e-05, +7.729163360e-04, +2.966312352e-04, +2.402804105e+00 + 3.485087e+00, +4.636173585e-03, +1.986006357e-05, +7.649793279e-04, +3.310044682e-04, +2.582857071e+00 + 3.522561e+00, +4.601086721e-03, +1.325948217e-05, +7.533876615e-04, +3.311061739e-04, +3.207230086e+00 + 3.560035e+00, +4.483565724e-03, +1.152020770e-05, +7.012870277e-04, +3.221707976e-04, +3.758897835e+00 + 3.597509e+00, +4.377095997e-03, +9.395424665e-06, +6.965628121e-04, +3.170519048e-04, +4.038904116e+00 + 3.634984e+00, +4.303366073e-03, +6.417194093e-06, +6.602556115e-04, +3.161137532e-04, +4.041099495e+00 + 3.672458e+00, +4.276935597e-03, +1.670096154e-05, +6.453484569e-04, +3.214659963e-04, +3.857051923e+00 + 3.709932e+00, +4.308136565e-03, +3.640909381e-05, +6.931352459e-04, +3.270196041e-04, +3.618537515e+00 + 3.747406e+00, +4.378529326e-03, +5.311975627e-05, +7.160619539e-04, +3.311462070e-04, +3.441274021e+00 + 3.784880e+00, +4.436833558e-03, +7.657101887e-05, +7.109920692e-04, +3.374734541e-04, +3.374125919e+00 + 3.822354e+00, +4.451429057e-03, +7.784195482e-05, +7.275739401e-04, +3.442812792e-04, +3.390782747e+00 + 3.859828e+00, +4.433648665e-03, +1.002326186e-04, +6.730611530e-04, +3.482987084e-04, +3.433261195e+00 + 3.897302e+00, +4.408866776e-03, +7.462561248e-05, +7.036042823e-04, +3.476429706e-04, +3.458316390e+00 + 3.934776e+00, +4.393189571e-03, +9.418059948e-05, +6.868885127e-04, +3.469907827e-04, +3.452477391e+00 + 3.972250e+00, +4.390673944e-03, +7.713722291e-05, +6.871560546e-04, +3.449738669e-04, +3.423913070e+00 + 4.009724e+00, +4.397384258e-03, +5.550722591e-05, +7.047701569e-04, +3.435886757e-04, +3.388071423e+00 + 4.047198e+00, +4.405052475e-03, +2.547452650e-05, +6.655516797e-04, +3.425904911e-04, +3.356481759e+00 + 4.084672e+00, +4.406309729e-03, +2.125363804e-05, +7.090142573e-04, +3.405259860e-04, +3.331811122e+00 + 4.122146e+00, +4.400740895e-03, +3.024243840e-05, +6.879534947e-04, +3.385591106e-04, +3.308561286e+00 + 4.159620e+00, +4.394348204e-03, +3.429148562e-05, +7.004321099e-04, +3.344706053e-04, +3.277077389e+00 + 4.197094e+00, +4.391653785e-03, +3.227299822e-05, +7.131522322e-04, +3.304012861e-04, +3.228672040e+00 + 4.234568e+00, +4.396612091e-03, +2.572046764e-05, +6.921198792e-04, +3.256250709e-04, +3.160462176e+00 + 4.272043e+00, +4.411971887e-03, +1.806419397e-05, +7.421598715e-04, +3.210648208e-04, +3.078621207e+00 + 4.309517e+00, +4.430500503e-03, +1.080824996e-05, +7.253476649e-04, +3.172436193e-04, +2.998083103e+00 + 4.346991e+00, +4.437101320e-03, +5.906229494e-06, +7.595525869e-04, +3.109230071e-04, +2.936632045e+00 + 4.384465e+00, +4.422367240e-03, +4.538408465e-06, +7.625569826e-04, +3.041998868e-04, +2.903960890e+00 + 4.421939e+00, +4.389149824e-03, +4.963427487e-06, +7.501297757e-04, +2.974805377e-04, +2.891846173e+00 + 4.459413e+00, +4.352146169e-03, +4.814431594e-06, +7.795870932e-04, +2.901570452e-04, +2.873914205e+00 + 4.496887e+00, +4.332453405e-03, +4.032824887e-06, +7.473165171e-04, +2.824857890e-04, +2.816758998e+00 + 4.534361e+00, +4.345668334e-03, +3.039511503e-06, +7.955955506e-04, +2.768278960e-04, +2.695681837e+00 + 4.571835e+00, +4.395506909e-03, +1.864446046e-06, +8.495917580e-04, +2.749055011e-04, +2.507645693e+00 + 4.609309e+00, +4.470084644e-03, +1.273532986e-06, +8.342727217e-04, +2.752504146e-04, +2.277803991e+00 + 4.646783e+00, +4.532846997e-03, +7.475180167e-07, +8.217625380e-04, +2.759682054e-04, +2.056633148e+00 + 4.684257e+00, +4.530713933e-03, +1.145253262e-06, +9.350513578e-04, +2.696959570e-04, +1.900910688e+00 + 4.721731e+00, +4.450362889e-03, +8.697930276e-08, +9.251042187e-04, +2.557978178e-04, +1.836469626e+00 + 4.759205e+00, +4.350936341e-03, +7.511915959e-07, +9.688092237e-04, +2.443824107e-04, +1.832767372e+00 + 4.796679e+00, +4.297690565e-03, +8.141429116e-07, +9.632222838e-04, +2.352004006e-04, +1.825304967e+00 + 4.834153e+00, +4.308844940e-03, +5.129102713e-07, +9.522437582e-04, +2.276977388e-04, +1.758563482e+00 + 4.871627e+00, +4.370157006e-03, +5.601185720e-07, +1.096010155e-03, +2.223631490e-04, +1.608482314e+00 + 4.909101e+00, +4.463581427e-03, +1.873403160e-07, +1.186054995e-03, +2.221821343e-04, +1.384441804e+00 + 4.946576e+00, +4.571461109e-03, +9.656986454e-07, +1.197067360e-03, +2.278830364e-04, +1.123464235e+00 + 4.984050e+00, +4.648007060e-03, +6.809479711e-07, +1.062733241e-03, +2.377213534e-04, +8.825893277e-01 + 5.021524e+00, +4.606824174e-03, +8.387731420e-07, +9.894893622e-04, +2.364047878e-04, +7.236586428e-01 + 5.058998e+00, +4.472937792e-03, +9.046810798e-07, +1.054022982e-03, +2.209844767e-04, +6.701071985e-01 + 5.096472e+00, +4.402554988e-03, +9.278912730e-08, +1.092198928e-03, +2.148883831e-04, +6.737184634e-01 + 5.133946e+00, +4.412222773e-03, +1.101376093e-06, +1.025832328e-03, +2.075392685e-04, +6.688498009e-01 + 5.171420e+00, +4.453268669e-03, +8.303709536e-07, +1.178145203e-03, +1.992887714e-04, +6.231396120e-01 + 5.208894e+00, +4.495822263e-03, +1.723687696e-06, +1.300762699e-03, +1.927303984e-04, +5.351006016e-01 + 5.246368e+00, +4.526093870e-03, +3.179489612e-06, +1.366802880e-03, +1.951785013e-04, +4.200215275e-01 + 5.283842e+00, +4.550083856e-03, +6.815021432e-06, +1.357183850e-03, +2.059465892e-04, +3.000669782e-01 + 5.321316e+00, +4.602439236e-03, +1.054611983e-05, +1.183929765e-03, +2.286449103e-04, +1.991688029e-01 + 5.358790e+00, +4.775687674e-03, +2.079224580e-05, +1.035715264e-03, +2.562132061e-04, +1.401126843e-01 + 5.396264e+00, +4.931276288e-03, +2.073422573e-05, +1.275293173e-03, +2.613929637e-04, +1.268528827e-01 + 5.433738e+00, +4.871529774e-03, +1.407855555e-05, +1.189747816e-03, +2.487492827e-04, +1.297292222e-01 + 5.471212e+00, +4.762234770e-03, +1.967185611e-05, +1.246169870e-03, +2.383336263e-04, +1.254260012e-01 + 5.508686e+00, +4.650373164e-03, +6.137676836e-06, +1.332329950e-03, +2.319775652e-04, +1.099715010e-01 + 5.546160e+00, +4.517149734e-03, +2.590729520e-05, +1.346192794e-03, +2.439991867e-04, +8.751561585e-02 + 5.583635e+00, +4.370148920e-03, +6.424283573e-05, +1.278366947e-03, +2.701713125e-04, +6.404749111e-02 + 5.621109e+00, +4.356302923e-03, +1.191632599e-04, +1.077307522e-03, +3.143008659e-04, +4.499882977e-02 + 5.658583e+00, +4.613690342e-03, +1.594292599e-04, +7.146845024e-04, +3.737522664e-04, +3.378351251e-02 + 5.696057e+00, +4.755727822e-03, +8.091549876e-05, +7.131673371e-04, +3.781535315e-04, +2.942388680e-02 + 5.733531e+00, +4.788740493e-03, +6.469339425e-05, +6.979902289e-04, +3.754147501e-04, +2.730232945e-02 + 5.771005e+00, +4.753486242e-03, +8.952992399e-05, +7.287758631e-04, +3.702480182e-04, +2.433933672e-02 + 5.808479e+00, +4.693920007e-03, +8.931037214e-05, +7.514977764e-04, +3.656159433e-04, +2.048163582e-02 + 5.845953e+00, +4.441961570e-03, +1.133994992e-04, +7.113551187e-04, +3.352214057e-04, +1.785036816e-02 + 5.883427e+00, +4.213401089e-03, +5.045887846e-05, +7.327108334e-04, +3.027224837e-04, +1.905047360e-02 + 5.920901e+00, +4.155172452e-03, +6.862668313e-05, +8.003306103e-04, +2.936820151e-04, +2.355153006e-02 + 5.958375e+00, +4.173810775e-03, +4.367577612e-05, +8.318489635e-04, +2.701900930e-04, +2.860371125e-02 + 5.995849e+00, +4.189257368e-03, +1.273253176e-05, +8.357800836e-04, +2.577949440e-04, +3.211074890e-02 + 6.033323e+00, +4.209961328e-03, +3.491479059e-05, +8.740559580e-04, +2.568575260e-04, +3.288978387e-02 + 6.070797e+00, +4.266400536e-03, +2.933453686e-05, +9.043412384e-04, +2.567577885e-04, +3.044108952e-02 + 6.108271e+00, +4.244289444e-03, +2.943097819e-05, +9.102644066e-04, +2.498315439e-04, +2.486884038e-02 + 6.145745e+00, +4.245171421e-03, +1.994132419e-05, +9.220440563e-04, +2.497784945e-04, +1.688201114e-02 + 6.183219e+00, +4.248747709e-03, +8.402724927e-05, +8.384360570e-04, +2.892884387e-04, +8.222464850e-03 + 6.220694e+00, +4.662283813e-03, +1.118597640e-04, +8.181924633e-04, +3.596857314e-04, +7.112352809e-03 + 6.258168e+00, +4.410589773e-03, +3.305698761e-05, +7.840836535e-04, +3.073818879e-04, +1.487454424e-02 + 6.295642e+00, +4.377957436e-03, +1.377863574e-05, +8.517837018e-04, +2.889232646e-04, +2.195967174e-02 + 6.333116e+00, +4.305122550e-03, +3.921198533e-05, +8.571540306e-04, +2.821091473e-04, +2.658824072e-02 + 6.370590e+00, +4.230789036e-03, +2.652881798e-05, +8.339275124e-04, +2.765798602e-04, +2.836687987e-02 + 6.408064e+00, +4.199832172e-03, +9.926594891e-06, +7.985209427e-04, +2.763883637e-04, +2.754736754e-02 + 6.445538e+00, +4.126903599e-03, +4.538717506e-05, +7.605100582e-04, +2.835650616e-04, +2.492401183e-02 + 6.483012e+00, +4.140333646e-03, +6.041048507e-05, +7.185311096e-04, +2.993597242e-04, +2.173369821e-02 + 6.520486e+00, +4.214591715e-03, +3.618561958e-05, +6.989334694e-04, +3.044662280e-04, +1.939616532e-02 + 6.557960e+00, +4.355660821e-03, +6.099227654e-05, +7.153688477e-04, +3.151751270e-04, +1.881263997e-02 + 6.595434e+00, +4.382990654e-03, +8.300039336e-05, +7.503679033e-04, +3.283099014e-04, +1.965680357e-02 + 6.632908e+00, +4.355663519e-03, +4.288532409e-05, +7.134074328e-04, +3.325441405e-04, +2.089060758e-02 + 6.670382e+00, +4.303437062e-03, +3.241395838e-05, +6.722645795e-04, +3.300449843e-04, +2.174333892e-02 + 6.707856e+00, +4.264692455e-03, +1.014047622e-04, +6.905521540e-04, +3.324644020e-04, +2.197662468e-02 + 6.745330e+00, +4.248405317e-03, +7.669202227e-05, +6.750942166e-04, +3.299645328e-04, +2.172186153e-02 + 6.782804e+00, +4.251482241e-03, +1.015410938e-04, +6.984757583e-04, +3.318296940e-04, +2.125715216e-02 + 6.820278e+00, +4.271324850e-03, +5.770536523e-05, +6.999799761e-04, +3.315638217e-04, +2.082496930e-02 + 6.857752e+00, +4.284078031e-03, +1.059437925e-05, +6.882047065e-04, +3.261040588e-04, +2.053509651e-02 + 6.895227e+00, +4.329168343e-03, +9.181770397e-05, +7.196668532e-04, +3.349631754e-04, +2.036976733e-02 + 6.932701e+00, +4.271394037e-03, +6.403725578e-05, +6.886072050e-04, +3.253227751e-04, +2.025681153e-02 + 6.970175e+00, +4.297083709e-03, +6.466712397e-05, +7.103696923e-04, +3.300412412e-04, +2.014092972e-02 + 7.007649e+00, +4.297857164e-03, +8.160125963e-05, +7.020581941e-04, +3.278824609e-04, +2.000770004e-02 + 7.045123e+00, +4.281025044e-03, +4.035378758e-05, +7.013606369e-04, +3.251174596e-04, +1.987163031e-02 + 7.082597e+00, +4.307887142e-03, +3.176045967e-05, +7.235813063e-04, +3.233738031e-04, +1.974790284e-02 + 7.120071e+00, +4.252117844e-03, +1.261328439e-05, +6.753896286e-04, +3.181003476e-04, +1.962790928e-02 + 7.157545e+00, +4.324803094e-03, +2.257048588e-05, +7.139585186e-04, +3.228143507e-04, +1.947211778e-02 + 7.195019e+00, +4.251210265e-03, +1.048381435e-05, +7.107750035e-04, +3.099075977e-04, +1.921995461e-02 + 7.232493e+00, +4.259019210e-03, +3.785736092e-05, +7.265454218e-04, +3.091292132e-04, +1.881509895e-02 + 7.269967e+00, +4.349497924e-03, +1.160062016e-05, +7.519212558e-04, +3.068929252e-04, +1.823254307e-02 + 7.307441e+00, +4.241816299e-03, +6.636178540e-06, +7.295626682e-04, +2.945637325e-04, +1.750631517e-02 + 7.344915e+00, +4.354318396e-03, +1.529053678e-05, +8.106289789e-04, +2.987514777e-04, +1.673913653e-02 + 7.382389e+00, +4.342203028e-03, +2.638061582e-05, +8.099402667e-04, +2.965480096e-04, +1.607581245e-02 + 7.419863e+00, +4.383969626e-03, +2.628342216e-05, +8.503681027e-04, +2.886649966e-04, +1.564415606e-02 + 7.457337e+00, +4.316254351e-03, +1.210537923e-05, +8.497623089e-04, +2.748196188e-04, +1.546991691e-02 + 7.494811e+00, +4.233213896e-03, +1.243198824e-05, +8.379629100e-04, +2.703016161e-04, +1.543440789e-02 From 386f087477d90162b7d72cf6f6ba044f0cb21b58 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Thu, 5 Oct 2023 17:36:24 -0400 Subject: [PATCH 26/39] Remove inplace flux projector Mult calls --- palace/linalg/errorestimator.cpp | 2 -- palace/linalg/errorestimator.hpp | 17 ++--------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 42b0aedfc..e207fc1ef 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -92,8 +92,6 @@ FluxProjector::FluxProjector( ksp = std::make_unique(std::move(pcg), std::move(pc)); ksp->SetOperators(*M, *M); - - tmp.SetSize(fespaces.GetFinestFESpace().GetTrueVSize()); } CurlFluxErrorEstimator::CurlFluxErrorEstimator( diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 1eb613f0b..cbc52cd46 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -33,28 +33,15 @@ class FluxProjector // Linear solver and preconditioner for the projected linear system M σ = σ̂. std::unique_ptr ksp; - // Intermediate storage vector used in Mult. - mutable Vector tmp; - public: FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, double tol, int max_it, int print_level, int pa_order_threshold); - inline void Mult(Vector &x) const - { - tmp = x; - Mult(tmp, x); - } inline void Mult(const Vector &x, Vector &y) const { ksp->Mult(x, y); } - inline void Mult(ComplexVector &x) const - { - Mult(x.Real()); - Mult(x.Imag()); - } inline void Mult(const ComplexVector &x, ComplexVector &y) const { - y = x; - Mult(y); + Mult(x.Real(), y.Real()); + Mult(x.Imag(), y.Imag()); } }; From 3202fdadda42e15e14f8e25c3f67851eba9036d8 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Thu, 5 Oct 2023 17:03:33 -0400 Subject: [PATCH 27/39] Compute vector of indicators and then postprocess combined --- palace/drivers/drivensolver.cpp | 2 ++ palace/drivers/eigensolver.cpp | 25 ++++++------- palace/drivers/electrostaticsolver.cpp | 46 +++++++++++------------- palace/drivers/electrostaticsolver.hpp | 7 ++-- palace/drivers/magnetostaticsolver.cpp | 49 +++++++++++--------------- palace/drivers/magnetostaticsolver.hpp | 7 ++-- palace/drivers/transientsolver.cpp | 1 + palace/linalg/errorestimator.cpp | 6 ++-- palace/utils/errorindicators.hpp | 9 ++++- 9 files changed, 71 insertions(+), 81 deletions(-) diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 12322f3a3..e43eba945 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -152,6 +152,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator BlockTimer bt0(Timer::ESTIMATION); auto sample_indicators = estimator.ComputeIndicators(E); postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); + BlockTimer bt_post(Timer::POSTPRO); PostprocessErrorIndicators("f (GHz)", step, frequency, sample_indicators.GetPostprocessData(spaceop.GetComm())); indicators.AddIndicators(sample_indicators); @@ -281,6 +282,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator { BlockTimer bt0(Timer::ESTIMATION); auto sample_indicators = estimator.ComputeIndicators(E); + BlockTimer bt_post(Timer::POSTPRO); PostprocessErrorIndicators("f (GHz)", step, frequency, sample_indicators.GetPostprocessData(spaceop.GetComm())); indicators.AddIndicators(sample_indicators); diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index ee4e54677..cc35328a8 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -263,26 +263,16 @@ EigenSolver::Solve(const std::vector> &mesh) cons int num_conv = eigen->Solve(); SaveMetadata(*ksp); - // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators indicators; - auto UpdateErrorIndicators = - [this, &estimator, &indicators, &postop, &spaceop](const auto &E, int i) - { - BlockTimer bt0(Timer::ESTIMATION); - auto sample_indicators = estimator.ComputeIndicators(E); - postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators( - "m", i, i + 1, sample_indicators.GetPostprocessData(spaceop.GetComm())); - indicators.AddIndicators(sample_indicators); - }; - // Save the eigenvalue estimates. + BlockTimer bt_est(Timer::ESTIMATION); + std::vector indicators; + indicators.reserve(num_conv); for (int i = 0; i < iodata.solver.eigenmode.n; i++) { eigen->GetEigenvector(i, E); // Only update the error indicator for targeted modes. Mpi::Print("\nComputing error estimates for mode {:d}\n", i); - UpdateErrorIndicators(E, i); + indicators.emplace_back(estimator.ComputeIndicators(E)); } // Postprocess the results. @@ -322,6 +312,13 @@ EigenSolver::Solve(const std::vector> &mesh) cons // Postprocess the mode. Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv); + + if (i < iodata.solver.eigenmode.n) + { + postop.SetIndicatorGridFunction(indicators[i].GetLocalErrorIndicators()); + PostprocessErrorIndicators("m", i, i + 1, + indicators[i].GetPostprocessData(spaceop.GetComm())); + } } return indicators; } diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index f3cee50cf..722cf81a8 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -30,6 +30,14 @@ ElectrostaticSolver::Solve(const std::vector> &me auto K = laplaceop.GetStiffnessMatrix(); SaveMetadata(laplaceop.GetH1Spaces()); + // Construct the error estimator. + auto estimator = [&]() + { + BlockTimer bt(Timer::ESTCONSTRUCT); + return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), + laplaceop.GetH1Spaces()); + }(); + // Set up the linear solver. KspSolver ksp(iodata, laplaceop.GetH1Spaces()); ksp.SetOperators(*K, *K); @@ -43,6 +51,7 @@ ElectrostaticSolver::Solve(const std::vector> &me // Right-hand side term and solution vector storage. Vector RHS(K->Height()); std::vector V(nstep); + std::vector indicators(nstep); // Main loop over terminal boundaries. Mpi::Print("\nComputing electrostatic fields for {:d} terminal boundar{}\n", nstep, @@ -62,7 +71,10 @@ ElectrostaticSolver::Solve(const std::vector> &me BlockTimer bt1(Timer::SOLVE); ksp.Mult(RHS, V[step]); - BlockTimer bt2(Timer::POSTPRO); + BlockTimer bt2(Timer::ESTIMATION); + indicators[step] = estimator.ComputeIndicators(V[step]); + + BlockTimer bt3(Timer::POSTPRO); Mpi::Print(" Sol. ||V|| = {:.6e} (||RHS|| = {:.6e})\n", linalg::Norml2(laplaceop.GetComm(), V[step]), linalg::Norml2(laplaceop.GetComm(), RHS)); @@ -70,23 +82,17 @@ ElectrostaticSolver::Solve(const std::vector> &me // Next terminal. step++; } - auto estimator = [&]() - { - BlockTimer bt(Timer::ESTCONSTRUCT); - return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), - laplaceop.GetH1Spaces()); - }(); // Postprocess the capacitance matrix from the computed field solutions. BlockTimer bt1(Timer::POSTPRO); SaveMetadata(ksp); - return Postprocess(laplaceop, postop, estimator, V); + Postprocess(laplaceop, postop, V, indicators); + return indicators; } -ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, - PostOperator &postop, - GradFluxErrorEstimator &estimator, - const std::vector &V) const +void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, + const std::vector &V, + const std::vector &indicators) const { // Postprocess the Maxwell capacitance matrix. See p. 97 of the COMSOL AC/DC Module manual // for the associated formulas based on the electric field energy based on a unit voltage @@ -100,17 +106,6 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, int nstep = static_cast(terminal_sources.size()); mfem::DenseMatrix C(nstep), Cm(nstep); Vector E(Grad->Height()), Vij(Grad->Width()); - ErrorIndicators indicators; - auto UpdateErrorIndicators = - [this, &estimator, &indicators, &postop, &laplaceop](const auto &V, int i, double idx) - { - BlockTimer bt0(Timer::ESTIMATION); - auto sample_indicators = estimator.ComputeIndicators(V); - postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators("i", i, i + 1, - sample_indicators.GetPostprocessData(laplaceop.GetComm())); - indicators.AddIndicators(sample_indicators); - }; if (iodata.solver.electrostatic.n_post > 0) { Mpi::Print("\n"); @@ -128,8 +123,8 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostprocessDomains(postop, "i", i, idx, Ue, 0.0, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, Ue, 0.0, 1.0, 0.0); PostprocessProbes(postop, "i", i, idx); - Mpi::Print("Computing error estimates for terminal {:d}\n", idx); - UpdateErrorIndicators(V[i], i, idx); + PostprocessErrorIndicators("i", i, idx, + indicators[i].GetPostprocessData(laplaceop.GetComm())); if (i < iodata.solver.electrostatic.n_post) { PostprocessFields(postop, i, idx); @@ -169,7 +164,6 @@ ErrorIndicators ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, mfem::DenseMatrix Cinv(C); Cinv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(terminal_sources, C, Cinv, Cm); - return indicators; } void ElectrostaticSolver::PostprocessTerminals( diff --git a/palace/drivers/electrostaticsolver.hpp b/palace/drivers/electrostaticsolver.hpp index f8070553f..9b584f2de 100644 --- a/palace/drivers/electrostaticsolver.hpp +++ b/palace/drivers/electrostaticsolver.hpp @@ -24,7 +24,6 @@ namespace palace { class ErrorIndicators; -class GradFluxErrorEstimator; class IoData; class LaplaceOperator; class PostOperator; @@ -36,9 +35,9 @@ class Timer; class ElectrostaticSolver : public BaseSolver { private: - ErrorIndicators Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, - GradFluxErrorEstimator &estimator, - const std::vector &V) const; + void Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, + const std::vector &V, + const std::vector &indicators) const; void PostprocessTerminals(const std::map> &terminal_sources, const mfem::DenseMatrix &C, const mfem::DenseMatrix &Cinv, diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index b86835963..f9d844c8d 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -29,6 +29,14 @@ MagnetostaticSolver::Solve(const std::vector> &me auto K = curlcurlop.GetStiffnessMatrix(); SaveMetadata(curlcurlop.GetNDSpaces()); + // Construct the error estimator. + auto estimator = [&]() + { + BlockTimer bt(Timer::ESTCONSTRUCT); + return CurlFluxErrorEstimator(iodata, curlcurlop.GetMaterialOp(), + curlcurlop.GetNDSpaces()); + }(); + // Set up the linear solver. KspSolver ksp(iodata, curlcurlop.GetNDSpaces(), &curlcurlop.GetH1Spaces()); ksp.SetOperators(*K, *K); @@ -42,6 +50,7 @@ MagnetostaticSolver::Solve(const std::vector> &me // Source term and solution vector storage. Vector RHS(K->Height()); std::vector A(nstep); + std::vector indicators(nstep); // Main loop over current source boundaries. Mpi::Print("\nComputing magnetostatic fields for {:d} source boundar{}\n", nstep, @@ -62,7 +71,10 @@ MagnetostaticSolver::Solve(const std::vector> &me BlockTimer bt1(Timer::SOLVE); ksp.Mult(RHS, A[step]); - BlockTimer bt2(Timer::POSTPRO); + BlockTimer bt2(Timer::ESTIMATION); + indicators[step] = estimator.ComputeIndicators(A[step]); + + BlockTimer bt3(Timer::POSTPRO); Mpi::Print(" Sol. ||A|| = {:.6e} (||RHS|| = {:.6e})\n", linalg::Norml2(curlcurlop.GetComm(), A[step]), linalg::Norml2(curlcurlop.GetComm(), RHS)); @@ -71,23 +83,16 @@ MagnetostaticSolver::Solve(const std::vector> &me step++; } - auto estimator = [&]() - { - BlockTimer bt(Timer::ESTCONSTRUCT); - return CurlFluxErrorEstimator(iodata, curlcurlop.GetMaterialOp(), - curlcurlop.GetNDSpaces()); - }(); - // Postprocess the capacitance matrix from the computed field solutions. BlockTimer bt1(Timer::POSTPRO); SaveMetadata(ksp); - return Postprocess(curlcurlop, postop, estimator, A); + Postprocess(curlcurlop, postop, A, indicators); + return indicators; } -ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, - PostOperator &postop, - CurlFluxErrorEstimator &estimator, - const std::vector &A) const +void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, + const std::vector &A, + const std::vector &indicators) const { // Postprocess the Maxwell inductance matrix. See p. 97 of the COMSOL AC/DC Module manual // for the associated formulas based on the magnetic field energy based on a current @@ -102,19 +107,6 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, mfem::DenseMatrix M(nstep), Mm(nstep); Vector B(Curl->Height()), Aij(Curl->Width()); Vector Iinc(nstep); - - // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators indicators; - auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop, - &curlcurlop](const auto &A, int i, double idx) - { - BlockTimer bt0(Timer::ESTIMATION); - auto sample_indicators = estimator.ComputeIndicators(A); - postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); - PostprocessErrorIndicators("i", i, i + 1, - sample_indicators.GetPostprocessData(curlcurlop.GetComm())); - indicators.AddIndicators(sample_indicators); - }; if (iodata.solver.magnetostatic.n_post > 0) { Mpi::Print("\n"); @@ -136,8 +128,8 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostprocessDomains(postop, "i", i, idx, 0.0, Um, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, 0.0, Um, 0.0, Iinc(i)); PostprocessProbes(postop, "i", i, idx); - Mpi::Print("Computing error estimates for terminal {:d}\n", idx); - UpdateErrorIndicators(A[i], i, idx); + PostprocessErrorIndicators("i", i, idx, + indicators[i].GetPostprocessData(curlcurlop.GetComm())); if (i < iodata.solver.magnetostatic.n_post) { PostprocessFields(postop, i, idx); @@ -177,7 +169,6 @@ ErrorIndicators MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, mfem::DenseMatrix Minv(M); Minv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(surf_j_op, M, Minv, Mm); - return indicators; } void MagnetostaticSolver::PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, diff --git a/palace/drivers/magnetostaticsolver.hpp b/palace/drivers/magnetostaticsolver.hpp index 29150f2cf..e32497472 100644 --- a/palace/drivers/magnetostaticsolver.hpp +++ b/palace/drivers/magnetostaticsolver.hpp @@ -21,7 +21,6 @@ namespace palace { class CurlCurlOperator; -class CurlFluxErrorEstimator; class ErrorIndicators; class IoData; class PostOperator; @@ -34,9 +33,9 @@ class Timer; class MagnetostaticSolver : public BaseSolver { private: - ErrorIndicators Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, - const std::vector &A) const; + void Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, + const std::vector &A, + const std::vector &indicators) const; void PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, const mfem::DenseMatrix &M, const mfem::DenseMatrix &Minv, diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index d96efc4fc..b9effee71 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -90,6 +90,7 @@ TransientSolver::Solve(const std::vector> &mesh) BlockTimer bt0(Timer::ESTIMATION); auto sample_indicators = estimator.ComputeIndicators(E); postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); + BlockTimer bt_post(Timer::POSTPRO); PostprocessErrorIndicators("t (ns)", step, time, sample_indicators.GetPostprocessData(spaceop.GetComm())); indicators.AddIndicators(sample_indicators); diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index e207fc1ef..c13cc5c1b 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -173,7 +173,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); } - return {estimates, normalization}; + return ErrorIndicators(std::move(estimates), normalization); } template <> @@ -235,7 +235,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); } - return {estimates, normalization}; + return ErrorIndicators(std::move(estimates), normalization); } GradFluxErrorEstimator::GradFluxErrorEstimator( @@ -326,7 +326,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const std::for_each(estimates.begin(), estimates.end(), [&normalization](auto &x) { x /= normalization; }); } - return {estimates, normalization}; + return ErrorIndicators(std::move(estimates), normalization); } } // namespace palace diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index a6b0e3c21..9be3a050e 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -15,10 +15,17 @@ namespace palace class ErrorIndicators { public: - ErrorIndicators(Vector local, double normalization) + ErrorIndicators(Vector &&local, double normalization) : local(std::move(local)), normalization(normalization), n(1) { } + ErrorIndicators(const std::vector &ind) + { + for (const auto &i : ind) + { + AddIndicators(i); + } + } ErrorIndicators() = default; ErrorIndicators(const ErrorIndicators &) = default; From a0012108e7718d29797a8af305bdf439025b4249 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Fri, 6 Oct 2023 11:31:45 -0400 Subject: [PATCH 28/39] Remove unneeded defaults and use Norml2 --- palace/utils/errorindicators.hpp | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index 9be3a050e..e760487a8 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -26,25 +26,14 @@ class ErrorIndicators AddIndicators(i); } } - ErrorIndicators() = default; - ErrorIndicators(const ErrorIndicators &) = default; - ErrorIndicators(ErrorIndicators &&) = default; - ErrorIndicators &operator=(const ErrorIndicators &) = default; - ErrorIndicators &operator=(ErrorIndicators &&) = default; - ~ErrorIndicators() = default; // Return the local error indicators. const auto &GetLocalErrorIndicators() const { return local; } // Return the global error indicator. inline auto GetGlobalErrorIndicator(MPI_Comm comm) const { - constexpr int p = 2; - double global_error_indicator = - std::transform_reduce(local.begin(), local.end(), 0.0, std::plus(), - [](auto val) { return std::pow(val, p); }); - Mpi::GlobalSum(1, &global_error_indicator, comm); - return std::pow(global_error_indicator, 1.0 / p); + return linalg::Norml2(comm, local); } // Return the largest local error indicator. inline auto GetMaxErrorIndicator(MPI_Comm comm) const From 78f09464957d810f215f2cba18788482a2f605da Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Fri, 6 Oct 2023 12:32:22 -0400 Subject: [PATCH 29/39] Delete some unused methods/debris, use member functions on Vector, and add config file arguments for the estimator. --- docs/src/config/solver.md | 6 ++++++ palace/fem/coefficient.hpp | 2 +- palace/linalg/errorestimator.cpp | 22 +++++++--------------- palace/main.cpp | 2 +- palace/utils/configfile.cpp | 11 +++++++++-- palace/utils/configfile.hpp | 6 ++++++ palace/utils/errorindicators.hpp | 9 +-------- scripts/schema/config/solver.json | 2 ++ 8 files changed, 33 insertions(+), 27 deletions(-) diff --git a/docs/src/config/solver.md b/docs/src/config/solver.md index 2f0d05de6..7890c61b0 100644 --- a/docs/src/config/solver.md +++ b/docs/src/config/solver.md @@ -420,6 +420,12 @@ eigenmode simulation type. `"DivFreeMaxIts" [100]` : Maximum number of iterations for divergence-free cleaning use in the eigenmode simulation type. +`"EstimatorTol" [1e-6]` : Relative tolerance for flux projection used in the +error estimate calculation. + +`"EstimatorMaxIts" [100]` : Maximum number of iterations for flux projection use in +the error estimate calculation. + `"GSOrthogonalization" ["MGS"]` : Gram-Schmidt variant used to explicitly orthogonalize vectors in Krylov subspace methods or other parts of the code. diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index 0c1ffc047..3ff76e661 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -489,7 +489,7 @@ class CurlFluxCoefficient : public mfem::VectorCoefficient public: CurlFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) : mfem::VectorCoefficient(gf.ParFESpace()->GetParMesh()->SpaceDimension()), U(gf), - mat_op(mat_op), curl(3) + mat_op(mat_op), curl(gf.ParFESpace()->GetParMesh()->SpaceDimension()) { } diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index c13cc5c1b..222172c18 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -98,7 +98,8 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &nd_fespaces) : mat_op(mat_op), nd_fespaces(nd_fespaces), - smooth_projector(nd_fespaces, iodata.solver.linear.tol, 200, 1, + smooth_projector(nd_fespaces, iodata.solver.linear.estimator_tol, + iodata.solver.linear.estimator_max_it, iodata.problem.verbose, iodata.solver.pa_order_threshold), smooth_flux(nd_fespaces.GetFinestFESpace().GetTrueVSize()), flux_rhs(nd_fespaces.GetFinestFESpace().GetTrueVSize()), @@ -170,8 +171,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v normalization = std::sqrt(normalization); if (normalization > 0) { - std::for_each(estimates.begin(), estimates.end(), - [&normalization](auto &x) { x /= normalization; }); + estimates /= normalization; } return ErrorIndicators(std::move(estimates), normalization); } @@ -232,8 +232,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const normalization = std::sqrt(normalization); if (normalization > 0) { - std::for_each(estimates.begin(), estimates.end(), - [&normalization](auto &x) { x /= normalization; }); + estimates /= normalization; } return ErrorIndicators(std::move(estimates), normalization); } @@ -242,7 +241,8 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &h1_fespaces) : mat_op(mat_op), h1_fespaces(h1_fespaces), - smooth_projector(h1_fespaces, iodata.solver.linear.tol, 200, 1, + smooth_projector(h1_fespaces, iodata.solver.linear.estimator_tol, + iodata.solver.linear.estimator_max_it, iodata.problem.verbose, iodata.solver.pa_order_threshold), smooth_flux(h1_fespaces.GetFinestFESpace().GetTrueVSize()), flux_rhs(h1_fespaces.GetFinestFESpace().GetTrueVSize()), @@ -303,13 +303,6 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const h1_fespace.GetParMesh()); paraview.RegisterCoeffField("Flux", &coef); paraview.RegisterField("SmoothFlux", &smooth_flux_gf); - - mfem::L2_FECollection est_fec(0, sdim); - mfem::ParFiniteElementSpace est_fespace(h1_fespace.GetParMesh(), &est_fec); - mfem::ParGridFunction est_field(&est_fespace); - est_field.SetFromTrueDofs(estimates); - - paraview.RegisterField("ErrorIndicator", &est_field); paraview.Save(); } } @@ -323,8 +316,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const if (normalization > 0) { - std::for_each(estimates.begin(), estimates.end(), - [&normalization](auto &x) { x /= normalization; }); + estimates /= normalization; } return ErrorIndicators(std::move(estimates), normalization); } diff --git a/palace/main.cpp b/palace/main.cpp index 4609f977a..a16cd6ea4 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -185,7 +185,7 @@ int main(int argc, char *argv[]) mesh::RefineMesh(iodata, mesh); // Run the problem driver. - auto solver_output = solver->Solve(mesh); + auto indicators = solver->Solve(mesh); // Print timing summary. BlockTimer::Print(world_comm); diff --git a/palace/utils/configfile.cpp b/palace/utils/configfile.cpp index 34803c400..94ec6298d 100644 --- a/palace/utils/configfile.cpp +++ b/palace/utils/configfile.cpp @@ -1639,7 +1639,7 @@ void LinearSolverData::SetUp(json &solver) max_size = linear->value("MaxSize", max_size); initial_guess = linear->value("InitialGuess", initial_guess); - // Options related to multigrid + // Options related to multigrid. mg_max_levels = linear->value("MGMaxLevels", mg_max_levels); mg_coarsen_type = linear->value("MGCoarsenType", mg_coarsen_type); mg_legacy_transfer = linear->value("MGLegacyTransfer", mg_legacy_transfer); @@ -1651,7 +1651,7 @@ void LinearSolverData::SetUp(json &solver) mg_smooth_sf_min = linear->value("MGSmoothEigScaleMin", mg_smooth_sf_min); mg_smooth_cheby_4th = linear->value("MGSmoothChebyshev4th", mg_smooth_cheby_4th); - // Preconditioner-specific options + // Preconditioner-specific options. pc_mat_real = linear->value("PCMatReal", pc_mat_real); pc_mat_shifted = linear->value("PCMatShifted", pc_mat_shifted); pc_mat_lor = linear->value("PCLowOrderRefined", pc_mat_lor); @@ -1669,6 +1669,10 @@ void LinearSolverData::SetUp(json &solver) divfree_max_it = linear->value("DivFreeMaxIts", divfree_max_it); gs_orthog_type = linear->value("GSOrthogonalization", gs_orthog_type); + // Options related to the estimator. + estimator_tol = linear->value("EstimatorTol", estimator_tol); + estimator_max_it = linear->value("EstimatorMaxIts", estimator_max_it); + // Cleanup linear->erase("Type"); linear->erase("KSPType"); @@ -1702,6 +1706,9 @@ void LinearSolverData::SetUp(json &solver) linear->erase("DivFreeTol"); linear->erase("DivFreeMaxIts"); linear->erase("GSOrthogonalization"); + + linear->erase("EstimatorTol"); + linear->erase("EstimatorMaxIts"); MFEM_VERIFY(linear->empty(), "Found an unsupported configuration file keyword under \"Linear\"!\n" << linear->dump(2)); diff --git a/palace/utils/configfile.hpp b/palace/utils/configfile.hpp index 17c55ae17..fadae6080 100644 --- a/palace/utils/configfile.hpp +++ b/palace/utils/configfile.hpp @@ -819,6 +819,12 @@ struct LinearSolverData // Maximum number of iterations for solving linear systems in divergence-free projector. int divfree_max_it = 1000; + // Relative tolerance for solving linear systems in the error estimator. + double estimator_tol = tol; + + // Maximum number of iterations for solving linear systems in the error estimator. + int estimator_max_it = max_it; + // Enable different variants of Gram-Schmidt orthogonalization for GMRES/FGMRES iterative // solvers and SLEPc eigenvalue solver. enum class OrthogType diff --git a/palace/utils/errorindicators.hpp b/palace/utils/errorindicators.hpp index e760487a8..78f858177 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/utils/errorindicators.hpp @@ -54,7 +54,7 @@ class ErrorIndicators { int size = local.Size(); Mpi::GlobalSum(1, &size, comm); - auto sum = std::accumulate(local.begin(), local.end(), 0.0); + auto sum = local.Sum(); Mpi::GlobalSum(1, &sum, comm); return sum / size; } @@ -62,13 +62,6 @@ class ErrorIndicators inline auto GetNormalization() const { return normalization; } // Add a set of indicators to the running totals. void AddIndicators(const ErrorIndicators &indicators); - // Reset a running total of error indicators ready for computing a new running average. - inline void Reset() - { - n = 0; - local = Vector(); - normalization = 0; - } // Return a vector of postprocess data. std::array GetPostprocessData(MPI_Comm comm) const { diff --git a/scripts/schema/config/solver.json b/scripts/schema/config/solver.json index bae4e1ae1..066e186dd 100644 --- a/scripts/schema/config/solver.json +++ b/scripts/schema/config/solver.json @@ -126,6 +126,8 @@ "AMSVector": { "type": "boolean" }, "DivFreeTol": { "type": "number", "minimum": 0.0 }, "DivFreeMaxIts": { "type": "integer", "minimum": 0 }, + "EstimatorTol": { "type": "number", "minimum": 0.0 }, + "EstimatorMaxIts": { "type": "integer", "minimum": 0 }, "GSOrthogonalization": { "type": "string" } } } From f536eecbf0d7e94deb46e7a549cf2902e13f3990 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Fri, 6 Oct 2023 14:02:50 -0400 Subject: [PATCH 30/39] Move from utils to fem, CwiseSqrt, and more debris --- palace/drivers/basesolver.cpp | 2 +- palace/drivers/drivensolver.cpp | 2 +- palace/drivers/eigensolver.cpp | 2 +- palace/drivers/electrostaticsolver.cpp | 2 +- palace/drivers/magnetostaticsolver.cpp | 2 +- palace/drivers/transientsolver.cpp | 2 +- palace/fem/CMakeLists.txt | 7 ++++--- palace/{utils => fem}/errorindicators.cpp | 3 +-- palace/{utils => fem}/errorindicators.hpp | 6 +++--- palace/linalg/errorestimator.cpp | 12 +++--------- palace/linalg/errorestimator.hpp | 2 +- palace/linalg/vector.cpp | 7 +++++++ palace/linalg/vector.hpp | 3 +++ palace/main.cpp | 2 +- palace/models/postoperator.cpp | 2 +- palace/utils/CMakeLists.txt | 1 - 16 files changed, 30 insertions(+), 27 deletions(-) rename palace/{utils => fem}/errorindicators.cpp (98%) rename palace/{utils => fem}/errorindicators.hpp (94%) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index 966d5e7d4..c15191c7f 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -6,12 +6,12 @@ #include #include #include +#include "fem/errorindicators.hpp" #include "linalg/ksp.hpp" #include "models/domainpostoperator.hpp" #include "models/postoperator.hpp" #include "models/surfacepostoperator.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/filesystem.hpp" #include "utils/iodata.hpp" #include "utils/timer.hpp" diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index e43eba945..2321119a8 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -5,6 +5,7 @@ #include #include +#include "fem/errorindicators.hpp" #include "linalg/errorestimator.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" @@ -16,7 +17,6 @@ #include "models/surfacecurrentoperator.hpp" #include "models/waveportoperator.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/iodata.hpp" #include "utils/prettyprint.hpp" #include "utils/timer.hpp" diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index cc35328a8..db93413a3 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -4,6 +4,7 @@ #include "eigensolver.hpp" #include +#include "fem/errorindicators.hpp" #include "linalg/arpack.hpp" #include "linalg/divfree.hpp" #include "linalg/errorestimator.hpp" @@ -15,7 +16,6 @@ #include "models/postoperator.hpp" #include "models/spaceoperator.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/iodata.hpp" #include "utils/timer.hpp" diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 722cf81a8..714d27517 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -4,6 +4,7 @@ #include "electrostaticsolver.hpp" #include +#include "fem/errorindicators.hpp" #include "linalg/errorestimator.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" @@ -11,7 +12,6 @@ #include "models/laplaceoperator.hpp" #include "models/postoperator.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/iodata.hpp" #include "utils/timer.hpp" diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index f9d844c8d..b7ddca484 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -4,6 +4,7 @@ #include "magnetostaticsolver.hpp" #include +#include "fem/errorindicators.hpp" #include "linalg/errorestimator.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" @@ -11,7 +12,6 @@ #include "models/postoperator.hpp" #include "models/surfacecurrentoperator.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/iodata.hpp" #include "utils/timer.hpp" diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index b9effee71..4a7fee463 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -4,6 +4,7 @@ #include "transientsolver.hpp" #include +#include "fem/errorindicators.hpp" #include "linalg/errorestimator.hpp" #include "linalg/vector.hpp" #include "models/lumpedportoperator.hpp" @@ -12,7 +13,6 @@ #include "models/surfacecurrentoperator.hpp" #include "models/timeoperator.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/excitations.hpp" #include "utils/iodata.hpp" #include "utils/timer.hpp" diff --git a/palace/fem/CMakeLists.txt b/palace/fem/CMakeLists.txt index a912c6aad..8ae6d7757 100644 --- a/palace/fem/CMakeLists.txt +++ b/palace/fem/CMakeLists.txt @@ -5,6 +5,7 @@ # Add source files and subdirectories. # -# target_sources(${TARGET_NAME} -# PRIVATE -# ) +target_sources(${TARGET_NAME} + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/errorindicators.cpp +) diff --git a/palace/utils/errorindicators.cpp b/palace/fem/errorindicators.cpp similarity index 98% rename from palace/utils/errorindicators.cpp rename to palace/fem/errorindicators.cpp index 960221489..3ad7b781d 100644 --- a/palace/utils/errorindicators.cpp +++ b/palace/fem/errorindicators.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "errorindicators.hpp" - +#include #include "utils/communication.hpp" namespace palace @@ -48,5 +48,4 @@ void ErrorIndicators::AddIndicators(const ErrorIndicators &error_indicators) (*this) = error_indicators; } } - } // namespace palace diff --git a/palace/utils/errorindicators.hpp b/palace/fem/errorindicators.hpp similarity index 94% rename from palace/utils/errorindicators.hpp rename to palace/fem/errorindicators.hpp index 78f858177..35d7e0ee5 100644 --- a/palace/utils/errorindicators.hpp +++ b/palace/fem/errorindicators.hpp @@ -1,8 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -#ifndef PALACE_UTILS_ERROR_INDICATORS_HPP -#define PALACE_UTILS_ERROR_INDICATORS_HPP +#ifndef PALACE_FEM_ERROR_INDICATORS_HPP +#define PALACE_FEM_ERROR_INDICATORS_HPP #include "linalg/vector.hpp" #include "utils/communication.hpp" @@ -81,4 +81,4 @@ class ErrorIndicators } // namespace palace -#endif // PALACE_UTILS_ERROR_INDICATORS_HPP +#endif // PALACE_FEM_ERROR_INDICATORS_HPP diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 222172c18..fe3e01fb9 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -4,6 +4,7 @@ #include "errorestimator.hpp" #include #include "fem/coefficient.hpp" +#include "fem/errorindicators.hpp" #include "fem/integrator.hpp" #include "fem/multigrid.hpp" #include "linalg/amg.hpp" @@ -12,7 +13,6 @@ #include "linalg/rap.hpp" #include "models/materialoperator.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/iodata.hpp" #include "utils/timer.hpp" @@ -162,10 +162,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v } } } - for (auto &e : estimates) - { - e = std::sqrt(e); - } + linalg::CwiseSqrt(estimates); Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm()); normalization = std::sqrt(normalization); @@ -306,10 +303,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const paraview.Save(); } } - for (auto &e : estimates) - { - e = std::sqrt(e); - } + linalg::CwiseSqrt(estimates); Mpi::GlobalSum(1, &normalization, h1_fespace.GetComm()); normalization = std::sqrt(normalization); diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index cbc52cd46..569020820 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -6,10 +6,10 @@ #include #include +#include "fem/errorindicators.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" #include "linalg/vector.hpp" -#include "utils/errorindicators.hpp" namespace palace { diff --git a/palace/linalg/vector.cpp b/palace/linalg/vector.cpp index 5f4b57f34..456079489 100644 --- a/palace/linalg/vector.cpp +++ b/palace/linalg/vector.cpp @@ -356,6 +356,13 @@ void SetRandomSign(MPI_Comm comm, ComplexVector &x, int seed) { XI[i] = (XI[i] > 0.0) ? 1.0 : ((XI[i] < 0.0) ? -1.0 : 0.0); }); } +void CwiseSqrt(Vector &x) +{ + const int N = x.Size(); + auto *X = x.ReadWrite(); + mfem::forall(N, [=] MFEM_HOST_DEVICE(int i) { X[i] = std::sqrt(X[i]); }); +} + template <> void SetSubVector(Vector &x, const mfem::Array &rows, double s) { diff --git a/palace/linalg/vector.hpp b/palace/linalg/vector.hpp index 8f25fb2d7..8ed4f9634 100644 --- a/palace/linalg/vector.hpp +++ b/palace/linalg/vector.hpp @@ -139,6 +139,9 @@ void SetRandomReal(MPI_Comm comm, VecType &x, int seed = 0); template void SetRandomSign(MPI_Comm comm, VecType &x, int seed = 0); +// Component-wise square root. +void CwiseSqrt(Vector &x); + // Calculate the inner product yᴴ x or yᵀ x. template inline auto Dot(MPI_Comm comm, const VecType &x, const VecType &y) diff --git a/palace/main.cpp b/palace/main.cpp index a16cd6ea4..67a335d97 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -12,9 +12,9 @@ #include "drivers/electrostaticsolver.hpp" #include "drivers/magnetostaticsolver.hpp" #include "drivers/transientsolver.hpp" +#include "fem/errorindicators.hpp" #include "linalg/slepc.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/geodata.hpp" #include "utils/iodata.hpp" #include "utils/timer.hpp" diff --git a/palace/models/postoperator.cpp b/palace/models/postoperator.cpp index 49e7c2f5b..4d4f0d740 100644 --- a/palace/models/postoperator.cpp +++ b/palace/models/postoperator.cpp @@ -4,6 +4,7 @@ #include "postoperator.hpp" #include "fem/coefficient.hpp" +#include "fem/errorindicators.hpp" #include "models/curlcurloperator.hpp" #include "models/laplaceoperator.hpp" #include "models/lumpedportoperator.hpp" @@ -12,7 +13,6 @@ #include "models/surfacecurrentoperator.hpp" #include "models/waveportoperator.hpp" #include "utils/communication.hpp" -#include "utils/errorindicators.hpp" #include "utils/geodata.hpp" #include "utils/iodata.hpp" diff --git a/palace/utils/CMakeLists.txt b/palace/utils/CMakeLists.txt index 3c55247eb..0f971601c 100644 --- a/palace/utils/CMakeLists.txt +++ b/palace/utils/CMakeLists.txt @@ -8,7 +8,6 @@ target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/configfile.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/errorindicators.cpp ${CMAKE_CURRENT_SOURCE_DIR}/geodata.cpp ${CMAKE_CURRENT_SOURCE_DIR}/iodata.cpp ${CMAKE_CURRENT_SOURCE_DIR}/meshio.cpp From 75cc87a809f8aaf12e4f34e6f620c7c1e5b488f2 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Fri, 6 Oct 2023 17:57:49 -0400 Subject: [PATCH 31/39] Rename ErrorIndicators to ErrorIndicator. Small cleanups --- docs/src/config/solver.md | 2 +- palace/drivers/basesolver.cpp | 14 ++--- palace/drivers/basesolver.hpp | 8 +-- palace/drivers/drivensolver.cpp | 56 +++++++++---------- palace/drivers/drivensolver.hpp | 13 ++--- palace/drivers/eigensolver.cpp | 10 ++-- palace/drivers/eigensolver.hpp | 6 +- palace/drivers/electrostaticsolver.cpp | 12 ++-- palace/drivers/electrostaticsolver.hpp | 8 +-- palace/drivers/magnetostaticsolver.cpp | 12 ++-- palace/drivers/magnetostaticsolver.hpp | 8 +-- palace/drivers/transientsolver.cpp | 20 +++---- palace/drivers/transientsolver.hpp | 6 +- palace/fem/CMakeLists.txt | 2 +- ...errorindicators.cpp => errorindicator.cpp} | 4 +- ...errorindicators.hpp => errorindicator.hpp} | 28 ++++------ palace/linalg/errorestimator.cpp | 18 +++--- palace/linalg/errorestimator.hpp | 6 +- palace/linalg/vector.cpp | 2 +- palace/linalg/vector.hpp | 2 +- palace/main.cpp | 2 +- palace/models/postoperator.cpp | 2 +- palace/models/postoperator.hpp | 2 +- palace/utils/configfile.cpp | 10 ++-- palace/utils/configfile.hpp | 4 +- 25 files changed, 126 insertions(+), 131 deletions(-) rename palace/fem/{errorindicators.cpp => errorindicator.cpp} (94%) rename palace/fem/{errorindicators.hpp => errorindicator.hpp} (70%) diff --git a/docs/src/config/solver.md b/docs/src/config/solver.md index 7890c61b0..d0f9e7e65 100644 --- a/docs/src/config/solver.md +++ b/docs/src/config/solver.md @@ -417,7 +417,7 @@ iterative solver choices, and the default choice depends on the iterative solver `"DivFreeTol" [1.0e-12]` : Relative tolerance for divergence-free cleaning used in the eigenmode simulation type. -`"DivFreeMaxIts" [100]` : Maximum number of iterations for divergence-free cleaning use in +`"DivFreeMaxIts" [1000]` : Maximum number of iterations for divergence-free cleaning use in the eigenmode simulation type. `"EstimatorTol" [1e-6]` : Relative tolerance for flux projection used in the diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index c15191c7f..6c243900e 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -6,7 +6,7 @@ #include #include #include -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "linalg/ksp.hpp" #include "models/domainpostoperator.hpp" #include "models/postoperator.hpp" @@ -566,8 +566,8 @@ void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double Mpi::Barrier(); } -void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, double time, - const std::array &data) const +void BaseSolver::PostprocessErrorIndicator(const std::string &name, int step, double time, + const std::array &data) const { if (post_dir.length() == 0) { @@ -584,10 +584,10 @@ void BaseSolver::PostprocessErrorIndicators(const std::string &name, int step, d // clang-format off output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", name, table.w1, - "Sum Rel.", table.w, - "Min Elem Rel.", table.w, - "Max Elem Rel.", table.w, - "Mean Elem Rel.", table.w, + "Sum", table.w, + "Min", table.w, + "Max", table.w, + "Mean", table.w, "Normalization", table.w); // clang-format on } diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index 6bb39d565..bcc485d79 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -20,7 +20,7 @@ class ParMesh; namespace palace { -class ErrorIndicators; +class ErrorIndicator; class IoData; class PostOperator; class Timer; @@ -71,8 +71,8 @@ class BaseSolver // Postprocess granular error indicator to file. The argument normalized indicates if // supplied indicators have already been normalized. - void PostprocessErrorIndicators(const std::string &name, int step, double time, - const std::array &data) const; + void PostprocessErrorIndicator(const std::string &name, int step, double time, + const std::array &data) const; public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, @@ -80,7 +80,7 @@ class BaseSolver virtual ~BaseSolver() = default; // Performs a solve using the mesh sequence, then reports error indicators. - virtual ErrorIndicators + virtual ErrorIndicator Solve(const std::vector> &mesh) const = 0; // These methods write different simulation metadata to a JSON file in post_dir. diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 2321119a8..78669b00b 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -5,7 +5,7 @@ #include #include -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "linalg/errorestimator.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" @@ -26,7 +26,7 @@ namespace palace using namespace std::complex_literals; -ErrorIndicators +ErrorIndicator DrivenSolver::Solve(const std::vector> &mesh) const { // Set up the spatial discretization and frequency sweep. @@ -109,10 +109,10 @@ DrivenSolver::Solve(const std::vector> &mesh) con : SweepUniform(spaceop, postop, estimator, nstep, step0, omega0, delta_omega); } -ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, - int step0, double omega0, - double delta_omega) const +ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &postop, + CurlFluxErrorEstimator &estimator, int nstep, + int step0, double omega0, + double delta_omega) const { // Construct the system matrices defining the linear operator. PEC boundaries are handled // simply by setting diagonal entries of the system matrix for the corresponding dofs. @@ -145,17 +145,17 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator B = 0.0; // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicators indicators; - auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop, - &spaceop](const auto &E, int step, double frequency) + ErrorIndicator indicators; + auto UpdateErrorIndicator = [this, &estimator, &indicators, &postop, + &spaceop](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTIMATION); - auto sample_indicators = estimator.ComputeIndicators(E); - postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); + ErrorIndicator sample = estimator.ComputeIndicators(E); + postop.SetIndicatorGridFunction(sample.Local()); BlockTimer bt_post(Timer::POSTPRO); - PostprocessErrorIndicators("f (GHz)", step, frequency, - sample_indicators.GetPostprocessData(spaceop.GetComm())); - indicators.AddIndicators(sample_indicators); + PostprocessErrorIndicator("f (GHz)", step, frequency, + sample.GetPostprocessData(spaceop.GetComm())); + indicators.AddIndicator(sample); }; // Main frequency sweep loop. @@ -187,7 +187,7 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator ksp.Mult(RHS, E); // Compute the error indicators, and post process the indicator field. - UpdateErrorIndicators(E, step, freq); + UpdateErrorIndicator(E, step, freq); // Compute B = -1/(iω) ∇ x E on the true dofs, and set the internal GridFunctions in // PostOperator for all postprocessing operations. @@ -221,10 +221,10 @@ ErrorIndicators DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator return indicators; } -ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, - int step0, double omega0, - double delta_omega) const +ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, + CurlFluxErrorEstimator &estimator, int nstep, + int step0, double omega0, + double delta_omega) const { // Configure default parameters if not specified. BlockTimer bt0(Timer::CONSTRUCT); @@ -276,16 +276,16 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // The error indicators will be calculated for each HDM sample rather than for // the online stage. - ErrorIndicators indicators; - auto UpdateErrorIndicators = + ErrorIndicator indicators; + auto UpdateErrorIndicator = [this, &estimator, &indicators, &spaceop](const auto &E, int step, double frequency) { BlockTimer bt0(Timer::ESTIMATION); auto sample_indicators = estimator.ComputeIndicators(E); BlockTimer bt_post(Timer::POSTPRO); - PostprocessErrorIndicators("f (GHz)", step, frequency, - sample_indicators.GetPostprocessData(spaceop.GetComm())); - indicators.AddIndicators(sample_indicators); + PostprocessErrorIndicator("f (GHz)", step, frequency, + sample_indicators.GetPostprocessData(spaceop.GetComm())); + indicators.AddIndicator(sample_indicators); }; // Initialize the basis with samples from the top and bottom of the frequency @@ -294,10 +294,10 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // of the RomOperator. BlockTimer bt1(Timer::CONSTRUCTPROM); prom.SolveHDM(omega0, E); // Print matrix stats at first HDM solve - UpdateErrorIndicators(E, 0, omega0 * f0); + UpdateErrorIndicator(E, 0, omega0 * f0); prom.AddHDMSample(omega0, E); prom.SolveHDM(omega0 + (nstep - step0 - 1) * delta_omega, E); - UpdateErrorIndicators(E, 1, (omega0 + (nstep - step0 - 1) * delta_omega) * f0); + UpdateErrorIndicator(E, 1, (omega0 + (nstep - step0 - 1) * delta_omega) * f0); prom.AddHDMSample(omega0 + (nstep - step0 - 1) * delta_omega, E); // Greedy procedure for basis construction (offline phase). Basis is initialized with @@ -321,7 +321,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator max_error); prom.SolveHDM(omega_star, E); Mpi::Print("Computing error estimates\n"); - UpdateErrorIndicators(E, iter - iter0 + 1, omega_star * f0); + UpdateErrorIndicator(E, iter - iter0 + 1, omega_star * f0); prom.AddHDMSample(omega_star, E); iter++; } @@ -335,7 +335,7 @@ ErrorIndicators DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator SaveMetadata(prom.GetLinearSolver()); // Set the indicator field to the combined field for postprocessing. - postop.SetIndicatorGridFunction(indicators.GetLocalErrorIndicators()); + postop.SetIndicatorGridFunction(indicators.Local()); // Main fast frequency sweep loop (online phase). BlockTimer bt2(Timer::CONSTRUCT); diff --git a/palace/drivers/drivensolver.hpp b/palace/drivers/drivensolver.hpp index ff82bd684..9e53fdd0d 100644 --- a/palace/drivers/drivensolver.hpp +++ b/palace/drivers/drivensolver.hpp @@ -19,7 +19,7 @@ namespace palace { class CurlFluxErrorEstimator; -class ErrorIndicators; +class ErrorIndicator; class IoData; class LumpedPortOperator; class PostOperator; @@ -36,12 +36,12 @@ class DrivenSolver : public BaseSolver private: int GetNumSteps(double start, double end, double delta) const; - ErrorIndicators SweepUniform(SpaceOperator &spaceop, PostOperator &postop, + ErrorIndicator SweepUniform(SpaceOperator &spaceop, PostOperator &postop, + CurlFluxErrorEstimator &estimator, int nstep, int step0, + double omega0, double delta_omega) const; + ErrorIndicator SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, CurlFluxErrorEstimator &estimator, int nstep, int step0, double omega0, double delta_omega) const; - ErrorIndicators SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, int step0, - double omega0, double delta_omega) const; void Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, const WavePortOperator &wave_port_op, @@ -62,8 +62,7 @@ class DrivenSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators - Solve(const std::vector> &mesh) const final; + ErrorIndicator Solve(const std::vector> &mesh) const final; }; } // namespace palace diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index db93413a3..cdb902b1e 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -4,7 +4,7 @@ #include "eigensolver.hpp" #include -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "linalg/arpack.hpp" #include "linalg/divfree.hpp" #include "linalg/errorestimator.hpp" @@ -24,7 +24,7 @@ namespace palace using namespace std::complex_literals; -ErrorIndicators +ErrorIndicator EigenSolver::Solve(const std::vector> &mesh) const { // Construct and extract the system matrices defining the eigenvalue problem. The diagonal @@ -265,7 +265,7 @@ EigenSolver::Solve(const std::vector> &mesh) cons // Save the eigenvalue estimates. BlockTimer bt_est(Timer::ESTIMATION); - std::vector indicators; + std::vector indicators; indicators.reserve(num_conv); for (int i = 0; i < iodata.solver.eigenmode.n; i++) { @@ -315,8 +315,8 @@ EigenSolver::Solve(const std::vector> &mesh) cons if (i < iodata.solver.eigenmode.n) { - postop.SetIndicatorGridFunction(indicators[i].GetLocalErrorIndicators()); - PostprocessErrorIndicators("m", i, i + 1, + postop.SetIndicatorGridFunction(indicators[i].Local()); + PostprocessErrorIndicator("m", i, i + 1, indicators[i].GetPostprocessData(spaceop.GetComm())); } } diff --git a/palace/drivers/eigensolver.hpp b/palace/drivers/eigensolver.hpp index 0194b69b6..9e935c7f1 100644 --- a/palace/drivers/eigensolver.hpp +++ b/palace/drivers/eigensolver.hpp @@ -19,7 +19,7 @@ class ParMesh; namespace palace { -class ErrorIndicators; +class ErrorIndicator; class IoData; class LumpedPortOperator; class PostOperator; @@ -43,8 +43,8 @@ class EigenSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators - Solve(const std::vector> &mesh) const final; + ErrorIndicator + Solve(const std::vector> &mesh) const override; }; } // namespace palace diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 714d27517..25c1ded92 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -4,7 +4,7 @@ #include "electrostaticsolver.hpp" #include -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "linalg/errorestimator.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" @@ -18,7 +18,7 @@ namespace palace { -ErrorIndicators +ErrorIndicator ElectrostaticSolver::Solve(const std::vector> &mesh) const { // Construct the system matrix defining the linear operator. Dirichlet boundaries are @@ -51,7 +51,7 @@ ElectrostaticSolver::Solve(const std::vector> &me // Right-hand side term and solution vector storage. Vector RHS(K->Height()); std::vector V(nstep); - std::vector indicators(nstep); + std::vector indicators(nstep); // Main loop over terminal boundaries. Mpi::Print("\nComputing electrostatic fields for {:d} terminal boundar{}\n", nstep, @@ -92,7 +92,7 @@ ElectrostaticSolver::Solve(const std::vector> &me void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, const std::vector &V, - const std::vector &indicators) const + const std::vector &indicators) const { // Postprocess the Maxwell capacitance matrix. See p. 97 of the COMSOL AC/DC Module manual // for the associated formulas based on the electric field energy based on a unit voltage @@ -123,8 +123,8 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & PostprocessDomains(postop, "i", i, idx, Ue, 0.0, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, Ue, 0.0, 1.0, 0.0); PostprocessProbes(postop, "i", i, idx); - PostprocessErrorIndicators("i", i, idx, - indicators[i].GetPostprocessData(laplaceop.GetComm())); + PostprocessErrorIndicator("i", i, idx, + indicators[i].GetPostprocessData(laplaceop.GetComm())); if (i < iodata.solver.electrostatic.n_post) { PostprocessFields(postop, i, idx); diff --git a/palace/drivers/electrostaticsolver.hpp b/palace/drivers/electrostaticsolver.hpp index 9b584f2de..61464e68c 100644 --- a/palace/drivers/electrostaticsolver.hpp +++ b/palace/drivers/electrostaticsolver.hpp @@ -23,7 +23,7 @@ class ParMesh; namespace palace { -class ErrorIndicators; +class ErrorIndicator; class IoData; class LaplaceOperator; class PostOperator; @@ -37,7 +37,7 @@ class ElectrostaticSolver : public BaseSolver private: void Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, const std::vector &V, - const std::vector &indicators) const; + const std::vector &indicators) const; void PostprocessTerminals(const std::map> &terminal_sources, const mfem::DenseMatrix &C, const mfem::DenseMatrix &Cinv, @@ -46,8 +46,8 @@ class ElectrostaticSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators - Solve(const std::vector> &mesh) const final; + ErrorIndicator + Solve(const std::vector> &mesh) const override; }; } // namespace palace diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index b7ddca484..7c3a9abad 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -4,7 +4,7 @@ #include "magnetostaticsolver.hpp" #include -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "linalg/errorestimator.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" @@ -18,7 +18,7 @@ namespace palace { -ErrorIndicators +ErrorIndicator MagnetostaticSolver::Solve(const std::vector> &mesh) const { // Construct the system matrix defining the linear operator. Dirichlet boundaries are @@ -50,7 +50,7 @@ MagnetostaticSolver::Solve(const std::vector> &me // Source term and solution vector storage. Vector RHS(K->Height()); std::vector A(nstep); - std::vector indicators(nstep); + std::vector indicators(nstep); // Main loop over current source boundaries. Mpi::Print("\nComputing magnetostatic fields for {:d} source boundar{}\n", nstep, @@ -92,7 +92,7 @@ MagnetostaticSolver::Solve(const std::vector> &me void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, const std::vector &A, - const std::vector &indicators) const + const std::vector &indicators) const { // Postprocess the Maxwell inductance matrix. See p. 97 of the COMSOL AC/DC Module manual // for the associated formulas based on the magnetic field energy based on a current @@ -128,8 +128,8 @@ void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator PostprocessDomains(postop, "i", i, idx, 0.0, Um, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, 0.0, Um, 0.0, Iinc(i)); PostprocessProbes(postop, "i", i, idx); - PostprocessErrorIndicators("i", i, idx, - indicators[i].GetPostprocessData(curlcurlop.GetComm())); + PostprocessErrorIndicator("i", i, idx, + indicators[i].GetPostprocessData(curlcurlop.GetComm())); if (i < iodata.solver.magnetostatic.n_post) { PostprocessFields(postop, i, idx); diff --git a/palace/drivers/magnetostaticsolver.hpp b/palace/drivers/magnetostaticsolver.hpp index e32497472..45cce6c23 100644 --- a/palace/drivers/magnetostaticsolver.hpp +++ b/palace/drivers/magnetostaticsolver.hpp @@ -21,7 +21,7 @@ namespace palace { class CurlCurlOperator; -class ErrorIndicators; +class ErrorIndicator; class IoData; class PostOperator; class SurfaceCurrentOperator; @@ -35,7 +35,7 @@ class MagnetostaticSolver : public BaseSolver private: void Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, const std::vector &A, - const std::vector &indicators) const; + const std::vector &indicators) const; void PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, const mfem::DenseMatrix &M, const mfem::DenseMatrix &Minv, @@ -44,8 +44,8 @@ class MagnetostaticSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators - Solve(const std::vector> &mesh) const final; + ErrorIndicator + Solve(const std::vector> &mesh) const override; }; } // namespace palace diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 4a7fee463..f1ec31788 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -4,7 +4,7 @@ #include "transientsolver.hpp" #include -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "linalg/errorestimator.hpp" #include "linalg/vector.hpp" #include "models/lumpedportoperator.hpp" @@ -20,7 +20,7 @@ namespace palace { -ErrorIndicators +ErrorIndicator TransientSolver::Solve(const std::vector> &mesh) const { // Set up the spatial discretization and time integrators for the E and B fields. @@ -83,17 +83,17 @@ TransientSolver::Solve(const std::vector> &mesh) BlockTimer bt(Timer::ESTCONSTRUCT); return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); - ErrorIndicators indicators; - auto UpdateErrorIndicators = [this, &estimator, &indicators, &postop, - &spaceop](const auto &E, int step, double time) + ErrorIndicator indicators; + auto UpdateErrorIndicator = [this, &estimator, &indicators, &postop, + &spaceop](const auto &E, int step, double time) { BlockTimer bt0(Timer::ESTIMATION); auto sample_indicators = estimator.ComputeIndicators(E); - postop.SetIndicatorGridFunction(sample_indicators.GetLocalErrorIndicators()); + postop.SetIndicatorGridFunction(sample_indicators.Local()); BlockTimer bt_post(Timer::POSTPRO); - PostprocessErrorIndicators("t (ns)", step, time, - sample_indicators.GetPostprocessData(spaceop.GetComm())); - indicators.AddIndicators(sample_indicators); + PostprocessErrorIndicator("t (ns)", step, time, + sample_indicators.GetPostprocessData(spaceop.GetComm())); + indicators.AddIndicator(sample_indicators); }; // Main time integration loop. @@ -138,7 +138,7 @@ TransientSolver::Solve(const std::vector> &mesh) } // Calculate and record the error indicators. - UpdateErrorIndicators(E, step, t); + UpdateErrorIndicator(E, step, t); // Postprocess port voltages/currents and optionally write solution to disk. Postprocess(postop, spaceop.GetLumpedPortOp(), spaceop.GetSurfaceCurrentOp(), step, t, diff --git a/palace/drivers/transientsolver.hpp b/palace/drivers/transientsolver.hpp index 20a17ac5b..19f0286df 100644 --- a/palace/drivers/transientsolver.hpp +++ b/palace/drivers/transientsolver.hpp @@ -19,7 +19,7 @@ class ParMesh; namespace palace { -class ErrorIndicators; +class ErrorIndicator; class IoData; class LumpedPortOperator; class PostOperator; @@ -50,8 +50,8 @@ class TransientSolver : public BaseSolver public: using BaseSolver::BaseSolver; - ErrorIndicators - Solve(const std::vector> &mesh) const final; + ErrorIndicator + Solve(const std::vector> &mesh) const override; }; } // namespace palace diff --git a/palace/fem/CMakeLists.txt b/palace/fem/CMakeLists.txt index 8ae6d7757..4072b0b1e 100644 --- a/palace/fem/CMakeLists.txt +++ b/palace/fem/CMakeLists.txt @@ -7,5 +7,5 @@ target_sources(${TARGET_NAME} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/errorindicators.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/errorindicator.cpp ) diff --git a/palace/fem/errorindicators.cpp b/palace/fem/errorindicator.cpp similarity index 94% rename from palace/fem/errorindicators.cpp rename to palace/fem/errorindicator.cpp index 3ad7b781d..742f51f6b 100644 --- a/palace/fem/errorindicators.cpp +++ b/palace/fem/errorindicator.cpp @@ -1,13 +1,13 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -#include "errorindicators.hpp" +#include "errorindicator.hpp" #include #include "utils/communication.hpp" namespace palace { -void ErrorIndicators::AddIndicators(const ErrorIndicators &error_indicators) +void ErrorIndicator::AddIndicator(const ErrorIndicator &error_indicators) { // The average local indicator is used rather than the indicator for the maximum // error to drive the adaptation, to account for a local error that might be marginally diff --git a/palace/fem/errorindicators.hpp b/palace/fem/errorindicator.hpp similarity index 70% rename from palace/fem/errorindicators.hpp rename to palace/fem/errorindicator.hpp index 35d7e0ee5..627c5a33a 100644 --- a/palace/fem/errorindicators.hpp +++ b/palace/fem/errorindicator.hpp @@ -12,45 +12,42 @@ namespace palace // Storage for error estimation results from the solve. Required in the AMR loop. This is // richer than the IndicatorsAndNormalization because it stores derived quantities, and a // communicator for use in adaptation. -class ErrorIndicators +class ErrorIndicator { public: - ErrorIndicators(Vector &&local, double normalization) + ErrorIndicator(Vector &&local, double normalization) : local(std::move(local)), normalization(normalization), n(1) { } - ErrorIndicators(const std::vector &ind) + ErrorIndicator(const std::vector &ind) { for (const auto &i : ind) { - AddIndicators(i); + AddIndicator(i); } } - ErrorIndicators() = default; + ErrorIndicator() = default; // Return the local error indicators. - const auto &GetLocalErrorIndicators() const { return local; } + const auto &Local() const { return local; } // Return the global error indicator. - inline auto GetGlobalErrorIndicator(MPI_Comm comm) const - { - return linalg::Norml2(comm, local); - } + inline auto Norml2(MPI_Comm comm) const { return linalg::Norml2(comm, local); } // Return the largest local error indicator. - inline auto GetMaxErrorIndicator(MPI_Comm comm) const + inline auto Max(MPI_Comm comm) const { double max = local.Max(); Mpi::GlobalMax(1, &max, comm); return max; } // Return the smallest local error indicator. - inline auto GetMinErrorIndicator(MPI_Comm comm) const + inline auto Min(MPI_Comm comm) const { double min = local.Min(); Mpi::GlobalMin(1, &min, comm); return min; } // Return the mean local error indicator. - inline auto GetMeanErrorIndicator(MPI_Comm comm) const + inline auto Mean(MPI_Comm comm) const { int size = local.Size(); Mpi::GlobalSum(1, &size, comm); @@ -61,12 +58,11 @@ class ErrorIndicators // Return the normalization constant for the absolute error. inline auto GetNormalization() const { return normalization; } // Add a set of indicators to the running totals. - void AddIndicators(const ErrorIndicators &indicators); + void AddIndicator(const ErrorIndicator &indicators); // Return a vector of postprocess data. std::array GetPostprocessData(MPI_Comm comm) const { - return {GetGlobalErrorIndicator(comm), GetMinErrorIndicator(comm), - GetMaxErrorIndicator(comm), GetMeanErrorIndicator(comm), GetNormalization()}; + return {Norml2(comm), Min(comm), Max(comm), Mean(comm), GetNormalization()}; } protected: diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index fe3e01fb9..b3c91d5c9 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -4,7 +4,7 @@ #include "errorestimator.hpp" #include #include "fem/coefficient.hpp" -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "fem/integrator.hpp" #include "fem/multigrid.hpp" #include "linalg/amg.hpp" @@ -109,7 +109,7 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( } template <> -ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v) const +ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v) const { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); const int nelem = nd_fespace.GetNE(); @@ -162,7 +162,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v } } } - linalg::CwiseSqrt(estimates); + linalg::Sqrt(estimates); Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm()); normalization = std::sqrt(normalization); @@ -170,11 +170,11 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v { estimates /= normalization; } - return ErrorIndicators(std::move(estimates), normalization); + return ErrorIndicator(std::move(estimates), normalization); } template <> -ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const +ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const { auto &nd_fespace = nd_fespaces.GetFinestFESpace(); field_gf.SetFromTrueDofs(v); @@ -231,7 +231,7 @@ ErrorIndicators CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const { estimates /= normalization; } - return ErrorIndicators(std::move(estimates), normalization); + return ErrorIndicator(std::move(estimates), normalization); } GradFluxErrorEstimator::GradFluxErrorEstimator( @@ -248,7 +248,7 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( { } -ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const +ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const { auto &h1_fespace = h1_fespaces.GetFinestFESpace(); const int sdim = h1_fespace.GetMesh()->SpaceDimension(); @@ -303,7 +303,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const paraview.Save(); } } - linalg::CwiseSqrt(estimates); + linalg::Sqrt(estimates); Mpi::GlobalSum(1, &normalization, h1_fespace.GetComm()); normalization = std::sqrt(normalization); @@ -312,7 +312,7 @@ ErrorIndicators GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const { estimates /= normalization; } - return ErrorIndicators(std::move(estimates), normalization); + return ErrorIndicator(std::move(estimates), normalization); } } // namespace palace diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 569020820..c811833f9 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -6,7 +6,7 @@ #include #include -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" #include "linalg/vector.hpp" @@ -63,7 +63,7 @@ class CurlFluxErrorEstimator // Compute elemental error indicators given a complex vector of true DOF. template - ErrorIndicators ComputeIndicators(const VectorType &v) const; + ErrorIndicator ComputeIndicators(const VectorType &v) const; }; // Class used for computing grad flux error estimate, i.e. || ϵ ∇ ϕₕ - F ||_K where F @@ -84,7 +84,7 @@ class GradFluxErrorEstimator mfem::ParFiniteElementSpaceHierarchy &h1_fespaces); // Compute elemental error indicators given a vector of true DOF. - ErrorIndicators ComputeIndicators(const Vector &v) const; + ErrorIndicator ComputeIndicators(const Vector &v) const; }; } // namespace palace diff --git a/palace/linalg/vector.cpp b/palace/linalg/vector.cpp index 456079489..62e5ac91f 100644 --- a/palace/linalg/vector.cpp +++ b/palace/linalg/vector.cpp @@ -356,7 +356,7 @@ void SetRandomSign(MPI_Comm comm, ComplexVector &x, int seed) { XI[i] = (XI[i] > 0.0) ? 1.0 : ((XI[i] < 0.0) ? -1.0 : 0.0); }); } -void CwiseSqrt(Vector &x) +void Sqrt(Vector &x) { const int N = x.Size(); auto *X = x.ReadWrite(); diff --git a/palace/linalg/vector.hpp b/palace/linalg/vector.hpp index 8ed4f9634..cc36296c1 100644 --- a/palace/linalg/vector.hpp +++ b/palace/linalg/vector.hpp @@ -140,7 +140,7 @@ template void SetRandomSign(MPI_Comm comm, VecType &x, int seed = 0); // Component-wise square root. -void CwiseSqrt(Vector &x); +void Sqrt(Vector &x); // Calculate the inner product yᴴ x or yᵀ x. template diff --git a/palace/main.cpp b/palace/main.cpp index 67a335d97..b68dc4f13 100644 --- a/palace/main.cpp +++ b/palace/main.cpp @@ -12,7 +12,7 @@ #include "drivers/electrostaticsolver.hpp" #include "drivers/magnetostaticsolver.hpp" #include "drivers/transientsolver.hpp" -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "linalg/slepc.hpp" #include "utils/communication.hpp" #include "utils/geodata.hpp" diff --git a/palace/models/postoperator.cpp b/palace/models/postoperator.cpp index 4d4f0d740..47b44f095 100644 --- a/palace/models/postoperator.cpp +++ b/palace/models/postoperator.cpp @@ -4,7 +4,7 @@ #include "postoperator.hpp" #include "fem/coefficient.hpp" -#include "fem/errorindicators.hpp" +#include "fem/errorindicator.hpp" #include "models/curlcurloperator.hpp" #include "models/laplaceoperator.hpp" #include "models/lumpedportoperator.hpp" diff --git a/palace/models/postoperator.hpp b/palace/models/postoperator.hpp index ff0e6b898..8ca93b0a6 100644 --- a/palace/models/postoperator.hpp +++ b/palace/models/postoperator.hpp @@ -21,7 +21,7 @@ namespace palace { class CurlCurlOperator; -class ErrorIndicators; +class ErrorIndicator; class IoData; class LaplaceOperator; class LumpedPortOperator; diff --git a/palace/utils/configfile.cpp b/palace/utils/configfile.cpp index 94ec6298d..f232cef33 100644 --- a/palace/utils/configfile.cpp +++ b/palace/utils/configfile.cpp @@ -1665,13 +1665,13 @@ void LinearSolverData::SetUp(json &solver) strumpack_butterfly_l = linear->value("STRUMPACKButterflyLevels", strumpack_butterfly_l); superlu_3d = linear->value("SuperLU3D", superlu_3d); ams_vector = linear->value("AMSVector", ams_vector); + + // Other linear solver options. divfree_tol = linear->value("DivFreeTol", divfree_tol); divfree_max_it = linear->value("DivFreeMaxIts", divfree_max_it); - gs_orthog_type = linear->value("GSOrthogonalization", gs_orthog_type); - - // Options related to the estimator. estimator_tol = linear->value("EstimatorTol", estimator_tol); estimator_max_it = linear->value("EstimatorMaxIts", estimator_max_it); + gs_orthog_type = linear->value("GSOrthogonalization", gs_orthog_type); // Cleanup linear->erase("Type"); @@ -1703,12 +1703,12 @@ void LinearSolverData::SetUp(json &solver) linear->erase("STRUMPACKButterflyLevels"); linear->erase("SuperLU3D"); linear->erase("AMSVector"); + linear->erase("DivFreeTol"); linear->erase("DivFreeMaxIts"); - linear->erase("GSOrthogonalization"); - linear->erase("EstimatorTol"); linear->erase("EstimatorMaxIts"); + linear->erase("GSOrthogonalization"); MFEM_VERIFY(linear->empty(), "Found an unsupported configuration file keyword under \"Linear\"!\n" << linear->dump(2)); diff --git a/palace/utils/configfile.hpp b/palace/utils/configfile.hpp index fadae6080..2b42fde30 100644 --- a/palace/utils/configfile.hpp +++ b/palace/utils/configfile.hpp @@ -820,10 +820,10 @@ struct LinearSolverData int divfree_max_it = 1000; // Relative tolerance for solving linear systems in the error estimator. - double estimator_tol = tol; + double estimator_tol = 1e-6; // Maximum number of iterations for solving linear systems in the error estimator. - int estimator_max_it = max_it; + int estimator_max_it = 100; // Enable different variants of Gram-Schmidt orthogonalization for GMRES/FGMRES iterative // solvers and SLEPc eigenvalue solver. From 045043d48da922fdd06af644e5bfa1f38ef48b70 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Mon, 9 Oct 2023 11:56:57 -0400 Subject: [PATCH 32/39] Add a folding method to error estimator class. Change to only dumping the reduced form to the csv --- palace/drivers/basesolver.cpp | 29 ++- palace/drivers/basesolver.hpp | 6 +- palace/drivers/drivensolver.cpp | 47 ++-- palace/drivers/eigensolver.cpp | 17 +- palace/drivers/electrostaticsolver.cpp | 9 +- palace/drivers/magnetostaticsolver.cpp | 11 +- palace/drivers/transientsolver.cpp | 18 +- palace/linalg/errorestimator.cpp | 35 +++ palace/linalg/errorestimator.hpp | 18 +- .../ref/cavity/impedance/error-indicators.csv | 18 +- test/ref/cavity/pec/error-indicators.csv | 18 +- test/ref/coaxial/matched/error-indicators.csv | 204 +----------------- test/ref/coaxial/open/error-indicators.csv | 204 +----------------- .../cpw/lumped_adaptive/error-indicators.csv | 13 +- .../cpw/lumped_uniform/error-indicators.csv | 18 +- .../cpw/wave_adaptive/error-indicators.csv | 13 +- .../ref/cpw/wave_uniform/error-indicators.csv | 18 +- test/ref/rings/error-indicators.csv | 5 +- test/ref/spheres/error-indicators.csv | 5 +- test/runtests.jl | 18 +- test/testcase.jl | 10 +- 21 files changed, 136 insertions(+), 598 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index 6c243900e..731dad8e9 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -566,8 +566,7 @@ void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double Mpi::Barrier(); } -void BaseSolver::PostprocessErrorIndicator(const std::string &name, int step, double time, - const std::array &data) const +void BaseSolver::PostprocessErrorIndicator(const std::array &data) const { if (post_dir.length() == 0) { @@ -578,23 +577,19 @@ void BaseSolver::PostprocessErrorIndicator(const std::string &name, int step, do if (root) { std::string path = post_dir + "error-indicators.csv"; - auto output = OutputFile(path, (step > 0)); - if (step == 0) - { - // clang-format off - output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", - name, table.w1, - "Sum", table.w, - "Min", table.w, - "Max", table.w, - "Mean", table.w, - "Normalization", table.w); - // clang-format on - } + auto output = OutputFile(path, false); + + // clang-format off + output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", + "Sum", table.w, + "Min", table.w, + "Max", table.w, + "Mean", table.w, + "Normalization", table.w); + // clang-format on // clang-format off - output.print("{:{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", - time, table.w1, table.p1, + output.print("{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", data[0], table.w, table.p, data[1], table.w, table.p, data[2], table.w, table.p, diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index bcc485d79..6938b13b5 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -68,11 +68,7 @@ class BaseSolver void PostprocessProbes(const PostOperator &postop, const std::string &name, int step, double time) const; void PostprocessFields(const PostOperator &postop, int step, double time) const; - - // Postprocess granular error indicator to file. The argument normalized indicates if - // supplied indicators have already been normalized. - void PostprocessErrorIndicator(const std::string &name, int step, double time, - const std::array &data) const; + void PostprocessErrorIndicator(const std::array &data) const; public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 78669b00b..69ee10ff6 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -144,19 +144,8 @@ ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator & E = 0.0; B = 0.0; - // Initialize structures for storing and reducing the results of error estimation. - ErrorIndicator indicators; - auto UpdateErrorIndicator = [this, &estimator, &indicators, &postop, - &spaceop](const auto &E, int step, double frequency) - { - BlockTimer bt0(Timer::ESTIMATION); - ErrorIndicator sample = estimator.ComputeIndicators(E); - postop.SetIndicatorGridFunction(sample.Local()); - BlockTimer bt_post(Timer::POSTPRO); - PostprocessErrorIndicator("f (GHz)", step, frequency, - sample.GetPostprocessData(spaceop.GetComm())); - indicators.AddIndicator(sample); - }; + // Initialize structures for storing the error indicator. + ErrorIndicator indicator; // Main frequency sweep loop. double omega = omega0; @@ -187,7 +176,7 @@ ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator & ksp.Mult(RHS, E); // Compute the error indicators, and post process the indicator field. - UpdateErrorIndicator(E, step, freq); + estimator.AddErrorIndicator(indicator, postop, E); // Compute B = -1/(iω) ∇ x E on the true dofs, and set the internal GridFunctions in // PostOperator for all postprocessing operations. @@ -218,7 +207,9 @@ ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator & omega += delta_omega; } SaveMetadata(ksp); - return indicators; + BlockTimer bt_post(Timer::POSTPRO); + PostprocessErrorIndicator(indicator.GetPostprocessData(spaceop.GetComm())); + return indicator; } ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, @@ -276,17 +267,7 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // The error indicators will be calculated for each HDM sample rather than for // the online stage. - ErrorIndicator indicators; - auto UpdateErrorIndicator = - [this, &estimator, &indicators, &spaceop](const auto &E, int step, double frequency) - { - BlockTimer bt0(Timer::ESTIMATION); - auto sample_indicators = estimator.ComputeIndicators(E); - BlockTimer bt_post(Timer::POSTPRO); - PostprocessErrorIndicator("f (GHz)", step, frequency, - sample_indicators.GetPostprocessData(spaceop.GetComm())); - indicators.AddIndicator(sample_indicators); - }; + ErrorIndicator indicator; // Initialize the basis with samples from the top and bottom of the frequency // range of interest. Each call for an HDM solution adds the frequency sample to P_S and @@ -294,10 +275,10 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // of the RomOperator. BlockTimer bt1(Timer::CONSTRUCTPROM); prom.SolveHDM(omega0, E); // Print matrix stats at first HDM solve - UpdateErrorIndicator(E, 0, omega0 * f0); + estimator.AddErrorIndicator(indicator, E); prom.AddHDMSample(omega0, E); prom.SolveHDM(omega0 + (nstep - step0 - 1) * delta_omega, E); - UpdateErrorIndicator(E, 1, (omega0 + (nstep - step0 - 1) * delta_omega) * f0); + estimator.AddErrorIndicator(indicator, E); prom.AddHDMSample(omega0 + (nstep - step0 - 1) * delta_omega, E); // Greedy procedure for basis construction (offline phase). Basis is initialized with @@ -321,7 +302,7 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator max_error); prom.SolveHDM(omega_star, E); Mpi::Print("Computing error estimates\n"); - UpdateErrorIndicator(E, iter - iter0 + 1, omega_star * f0); + estimator.AddErrorIndicator(indicator, E); prom.AddHDMSample(omega_star, E); iter++; } @@ -334,8 +315,10 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator Timer::Duration(Timer::Now() - t0).count()); // Timing on root SaveMetadata(prom.GetLinearSolver()); - // Set the indicator field to the combined field for postprocessing. - postop.SetIndicatorGridFunction(indicators.Local()); + // Postprocess the indicator. + BlockTimer bt_post(Timer::POSTPRO); + postop.SetIndicatorGridFunction(indicator.Local()); + PostprocessErrorIndicator(indicator.GetPostprocessData(spaceop.GetComm())); // Main fast frequency sweep loop (online phase). BlockTimer bt2(Timer::CONSTRUCT); @@ -384,7 +367,7 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator step++; omega += delta_omega; } - return indicators; + return indicator; } int DrivenSolver::GetNumSteps(double start, double end, double delta) const diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index cdb902b1e..22e07d519 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -266,12 +266,12 @@ EigenSolver::Solve(const std::vector> &mesh) cons // Save the eigenvalue estimates. BlockTimer bt_est(Timer::ESTIMATION); std::vector indicators; - indicators.reserve(num_conv); + indicators.reserve(iodata.solver.eigenmode.n); for (int i = 0; i < iodata.solver.eigenmode.n; i++) { - eigen->GetEigenvector(i, E); // Only update the error indicator for targeted modes. Mpi::Print("\nComputing error estimates for mode {:d}\n", i); + eigen->GetEigenvector(i, E); indicators.emplace_back(estimator.ComputeIndicators(E)); } @@ -309,17 +309,16 @@ EigenSolver::Solve(const std::vector> &mesh) cons postop.SetEGridFunction(E); postop.SetBGridFunction(B); postop.UpdatePorts(spaceop.GetLumpedPortOp(), omega.real()); - - // Postprocess the mode. - Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv); - - if (i < iodata.solver.eigenmode.n) + if (i < iodata.solver.eigenmode.n_post) { postop.SetIndicatorGridFunction(indicators[i].Local()); - PostprocessErrorIndicator("m", i, i + 1, - indicators[i].GetPostprocessData(spaceop.GetComm())); } + + // Postprocess the mode. + Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv); } + PostprocessErrorIndicator( + ErrorIndicator(indicators).GetPostprocessData(spaceop.GetComm())); return indicators; } diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 25c1ded92..6ac7e7861 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -70,11 +70,9 @@ ElectrostaticSolver::Solve(const std::vector> &me BlockTimer bt1(Timer::SOLVE); ksp.Mult(RHS, V[step]); - - BlockTimer bt2(Timer::ESTIMATION); indicators[step] = estimator.ComputeIndicators(V[step]); - BlockTimer bt3(Timer::POSTPRO); + BlockTimer bt2(Timer::POSTPRO); Mpi::Print(" Sol. ||V|| = {:.6e} (||RHS|| = {:.6e})\n", linalg::Norml2(laplaceop.GetComm(), V[step]), linalg::Norml2(laplaceop.GetComm(), RHS)); @@ -87,6 +85,8 @@ ElectrostaticSolver::Solve(const std::vector> &me BlockTimer bt1(Timer::POSTPRO); SaveMetadata(ksp); Postprocess(laplaceop, postop, V, indicators); + PostprocessErrorIndicator( + ErrorIndicator(indicators).GetPostprocessData(laplaceop.GetComm())); return indicators; } @@ -123,10 +123,9 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & PostprocessDomains(postop, "i", i, idx, Ue, 0.0, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, Ue, 0.0, 1.0, 0.0); PostprocessProbes(postop, "i", i, idx); - PostprocessErrorIndicator("i", i, idx, - indicators[i].GetPostprocessData(laplaceop.GetComm())); if (i < iodata.solver.electrostatic.n_post) { + postop.SetIndicatorGridFunction(indicators[i].Local()); PostprocessFields(postop, i, idx); Mpi::Print(" Wrote fields to disk for terminal {:d}\n", idx); } diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index 7c3a9abad..d6fbc5821 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -47,7 +47,7 @@ MagnetostaticSolver::Solve(const std::vector> &me MFEM_VERIFY(nstep > 0, "No surface current boundaries specified for magnetostatic simulation!"); - // Source term and solution vector storage. + // Source term, solution vector storage and error indicators storage. Vector RHS(K->Height()); std::vector A(nstep); std::vector indicators(nstep); @@ -70,11 +70,9 @@ MagnetostaticSolver::Solve(const std::vector> &me BlockTimer bt1(Timer::SOLVE); ksp.Mult(RHS, A[step]); - - BlockTimer bt2(Timer::ESTIMATION); indicators[step] = estimator.ComputeIndicators(A[step]); - BlockTimer bt3(Timer::POSTPRO); + BlockTimer bt2(Timer::POSTPRO); Mpi::Print(" Sol. ||A|| = {:.6e} (||RHS|| = {:.6e})\n", linalg::Norml2(curlcurlop.GetComm(), A[step]), linalg::Norml2(curlcurlop.GetComm(), RHS)); @@ -87,6 +85,8 @@ MagnetostaticSolver::Solve(const std::vector> &me BlockTimer bt1(Timer::POSTPRO); SaveMetadata(ksp); Postprocess(curlcurlop, postop, A, indicators); + PostprocessErrorIndicator( + ErrorIndicator(indicators).GetPostprocessData(curlcurlop.GetComm())); return indicators; } @@ -128,10 +128,9 @@ void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator PostprocessDomains(postop, "i", i, idx, 0.0, Um, 0.0, 0.0); PostprocessSurfaces(postop, "i", i, idx, 0.0, Um, 0.0, Iinc(i)); PostprocessProbes(postop, "i", i, idx); - PostprocessErrorIndicator("i", i, idx, - indicators[i].GetPostprocessData(curlcurlop.GetComm())); if (i < iodata.solver.magnetostatic.n_post) { + postop.SetIndicatorGridFunction(indicators[i].Local()); PostprocessFields(postop, i, idx); Mpi::Print(" Wrote fields to disk for terminal {:d}\n", idx); } diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index f1ec31788..2f3c7ec86 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -83,18 +83,7 @@ TransientSolver::Solve(const std::vector> &mesh) BlockTimer bt(Timer::ESTCONSTRUCT); return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); - ErrorIndicator indicators; - auto UpdateErrorIndicator = [this, &estimator, &indicators, &postop, - &spaceop](const auto &E, int step, double time) - { - BlockTimer bt0(Timer::ESTIMATION); - auto sample_indicators = estimator.ComputeIndicators(E); - postop.SetIndicatorGridFunction(sample_indicators.Local()); - BlockTimer bt_post(Timer::POSTPRO); - PostprocessErrorIndicator("t (ns)", step, time, - sample_indicators.GetPostprocessData(spaceop.GetComm())); - indicators.AddIndicator(sample_indicators); - }; + ErrorIndicator indicator; // Main time integration loop. int step = 0; @@ -138,7 +127,7 @@ TransientSolver::Solve(const std::vector> &mesh) } // Calculate and record the error indicators. - UpdateErrorIndicator(E, step, t); + estimator.AddErrorIndicator(indicator, postop, E); // Postprocess port voltages/currents and optionally write solution to disk. Postprocess(postop, spaceop.GetLumpedPortOp(), spaceop.GetSurfaceCurrentOp(), step, t, @@ -148,7 +137,8 @@ TransientSolver::Solve(const std::vector> &mesh) step++; } SaveMetadata(timeop.GetLinearSolver()); - return indicators; + PostprocessErrorIndicator(indicator.GetPostprocessData(spaceop.GetComm())); + return indicator; } std::function TransientSolver::GetTimeExcitation(bool dot) const { diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index b3c91d5c9..34d414821 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -12,6 +12,7 @@ #include "linalg/iterative.hpp" #include "linalg/rap.hpp" #include "models/materialoperator.hpp" +#include "models/postoperator.hpp" #include "utils/communication.hpp" #include "utils/iodata.hpp" #include "utils/timer.hpp" @@ -111,6 +112,7 @@ CurlFluxErrorEstimator::CurlFluxErrorEstimator( template <> ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v) const { + BlockTimer bt_est(Timer::ESTIMATION); auto &nd_fespace = nd_fespaces.GetFinestFESpace(); const int nelem = nd_fespace.GetNE(); @@ -176,6 +178,7 @@ ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v) template <> ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const { + BlockTimer bt_est(Timer::ESTIMATION); auto &nd_fespace = nd_fespaces.GetFinestFESpace(); field_gf.SetFromTrueDofs(v); const int nelem = nd_fespace.GetNE(); @@ -234,6 +237,37 @@ ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const return ErrorIndicator(std::move(estimates), normalization); } +template +void CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &indicator, + PostOperator &postop, + const VectorType &v) const +{ + auto i = ComputeIndicators(v); + BlockTimer bt(Timer::POSTPRO); + postop.SetIndicatorGridFunction(i.Local()); + indicator.AddIndicator(i); +} + +template +void CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &indicator, + const VectorType &v) const +{ + BlockTimer bt(Timer::POSTPRO); + indicator.AddIndicator(ComputeIndicators(v)); +} + +template void +CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &, PostOperator &, + const ComplexVector &) const; +template void CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &, + PostOperator &, + const Vector &) const; +template void +CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &, + const ComplexVector &) const; +template void CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &, + const Vector &) const; + GradFluxErrorEstimator::GradFluxErrorEstimator( const IoData &iodata, const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &h1_fespaces) @@ -250,6 +284,7 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const { + BlockTimer bt_est(Timer::ESTIMATION); auto &h1_fespace = h1_fespaces.GetFinestFESpace(); const int sdim = h1_fespace.GetMesh()->SpaceDimension(); field_gf.SetFromTrueDofs(v); diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index c811833f9..872ec9f58 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -16,6 +16,7 @@ namespace palace class IoData; class MaterialOperator; +class PostOperator; // // This solver computes a smooth reconstruction of a discontinuous flux. The difference @@ -61,9 +62,17 @@ class CurlFluxErrorEstimator CurlFluxErrorEstimator(const IoData &iodata, const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &nd_fespaces); - // Compute elemental error indicators given a complex vector of true DOF. + // Compute elemental error indicators given a vector of true DOF. template ErrorIndicator ComputeIndicators(const VectorType &v) const; + + // Compute elemental error indicators given a vector of true DOF, v, and fold into an + // existing indicator. Optionally set the error indicator field within a PostOperator. + template + void AddErrorIndicator(ErrorIndicator &indicator, PostOperator &postop, + const VectorType &v) const; + template + void AddErrorIndicator(ErrorIndicator &indicator, const VectorType &v) const; }; // Class used for computing grad flux error estimate, i.e. || ϵ ∇ ϕₕ - F ||_K where F @@ -85,6 +94,13 @@ class GradFluxErrorEstimator // Compute elemental error indicators given a vector of true DOF. ErrorIndicator ComputeIndicators(const Vector &v) const; + + // Compute elemental error indicators given a vector of true DOF, v, and fold into an + // existing indicator. + void AddErrorIndicator(ErrorIndicator &indicator, const Vector &v) const + { + indicator.AddIndicator(ComputeIndicators(v)); + } }; } // namespace palace diff --git a/test/ref/cavity/impedance/error-indicators.csv b/test/ref/cavity/impedance/error-indicators.csv index a81f09c9f..fb6548b0f 100644 --- a/test/ref/cavity/impedance/error-indicators.csv +++ b/test/ref/cavity/impedance/error-indicators.csv @@ -1,16 +1,2 @@ - m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +8.196374239e-03, +9.214842342e-05, +6.510932304e-04, +3.254431675e-04, +6.964480813e-01 - 2.000000e+00, +7.387353181e-03, +1.120183148e-04, +5.134253635e-04, +2.965305671e-04, +4.545494361e-01 - 3.000000e+00, +7.977895730e-03, +1.062690571e-04, +6.123658757e-04, +3.123956087e-04, +5.169059866e-01 - 4.000000e+00, +8.274201489e-03, +6.590964834e-05, +8.966948182e-04, +3.045023806e-04, +7.254767932e-01 - 5.000000e+00, +1.685960941e-02, +2.820942104e-04, +1.153640116e-03, +6.780447656e-04, +7.021998742e-01 - 6.000000e+00, +1.730399293e-02, +2.679568469e-04, +1.285571611e-03, +6.941394542e-04, +6.917741537e-01 - 7.000000e+00, +1.541951446e-02, +3.015087299e-04, +1.088775657e-03, +6.184944349e-04, +6.837241462e-01 - 8.000000e+00, +1.589962692e-02, +2.715177297e-04, +1.182648337e-03, +6.283845490e-04, +7.775219321e-01 - 9.000000e+00, +2.159859358e-02, +3.943793250e-04, +1.605016900e-03, +8.505673899e-04, +1.082720492e+00 - 1.000000e+01, +2.566043946e-02, +3.654560100e-04, +2.348156651e-03, +9.735827558e-04, +1.143440851e+00 - 1.100000e+01, +1.170857739e-02, +2.548269260e-04, +8.762332197e-04, +4.764991796e-04, +8.880913705e-01 - 1.200000e+01, +2.430340822e-02, +3.361537088e-04, +2.235911969e-03, +9.591480064e-04, +8.564002177e-01 - 1.300000e+01, +2.162823528e-02, +1.439773671e-04, +2.211508889e-03, +7.807885965e-04, +1.123798284e+00 - 1.400000e+01, +2.568539055e-02, +1.169360347e-04, +3.234035290e-03, +8.927006761e-04, +1.125061440e+00 - 1.500000e+01, +2.696836169e-02, +4.629929123e-04, +1.772177016e-03, +1.080937100e-03, +8.955419511e-01 + Sum, Min, Max, Mean, Normalization + +1.832232364e-02, +4.343456836e-04, +1.399358786e-03, +7.453986746e-04, +8.242436674e-01 diff --git a/test/ref/cavity/pec/error-indicators.csv b/test/ref/cavity/pec/error-indicators.csv index afe734cbc..3e8f88db3 100644 --- a/test/ref/cavity/pec/error-indicators.csv +++ b/test/ref/cavity/pec/error-indicators.csv @@ -1,16 +1,2 @@ - m, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +8.196373525e-03, +9.214821861e-05, +6.510933257e-04, +3.254431264e-04, +6.964480689e-01 - 2.000000e+00, +7.387353222e-03, +1.120182421e-04, +5.134245794e-04, +2.965305493e-04, +4.545494456e-01 - 3.000000e+00, +7.977895796e-03, +1.062685104e-04, +6.123649465e-04, +3.123955986e-04, +5.169059367e-01 - 4.000000e+00, +8.274201169e-03, +6.590958800e-05, +8.966948963e-04, +3.045023647e-04, +7.254767786e-01 - 5.000000e+00, +1.685960472e-02, +2.820836869e-04, +1.153598318e-03, +6.780453428e-04, +7.021989802e-01 - 6.000000e+00, +1.730399491e-02, +2.679493494e-04, +1.285568826e-03, +6.941391330e-04, +6.917744665e-01 - 7.000000e+00, +1.541950856e-02, +3.013866066e-04, +1.088523264e-03, +6.184933404e-04, +6.837273782e-01 - 8.000000e+00, +1.589962884e-02, +2.714996993e-04, +1.182628298e-03, +6.283841145e-04, +7.775204071e-01 - 9.000000e+00, +2.159859353e-02, +3.943791856e-04, +1.605017990e-03, +8.505674052e-04, +1.082720471e+00 - 1.000000e+01, +2.566044031e-02, +3.654560313e-04, +2.348157389e-03, +9.735827810e-04, +1.143440852e+00 - 1.100000e+01, +1.170857794e-02, +2.548269622e-04, +8.762333072e-04, +4.764992002e-04, +8.880913583e-01 - 1.200000e+01, +2.430340709e-02, +3.361540626e-04, +2.235911950e-03, +9.591479625e-04, +8.564001863e-01 - 1.300000e+01, +2.162823566e-02, +1.439781616e-04, +2.211512532e-03, +7.807886467e-04, +1.123798302e+00 - 1.400000e+01, +2.568539163e-02, +1.169360360e-04, +3.234036921e-03, +8.927007077e-04, +1.125061428e+00 - 1.500000e+01, +2.696828680e-02, +4.629949994e-04, +1.772157010e-03, +1.080939278e-03, +8.955391704e-01 + Sum, Min, Max, Mean, Normalization + +1.832231602e-02, +4.343391518e-04, +1.399358147e-03, +7.453990858e-04, +8.242435486e-01 diff --git a/test/ref/coaxial/matched/error-indicators.csv b/test/ref/coaxial/matched/error-indicators.csv index eedd72de9..9cce1c0b0 100644 --- a/test/ref/coaxial/matched/error-indicators.csv +++ b/test/ref/coaxial/matched/error-indicators.csv @@ -1,202 +1,2 @@ - t (ns), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 - 3.747406e-02, +5.572991710e-02, +4.767755798e-15, +1.407880294e-02, +1.856308518e-03, +1.015013075e-04 - 7.494811e-02, +1.034912856e-01, +9.719720872e-15, +2.607229698e-02, +3.523283344e-03, +1.286451027e-04 - 1.124222e-01, +3.324496389e-02, +6.365619917e-15, +8.388885356e-03, +1.164706330e-03, +1.363415075e-04 - 1.498962e-01, +2.375714211e-02, +2.397965370e-15, +6.044798885e-03, +8.245600591e-04, +2.237425263e-04 - 1.873703e-01, +3.819179106e-02, +5.024345578e-15, +9.668211450e-03, +1.299554229e-03, +5.988175109e-04 - 2.248443e-01, +1.297340613e-02, +1.781314293e-15, +3.303352868e-03, +4.708062401e-04, +1.231303285e-03 - 2.623184e-01, +5.246709261e-03, +2.383679123e-15, +1.405616340e-03, +2.071756999e-04, +2.183486767e-03 - 2.997925e-01, +6.058354891e-03, +3.690614455e-15, +1.493506908e-03, +2.376336055e-04, +3.443837437e-03 - 3.372665e-01, +6.156851514e-03, +4.074907959e-15, +1.749181241e-03, +2.239413758e-04, +4.896408733e-03 - 3.747406e-01, +4.694296649e-03, +8.973267467e-15, +1.289674681e-03, +1.823298192e-04, +6.288158078e-03 - 4.122146e-01, +5.417259350e-03, +1.949534383e-15, +1.436619200e-03, +2.081413126e-04, +7.236866239e-03 - 4.496887e-01, +7.374336957e-03, +1.897219364e-14, +1.846629384e-03, +2.874502795e-04, +7.438562524e-03 - 4.871627e-01, +6.930576439e-03, +2.280046795e-14, +1.752840045e-03, +2.931825986e-04, +7.655638548e-03 - 5.246368e-01, +5.525646929e-03, +3.091446252e-14, +1.406681124e-03, +2.243733864e-04, +1.144492786e-02 - 5.621109e-01, +4.673784424e-03, +3.121340021e-14, +1.348460932e-03, +1.868352062e-04, +2.129721915e-02 - 5.995849e-01, +4.253659682e-03, +1.861613505e-14, +1.368615134e-03, +1.587836337e-04, +3.669178320e-02 - 6.370590e-01, +4.178197277e-03, +5.274944314e-14, +1.375975768e-03, +1.455795304e-04, +5.641443987e-02 - 6.745330e-01, +4.365910098e-03, +2.727376788e-14, +1.376592666e-03, +1.531072976e-04, +7.828637698e-02 - 7.120071e-01, +4.532195073e-03, +8.520431431e-14, +1.342611942e-03, +1.672385806e-04, +9.885177909e-02 - 7.494811e-01, +4.713112853e-03, +1.938911398e-13, +1.277228489e-03, +1.799446727e-04, +1.137531996e-01 - 7.869552e-01, +4.875763465e-03, +3.075861075e-13, +1.187769851e-03, +1.938215810e-04, +1.195346821e-01 - 8.244293e-01, +4.981324068e-03, +1.288555229e-12, +1.301107022e-03, +2.128512979e-04, +1.188340974e-01 - 8.619033e-01, +4.771406979e-03, +6.396990220e-12, +1.099709362e-03, +2.160907931e-04, +1.309553939e-01 - 8.993774e-01, +4.387618618e-03, +1.891176183e-11, +1.104046666e-03, +1.969683895e-04, +1.857760966e-01 - 9.368514e-01, +4.317746826e-03, +4.265185878e-11, +1.315698936e-03, +1.778612664e-04, +2.867307084e-01 - 9.743255e-01, +4.352783980e-03, +9.207041359e-11, +1.367694242e-03, +1.732486187e-04, +4.167673658e-01 - 1.011800e+00, +4.392520145e-03, +1.998889274e-10, +1.347497137e-03, +1.702743981e-04, +5.549049357e-01 - 1.049274e+00, +4.412402763e-03, +4.284702403e-10, +1.276609665e-03, +1.748952061e-04, +6.782106751e-01 - 1.086748e+00, +4.409231342e-03, +8.771686060e-10, +1.151820795e-03, +1.841348266e-04, +7.648957645e-01 - 1.124222e+00, +4.385361349e-03, +1.640994893e-09, +1.054162063e-03, +1.931535836e-04, +8.022489576e-01 - 1.161696e+00, +4.359684491e-03, +2.540940898e-09, +1.097370972e-03, +2.015036517e-04, +8.009704160e-01 - 1.199170e+00, +4.376985093e-03, +2.358239072e-09, +9.962793898e-04, +2.076058305e-04, +8.143229074e-01 - 1.236644e+00, +4.452602827e-03, +3.901518113e-09, +9.301914663e-04, +2.236743970e-04, +9.294380600e-01 - 1.274118e+00, +4.490626319e-03, +1.578140786e-08, +1.141847583e-03, +2.193640151e-04, +1.185764957e+00 - 1.311592e+00, +4.461869998e-03, +3.635560929e-08, +1.253108436e-03, +2.106610575e-04, +1.534641583e+00 - 1.349066e+00, +4.401936395e-03, +6.699205042e-08, +1.237832414e-03, +2.050874105e-04, +1.896337013e+00 - 1.386540e+00, +4.333203621e-03, +1.076793012e-07, +1.149104164e-03, +2.051917039e-04, +2.199936335e+00 - 1.424014e+00, +4.274237239e-03, +1.530629529e-07, +1.003500853e-03, +2.108710395e-04, +2.397678575e+00 - 1.461488e+00, +4.247523108e-03, +1.896601883e-07, +9.868601873e-04, +2.177096147e-04, +2.477217295e+00 - 1.498962e+00, +4.280032390e-03, +2.092002356e-07, +9.955121384e-04, +2.271285516e-04, +2.474013829e+00 - 1.536436e+00, +4.376655840e-03, +2.839365838e-07, +9.435084758e-04, +2.375100394e-04, +2.473624483e+00 - 1.573910e+00, +4.478796254e-03, +5.266487239e-07, +9.452199888e-04, +2.535903206e-04, +2.579178593e+00 - 1.611384e+00, +4.504195280e-03, +8.670090215e-07, +8.305664715e-04, +2.598806764e-04, +2.837660777e+00 - 1.648859e+00, +4.449741379e-03, +1.200124989e-06, +9.532357119e-04, +2.571489935e-04, +3.200525521e+00 - 1.686333e+00, +4.368989074e-03, +1.481583714e-06, +9.657314632e-04, +2.553327470e-04, +3.569943474e+00 - 1.723807e+00, +4.305814700e-03, +1.685010880e-06, +9.004533391e-04, +2.557437501e-04, +3.860928050e+00 - 1.761281e+00, +4.281235699e-03, +1.743569614e-06, +8.111212810e-04, +2.616480310e-04, +4.031057327e+00 - 1.798755e+00, +4.299570361e-03, +1.539868619e-06, +8.541153812e-04, +2.692315782e-04, +4.086183103e+00 - 1.836229e+00, +4.350193371e-03, +1.240981512e-06, +8.333310469e-04, +2.775510898e-04, +4.072063131e+00 - 1.873703e+00, +4.406139686e-03, +2.397908290e-06, +8.321882404e-04, +2.865840244e-04, +4.053029029e+00 - 1.911177e+00, +4.436326424e-03, +5.244203576e-06, +8.330079752e-04, +2.954125346e-04, +4.080832967e+00 - 1.948651e+00, +4.428790225e-03, +9.058143034e-06, +7.711842311e-04, +3.027223741e-04, +4.169084349e+00 - 1.986125e+00, +4.399307303e-03, +1.325746399e-05, +7.862986976e-04, +3.057197152e-04, +4.292757706e+00 - 2.023599e+00, +4.372194815e-03, +1.510320229e-05, +7.323360545e-04, +3.100933893e-04, +4.411208547e+00 - 2.061073e+00, +4.361026041e-03, +1.884890795e-05, +7.327459971e-04, +3.150197546e-04, +4.493075573e+00 - 2.098547e+00, +4.366509432e-03, +1.374550555e-05, +7.246019676e-04, +3.202936759e-04, +4.527901974e+00 - 2.136021e+00, +4.381912029e-03, +1.715811914e-05, +6.970023574e-04, +3.265541668e-04, +4.524236977e+00 - 2.173495e+00, +4.397351815e-03, +1.079978217e-05, +7.272899437e-04, +3.301454651e-04, +4.500275525e+00 - 2.210969e+00, +4.405183345e-03, +1.447955334e-05, +6.922741206e-04, +3.337723250e-04, +4.473334986e+00 - 2.248443e+00, +4.405486375e-03, +3.446567370e-05, +7.204423006e-04, +3.362065933e-04, +4.453002776e+00 - 2.285917e+00, +4.402807453e-03, +6.095148461e-05, +7.118317587e-04, +3.394811402e-04, +4.440058051e+00 - 2.323392e+00, +4.399850845e-03, +8.934753195e-05, +6.971610885e-04, +3.421644326e-04, +4.429819755e+00 - 2.360866e+00, +4.398068699e-03, +9.030215537e-05, +7.175095818e-04, +3.437942479e-04, +4.416620275e+00 - 2.398340e+00, +4.398861137e-03, +1.235780509e-04, +6.740707350e-04, +3.459275998e-04, +4.396813054e+00 - 2.435814e+00, +4.400980771e-03, +9.467385267e-05, +7.142802747e-04, +3.454258513e-04, +4.369717681e+00 - 2.473288e+00, +4.402460543e-03, +1.022104767e-04, +7.009001488e-04, +3.451696172e-04, +4.337243705e+00 - 2.510762e+00, +4.403895760e-03, +7.877147797e-05, +7.018112881e-04, +3.433220166e-04, +4.302986459e+00 - 2.548236e+00, +4.405792751e-03, +4.246249635e-05, +7.172947620e-04, +3.411869495e-04, +4.270950549e+00 - 2.585710e+00, +4.405598720e-03, +1.994809524e-05, +6.763893213e-04, +3.399230388e-04, +4.243663227e+00 - 2.623184e+00, +4.400694419e-03, +2.364148071e-05, +7.186146004e-04, +3.378805735e-04, +4.219874234e+00 - 2.660658e+00, +4.392485708e-03, +2.844058393e-05, +6.965240664e-04, +3.355645968e-04, +4.193027513e+00 - 2.698132e+00, +4.384585819e-03, +2.571478454e-05, +7.103855285e-04, +3.307865051e-04, +4.152202300e+00 - 2.735606e+00, +4.380780471e-03, +3.213583979e-05, +7.236362645e-04, +3.262575401e-04, +4.086406465e+00 - 2.773080e+00, +4.386554361e-03, +2.373514849e-05, +7.046036970e-04, +3.212456071e-04, +3.991182680e+00 - 2.810554e+00, +4.404814760e-03, +1.878884496e-05, +7.577059295e-04, +3.166360002e-04, +3.874510868e+00 - 2.848028e+00, +4.426459055e-03, +1.152351783e-05, +7.432128881e-04, +3.128601851e-04, +3.757750265e+00 - 2.885502e+00, +4.433150133e-03, +5.448998919e-06, +7.815790047e-04, +3.061679058e-04, +3.667691740e+00 - 2.922976e+00, +4.414697950e-03, +1.997860698e-06, +7.857457169e-04, +2.988084941e-04, +3.620168807e+00 - 2.960451e+00, +4.375682264e-03, +2.690972492e-06, +7.747290766e-04, +2.912160446e-04, +3.604784042e+00 - 2.997925e+00, +4.333071394e-03, +3.554263518e-06, +8.038401126e-04, +2.835943343e-04, +3.584400783e+00 - 3.035399e+00, +4.311111757e-03, +3.499465101e-06, +7.701958486e-04, +2.758686787e-04, +3.511864377e+00 - 3.072873e+00, +4.325993916e-03, +3.007849963e-06, +8.245577518e-04, +2.700227386e-04, +3.352677539e+00 - 3.110347e+00, +4.379396657e-03, +2.341254635e-06, +8.846266414e-04, +2.683847956e-04, +3.102447387e+00 - 3.147821e+00, +4.454636199e-03, +1.573502276e-06, +8.735639996e-04, +2.689007160e-04, +2.794852318e+00 - 3.185295e+00, +4.515021137e-03, +9.659168234e-07, +8.139447345e-04, +2.703820992e-04, +2.497474676e+00 - 3.222769e+00, +4.509445159e-03, +4.733078592e-07, +9.371588790e-04, +2.641431162e-04, +2.287003000e+00 - 3.260243e+00, +4.420580954e-03, +3.531254160e-07, +9.328068147e-04, +2.495118539e-04, +2.199280716e+00 - 3.297717e+00, +4.319453795e-03, +2.783172778e-07, +9.755430751e-04, +2.384117920e-04, +2.194256365e+00 - 3.335191e+00, +4.271601222e-03, +3.754524310e-07, +9.704484767e-04, +2.295039199e-04, +2.187349161e+00 - 3.372665e+00, +4.285381211e-03, +2.365562917e-07, +9.618492656e-04, +2.221995959e-04, +2.107609658e+00 - 3.410139e+00, +4.341461532e-03, +1.107883651e-07, +1.107163853e-03, +2.161560933e-04, +1.925381439e+00 - 3.447613e+00, +4.421259555e-03, +1.341227219e-07, +1.199144103e-03, +2.154776888e-04, +1.652331416e+00 - 3.485087e+00, +4.515090823e-03, +2.506405354e-07, +1.219011284e-03, +2.200093999e-04, +1.332405222e+00 - 3.522561e+00, +4.575781860e-03, +5.021636995e-07, +1.102521402e-03, +2.284250880e-04, +1.031973346e+00 - 3.560035e+00, +4.546571541e-03, +8.814336894e-07, +9.669912103e-04, +2.307163851e-04, +8.248149186e-01 - 3.597509e+00, +4.439546624e-03, +9.644046739e-07, +1.029526458e-03, +2.166721858e-04, +7.467546178e-01 - 3.634984e+00, +4.373608421e-03, +7.723624291e-07, +1.091750868e-03, +2.116976978e-04, +7.476143297e-01 - 3.672458e+00, +4.382050521e-03, +1.067257149e-06, +1.035051104e-03, +2.048938622e-04, +7.473446775e-01 - 3.709932e+00, +4.404293078e-03, +6.653011939e-07, +1.149806513e-03, +1.974798896e-04, +7.038738376e-01 - 3.747406e+00, +4.430911334e-03, +8.595315846e-07, +1.275090226e-03, +1.900683641e-04, +6.126704622e-01 - 3.784880e+00, +4.434943455e-03, +2.471155334e-06, +1.346868556e-03, +1.892394276e-04, +4.890429311e-01 - 3.822354e+00, +4.432465684e-03, +6.662846327e-06, +1.361827014e-03, +1.977109325e-04, +3.558817348e-01 - 3.859828e+00, +4.472946341e-03, +1.471115194e-05, +1.270690029e-03, +2.192143040e-04, +2.372244458e-01 - 3.897302e+00, +4.608050556e-03, +2.995805064e-05, +9.306446641e-04, +2.555399562e-04, +1.561822526e-01 - 3.934776e+00, +4.881684955e-03, +2.811035769e-05, +1.216238723e-03, +2.725953442e-04, +1.265095408e-01 - 3.972250e+00, +4.866553825e-03, +3.795415085e-05, +1.220274883e-03, +2.666603348e-04, +1.273673799e-01 - 4.009724e+00, +4.748277124e-03, +3.192103916e-05, +1.158713078e-03, +2.539291412e-04, +1.276014323e-01 - 4.047198e+00, +4.585078550e-03, +3.182645796e-05, +1.255153535e-03, +2.447044579e-04, +1.171764854e-01 - 4.084672e+00, +4.459956643e-03, +4.434580877e-05, +1.290534216e-03, +2.465663493e-04, +9.815025303e-02 - 4.122146e+00, +4.357849031e-03, +4.451221836e-05, +1.261671269e-03, +2.674618203e-04, +7.584704242e-02 - 4.159620e+00, +4.242745441e-03, +8.663524762e-05, +1.134062116e-03, +2.934100756e-04, +5.552946065e-02 - 4.197094e+00, +4.375605406e-03, +1.365338261e-04, +8.763229146e-04, +3.425542935e-04, +4.116638402e-02 - 4.234568e+00, +4.535627597e-03, +7.791910123e-05, +6.462325798e-04, +3.669813290e-04, +3.415469948e-02 - 4.272043e+00, +4.532610005e-03, +5.193406384e-05, +6.707342473e-04, +3.562439774e-04, +3.215669992e-02 - 4.309517e+00, +4.422878873e-03, +6.066243178e-05, +6.757720455e-04, +3.445782429e-04, +3.160208211e-02 - 4.346991e+00, +4.405074263e-03, +5.218897198e-05, +7.105897121e-04, +3.336881563e-04, +3.065433292e-02 - 4.384465e+00, +4.308113084e-03, +4.409648491e-05, +7.002029226e-04, +3.182689189e-04, +2.904485957e-02 - 4.421939e+00, +4.285526963e-03, +4.707655638e-05, +7.905553822e-04, +3.035324297e-04, +2.713891437e-02 - 4.459413e+00, +4.343692498e-03, +5.926643851e-05, +8.126272690e-04, +3.060319714e-04, +2.542525795e-02 - 4.496887e+00, +4.333152821e-03, +2.383821210e-05, +8.599452557e-04, +2.888604032e-04, +2.427020011e-02 - 4.534361e+00, +4.293537928e-03, +3.286058120e-05, +8.848851674e-04, +2.729569059e-04, +2.376332646e-02 - 4.571835e+00, +4.249950706e-03, +2.519408005e-05, +8.591102042e-04, +2.630769361e-04, +2.366890478e-02 - 4.609309e+00, +4.228465151e-03, +3.800292504e-05, +9.007685395e-04, +2.634801714e-04, +2.354601494e-02 - 4.646783e+00, +4.202837257e-03, +1.507852251e-05, +8.793042581e-04, +2.497816406e-04, +2.295709422e-02 - 4.684257e+00, +4.213544516e-03, +1.795495441e-05, +9.494334056e-04, +2.388208750e-04, +2.162669198e-02 - 4.721731e+00, +4.271063337e-03, +1.116033951e-05, +1.048741666e-03, +2.402208066e-04, +1.951405874e-02 - 4.759205e+00, +4.361823065e-03, +1.679035723e-05, +1.096732269e-03, +2.530489235e-04, +1.682482853e-02 - 4.796679e+00, +4.430739622e-03, +2.817473658e-05, +1.058445681e-03, +2.571286426e-04, +1.398738199e-02 - 4.834153e+00, +4.468650907e-03, +8.016321433e-06, +8.857266847e-04, +2.568817235e-04, +1.158616965e-02 - 4.871627e+00, +4.472242319e-03, +2.903955281e-05, +9.739320412e-04, +2.681645474e-04, +1.014926309e-02 - 4.909101e+00, +4.377306905e-03, +2.485303321e-05, +9.696690666e-04, +2.579063507e-04, +9.727291773e-03 - 4.946576e+00, +4.290251676e-03, +3.215230353e-05, +1.027097834e-03, +2.468654140e-04, +9.783162092e-03 - 4.984050e+00, +4.298852002e-03, +1.343957356e-05, +9.906713457e-04, +2.331045920e-04, +9.697365188e-03 - 5.021524e+00, +4.334948003e-03, +2.414041116e-05, +1.079954519e-03, +2.288631376e-04, +9.137158595e-03 - 5.058998e+00, +4.391053222e-03, +5.546421606e-05, +1.210551035e-03, +2.385691775e-04, +8.056788668e-03 - 5.096472e+00, +4.486289474e-03, +5.622452948e-05, +1.302320657e-03, +2.477799587e-04, +6.593799082e-03 - 5.133946e+00, +4.561488101e-03, +3.412102235e-05, +1.334140103e-03, +2.568804178e-04, +4.979274167e-03 - 5.171420e+00, +4.762251584e-03, +9.052984162e-05, +1.277977765e-03, +3.005184401e-04, +3.482289178e-03 - 5.208894e+00, +5.224459873e-03, +6.562928293e-05, +9.942272374e-04, +3.837846386e-04, +2.384265177e-03 - 5.246368e+00, +5.899621480e-03, +1.120080911e-04, +1.072941632e-03, +4.544766768e-04, +1.893843419e-03 - 5.283842e+00, +5.767016137e-03, +1.073061804e-04, +1.220911558e-03, +4.204603009e-04, +1.857438703e-03 - 5.321316e+00, +5.314092626e-03, +1.599609750e-04, +1.081665938e-03, +3.832096305e-04, +1.884002961e-03 - 5.358790e+00, +5.750003298e-03, +1.097718618e-04, +1.198336766e-03, +4.085534182e-04, +1.781436184e-03 - 5.396264e+00, +6.473457644e-03, +1.874973518e-04, +1.304272440e-03, +5.029876221e-04, +1.539642657e-03 - 5.433738e+00, +6.670885584e-03, +7.486683651e-05, +1.344655616e-03, +5.206861680e-04, +1.215798114e-03 - 5.471212e+00, +6.890518082e-03, +1.680465513e-04, +1.295438982e-03, +5.273174727e-04, +8.794540096e-04 - 5.508686e+00, +1.129388144e-02, +1.632579776e-04, +1.643767035e-03, +8.852734490e-04, +5.925691539e-04 - 5.546160e+00, +1.838113858e-02, +7.633509650e-04, +2.717852868e-03, +1.496740809e-03, +4.025824102e-04 - 5.583635e+00, +1.898308683e-02, +4.137337581e-04, +3.003513057e-03, +1.531899256e-03, +3.249971140e-04 - 5.621109e+00, +1.669982550e-02, +2.931819740e-04, +2.394492731e-03, +1.286154385e-03, +3.144512592e-04 - 5.658583e+00, +1.919434007e-02, +4.828840707e-04, +2.665016049e-03, +1.558940214e-03, +3.143193611e-04 - 5.696057e+00, +2.277029501e-02, +5.669600759e-04, +3.381616077e-03, +1.770994274e-03, +3.053742378e-04 - 5.733531e+00, +2.277559895e-02, +4.797231463e-04, +3.341241825e-03, +1.823991135e-03, +2.905237722e-04 - 5.771005e+00, +1.799965499e-02, +2.794507628e-04, +2.875729474e-03, +1.412928776e-03, +2.767811258e-04 - 5.808479e+00, +2.056688477e-02, +2.026163128e-04, +3.018976396e-03, +1.556103223e-03, +2.672806062e-04 - 5.845953e+00, +2.667317497e-02, +5.823552172e-04, +3.853745073e-03, +2.097198542e-03, +2.606010338e-04 - 5.883427e+00, +2.494176114e-02, +3.598671841e-04, +3.321745627e-03, +1.970481530e-03, +2.537119531e-04 - 5.920901e+00, +2.137345270e-02, +8.293427173e-05, +3.356201503e-03, +1.621854163e-03, +2.439717849e-04 - 5.958375e+00, +2.318354549e-02, +7.422870817e-04, +2.941740678e-03, +1.931362441e-03, +2.287977096e-04 - 5.995849e+00, +2.952142379e-02, +6.252541959e-04, +4.583053556e-03, +2.338847088e-03, +2.091943852e-04 - 6.033323e+00, +3.555519752e-02, +4.573786887e-05, +4.813807246e-03, +2.849462755e-03, +1.868935198e-04 - 6.070797e+00, +3.292932048e-02, +9.200970879e-04, +5.209752704e-03, +2.597454364e-03, +1.639087117e-04 - 6.108271e+00, +3.192086283e-02, +1.004172293e-03, +4.022460541e-03, +2.693222938e-03, +1.453340940e-04 - 6.145745e+00, +4.585681692e-02, +6.149629464e-04, +6.824695798e-03, +3.691967369e-03, +1.340064667e-04 - 6.183219e+00, +4.987215286e-02, +4.811157598e-04, +7.021275439e-03, +4.037545247e-03, +1.300333935e-04 - 6.220694e+00, +4.260084454e-02, +6.591544682e-04, +6.013065495e-03, +3.525643664e-03, +1.300855088e-04 - 6.258168e+00, +3.842681872e-02, +8.303456662e-04, +5.473324578e-03, +3.054677461e-03, +1.297238444e-04 - 6.295642e+00, +4.254919607e-02, +3.750645263e-04, +6.364911037e-03, +3.445730899e-03, +1.253570119e-04 - 6.333116e+00, +5.540288504e-02, +2.501391170e-03, +7.942184088e-03, +4.659687721e-03, +1.158951038e-04 - 6.370590e+00, +5.981388626e-02, +8.703291313e-04, +7.663739587e-03, +4.914756428e-03, +1.015394549e-04 - 6.408064e+00, +5.142942118e-02, +9.447762178e-04, +7.628766072e-03, +3.861924917e-03, +8.295320962e-05 - 6.445538e+00, +7.833604887e-02, +1.490927129e-03, +1.020652767e-02, +6.502611406e-03, +6.413630695e-05 - 6.483012e+00, +1.351682354e-01, +4.725368493e-03, +1.996264790e-02, +1.122421850e-02, +4.719566269e-05 - 6.520486e+00, +1.660567873e-01, +4.237671924e-03, +2.090921607e-02, +1.371474514e-02, +3.428678049e-05 - 6.557960e+00, +1.550267507e-01, +2.313466579e-03, +2.476527883e-02, +1.181285584e-02, +2.871600307e-05 - 6.595434e+00, +1.730841742e-01, +5.141557630e-03, +2.592193488e-02, +1.435443590e-02, +2.830677177e-05 - 6.632908e+00, +1.992112658e-01, +4.749548819e-03, +3.099782108e-02, +1.586629581e-02, +2.868893844e-05 - 6.670382e+00, +2.040562846e-01, +5.463375103e-03, +2.827636340e-02, +1.682686890e-02, +2.757634084e-05 - 6.707856e+00, +1.996589384e-01, +5.692265472e-03, +3.072940824e-02, +1.604490807e-02, +2.478347455e-05 - 6.745330e+00, +2.220601568e-01, +4.384969989e-03, +3.844511254e-02, +1.738270091e-02, +2.028527677e-05 - 6.782804e+00, +3.127899706e-01, +5.466747267e-03, +4.990321440e-02, +2.436677342e-02, +1.596573591e-05 - 6.820278e+00, +4.554309472e-01, +1.116607680e-02, +6.606498094e-02, +3.692086390e-02, +1.262062648e-05 - 6.857752e+00, +5.476583894e-01, +1.892382880e-02, +8.063201468e-02, +4.500018629e-02, +9.582225142e-06 - 6.895227e+00, +5.348274640e-01, +4.863351168e-04, +8.471730342e-02, +4.166993486e-02, +7.451235411e-06 - 6.932701e+00, +5.945685356e-01, +1.093208659e-02, +8.896160366e-02, +4.741098762e-02, +7.713935361e-06 - 6.970175e+00, +6.929449310e-01, +1.224113099e-02, +1.034483512e-01, +5.575434489e-02, +8.153341350e-06 - 7.007649e+00, +6.555652393e-01, +7.881293768e-03, +8.681919994e-02, +5.444965786e-02, +8.013144415e-06 - 7.045123e+00, +5.571370132e-01, +5.480258866e-03, +9.034180164e-02, +3.982552376e-02, +7.103058561e-06 - 7.082597e+00, +6.506018316e-01, +1.647729060e-02, +8.770112806e-02, +5.317402765e-02, +6.251908497e-06 - 7.120071e+00, +6.995608325e-01, +5.152246213e-03, +1.104926161e-01, +5.682517846e-02, +7.632185039e-06 - 7.157545e+00, +7.269145411e-01, +1.606521425e-02, +9.466708003e-02, +5.754811580e-02, +7.329401859e-06 - 7.195019e+00, +6.566184307e-01, +1.315114976e-02, +9.984999884e-02, +4.894605762e-02, +6.045703988e-06 - 7.232493e+00, +6.414695957e-01, +2.424328558e-02, +9.657678114e-02, +5.297373021e-02, +5.792304398e-06 - 7.269967e+00, +7.572689483e-01, +2.435442014e-02, +1.170802395e-01, +5.981027653e-02, +6.622353708e-06 - 7.307441e+00, +7.333739211e-01, +1.622387621e-02, +1.010460998e-01, +5.809664807e-02, +7.298735179e-06 - 7.344915e+00, +7.499337130e-01, +8.090180525e-03, +1.105666642e-01, +5.782933693e-02, +5.538378559e-06 - 7.382389e+00, +6.549346065e-01, +6.641746123e-03, +9.909115634e-02, +5.046346915e-02, +5.496635574e-06 - 7.419863e+00, +7.196831554e-01, +8.766982205e-03, +1.068912596e-01, +5.713708497e-02, +6.712265589e-06 - 7.457337e+00, +8.006151360e-01, +1.809409626e-02, +1.175250465e-01, +6.486644349e-02, +6.428561978e-06 - 7.494811e+00, +7.048734441e-01, +1.731893931e-02, +9.294134442e-02, +5.885149257e-02, +6.238486822e-06 + Sum, Min, Max, Mean, Normalization + +2.096263303e-01, +1.115162750e-02, +2.559512195e-02, +1.801842180e-02, +1.099244137e+00 diff --git a/test/ref/coaxial/open/error-indicators.csv b/test/ref/coaxial/open/error-indicators.csv index 741fd9506..8fc67cb44 100644 --- a/test/ref/coaxial/open/error-indicators.csv +++ b/test/ref/coaxial/open/error-indicators.csv @@ -1,202 +1,2 @@ - t (ns), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 0.000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00, +0.000000000e+00 - 3.747406e-02, +5.572991711e-02, +4.780005949e-15, +1.407880294e-02, +1.856308518e-03, +1.015013075e-04 - 7.494811e-02, +1.034912856e-01, +9.732567615e-15, +2.607229698e-02, +3.523283343e-03, +1.286451027e-04 - 1.124222e-01, +3.324496388e-02, +6.375326658e-15, +8.388885354e-03, +1.164706330e-03, +1.363415075e-04 - 1.498962e-01, +2.375714211e-02, +2.405820589e-15, +6.044798887e-03, +8.245600593e-04, +2.237425263e-04 - 1.873703e-01, +3.819179106e-02, +5.021614958e-15, +9.668211449e-03, +1.299554229e-03, +5.988175109e-04 - 2.248443e-01, +1.297340613e-02, +1.776716125e-15, +3.303352868e-03, +4.708062400e-04, +1.231303285e-03 - 2.623184e-01, +5.246709260e-03, +2.385106704e-15, +1.405616340e-03, +2.071756999e-04, +2.183486767e-03 - 2.997925e-01, +6.058354891e-03, +3.688072737e-15, +1.493506908e-03, +2.376336055e-04, +3.443837437e-03 - 3.372665e-01, +6.156851514e-03, +4.075182156e-15, +1.749181241e-03, +2.239413758e-04, +4.896408733e-03 - 3.747406e-01, +4.694296650e-03, +8.972823355e-15, +1.289674681e-03, +1.823298192e-04, +6.288158078e-03 - 4.122146e-01, +5.417259349e-03, +1.948301910e-15, +1.436619200e-03, +2.081413126e-04, +7.236866239e-03 - 4.496887e-01, +7.374336957e-03, +1.897079415e-14, +1.846629384e-03, +2.874502795e-04, +7.438562524e-03 - 4.871627e-01, +6.930576439e-03, +2.279830021e-14, +1.752840045e-03, +2.931825986e-04, +7.655638548e-03 - 5.246368e-01, +5.525646929e-03, +3.091423690e-14, +1.406681124e-03, +2.243733865e-04, +1.144492786e-02 - 5.621109e-01, +4.673784425e-03, +3.121350320e-14, +1.348460932e-03, +1.868352062e-04, +2.129721915e-02 - 5.995849e-01, +4.253659682e-03, +1.861559471e-14, +1.368615134e-03, +1.587836337e-04, +3.669178320e-02 - 6.370590e-01, +4.178197277e-03, +5.274962070e-14, +1.375975768e-03, +1.455795304e-04, +5.641443987e-02 - 6.745330e-01, +4.365910098e-03, +2.726784104e-14, +1.376592666e-03, +1.531072976e-04, +7.828637698e-02 - 7.120071e-01, +4.532195073e-03, +8.525225747e-14, +1.342611942e-03, +1.672385806e-04, +9.885177909e-02 - 7.494811e-01, +4.713112853e-03, +1.941937091e-13, +1.277228489e-03, +1.799446727e-04, +1.137531996e-01 - 7.869552e-01, +4.875763465e-03, +3.094044938e-13, +1.187769851e-03, +1.938215810e-04, +1.195346821e-01 - 8.244293e-01, +4.981324068e-03, +1.298916160e-12, +1.301107022e-03, +2.128512979e-04, +1.188340974e-01 - 8.619033e-01, +4.771406979e-03, +6.445150636e-12, +1.099709362e-03, +2.160907931e-04, +1.309553939e-01 - 8.993774e-01, +4.387618618e-03, +1.907002776e-11, +1.104046666e-03, +1.969683896e-04, +1.857760966e-01 - 9.368514e-01, +4.317746827e-03, +4.308903833e-11, +1.315698937e-03, +1.778612665e-04, +2.867307084e-01 - 9.743255e-01, +4.352783979e-03, +9.324588533e-11, +1.367694242e-03, +1.732486187e-04, +4.167673658e-01 - 1.011800e+00, +4.392520145e-03, +2.030563517e-10, +1.347497137e-03, +1.702743984e-04, +5.549049357e-01 - 1.049274e+00, +4.412402764e-03, +4.369999452e-10, +1.276609666e-03, +1.748952068e-04, +6.782106751e-01 - 1.086748e+00, +4.409231341e-03, +8.999345643e-10, +1.151820795e-03, +1.841348285e-04, +7.648957645e-01 - 1.124222e+00, +4.385361349e-03, +1.700272247e-09, +1.054162064e-03, +1.931535896e-04, +8.022489576e-01 - 1.161696e+00, +4.359684491e-03, +2.684499850e-09, +1.097370972e-03, +2.015036618e-04, +8.009704160e-01 - 1.199170e+00, +4.376985093e-03, +2.607577158e-09, +9.962793897e-04, +2.076058414e-04, +8.143229074e-01 - 1.236644e+00, +4.452602826e-03, +3.312088983e-09, +9.301914660e-04, +2.236743467e-04, +9.294380600e-01 - 1.274118e+00, +4.490626318e-03, +1.499013876e-08, +1.141847583e-03, +2.193639410e-04, +1.185764957e+00 - 1.311592e+00, +4.461869998e-03, +3.572079499e-08, +1.253108436e-03, +2.106609832e-04, +1.534641584e+00 - 1.349066e+00, +4.401936395e-03, +6.722317675e-08, +1.237832414e-03, +2.050873994e-04, +1.896337013e+00 - 1.386540e+00, +4.333203622e-03, +1.103743704e-07, +1.149104164e-03, +2.051919384e-04, +2.199936335e+00 - 1.424014e+00, +4.274237244e-03, +1.614556678e-07, +1.003500852e-03, +2.108719056e-04, +2.397678575e+00 - 1.461488e+00, +4.247523124e-03, +2.087791015e-07, +9.868601870e-04, +2.177114260e-04, +2.477217296e+00 - 1.498962e+00, +4.280032410e-03, +2.381229541e-07, +9.955121378e-04, +2.271307112e-04, +2.474013830e+00 - 1.536436e+00, +4.376655811e-03, +2.787897261e-07, +9.435084780e-04, +2.375088848e-04, +2.473624477e+00 - 1.573910e+00, +4.478796048e-03, +4.348188822e-07, +9.452200051e-04, +2.535812295e-04, +2.579178548e+00 - 1.611384e+00, +4.504194805e-03, +6.949622119e-07, +8.305665204e-04, +2.598628685e-04, +2.837660610e+00 - 1.648859e+00, +4.449740742e-03, +9.686036472e-07, +9.532358385e-04, +2.571234946e-04, +3.200525096e+00 - 1.686333e+00, +4.368988747e-03, +1.222520265e-06, +9.657316748e-04, +2.553060052e-04, +3.569942692e+00 - 1.723807e+00, +4.305815658e-03, +1.463913610e-06, +9.004535416e-04, +2.557293609e-04, +3.860927182e+00 - 1.761281e+00, +4.281239183e-03, +1.695622917e-06, +8.111211761e-04, +2.616688934e-04, +4.031057848e+00 - 1.798755e+00, +4.299576106e-03, +1.914307365e-06, +8.541141264e-04, +2.692992642e-04, +4.086189106e+00 - 1.836229e+00, +4.350195471e-03, +2.265209390e-06, +8.333271263e-04, +2.776670855e-04, +4.072082288e+00 - 1.873703e+00, +4.406123355e-03, +3.259183597e-06, +8.321799456e-04, +2.866860335e-04, +4.053069427e+00 - 1.911177e+00, +4.436275347e-03, +5.260285777e-06, +8.329962539e-04, +2.954441289e-04, +4.080890391e+00 - 1.948651e+00, +4.428710083e-03, +8.002413959e-06, +7.711775732e-04, +3.026474470e-04, +4.169120342e+00 - 1.986125e+00, +4.399248505e-03, +1.094520855e-05, +7.863124125e-04, +3.055255888e-04, +4.292682833e+00 - 2.023599e+00, +4.372255611e-03, +1.349310551e-05, +7.323874733e-04, +3.097794542e-04, +4.410898850e+00 - 2.061073e+00, +4.361315731e-03, +1.494909302e-05, +7.328463050e-04, +3.146496395e-04, +4.492460625e+00 - 2.098547e+00, +4.367030285e-03, +1.457796662e-05, +7.247211296e-04, +3.200442587e-04, +4.527157525e+00 - 2.136021e+00, +4.382374080e-03, +1.253784498e-05, +6.970316424e-04, +3.266236440e-04, +4.524046965e+00 - 2.173495e+00, +4.397034295e-03, +1.351026325e-05, +7.270181577e-04, +3.307469479e-04, +4.501957938e+00 - 2.210969e+00, +4.403170587e-03, +2.415199665e-05, +6.914860835e-04, +3.348204316e-04, +4.478434139e+00 - 2.248443e+00, +4.401493826e-03, +4.171539184e-05, +7.189816263e-04, +3.371262712e-04, +4.462008482e+00 - 2.285917e+00, +4.398225001e-03, +6.192280212e-05, +7.101556969e-04, +3.401173553e-04, +4.450486620e+00 - 2.323392e+00, +4.398269519e-03, +8.037723425e-05, +6.962721148e-04, +3.425770451e-04, +4.434865564e+00 - 2.360866e+00, +4.404017255e-03, +9.201831682e-05, +7.190225781e-04, +3.439997305e-04, +4.406388329e+00 - 2.398340e+00, +4.414405092e-03, +9.158558418e-05, +6.788819331e-04, +3.452286529e-04, +4.363973589e+00 - 2.435814e+00, +4.421143782e-03, +7.441843177e-05, +7.220381311e-04, +3.432555218e-04, +4.319369685e+00 - 2.473288e+00, +4.412692077e-03, +4.282061526e-05, +7.068656107e-04, +3.418175535e-04, +4.296269972e+00 - 2.510762e+00, +4.386278801e-03, +5.105127318e-05, +7.009204519e-04, +3.404021990e-04, +4.319012990e+00 - 2.548236e+00, +4.353215251e-03, +4.129387046e-05, +7.011585940e-04, +3.393297532e-04, +4.391832454e+00 - 2.585710e+00, +4.333456514e-03, +1.888982387e-05, +6.505948543e-04, +3.367477514e-04, +4.481474420e+00 - 2.623184e+00, +4.344230010e-03, +2.207235676e-05, +6.844735434e-04, +3.343510978e-04, +4.519833729e+00 - 2.660658e+00, +4.391992250e-03, +2.692671046e-05, +6.743976743e-04, +3.352654771e-04, +4.429683004e+00 - 2.698132e+00, +4.471042289e-03, +2.563970053e-05, +6.925965452e-04, +3.381059664e-04, +4.164305386e+00 - 2.735606e+00, +4.553202267e-03, +3.496086120e-05, +7.419389256e-04, +3.407149926e-04, +3.755743891e+00 - 2.773080e+00, +4.565896612e-03, +2.812102842e-05, +7.106904383e-04, +3.324536761e-04, +3.369620795e+00 - 2.810554e+00, +4.424048922e-03, +2.205191071e-05, +7.354060681e-04, +3.099287774e-04, +3.300078313e+00 - 2.848028e+00, +4.245521437e-03, +1.165298270e-05, +7.127638729e-04, +2.974966986e-04, +3.715663123e+00 - 2.885502e+00, +4.195356915e-03, +4.545975177e-06, +7.596043263e-04, +2.830996949e-04, +4.412696644e+00 - 2.922976e+00, +4.234424822e-03, +1.477205397e-06, +7.970248517e-04, +2.750210888e-04, +5.035062025e+00 - 2.960451e+00, +4.302783352e-03, +1.800762190e-06, +8.562397769e-04, +2.715342841e-04, +5.296751662e+00 - 2.997925e+00, +4.375166831e-03, +2.414631292e-06, +8.927687994e-04, +2.716622273e-04, +5.025294040e+00 - 3.035399e+00, +4.448792499e-03, +2.550087762e-06, +9.171595987e-04, +2.741412579e-04, +4.165707984e+00 - 3.072873e+00, +4.547392172e-03, +2.930191671e-06, +9.339070315e-04, +2.807584580e-04, +2.782735535e+00 - 3.110347e+00, +4.822285737e-03, +4.196452250e-06, +9.011497982e-04, +3.124124847e-04, +1.091908737e+00 - 3.147821e+00, +4.205292450e-03, +1.544693998e-06, +7.966267592e-04, +2.677084155e-04, +1.087897791e+00 - 3.185295e+00, +4.322357444e-03, +8.097038803e-07, +9.017757419e-04, +2.603103176e-04, +2.795340363e+00 - 3.222769e+00, +4.355601912e-03, +8.154563458e-07, +9.178576606e-04, +2.603565288e-04, +4.198205341e+00 - 3.260243e+00, +4.349122198e-03, +7.337744842e-07, +9.173532313e-04, +2.596715998e-04, +5.054782764e+00 - 3.297717e+00, +4.314425000e-03, +1.030321438e-06, +9.050681552e-04, +2.587874217e-04, +5.270954600e+00 - 3.335191e+00, +4.256320894e-03, +2.230359642e-06, +8.757309727e-04, +2.592544095e-04, +4.875664359e+00 - 3.372665e+00, +4.181108331e-03, +4.831601574e-06, +8.081073485e-04, +2.628800519e-04, +4.026124476e+00 - 3.410139e+00, +4.150992695e-03, +9.956774975e-06, +7.711844066e-04, +2.781228495e-04, +3.029466121e+00 - 3.447613e+00, +4.374155513e-03, +1.534747727e-05, +7.729163360e-04, +2.966312352e-04, +2.402804105e+00 - 3.485087e+00, +4.636173585e-03, +1.986006357e-05, +7.649793279e-04, +3.310044682e-04, +2.582857071e+00 - 3.522561e+00, +4.601086721e-03, +1.325948217e-05, +7.533876615e-04, +3.311061739e-04, +3.207230086e+00 - 3.560035e+00, +4.483565724e-03, +1.152020770e-05, +7.012870277e-04, +3.221707976e-04, +3.758897835e+00 - 3.597509e+00, +4.377095997e-03, +9.395424665e-06, +6.965628121e-04, +3.170519048e-04, +4.038904116e+00 - 3.634984e+00, +4.303366073e-03, +6.417194093e-06, +6.602556115e-04, +3.161137532e-04, +4.041099495e+00 - 3.672458e+00, +4.276935597e-03, +1.670096154e-05, +6.453484569e-04, +3.214659963e-04, +3.857051923e+00 - 3.709932e+00, +4.308136565e-03, +3.640909381e-05, +6.931352459e-04, +3.270196041e-04, +3.618537515e+00 - 3.747406e+00, +4.378529326e-03, +5.311975627e-05, +7.160619539e-04, +3.311462070e-04, +3.441274021e+00 - 3.784880e+00, +4.436833558e-03, +7.657101887e-05, +7.109920692e-04, +3.374734541e-04, +3.374125919e+00 - 3.822354e+00, +4.451429057e-03, +7.784195482e-05, +7.275739401e-04, +3.442812792e-04, +3.390782747e+00 - 3.859828e+00, +4.433648665e-03, +1.002326186e-04, +6.730611530e-04, +3.482987084e-04, +3.433261195e+00 - 3.897302e+00, +4.408866776e-03, +7.462561248e-05, +7.036042823e-04, +3.476429706e-04, +3.458316390e+00 - 3.934776e+00, +4.393189571e-03, +9.418059948e-05, +6.868885127e-04, +3.469907827e-04, +3.452477391e+00 - 3.972250e+00, +4.390673944e-03, +7.713722291e-05, +6.871560546e-04, +3.449738669e-04, +3.423913070e+00 - 4.009724e+00, +4.397384258e-03, +5.550722591e-05, +7.047701569e-04, +3.435886757e-04, +3.388071423e+00 - 4.047198e+00, +4.405052475e-03, +2.547452650e-05, +6.655516797e-04, +3.425904911e-04, +3.356481759e+00 - 4.084672e+00, +4.406309729e-03, +2.125363804e-05, +7.090142573e-04, +3.405259860e-04, +3.331811122e+00 - 4.122146e+00, +4.400740895e-03, +3.024243840e-05, +6.879534947e-04, +3.385591106e-04, +3.308561286e+00 - 4.159620e+00, +4.394348204e-03, +3.429148562e-05, +7.004321099e-04, +3.344706053e-04, +3.277077389e+00 - 4.197094e+00, +4.391653785e-03, +3.227299822e-05, +7.131522322e-04, +3.304012861e-04, +3.228672040e+00 - 4.234568e+00, +4.396612091e-03, +2.572046764e-05, +6.921198792e-04, +3.256250709e-04, +3.160462176e+00 - 4.272043e+00, +4.411971887e-03, +1.806419397e-05, +7.421598715e-04, +3.210648208e-04, +3.078621207e+00 - 4.309517e+00, +4.430500503e-03, +1.080824996e-05, +7.253476649e-04, +3.172436193e-04, +2.998083103e+00 - 4.346991e+00, +4.437101320e-03, +5.906229494e-06, +7.595525869e-04, +3.109230071e-04, +2.936632045e+00 - 4.384465e+00, +4.422367240e-03, +4.538408465e-06, +7.625569826e-04, +3.041998868e-04, +2.903960890e+00 - 4.421939e+00, +4.389149824e-03, +4.963427487e-06, +7.501297757e-04, +2.974805377e-04, +2.891846173e+00 - 4.459413e+00, +4.352146169e-03, +4.814431594e-06, +7.795870932e-04, +2.901570452e-04, +2.873914205e+00 - 4.496887e+00, +4.332453405e-03, +4.032824887e-06, +7.473165171e-04, +2.824857890e-04, +2.816758998e+00 - 4.534361e+00, +4.345668334e-03, +3.039511503e-06, +7.955955506e-04, +2.768278960e-04, +2.695681837e+00 - 4.571835e+00, +4.395506909e-03, +1.864446046e-06, +8.495917580e-04, +2.749055011e-04, +2.507645693e+00 - 4.609309e+00, +4.470084644e-03, +1.273532986e-06, +8.342727217e-04, +2.752504146e-04, +2.277803991e+00 - 4.646783e+00, +4.532846997e-03, +7.475180167e-07, +8.217625380e-04, +2.759682054e-04, +2.056633148e+00 - 4.684257e+00, +4.530713933e-03, +1.145253262e-06, +9.350513578e-04, +2.696959570e-04, +1.900910688e+00 - 4.721731e+00, +4.450362889e-03, +8.697930276e-08, +9.251042187e-04, +2.557978178e-04, +1.836469626e+00 - 4.759205e+00, +4.350936341e-03, +7.511915959e-07, +9.688092237e-04, +2.443824107e-04, +1.832767372e+00 - 4.796679e+00, +4.297690565e-03, +8.141429116e-07, +9.632222838e-04, +2.352004006e-04, +1.825304967e+00 - 4.834153e+00, +4.308844940e-03, +5.129102713e-07, +9.522437582e-04, +2.276977388e-04, +1.758563482e+00 - 4.871627e+00, +4.370157006e-03, +5.601185720e-07, +1.096010155e-03, +2.223631490e-04, +1.608482314e+00 - 4.909101e+00, +4.463581427e-03, +1.873403160e-07, +1.186054995e-03, +2.221821343e-04, +1.384441804e+00 - 4.946576e+00, +4.571461109e-03, +9.656986454e-07, +1.197067360e-03, +2.278830364e-04, +1.123464235e+00 - 4.984050e+00, +4.648007060e-03, +6.809479711e-07, +1.062733241e-03, +2.377213534e-04, +8.825893277e-01 - 5.021524e+00, +4.606824174e-03, +8.387731420e-07, +9.894893622e-04, +2.364047878e-04, +7.236586428e-01 - 5.058998e+00, +4.472937792e-03, +9.046810798e-07, +1.054022982e-03, +2.209844767e-04, +6.701071985e-01 - 5.096472e+00, +4.402554988e-03, +9.278912730e-08, +1.092198928e-03, +2.148883831e-04, +6.737184634e-01 - 5.133946e+00, +4.412222773e-03, +1.101376093e-06, +1.025832328e-03, +2.075392685e-04, +6.688498009e-01 - 5.171420e+00, +4.453268669e-03, +8.303709536e-07, +1.178145203e-03, +1.992887714e-04, +6.231396120e-01 - 5.208894e+00, +4.495822263e-03, +1.723687696e-06, +1.300762699e-03, +1.927303984e-04, +5.351006016e-01 - 5.246368e+00, +4.526093870e-03, +3.179489612e-06, +1.366802880e-03, +1.951785013e-04, +4.200215275e-01 - 5.283842e+00, +4.550083856e-03, +6.815021432e-06, +1.357183850e-03, +2.059465892e-04, +3.000669782e-01 - 5.321316e+00, +4.602439236e-03, +1.054611983e-05, +1.183929765e-03, +2.286449103e-04, +1.991688029e-01 - 5.358790e+00, +4.775687674e-03, +2.079224580e-05, +1.035715264e-03, +2.562132061e-04, +1.401126843e-01 - 5.396264e+00, +4.931276288e-03, +2.073422573e-05, +1.275293173e-03, +2.613929637e-04, +1.268528827e-01 - 5.433738e+00, +4.871529774e-03, +1.407855555e-05, +1.189747816e-03, +2.487492827e-04, +1.297292222e-01 - 5.471212e+00, +4.762234770e-03, +1.967185611e-05, +1.246169870e-03, +2.383336263e-04, +1.254260012e-01 - 5.508686e+00, +4.650373164e-03, +6.137676836e-06, +1.332329950e-03, +2.319775652e-04, +1.099715010e-01 - 5.546160e+00, +4.517149734e-03, +2.590729520e-05, +1.346192794e-03, +2.439991867e-04, +8.751561585e-02 - 5.583635e+00, +4.370148920e-03, +6.424283573e-05, +1.278366947e-03, +2.701713125e-04, +6.404749111e-02 - 5.621109e+00, +4.356302923e-03, +1.191632599e-04, +1.077307522e-03, +3.143008659e-04, +4.499882977e-02 - 5.658583e+00, +4.613690342e-03, +1.594292599e-04, +7.146845024e-04, +3.737522664e-04, +3.378351251e-02 - 5.696057e+00, +4.755727822e-03, +8.091549876e-05, +7.131673371e-04, +3.781535315e-04, +2.942388680e-02 - 5.733531e+00, +4.788740493e-03, +6.469339425e-05, +6.979902289e-04, +3.754147501e-04, +2.730232945e-02 - 5.771005e+00, +4.753486242e-03, +8.952992399e-05, +7.287758631e-04, +3.702480182e-04, +2.433933672e-02 - 5.808479e+00, +4.693920007e-03, +8.931037214e-05, +7.514977764e-04, +3.656159433e-04, +2.048163582e-02 - 5.845953e+00, +4.441961570e-03, +1.133994992e-04, +7.113551187e-04, +3.352214057e-04, +1.785036816e-02 - 5.883427e+00, +4.213401089e-03, +5.045887846e-05, +7.327108334e-04, +3.027224837e-04, +1.905047360e-02 - 5.920901e+00, +4.155172452e-03, +6.862668313e-05, +8.003306103e-04, +2.936820151e-04, +2.355153006e-02 - 5.958375e+00, +4.173810775e-03, +4.367577612e-05, +8.318489635e-04, +2.701900930e-04, +2.860371125e-02 - 5.995849e+00, +4.189257368e-03, +1.273253176e-05, +8.357800836e-04, +2.577949440e-04, +3.211074890e-02 - 6.033323e+00, +4.209961328e-03, +3.491479059e-05, +8.740559580e-04, +2.568575260e-04, +3.288978387e-02 - 6.070797e+00, +4.266400536e-03, +2.933453686e-05, +9.043412384e-04, +2.567577885e-04, +3.044108952e-02 - 6.108271e+00, +4.244289444e-03, +2.943097819e-05, +9.102644066e-04, +2.498315439e-04, +2.486884038e-02 - 6.145745e+00, +4.245171421e-03, +1.994132419e-05, +9.220440563e-04, +2.497784945e-04, +1.688201114e-02 - 6.183219e+00, +4.248747709e-03, +8.402724927e-05, +8.384360570e-04, +2.892884387e-04, +8.222464850e-03 - 6.220694e+00, +4.662283813e-03, +1.118597640e-04, +8.181924633e-04, +3.596857314e-04, +7.112352809e-03 - 6.258168e+00, +4.410589773e-03, +3.305698761e-05, +7.840836535e-04, +3.073818879e-04, +1.487454424e-02 - 6.295642e+00, +4.377957436e-03, +1.377863574e-05, +8.517837018e-04, +2.889232646e-04, +2.195967174e-02 - 6.333116e+00, +4.305122550e-03, +3.921198533e-05, +8.571540306e-04, +2.821091473e-04, +2.658824072e-02 - 6.370590e+00, +4.230789036e-03, +2.652881798e-05, +8.339275124e-04, +2.765798602e-04, +2.836687987e-02 - 6.408064e+00, +4.199832172e-03, +9.926594891e-06, +7.985209427e-04, +2.763883637e-04, +2.754736754e-02 - 6.445538e+00, +4.126903599e-03, +4.538717506e-05, +7.605100582e-04, +2.835650616e-04, +2.492401183e-02 - 6.483012e+00, +4.140333646e-03, +6.041048507e-05, +7.185311096e-04, +2.993597242e-04, +2.173369821e-02 - 6.520486e+00, +4.214591715e-03, +3.618561958e-05, +6.989334694e-04, +3.044662280e-04, +1.939616532e-02 - 6.557960e+00, +4.355660821e-03, +6.099227654e-05, +7.153688477e-04, +3.151751270e-04, +1.881263997e-02 - 6.595434e+00, +4.382990654e-03, +8.300039336e-05, +7.503679033e-04, +3.283099014e-04, +1.965680357e-02 - 6.632908e+00, +4.355663519e-03, +4.288532409e-05, +7.134074328e-04, +3.325441405e-04, +2.089060758e-02 - 6.670382e+00, +4.303437062e-03, +3.241395838e-05, +6.722645795e-04, +3.300449843e-04, +2.174333892e-02 - 6.707856e+00, +4.264692455e-03, +1.014047622e-04, +6.905521540e-04, +3.324644020e-04, +2.197662468e-02 - 6.745330e+00, +4.248405317e-03, +7.669202227e-05, +6.750942166e-04, +3.299645328e-04, +2.172186153e-02 - 6.782804e+00, +4.251482241e-03, +1.015410938e-04, +6.984757583e-04, +3.318296940e-04, +2.125715216e-02 - 6.820278e+00, +4.271324850e-03, +5.770536523e-05, +6.999799761e-04, +3.315638217e-04, +2.082496930e-02 - 6.857752e+00, +4.284078031e-03, +1.059437925e-05, +6.882047065e-04, +3.261040588e-04, +2.053509651e-02 - 6.895227e+00, +4.329168343e-03, +9.181770397e-05, +7.196668532e-04, +3.349631754e-04, +2.036976733e-02 - 6.932701e+00, +4.271394037e-03, +6.403725578e-05, +6.886072050e-04, +3.253227751e-04, +2.025681153e-02 - 6.970175e+00, +4.297083709e-03, +6.466712397e-05, +7.103696923e-04, +3.300412412e-04, +2.014092972e-02 - 7.007649e+00, +4.297857164e-03, +8.160125963e-05, +7.020581941e-04, +3.278824609e-04, +2.000770004e-02 - 7.045123e+00, +4.281025044e-03, +4.035378758e-05, +7.013606369e-04, +3.251174596e-04, +1.987163031e-02 - 7.082597e+00, +4.307887142e-03, +3.176045967e-05, +7.235813063e-04, +3.233738031e-04, +1.974790284e-02 - 7.120071e+00, +4.252117844e-03, +1.261328439e-05, +6.753896286e-04, +3.181003476e-04, +1.962790928e-02 - 7.157545e+00, +4.324803094e-03, +2.257048588e-05, +7.139585186e-04, +3.228143507e-04, +1.947211778e-02 - 7.195019e+00, +4.251210265e-03, +1.048381435e-05, +7.107750035e-04, +3.099075977e-04, +1.921995461e-02 - 7.232493e+00, +4.259019210e-03, +3.785736092e-05, +7.265454218e-04, +3.091292132e-04, +1.881509895e-02 - 7.269967e+00, +4.349497924e-03, +1.160062016e-05, +7.519212558e-04, +3.068929252e-04, +1.823254307e-02 - 7.307441e+00, +4.241816299e-03, +6.636178540e-06, +7.295626682e-04, +2.945637325e-04, +1.750631517e-02 - 7.344915e+00, +4.354318396e-03, +1.529053678e-05, +8.106289789e-04, +2.987514777e-04, +1.673913653e-02 - 7.382389e+00, +4.342203028e-03, +2.638061582e-05, +8.099402667e-04, +2.965480096e-04, +1.607581245e-02 - 7.419863e+00, +4.383969626e-03, +2.628342216e-05, +8.503681027e-04, +2.886649966e-04, +1.564415606e-02 - 7.457337e+00, +4.316254351e-03, +1.210537923e-05, +8.497623089e-04, +2.748196188e-04, +1.546991691e-02 - 7.494811e+00, +4.233213896e-03, +1.243198824e-05, +8.379629100e-04, +2.703016161e-04, +1.543440789e-02 + Sum, Min, Max, Mean, Normalization + +1.022347200e-02, +2.024010727e-04, +2.416110589e-03, +5.982341447e-04, +1.742421290e+00 diff --git a/test/ref/cpw/lumped_adaptive/error-indicators.csv b/test/ref/cpw/lumped_adaptive/error-indicators.csv index 9739c51b9..1ac73b6ee 100644 --- a/test/ref/cpw/lumped_adaptive/error-indicators.csv +++ b/test/ref/cpw/lumped_adaptive/error-indicators.csv @@ -1,11 +1,2 @@ - f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +8.806573475e-04, +2.186423623e-01 - 3.000000e+01, +7.048573140e-01, +1.801733889e-06, +1.911288392e-02, +8.604087499e-04, +4.552939315e+00 - 1.760000e+01, +7.085030395e-01, +1.269973795e-06, +1.968683336e-02, +8.578765318e-04, +2.496240545e+00 - 1.280000e+01, +7.050021999e-01, +1.023120281e-06, +1.779303441e-02, +8.631933312e-04, +1.992681982e+00 - 2.530000e+01, +7.063233638e-01, +1.119274219e-06, +1.918472307e-02, +8.641051749e-04, +3.585707764e+00 - 5.700000e+00, +7.030751823e-01, +7.236778828e-07, +1.762962944e-02, +8.676377055e-04, +7.089908259e-01 - 2.880000e+01, +7.050114123e-01, +1.148903905e-06, +1.905258155e-02, +8.593880474e-04, +4.413027198e+00 - 2.090000e+01, +7.085921616e-01, +7.822419313e-07, +1.961007783e-02, +8.645942925e-04, +2.739108293e+00 - 3.100000e+00, +7.042983087e-01, +9.303255016e-07, +1.650025839e-02, +8.775457687e-04, +3.510194976e-01 - 3.500000e+00, +7.040400654e-01, +8.844716811e-07, +1.682088474e-02, +8.760540565e-04, +4.019232247e-01 + Sum, Min, Max, Mean, Normalization + +7.058727272e-01, +1.275543633e-06, +1.731621567e-02, +8.748160640e-04, +2.521900498e+00 diff --git a/test/ref/cpw/lumped_uniform/error-indicators.csv b/test/ref/cpw/lumped_uniform/error-indicators.csv index 78281b387..fb63b8082 100644 --- a/test/ref/cpw/lumped_uniform/error-indicators.csv +++ b/test/ref/cpw/lumped_uniform/error-indicators.csv @@ -1,16 +1,2 @@ - f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +7.050139860e-01, +1.089714440e-06, +1.591585961e-02, +8.806573475e-04, +2.186423623e-01 - 4.000000e+00, +7.037534929e-01, +8.348638678e-07, +1.716164791e-02, +8.740225193e-04, +4.676837686e-01 - 6.000000e+00, +7.030062911e-01, +7.118689649e-07, +1.767838766e-02, +8.667794453e-04, +7.544551055e-01 - 8.000000e+00, +7.029165326e-01, +6.832337446e-07, +1.800232414e-02, +8.638803686e-04, +1.081001045e+00 - 1.000000e+01, +7.034192125e-01, +7.418014191e-07, +1.790795027e-02, +8.645952292e-04, +1.449355685e+00 - 1.200000e+01, +7.044599247e-01, +9.086471535e-07, +1.751260139e-02, +8.646185630e-04, +1.842149227e+00 - 1.400000e+01, +7.058985472e-01, +1.260914522e-06, +1.833900466e-02, +8.591277673e-04, +2.190892577e+00 - 1.600000e+01, +7.074564924e-01, +1.093986277e-06, +1.938386319e-02, +8.542665420e-04, +2.406792037e+00 - 1.800000e+01, +7.086746633e-01, +1.209024854e-06, +1.965872658e-02, +8.590913096e-04, +2.515530177e+00 - 2.000000e+01, +7.088448209e-01, +8.917186071e-07, +1.963851961e-02, +8.636924028e-04, +2.646279233e+00 - 2.200000e+01, +7.081043245e-01, +7.960606264e-07, +1.946623509e-02, +8.649306217e-04, +2.889633318e+00 - 2.400000e+01, +7.070103246e-01, +9.980955322e-07, +1.928974677e-02, +8.646218376e-04, +3.271855291e+00 - 2.600000e+01, +7.059908728e-01, +1.195377481e-06, +1.913395059e-02, +8.635749385e-04, +3.767453335e+00 - 2.800000e+01, +7.052267075e-01, +9.841023722e-07, +1.902600821e-02, +8.604645102e-04, +4.261952660e+00 - 3.000000e+01, +7.048573097e-01, +1.801758125e-06, +1.911288376e-02, +8.604087621e-04, +4.552939157e+00 + Sum, Min, Max, Mean, Normalization + +7.056448946e-01, +1.148429809e-06, +1.689680158e-02, +8.751153409e-04, +2.287774332e+00 diff --git a/test/ref/cpw/wave_adaptive/error-indicators.csv b/test/ref/cpw/wave_adaptive/error-indicators.csv index f896ced91..213baba81 100644 --- a/test/ref/cpw/wave_adaptive/error-indicators.csv +++ b/test/ref/cpw/wave_adaptive/error-indicators.csv @@ -1,11 +1,2 @@ - f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +9.263986181e-04, +2.614559004e-01 - 3.000000e+01, +6.802918578e-01, +5.681633835e-06, +1.845184430e-02, +1.069612630e-03, +4.266998699e+00 - 1.800000e+01, +6.988641990e-01, +1.805646533e-06, +1.748858742e-02, +9.321067007e-04, +2.396980554e+00 - 2.750000e+01, +6.989649048e-01, +3.352201085e-06, +1.700252668e-02, +9.410894391e-04, +3.692517431e+00 - 2.960000e+01, +6.281346087e-01, +8.180986748e-06, +1.580175763e-02, +1.177590100e-03, +5.361105316e+00 - 2.430000e+01, +6.989731036e-01, +2.948013356e-06, +1.729382892e-02, +9.310877420e-04, +3.243999699e+00 - 1.650000e+01, +6.962225150e-01, +5.006960530e-06, +1.720841009e-02, +9.616412940e-04, +2.212464633e+00 - 1.420000e+01, +6.995569363e-01, +1.848544401e-06, +1.709761753e-02, +9.315059097e-04, +1.903555185e+00 - 8.500000e+00, +6.993502755e-01, +2.435632799e-06, +1.676522029e-02, +9.259791653e-04, +1.135422506e+00 - 1.510000e+01, +6.964183126e-01, +4.767143173e-06, +1.723792019e-02, +9.938502198e-04, +2.042324804e+00 + Sum, Min, Max, Mean, Normalization + +6.809041632e-01, +6.042001464e-06, +1.659737114e-02, +1.093675108e-03, +3.034916068e+00 diff --git a/test/ref/cpw/wave_uniform/error-indicators.csv b/test/ref/cpw/wave_uniform/error-indicators.csv index a317b58a0..27f9dafff 100644 --- a/test/ref/cpw/wave_uniform/error-indicators.csv +++ b/test/ref/cpw/wave_uniform/error-indicators.csv @@ -1,16 +1,2 @@ - f (GHz), Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 2.000000e+00, +6.993092200e-01, +1.566265918e-06, +1.685105942e-02, +9.263986181e-04, +2.614559004e-01 - 4.000000e+00, +6.992890992e-01, +1.736035539e-06, +1.680419113e-02, +9.263152213e-04, +5.258892594e-01 - 6.000000e+00, +6.992881535e-01, +2.021152752e-06, +1.676125294e-02, +9.261540058e-04, +7.947256627e-01 - 8.000000e+00, +6.993312910e-01, +2.448759169e-06, +1.675682757e-02, +9.260015170e-04, +1.067084152e+00 - 1.000000e+01, +6.994243806e-01, +2.399252549e-06, +1.681663205e-02, +9.259792773e-04, +1.340172770e+00 - 1.200000e+01, +6.995448511e-01, +2.366910745e-06, +1.694287832e-02, +9.262810134e-04, +1.610652070e+00 - 1.400000e+01, +6.995942415e-01, +1.874364476e-06, +1.709174983e-02, +9.296637533e-04, +1.876886099e+00 - 1.600000e+01, +6.980880025e-01, +5.469569257e-06, +1.721279606e-02, +9.594361262e-04, +2.144504886e+00 - 1.800000e+01, +6.988641967e-01, +1.805639359e-06, +1.748858870e-02, +9.321067031e-04, +2.396980543e+00 - 2.000000e+01, +6.990669364e-01, +2.883025515e-06, +1.755245634e-02, +9.293916043e-04, +2.658455702e+00 - 2.200000e+01, +6.990046675e-01, +3.191459315e-06, +1.749703902e-02, +9.296276956e-04, +2.926893770e+00 - 2.400000e+01, +6.989710885e-01, +2.953254947e-06, +1.732841600e-02, +9.308023476e-04, +3.202248951e+00 - 2.600000e+01, +6.990090439e-01, +3.218910262e-06, +1.709437140e-02, +9.338867734e-04, +3.481668528e+00 - 2.800000e+01, +6.988244472e-01, +3.387084517e-06, +1.706546999e-02, +9.468574632e-04, +3.764267665e+00 - 3.000000e+01, +6.802918601e-01, +5.681637479e-06, +1.845184332e-02, +1.069612628e-03, +4.266998672e+00 + Sum, Min, Max, Mean, Normalization + +6.978759848e-01, +4.868944903e-06, +1.714751896e-02, +9.637101290e-04, +2.154592309e+00 diff --git a/test/ref/rings/error-indicators.csv b/test/ref/rings/error-indicators.csv index 855775f8c..006fd9db3 100644 --- a/test/ref/rings/error-indicators.csv +++ b/test/ref/rings/error-indicators.csv @@ -1,3 +1,2 @@ - i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +3.544895565e-01, +4.103268389e-09, +2.982074859e-02, +2.217169423e-04, +1.115529612e-01 - 2.000000e+00, +2.764894272e-01, +1.023265139e-07, +7.245458667e-03, +5.519522597e-04, +4.557585527e-01 + Sum, Min, Max, Mean, Normalization + +3.178909004e-01, +7.241787209e-08, +2.108657479e-02, +5.448306933e-04, +2.836557570e-01 diff --git a/test/ref/spheres/error-indicators.csv b/test/ref/spheres/error-indicators.csv index 2586439ba..8f56cd272 100644 --- a/test/ref/spheres/error-indicators.csv +++ b/test/ref/spheres/error-indicators.csv @@ -1,3 +1,2 @@ - i, Sum Rel., Min Elem Rel., Max Elem Rel., Mean Elem Rel., Normalization - 1.000000e+00, +7.835018957e-03, +7.728228292e-07, +5.659925890e-04, +4.310697912e-05, +3.052442134e-01 - 2.000000e+00, +7.529893122e-03, +8.146906715e-07, +7.245606637e-04, +4.195362244e-05, +4.319918577e-01 + Sum, Min, Max, Mean, Normalization + +7.683970730e-03, +1.578064762e-06, +5.153233092e-04, +4.815651013e-05, +3.686180355e-01 diff --git a/test/runtests.jl b/test/runtests.jl index f4c7fb906..41cb5f5ab 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,7 +34,7 @@ abstol = 1.0e-16 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max Elem", "Min Elem"] + excluded_columns=["Max", "Min"] ) @info "Testing rings..." @@ -46,7 +46,7 @@ abstol = 1.0e-16 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max Elem", "Min Elem"] + excluded_columns=["Max", "Min"] ) @info "Testing cavity (PEC)..." @@ -58,7 +58,8 @@ abstol = 1.0e-16 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max Elem", "Min Elem"] + excluded_columns=["Max", "Min"], + skip_rowcount=true ) @info "Testing cavity (impedance)..." @@ -70,7 +71,8 @@ abstol = 1.0e-16 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max Elem", "Min Elem"] + excluded_columns=["Max", "Min"], + skip_rowcount=true ) # Coarser test tolerances for driven simulations with ports @@ -86,7 +88,7 @@ abstol = 2.0e-12 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max Elem", "Min Elem"] + excluded_columns=["Max", "Min"] ) @info "Testing coaxial (matched)..." @@ -98,7 +100,7 @@ abstol = 2.0e-12 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max Elem", "Min Elem"] + excluded_columns=["Max", "Min"] ) @info "Testing CPW (lumped ports)" @@ -110,7 +112,7 @@ abstol = 2.0e-12 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max Elem", "Min Elem"] + excluded_columns=["Max", "Min"] ) @info "Testing CPW (wave ports)" @@ -122,7 +124,7 @@ abstol = 2.0e-12 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max Elem", "Min Elem"] + excluded_columns=["Max", "Min"] ) # Don't check accuracy for adaptive frequency sweep simulations diff --git a/test/testcase.jl b/test/testcase.jl index d26af19aa..673209f65 100644 --- a/test/testcase.jl +++ b/test/testcase.jl @@ -9,7 +9,8 @@ function testcase( np=1, rtol=1.0e-6, atol=1.0e-18, - excluded_columns=[] + excluded_columns=[], + skip_rowcount=false ) if isempty(testdir) @info "$testdir/ is empty, skipping tests" @@ -67,11 +68,10 @@ function testcase( for file in filesref dataref = CSV.File(joinpath(refpostprodir, file); header=1) |> DataFrame data = CSV.File(joinpath(postprodir, file); header=1) |> DataFrame - if (size(data, 1) < size(dataref, 1)) - # pad the data with duplicates of final row. - append!(data, [data[end, :] for _ ∈ 1:(size(dataref, 1) - size(data, 1))]) + if !skip_rowcount + @test size(dataref, 1) <= size(data, 1) end - data = data[1:size(dataref, 1), :] + data = data[1:min(size(dataref, 1), size(data, 1)), :] # Check the number of columns matches, before removing any excluded_columns @test ncol(data) == ncol(dataref) From 4d0dc702f1b51fd3bc4b9b0fd4f34f3603916480 Mon Sep 17 00:00:00 2001 From: Hugh Carson Date: Mon, 9 Oct 2023 14:15:26 -0400 Subject: [PATCH 33/39] New Eval method to avoid duplicate gradient evaluation --- palace/fem/coefficient.hpp | 13 +++++++-- palace/linalg/errorestimator.cpp | 47 +++++++++++++++----------------- palace/linalg/errorestimator.hpp | 3 +- 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index 3ff76e661..97bf711c4 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -508,14 +508,14 @@ class GradFluxCoefficient : public mfem::Coefficient private: const mfem::ParGridFunction &gf; const MaterialOperator &mat_op; - mfem::Vector grad, V; + mfem::Vector grad, tmp; int component; public: GradFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) : mfem::Coefficient(), gf(gf), mat_op(mat_op), grad(gf.ParFESpace()->GetParMesh()->SpaceDimension()), - V(gf.ParFESpace()->GetParMesh()->SpaceDimension()), component(-1) + tmp(gf.ParFESpace()->GetParMesh()->SpaceDimension()), component(-1) { } @@ -532,9 +532,16 @@ class GradFluxCoefficient : public mfem::Coefficient MFEM_ASSERT(component >= 0 && component < gf.ParFESpace()->GetParMesh()->SpaceDimension(), "Invalid component index, try calling SetComponent(int)!"); + Eval(tmp, T, ip); + return tmp(component); + } + + // VectorCoefficient style Eval method for use in elemental integral evaluation. + inline void Eval(mfem::Vector &V, mfem::ElementTransformation &T, + const mfem::IntegrationPoint &ip) + { gf.GetGradient(T, grad); mat_op.GetPermittivityReal(T.Attribute).Mult(grad, V); - return V(component); } }; diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 34d414821..57e07370f 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -278,7 +278,8 @@ GradFluxErrorEstimator::GradFluxErrorEstimator( smooth_flux(h1_fespaces.GetFinestFESpace().GetTrueVSize()), flux_rhs(h1_fespaces.GetFinestFESpace().GetTrueVSize()), field_gf(&h1_fespaces.GetFinestFESpace()), - smooth_flux_gf(&h1_fespaces.GetFinestFESpace()) + smooth_flux_gf{&h1_fespaces.GetFinestFESpace(), &h1_fespaces.GetFinestFESpace(), + &h1_fespaces.GetFinestFESpace()} { } @@ -310,39 +311,35 @@ ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const h1_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); smooth_projector.Mult(flux_rhs, smooth_flux); } - smooth_flux_gf.SetFromTrueDofs(smooth_flux); - smooth_flux_gf.ExchangeFaceNbrData(); - for (int e = 0; e < h1_fespace.GetNE(); e++) + smooth_flux_gf[c].SetFromTrueDofs(smooth_flux); + smooth_flux_gf[c].ExchangeFaceNbrData(); + } + + Vector coef_eval(3); + for (int e = 0; e < h1_fespace.GetNE(); e++) + { + auto &T = *h1_fespace.GetElementTransformation(e); + // integration order 2p + q + const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), + 2 * h1_fespace.GetFE(e)->GetOrder() + T.Order()); + for (const auto &ip : ir) { - auto &T = *h1_fespace.GetElementTransformation(e); - // integration order 2p + q - const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), - 2 * h1_fespace.GetFE(e)->GetOrder() + T.Order()); - for (const auto &ip : ir) + T.SetIntPoint(&ip); + coef.Eval(coef_eval, T, ip); + for (int c = 0; c < sdim; c++) { - T.SetIntPoint(&ip); - double smooth_val = smooth_flux_gf.GetValue(e, ip); - double coarse_val = coef.Eval(T, ip); + coef.SetComponent(c); + double smooth_val = smooth_flux_gf[c].GetValue(e, ip); const double w_i = ip.weight * T.Weight(); - estimates[e] += w_i * std::pow(smooth_val - coarse_val, 2.0); - normalization += w_i * std::pow(coarse_val, 2.0); + estimates[e] += w_i * std::pow(smooth_val - coef_eval(c), 2.0); + normalization += w_i * std::pow(coef_eval(c), 2.0); } } - if constexpr (false) - { - // Debugging branch generates some intermediate fields for paraview. - mfem::ParaViewDataCollection paraview("debug_coeff" + std::to_string(c), - h1_fespace.GetParMesh()); - paraview.RegisterCoeffField("Flux", &coef); - paraview.RegisterField("SmoothFlux", &smooth_flux_gf); - paraview.Save(); - } + estimates[e] = std::sqrt(estimates[e]); } - linalg::Sqrt(estimates); Mpi::GlobalSum(1, &normalization, h1_fespace.GetComm()); normalization = std::sqrt(normalization); - if (normalization > 0) { estimates /= normalization; diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 872ec9f58..c4f1b7c52 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -85,7 +85,8 @@ class GradFluxErrorEstimator FluxProjector smooth_projector; mutable Vector smooth_flux, flux_rhs; - mutable mfem::ParGridFunction field_gf, smooth_flux_gf; + mutable mfem::ParGridFunction field_gf; + mutable std::array smooth_flux_gf; public: // Constructor for using geometric and p multigrid. From a2c47c0ee4fcf97402ac4207491b04f5aa9045f6 Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Mon, 9 Oct 2023 16:08:55 -0700 Subject: [PATCH 34/39] Refactor postprocessing of an ErrorIndicator in BaseSolver and PostOperator now that only one field is written per Solve() Also some corresponding simplification of ErrorIndicator class some remaining style fixes/remove std::transform over mfem::Vector. --- palace/drivers/basesolver.cpp | 45 +++++++++-------- palace/drivers/basesolver.hpp | 16 ++++-- palace/drivers/drivensolver.cpp | 4 +- palace/fem/coefficient.hpp | 61 +++++++++++------------ palace/fem/errorindicator.cpp | 78 +++++++++++++++--------------- palace/fem/errorindicator.hpp | 74 ++++++++++++++-------------- palace/fem/integrator.hpp | 4 +- palace/models/materialoperator.cpp | 8 +-- palace/models/postoperator.cpp | 54 +++++++++++---------- palace/models/postoperator.hpp | 15 +++--- 10 files changed, 185 insertions(+), 174 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index 731dad8e9..a2de02afa 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -3,6 +3,7 @@ #include "basesolver.hpp" +#include #include #include #include @@ -552,49 +553,51 @@ void BaseSolver::PostprocessProbes(const PostOperator &postop, const std::string #endif } -void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double time) const +void BaseSolver::PostprocessFields(const PostOperator &postop, int step, double time, + const ErrorIndicator *indicator) const { // Save the computed fields in parallel in format for viewing with ParaView. BlockTimer bt(Timer::IO); if (post_dir.length() == 0) { - Mpi::Warning("No file specified under [\"Problem\"][\"Output\"]!\nSkipping saving of " + Mpi::Warning(postop.GetComm(), + "No file specified under [\"Problem\"][\"Output\"]!\nSkipping saving of " "fields to disk!\n"); return; } - postop.WriteFields(step, time); - Mpi::Barrier(); + postop.WriteFields(step, time, indicator); + Mpi::Barrier(postop.GetComm()); } -void BaseSolver::PostprocessErrorIndicator(const std::array &data) const +void BaseSolver::PostprocessErrorIndicator(const PostOperator &postop, + const ErrorIndicator &indicator) const { + // Write the indicator statistics. if (post_dir.length() == 0) { return; } - - // Write the indicator statistics + MPI_Comm comm = postop.GetComm(); + std::array data = {indicator.Sum(comm), indicator.Min(comm), + indicator.Max(comm), indicator.Mean(comm), + indicator.Normalization()}; if (root) { std::string path = post_dir + "error-indicators.csv"; auto output = OutputFile(path, false); - // clang-format off output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", - "Sum", table.w, - "Min", table.w, - "Max", table.w, - "Mean", table.w, - "Normalization", table.w); - // clang-format on - - // clang-format off + "Sum", table.w, + "Min.", table.w, + "Max.", table.w, + "Mean", table.w, + "Normalization", table.w); output.print("{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", - data[0], table.w, table.p, - data[1], table.w, table.p, - data[2], table.w, table.p, - data[3], table.w, table.p, - data[4], table.w, table.p); + data[0], table.w, table.p, + data[1], table.w, table.p, + data[2], table.w, table.p, + data[3], table.w, table.p, + data[4], table.w, table.p); // clang-format on } } diff --git a/palace/drivers/basesolver.hpp b/palace/drivers/basesolver.hpp index 6938b13b5..786c99ed4 100644 --- a/palace/drivers/basesolver.hpp +++ b/palace/drivers/basesolver.hpp @@ -58,17 +58,27 @@ class BaseSolver fmt::file::TRUNC); } - // Common postprocessing functions for all simulation types. + // Common domain postprocessing for all simulation types. void PostprocessDomains(const PostOperator &postop, const std::string &name, int step, double time, double E_elec, double E_mag, double E_cap, double E_ind) const; + + // Common surface postprocessing for all simulation types. void PostprocessSurfaces(const PostOperator &postop, const std::string &name, int step, double time, double E_elec, double E_mag, double Vinc, double Iinc) const; + + // Common probe postprocessing for all simulation types. void PostprocessProbes(const PostOperator &postop, const std::string &name, int step, double time) const; - void PostprocessFields(const PostOperator &postop, int step, double time) const; - void PostprocessErrorIndicator(const std::array &data) const; + + // Common field visualization postprocessing for all simulation types. + void PostprocessFields(const PostOperator &postop, int step, double time, + const ErrorIndicator *indicator = nullptr) const; + + // Common error indicator postprocessing for all simulation types. + void PostprocessErrorIndicator(const PostOperator &postop, + const ErrorIndicator &indicator) const; public: BaseSolver(const IoData &iodata, bool root, int size = 0, int num_thread = 0, diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 69ee10ff6..1fac11103 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -432,8 +432,8 @@ struct PortVIData struct PortSData { - int idx; // Port index - std::complex Sij; // Scattering parameter + const int idx; // Port index + const std::complex Sij; // Scattering parameter }; } // namespace diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index 97bf711c4..c488a02e1 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -209,7 +209,7 @@ class BdrCurrentVectorCoefficient : public mfem::VectorCoefficient, private: const mfem::ParGridFunction &B; const MaterialOperator &mat_op; - mutable mfem::Vector C1, W, VU, VL, nor; + mfem::Vector C1, W, VU, VL, nor; public: BdrCurrentVectorCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &op) @@ -267,7 +267,7 @@ class BdrChargeCoefficient : public mfem::Coefficient, public BdrGridFunctionCoe private: const mfem::ParGridFunction &E; const MaterialOperator &mat_op; - mutable mfem::Vector C1, W, VU, VL, nor; + mfem::Vector C1, W, VU, VL, nor; public: BdrChargeCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &op) @@ -309,7 +309,7 @@ class BdrFluxCoefficient : public mfem::Coefficient, public BdrGridFunctionCoeff private: const mfem::ParGridFunction &B; const mfem::Vector dir; - mutable mfem::Vector V, VL, nor; + mfem::Vector V, VL, nor; public: BdrFluxCoefficient(const mfem::ParGridFunction &gf, mfem::Vector d, @@ -367,7 +367,7 @@ class DielectricInterfaceCoefficient : public mfem::Coefficient, const MaterialOperator &mat_op; const double ts, epsilon; const mfem::Vector side; - mutable mfem::Vector C1, V, nor; + mfem::Vector C1, V, nor; int Initialize(mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip, mfem::Vector &V) @@ -433,7 +433,7 @@ inline double DielectricInterfaceCoefficient::Eval( Initialize(T, ip, V); GetNormal(T, ip, nor); - // Metal-air interface: 0.5 * t / ϵ_MA * |E_n|² . + // Metal-air interface: 0.5 * t / ε_MA * |E_n|² . double Vn = V * nor; return 0.5 * ts / epsilon * (Vn * Vn); } @@ -446,7 +446,7 @@ inline double DielectricInterfaceCoefficient::Eval( int attr = Initialize(T, ip, V); GetNormal(T, ip, nor); - // Metal-substrate interface: 0.5 * t * (ϵ_S)² / ϵ_MS * |E_n|² . + // Metal-substrate interface: 0.5 * t * (ε_S)² / ε_MS * |E_n|² . const double Vn = V * nor; const double epsilon_S = mat_op.GetPermittivityReal(attr).InnerProduct(nor, nor); return 0.5 * ts * std::pow(epsilon_S, 2) / epsilon * (Vn * Vn); @@ -460,7 +460,7 @@ inline double DielectricInterfaceCoefficient::Eval( Initialize(T, ip, V); GetNormal(T, ip, nor); - // Substrate-air interface: 0.5 * t * (ϵ_SA * |E_t|² + 1 / ϵ_MS * |E_n|²) . + // Substrate-air interface: 0.5 * t * (ε_SA * |E_t|² + 1 / ε_MS * |E_n|²) . double Vn = V * nor; V.Add(-Vn, nor); return 0.5 * ts * (epsilon * (V * V) + (Vn * Vn) / epsilon); @@ -473,7 +473,7 @@ inline double DielectricInterfaceCoefficient:: // Get single-sided solution and neighboring element attribute. Initialize(T, ip, V); - // No specific interface, use full field evaluation: 0.5 * t * ϵ * |E|² . + // No specific interface, use full field evaluation: 0.5 * t * ε * |E|² . return 0.5 * ts * epsilon * (V * V); } @@ -489,58 +489,53 @@ class CurlFluxCoefficient : public mfem::VectorCoefficient public: CurlFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) : mfem::VectorCoefficient(gf.ParFESpace()->GetParMesh()->SpaceDimension()), U(gf), - mat_op(mat_op), curl(gf.ParFESpace()->GetParMesh()->SpaceDimension()) + mat_op(mat_op) { } void Eval(mfem::Vector &V, mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip) override { - V.SetSize(3); + V.SetSize(vdim); U.GetCurl(T, curl); mat_op.GetInvPermeability(T.Attribute).Mult(curl, V); } }; -// Computes the flux, ϵ ∇ ϕ, of the electrostatic potential ϕ. +// Computes the flux, ε ∇ϕ, of the electrostatic potential ϕ. class GradFluxCoefficient : public mfem::Coefficient { private: - const mfem::ParGridFunction &gf; + const mfem::ParGridFunction Φ const MaterialOperator &mat_op; - mfem::Vector grad, tmp; + mfem::Vector grad, W; int component; public: GradFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) - : mfem::Coefficient(), gf(gf), mat_op(mat_op), - grad(gf.ParFESpace()->GetParMesh()->SpaceDimension()), - tmp(gf.ParFESpace()->GetParMesh()->SpaceDimension()), component(-1) + : mfem::Coefficient(), Phi(gf), mat_op(mat_op), component(-1) { } - // Specify the component void SetComponent(int i) { - MFEM_ASSERT(i >= 0 && i < gf.ParFESpace()->GetParMesh()->SpaceDimension(), - "Invalid component index!"); + MFEM_ASSERT(i >= 0 && i < mat_op.SpaceDimension(), "Invalid component index!"); component = i; } double Eval(mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip) override { - MFEM_ASSERT(component >= 0 && - component < gf.ParFESpace()->GetParMesh()->SpaceDimension(), - "Invalid component index, try calling SetComponent(int)!"); - Eval(tmp, T, ip); - return tmp(component); + MFEM_ASSERT(component >= 0 && component < mat_op.SpaceDimension(), + "Invalid component index, try calling SetComponent!"); + Eval(W, T, ip); + return W(component); } - // VectorCoefficient style Eval method for use in elemental integral evaluation. - inline void Eval(mfem::Vector &V, mfem::ElementTransformation &T, - const mfem::IntegrationPoint &ip) + void Eval(mfem::Vector &V, mfem::ElementTransformation &T, + const mfem::IntegrationPoint &ip) { - gf.GetGradient(T, grad); + V.SetSize(mat_op.SpaceDimension()); + Phi.GetGradient(T, grad); mat_op.GetPermittivityReal(T.Attribute).Mult(grad, V); } }; @@ -560,7 +555,7 @@ class EnergyDensityCoefficient : public mfem::Coefficient, public BdrGridFunctio private: const GridFunctionType &U; const MaterialOperator &mat_op; - mutable mfem::Vector V; + mfem::Vector V; double GetLocalEnergyDensity(mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip, int attr); @@ -728,8 +723,8 @@ class BdrFieldCoefficient : public mfem::Coefficient, public BdrGridFunctionCoef class NormalProjectedCoefficient : public mfem::Coefficient { std::unique_ptr c; - mutable mfem::DenseMatrix K; - mutable mfem::Vector nor; + mfem::DenseMatrix K; + mfem::Vector nor; public: NormalProjectedCoefficient(std::unique_ptr &&coef) @@ -850,7 +845,7 @@ class SumVectorCoefficient : public mfem::VectorCoefficient private: std::vector, const mfem::Array *>> c; - mutable mfem::Vector U; + mfem::Vector U; void AddCoefficient(std::unique_ptr &&coef, const mfem::Array *marker) @@ -924,7 +919,7 @@ class SumMatrixCoefficient : public mfem::MatrixCoefficient private: std::vector, const mfem::Array *>> c; - mutable mfem::DenseMatrix M; + mfem::DenseMatrix M; void AddCoefficient(std::unique_ptr &&coef, const mfem::Array *marker) diff --git a/palace/fem/errorindicator.cpp b/palace/fem/errorindicator.cpp index 742f51f6b..40b234bfa 100644 --- a/palace/fem/errorindicator.cpp +++ b/palace/fem/errorindicator.cpp @@ -2,50 +2,52 @@ // SPDX-License-Identifier: Apache-2.0 #include "errorindicator.hpp" -#include -#include "utils/communication.hpp" + +#include namespace palace { -void ErrorIndicator::AddIndicator(const ErrorIndicator &error_indicators) + +void ErrorIndicator::AddIndicator(const ErrorIndicator &indicator) { + if (n == 0) + { + local = indicator.local; + normalization = indicator.normalization; + n = indicator.n; + return; + } + // The average local indicator is used rather than the indicator for the maximum // error to drive the adaptation, to account for a local error that might be marginally // important to many solves, rather than only large in one solve. - if (n > 0) - { - normalization = - (n * normalization + error_indicators.normalization * error_indicators.n) / - (n + error_indicators.n); - - // The local indicators must be squared before combining, so that the global error - // calculation is valid: - // E = sqrt(1/N sum_N sum_K eta_{kn}^2) - // from which it follows that: - // E^2 = 1/N sum_N sum_K eta_{kn}^2 - // = 1/N sum_N E_n - // Namely the average of the global error indicators included in the reduction. - // Squaring both sides means the summation can be rearranged, and then the local error - // indicators become: - // e_K = sqrt(1/N sum_N eta_{Kn}^2) - auto running_average = [this, error_indicators](const auto &xbar, const auto &x) - { - return std::sqrt((xbar * xbar * n + x * x * error_indicators.n) / - (n + error_indicators.n)); - }; - MFEM_VERIFY(local.Size() == error_indicators.local.Size(), - "Local error indicator vectors mismatch."); - // Combine these error indicators into the current average. - std::transform(local.begin(), local.end(), error_indicators.local.begin(), - local.begin(), running_average); - - // More samples have been added, update for the running average lambda. - n += error_indicators.n; - } - else - { - // This indicator was empty, just steal. - (*this) = error_indicators; - } + MFEM_ASSERT(local.Size() == indicator.local.Size(), + "Unexpected size mismatch for ErrorIndicator::AddIndicator!"); + + // The local indicators must be squared before combining, so that the global error + // calculation is valid: + // E = √(1/N ∑ₙ ∑ₖ ηₖₙ²) + // from which it follows that: + // E² = 1/N ∑ₙ ∑ₖ ηₖₙ² + // = 1/N ∑ₙ Eₙ + // Namely the average of the global error indicators included in the reduction. + // Squaring both sides means the summation can be rearranged, and then the local error + // indicators become: + // eₖ = √(1/N ∑ₙ ηₖₙ²) + const int N = local.Size(); + const auto *DIL = indicator.local.Read(); + auto *DL = local.ReadWrite(); + mfem::forall(N, + [=] MFEM_HOST_DEVICE(int i) + { + DL[i] = std::sqrt((DL[i] * DL[i] * n + DIL[i] * DIL[i] * indicator.n) / + (n + indicator.n)); + }); + + // More samples have been added, update for the running average. + normalization = + (n * normalization + indicator.normalization * indicator.n) / (n + indicator.n); + n += indicator.n; } + } // namespace palace diff --git a/palace/fem/errorindicator.hpp b/palace/fem/errorindicator.hpp index 627c5a33a..0487b4821 100644 --- a/palace/fem/errorindicator.hpp +++ b/palace/fem/errorindicator.hpp @@ -4,75 +4,73 @@ #ifndef PALACE_FEM_ERROR_INDICATORS_HPP #define PALACE_FEM_ERROR_INDICATORS_HPP +#include +#include #include "linalg/vector.hpp" #include "utils/communication.hpp" namespace palace { -// Storage for error estimation results from the solve. Required in the AMR loop. This is -// richer than the IndicatorsAndNormalization because it stores derived quantities, and a -// communicator for use in adaptation. + +// +// Storage for error estimation results from a simulation which involves one or more solves, +// required in the AMR loop. +// class ErrorIndicator { +protected: + // Elemental localized error indicators. Used for marking elements for + // refinement and coarsening. + Vector local; + + // Normalization constant. + double normalization; + + // Number of samples. + int n; + public: ErrorIndicator(Vector &&local, double normalization) : local(std::move(local)), normalization(normalization), n(1) { } - ErrorIndicator(const std::vector &ind) - { - for (const auto &i : ind) - { - AddIndicator(i); - } - } - ErrorIndicator() = default; + ErrorIndicator() : normalization(0.0), n(0) {} + + // Add an indicator to the running total. + void AddIndicator(const ErrorIndicator &indicator); - // Return the local error indicators. + // Return the local error indicator. const auto &Local() const { return local; } + // Return the global error indicator. - inline auto Norml2(MPI_Comm comm) const { return linalg::Norml2(comm, local); } + auto Norml2(MPI_Comm comm) const { return linalg::Norml2(comm, local); } + // Return the largest local error indicator. - inline auto Max(MPI_Comm comm) const + auto Max(MPI_Comm comm) const { - double max = local.Max(); + auto max = local.Max(); Mpi::GlobalMax(1, &max, comm); return max; } + // Return the smallest local error indicator. - inline auto Min(MPI_Comm comm) const + auto Min(MPI_Comm comm) const { - double min = local.Min(); + auto min = local.Min(); Mpi::GlobalMin(1, &min, comm); return min; } + // Return the mean local error indicator. - inline auto Mean(MPI_Comm comm) const + auto Mean(MPI_Comm comm) const { - int size = local.Size(); - Mpi::GlobalSum(1, &size, comm); auto sum = local.Sum(); Mpi::GlobalSum(1, &sum, comm); - return sum / size; - } - // Return the normalization constant for the absolute error. - inline auto GetNormalization() const { return normalization; } - // Add a set of indicators to the running totals. - void AddIndicator(const ErrorIndicator &indicators); - // Return a vector of postprocess data. - std::array GetPostprocessData(MPI_Comm comm) const - { - return {Norml2(comm), Min(comm), Max(comm), Mean(comm), GetNormalization()}; + return sum / linalg::GlobalSize(comm, local); } -protected: - // Elemental localized error indicators. Used for marking elements for - // refinement and coarsening. - Vector local; - // Normalization constant. - double normalization; - // Number of samples. Mutability required to guarantee operation. - int n = 0; + // Return the normalization constant for the absolute error. + auto Normalization() const { return normalization; } }; } // namespace palace diff --git a/palace/fem/integrator.hpp b/palace/fem/integrator.hpp index 5cd6734b4..44a646b91 100644 --- a/palace/fem/integrator.hpp +++ b/palace/fem/integrator.hpp @@ -43,7 +43,7 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator, mfem::Vector f_loc, f_hat; public: - VectorFEBoundaryLFIntegrator(mfem::VectorCoefficient &QG) : Q(QG), f_loc(QG.GetVDim()) {} + VectorFEBoundaryLFIntegrator(mfem::VectorCoefficient &QG) : Q(QG) {} void AssembleRHSElementVect(const mfem::FiniteElement &fe, mfem::ElementTransformation &Tr, @@ -53,10 +53,10 @@ class VectorFEBoundaryLFIntegrator : public mfem::LinearFormIntegrator, const int dim = fe.GetDim(); const mfem::IntegrationRule *ir = (IntRule != nullptr) ? IntRule : GetDefaultRule(fe, Tr); + f_hat.SetSize(dim); vshape.SetSize(dof, dim); elvect.SetSize(dof); elvect = 0.0; - f_hat.SetSize(dim); for (int i = 0; i < ir->GetNPoints(); i++) { diff --git a/palace/models/materialoperator.cpp b/palace/models/materialoperator.cpp index 251feb038..01aa9cf2f 100644 --- a/palace/models/materialoperator.cpp +++ b/palace/models/materialoperator.cpp @@ -373,14 +373,14 @@ void MaterialOperator::SetUpMaterialProperties(const IoData &iodata, mfem::ParMe mfem::DenseMatrix mu_r = ToDenseMatrix(data.mu_r); mfem::DenseMatrixInverse(mu_r, true).GetInverseMatrix(mat_muinv.at(attr - 1)); - // Material permittivity: Im{ϵ} = - ϵ * tan(δ) + // Material permittivity: Im{ε} = - ε * tan(δ) mfem::DenseMatrix T(sdim, sdim); mat_epsilon.at(attr - 1) = ToDenseMatrix(data.epsilon_r); Mult(mat_epsilon.at(attr - 1), ToDenseMatrix(data.tandelta), T); T *= -1.0; mat_epsilon_imag.at(attr - 1) = T; - // ϵ * √(I + tan(δ) * tan(δ)ᵀ) + // ε * √(I + tan(δ) * tan(δ)ᵀ) MultAAt(ToDenseMatrix(data.tandelta), T); for (int i = 0; i < T.Height(); i++) { @@ -388,11 +388,11 @@ void MaterialOperator::SetUpMaterialProperties(const IoData &iodata, mfem::ParMe } Mult(mat_epsilon.at(attr - 1), MatrixSqrt(T), mat_epsilon_abs.at(attr - 1)); - // √μ⁻¹ ϵ + // √μ⁻¹ ε Mult(mat_muinv.at(attr - 1), mat_epsilon.at(attr - 1), mat_invz0.at(attr - 1)); mat_invz0.at(attr - 1) = MatrixSqrt(mat_invz0.at(attr - 1)); - // (√μ ϵ)⁻¹ + // (√μ ε)⁻¹ mfem::DenseMatrixInverse(mat_epsilon.at(attr - 1), true).GetInverseMatrix(T); Mult(mat_muinv.at(attr - 1), T, mat_c0.at(attr - 1)); mat_c0.at(attr - 1) = MatrixSqrt(mat_c0.at(attr - 1)); diff --git a/palace/models/postoperator.cpp b/palace/models/postoperator.cpp index 47b44f095..07fc9a444 100644 --- a/palace/models/postoperator.cpp +++ b/palace/models/postoperator.cpp @@ -45,9 +45,7 @@ PostOperator::PostOperator(const IoData &iodata, SpaceOperator &spaceop, &spaceop.GetRTSpace(), iodata.solver.pa_order_threshold), has_imaginary(iodata.problem.type != config::ProblemData::Type::TRANSIENT), E(&spaceop.GetNDSpace()), B(&spaceop.GetRTSpace()), V(std::nullopt), A(std::nullopt), - indicator_fec(0, spaceop.GetNDSpace().GetParMesh()->Dimension()), - indicator_fes(spaceop.GetNDSpace().GetParMesh(), &indicator_fec), - indicator_field(&indicator_fes), lumped_port_init(false), wave_port_init(false), + lumped_port_init(false), wave_port_init(false), paraview(CreateParaviewPath(iodata, name), spaceop.GetNDSpace().GetParMesh()), paraview_bdr(CreateParaviewPath(iodata, name) + "_boundary", spaceop.GetNDSpace().GetParMesh()), @@ -101,10 +99,8 @@ PostOperator::PostOperator(const IoData &iodata, LaplaceOperator &laplaceop, dom_post_op(iodata, laplaceop.GetMaterialOp(), &laplaceop.GetNDSpace(), nullptr, iodata.solver.pa_order_threshold), has_imaginary(false), E(&laplaceop.GetNDSpace()), B(std::nullopt), - V(&laplaceop.GetH1Space()), A(std::nullopt), - indicator_fec(0, laplaceop.GetH1Space().GetParMesh()->Dimension()), - indicator_fes(laplaceop.GetH1Space().GetParMesh(), &indicator_fec), - indicator_field(&indicator_fes), lumped_port_init(false), wave_port_init(false), + V(&laplaceop.GetH1Space()), A(std::nullopt), lumped_port_init(false), + wave_port_init(false), paraview(CreateParaviewPath(iodata, name), laplaceop.GetNDSpace().GetParMesh()), paraview_bdr(CreateParaviewPath(iodata, name) + "_boundary", laplaceop.GetNDSpace().GetParMesh()), @@ -131,10 +127,7 @@ PostOperator::PostOperator(const IoData &iodata, CurlCurlOperator &curlcurlop, dom_post_op(iodata, curlcurlop.GetMaterialOp(), nullptr, &curlcurlop.GetRTSpace(), iodata.solver.pa_order_threshold), has_imaginary(false), E(std::nullopt), B(&curlcurlop.GetRTSpace()), V(std::nullopt), - A(&curlcurlop.GetNDSpace()), - indicator_fec(0, curlcurlop.GetNDSpace().GetParMesh()->Dimension()), - indicator_fes(curlcurlop.GetNDSpace().GetParMesh(), &indicator_fec), - indicator_field(&indicator_fes), lumped_port_init(false), wave_port_init(false), + A(&curlcurlop.GetNDSpace()), lumped_port_init(false), wave_port_init(false), paraview(CreateParaviewPath(iodata, name), curlcurlop.GetNDSpace().GetParMesh()), paraview_bdr(CreateParaviewPath(iodata, name) + "_boundary", curlcurlop.GetNDSpace().GetParMesh()), @@ -231,7 +224,6 @@ void PostOperator::InitializeDataCollection(const IoData &iodata) paraview.RegisterField("A", &*A); paraview_bdr.RegisterVCoeffField("A", As.get()); } - paraview.RegisterField("ErrorIndicator", &*indicator_field); // Extract surface charge from normally discontinuous ND E-field. Also extract surface // currents from tangentially discontinuous RT B-field The surface charge and surface @@ -339,13 +331,6 @@ void PostOperator::SetAGridFunction(const Vector &a) A->ExchangeFaceNbrData(); } -void PostOperator::SetIndicatorGridFunction(const mfem::Vector &i) -{ - MFEM_VERIFY(indicator_field, - "Incorrect usage of PostOperator::SetIndicatorGridFunction!"); - indicator_field->SetFromTrueDofs(i); -} - void PostOperator::UpdatePorts(const LumpedPortOperator &lumped_port_op, double omega) { MFEM_VERIFY(E && B, "Incorrect usage of PostOperator::UpdatePorts!"); @@ -648,7 +633,7 @@ double PostOperator::GetSurfaceFlux(int idx) const return Phi; } -void PostOperator::WriteFields(int step, double time) const +void PostOperator::WriteFields(int step, double time, const ErrorIndicator *indicator) const { // Given the electric field and magnetic flux density, write the fields to disk for // visualization. @@ -657,17 +642,36 @@ void PostOperator::WriteFields(int step, double time) const paraview.SetTime(time); paraview_bdr.SetCycle(step); paraview_bdr.SetTime(time); - if (first_save) + if (first_save || indicator) { mfem::ParMesh &mesh = (E) ? *E->ParFESpace()->GetParMesh() : *B->ParFESpace()->GetParMesh(); mfem::L2_FECollection pwconst_fec(0, mesh.Dimension()); mfem::ParFiniteElementSpace pwconst_fespace(&mesh, &pwconst_fec); - mfem::ParGridFunction rank(&pwconst_fespace); - rank = mesh.GetMyRank() + 1; - paraview.RegisterField("rank", &rank); + std::unique_ptr rank, eta; + if (first_save) + { + rank = std::make_unique(&pwconst_fespace); + *rank = mesh.GetMyRank() + 1; + paraview.RegisterField("Rank", rank.get()); + } + if (indicator) + { + eta = std::make_unique(&pwconst_fespace); + MFEM_VERIFY(eta.Size() == indicator->Size(), + "Size mismatch for provided ErrorIndicator for postprocessing!"); + *eta = indicator->Local(); + paraview.RegisterField("Indicator", eta.get()); + } paraview.Save(); - paraview.DeregisterField("rank"); + if (rank) + { + paraview.DeregisterField("Rank"); + } + if (eta) + { + paraview.DeregisterField("Indicator"); + } } else { diff --git a/palace/models/postoperator.hpp b/palace/models/postoperator.hpp index 8ca93b0a6..8d49a2055 100644 --- a/palace/models/postoperator.hpp +++ b/palace/models/postoperator.hpp @@ -50,11 +50,6 @@ class PostOperator std::unique_ptr Esr, Esi, Bsr, Bsi, As, Jsr, Jsi; std::unique_ptr Vs, Ue, Um, Qsr, Qsi; - // Objects for storing the error indicator field for plotting - mfem::L2_FECollection indicator_fec; - mfem::ParFiniteElementSpace indicator_fes; - std::optional indicator_field; - // Lumped and wave port voltage and current (R, L, and C branches) caches updated when // the grid functions are set. struct PortPostData @@ -94,8 +89,6 @@ class PostOperator void SetVGridFunction(const Vector &v); void SetAGridFunction(const Vector &a); - void SetIndicatorGridFunction(const Vector &i); - // Update cached port voltages and currents for lumped and wave port operators. void UpdatePorts(const LumpedPortOperator &lumped_port_op, const WavePortOperator &wave_port_op, double omega = 0.0) @@ -169,7 +162,7 @@ class PostOperator // Write to disk the E- and B-fields extracted from the solution vectors. Note that fields // are not redimensionalized, to do so one needs to compute: B <= B * (μ₀ H₀), E <= E * // (Z₀ H₀), V <= V * (Z₀ H₀ L₀), etc. - void WriteFields(int step, double time) const; + void WriteFields(int step, double time, const ErrorIndicator *indicator = nullptr) const; // Probe the E- and B-fields for their vector-values at speceified locations in space. // Locations of probes are set up in constructor from configuration file data. If @@ -178,6 +171,12 @@ class PostOperator const auto &GetProbes() const { return interp_op.GetProbes(); } std::vector> ProbeEField() const; std::vector> ProbeBField() const; + + // Get the associated MPI communicator. + MPI_Comm GetComm() const + { + return (E) ? *E->ParFESpace()->GetComm() : *B->ParFESpace()->GetComm(); + } }; } // namespace palace From 9152d2ddf972d4f7c9e7ca4664fdceab3473ae5f Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Mon, 9 Oct 2023 16:44:28 -0700 Subject: [PATCH 35/39] Clean up for problem drivers with new simpler postprocessing --- palace/drivers/drivensolver.cpp | 61 ++++++++++++-------------- palace/drivers/drivensolver.hpp | 17 +++---- palace/drivers/eigensolver.cpp | 46 ++++++++----------- palace/drivers/eigensolver.hpp | 3 +- palace/drivers/electrostaticsolver.cpp | 44 ++++++++++--------- palace/drivers/electrostaticsolver.hpp | 5 +-- palace/drivers/magnetostaticsolver.cpp | 44 ++++++++++--------- palace/drivers/magnetostaticsolver.hpp | 5 +-- palace/drivers/transientsolver.cpp | 17 ++++--- palace/drivers/transientsolver.hpp | 4 +- palace/utils/timer.hpp | 14 +++--- 11 files changed, 132 insertions(+), 128 deletions(-) diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 1fac11103..2b097790b 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -96,12 +96,6 @@ DrivenSolver::Solve(const std::vector> &mesh) con MFEM_VERIFY(excitations > 0, "No excitation specified for driven simulation!"); } Mpi::Print("\n"); - // Initalize the error estimation operator. - auto estimator = [&]() - { - BlockTimer bt(Timer::ESTCONSTRUCT); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); - }(); // Main frequency sweep loop. return adaptive @@ -110,8 +104,7 @@ DrivenSolver::Solve(const std::vector> &mesh) con } ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, - int step0, double omega0, + int nstep, int step0, double omega0, double delta_omega) const { // Construct the system matrices defining the linear operator. PEC boundaries are handled @@ -144,13 +137,19 @@ ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator & E = 0.0; B = 0.0; - // Initialize structures for storing the error indicator. + // Initialize structures for storing and reducing the results of error estimation. + auto estimator = [&]() + { + BlockTimer bt(Timer::CONSTRUCTESTIMATE); + return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); + }(); ErrorIndicator indicator; // Main frequency sweep loop. + int step = step0; double omega = omega0; auto t0 = Timer::Now(); - for (int step = step0; step < nstep; step++) + while (step < nstep) { const double freq = iodata.DimensionalizeValue(IoData::ValueType::FREQUENCY, omega); Mpi::Print("\nIt {:d}/{:d}: ω/2π = {:.3e} GHz (elapsed time = {:.2e} s)\n", step + 1, @@ -175,9 +174,6 @@ ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator & Mpi::Print("\n"); ksp.Mult(RHS, E); - // Compute the error indicators, and post process the indicator field. - estimator.AddErrorIndicator(indicator, postop, E); - // Compute B = -1/(iω) ∇ x E on the true dofs, and set the internal GridFunctions in // PostOperator for all postprocessing operations. BlockTimer bt2(Timer::POSTPRO); @@ -198,23 +194,25 @@ ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator & E_elec + E_mag); } + // Calculate and record the error indicators. + estimator.AddErrorIndicator(E, indicator); + // Postprocess S-parameters and optionally write solution to disk. Postprocess(postop, spaceop.GetLumpedPortOp(), spaceop.GetWavePortOp(), spaceop.GetSurfaceCurrentOp(), step, omega, E_elec, E_mag, - !iodata.solver.driven.only_port_post); + !iodata.solver.driven.only_port_post, + (step == nstep - 1) ? &indicator : nullptr); // Increment frequency. + step++; omega += delta_omega; } SaveMetadata(ksp); - BlockTimer bt_post(Timer::POSTPRO); - PostprocessErrorIndicator(indicator.GetPostprocessData(spaceop.GetComm())); return indicator; } ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, - int step0, double omega0, + int nstep, int step0, double omega0, double delta_omega) const { // Configure default parameters if not specified. @@ -253,6 +251,14 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator E = 0.0; B = 0.0; + // Initialize structures for storing and reducing the results of error estimation. + auto estimator = [&]() + { + BlockTimer bt(Timer::CONSTRUCTESTIMATE); + return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); + }(); + ErrorIndicator indicator; + // Configure the PROM operator which performs the parameter space sampling and basis // construction during the offline phase as well as the PROM solution during the online // phase. @@ -265,21 +271,17 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator prom.Initialize(omega0, delta_omega, nstep - step0, nmax); spaceop.GetWavePortOp().SetSuppressOutput(true); // Suppress wave port output for offline - // The error indicators will be calculated for each HDM sample rather than for - // the online stage. - ErrorIndicator indicator; - // Initialize the basis with samples from the top and bottom of the frequency // range of interest. Each call for an HDM solution adds the frequency sample to P_S and // removes it from P \ P_S. Timing for the HDM construction and solve is handled inside // of the RomOperator. BlockTimer bt1(Timer::CONSTRUCTPROM); prom.SolveHDM(omega0, E); // Print matrix stats at first HDM solve - estimator.AddErrorIndicator(indicator, E); prom.AddHDMSample(omega0, E); + estimator.AddErrorIndicator(E, indicator); prom.SolveHDM(omega0 + (nstep - step0 - 1) * delta_omega, E); - estimator.AddErrorIndicator(indicator, E); prom.AddHDMSample(omega0 + (nstep - step0 - 1) * delta_omega, E); + estimator.AddErrorIndicator(E, indicator); // Greedy procedure for basis construction (offline phase). Basis is initialized with // solutions at frequency sweep endpoints. @@ -301,9 +303,8 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator iter - iter0 + 1, prom.GetReducedDimension(), omega_star * f0, omega_star, max_error); prom.SolveHDM(omega_star, E); - Mpi::Print("Computing error estimates\n"); - estimator.AddErrorIndicator(indicator, E); prom.AddHDMSample(omega_star, E); + estimator.AddErrorIndicator(E, indicator); iter++; } Mpi::Print("\nAdaptive sampling{} {:d} frequency samples:\n" @@ -315,11 +316,6 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator Timer::Duration(Timer::Now() - t0).count()); // Timing on root SaveMetadata(prom.GetLinearSolver()); - // Postprocess the indicator. - BlockTimer bt_post(Timer::POSTPRO); - postop.SetIndicatorGridFunction(indicator.Local()); - PostprocessErrorIndicator(indicator.GetPostprocessData(spaceop.GetComm())); - // Main fast frequency sweep loop (online phase). BlockTimer bt2(Timer::CONSTRUCT); Mpi::Print("\nBeginning fast frequency sweep online phase\n"); @@ -361,7 +357,8 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // Postprocess S-parameters and optionally write solution to disk. Postprocess(postop, spaceop.GetLumpedPortOp(), spaceop.GetWavePortOp(), spaceop.GetSurfaceCurrentOp(), step, omega, E_elec, E_mag, - !iodata.solver.driven.only_port_post); + !iodata.solver.driven.only_port_post, + (step == nstep - 1) ? &indicator : nullptr); // Increment frequency. step++; diff --git a/palace/drivers/drivensolver.hpp b/palace/drivers/drivensolver.hpp index 9e53fdd0d..7874b771f 100644 --- a/palace/drivers/drivensolver.hpp +++ b/palace/drivers/drivensolver.hpp @@ -18,7 +18,6 @@ class ParMesh; namespace palace { -class CurlFluxErrorEstimator; class ErrorIndicator; class IoData; class LumpedPortOperator; @@ -36,24 +35,26 @@ class DrivenSolver : public BaseSolver private: int GetNumSteps(double start, double end, double delta) const; - ErrorIndicator SweepUniform(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, int step0, - double omega0, double delta_omega) const; - ErrorIndicator SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, - CurlFluxErrorEstimator &estimator, int nstep, int step0, - double omega0, double delta_omega) const; + ErrorIndicator SweepUniform(SpaceOperator &spaceop, PostOperator &postop, int nstep, + int step0, double omega0, double delta_omega) const; + + ErrorIndicator SweepAdaptive(SpaceOperator &spaceop, PostOperator &postop, int nstep, + int step0, double omega0, double delta_omega) const; void Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, const WavePortOperator &wave_port_op, const SurfaceCurrentOperator &surf_j_op, int step, double omega, - double E_elec, double E_mag, bool full) const; + double E_elec, double E_mag, bool full, + const ErrorIndicator *indicator) const; void PostprocessCurrents(const PostOperator &postop, const SurfaceCurrentOperator &surf_j_op, int step, double omega) const; + void PostprocessPorts(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, int step, double omega) const; + void PostprocessSParameters(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, const WavePortOperator &wave_port_op, int step, diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 22e07d519..8dd97522f 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -251,28 +251,23 @@ EigenSolver::Solve(const std::vector> &mesh) cons ksp->SetOperators(*A, *P); eigen->SetLinearSolver(*ksp); - auto estimator = [&]() - { - BlockTimer bt(Timer::ESTCONSTRUCT); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); - }(); - // Eigenvalue problem solve. BlockTimer bt1(Timer::SOLVE); Mpi::Print("\n"); int num_conv = eigen->Solve(); SaveMetadata(*ksp); - // Save the eigenvalue estimates. - BlockTimer bt_est(Timer::ESTIMATION); - std::vector indicators; - indicators.reserve(iodata.solver.eigenmode.n); + // Calculate and record the error indicators. + auto estimator = [&]() + { + BlockTimer bt(Timer::CONSTRUCTESTIMATE); + return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); + }(); + ErrorIndicator indicator; for (int i = 0; i < iodata.solver.eigenmode.n; i++) { - // Only update the error indicator for targeted modes. - Mpi::Print("\nComputing error estimates for mode {:d}\n", i); eigen->GetEigenvector(i, E); - indicators.emplace_back(estimator.ComputeIndicators(E)); + estimator.AddErrorIndicator(E, indicator); } // Postprocess the results. @@ -295,9 +290,8 @@ EigenSolver::Solve(const std::vector> &mesh) cons } if (i == 0) { - Mpi::Print(" Found {:d} converged eigenvalue{} (first = {:.3e}{:+.3e}i)\n", num_conv, - (num_conv > 1) ? "s" : "", omega.real(), omega.imag()); - Mpi::Print("\n"); + Mpi::Print(" Found {:d} converged eigenvalue{} (first = {:.3e}{:+.3e}i)\n\n", + num_conv, (num_conv > 1) ? "s" : "", omega.real(), omega.imag()); } // Compute B = -1/(iω) ∇ x E on the true dofs, and set the internal GridFunctions in @@ -305,27 +299,21 @@ EigenSolver::Solve(const std::vector> &mesh) cons eigen->GetEigenvector(i, E); Curl->Mult(E, B); B *= -1.0 / (1i * omega); - postop.SetEGridFunction(E); postop.SetBGridFunction(B); postop.UpdatePorts(spaceop.GetLumpedPortOp(), omega.real()); - if (i < iodata.solver.eigenmode.n_post) - { - postop.SetIndicatorGridFunction(indicators[i].Local()); - } // Postprocess the mode. - Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv); + Postprocess(postop, spaceop.GetLumpedPortOp(), i, omega, error1, error2, num_conv, + (i == 0) ? &indicator : nullptr); } - PostprocessErrorIndicator( - ErrorIndicator(indicators).GetPostprocessData(spaceop.GetComm())); - return indicators; + return indicator; } void EigenSolver::Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, int i, std::complex omega, double error1, double error2, - int num_conv) const + int num_conv, const ErrorIndicator *indicator) const { // The internal GridFunctions for PostOperator have already been set from the E and B // solutions in the main loop over converged eigenvalues. Note: The energies output are @@ -343,9 +331,13 @@ void EigenSolver::Postprocess(const PostOperator &postop, PostprocessProbes(postop, "m", i, i + 1); if (i < iodata.solver.eigenmode.n_post) { - PostprocessFields(postop, i, i + 1); + PostprocessFields(postop, i, i + 1, indicator); Mpi::Print(" Wrote mode {:d} to disk\n", i + 1); } + if (indicator) + { + PostprocessErrorIndicator(postop, *indicator); + } } namespace diff --git a/palace/drivers/eigensolver.hpp b/palace/drivers/eigensolver.hpp index 9e935c7f1..2b717e7ae 100644 --- a/palace/drivers/eigensolver.hpp +++ b/palace/drivers/eigensolver.hpp @@ -33,10 +33,11 @@ class EigenSolver : public BaseSolver private: void Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, int i, std::complex omega, double error1, double error2, - int num_conv) const; + int num_conv, const ErrorIndicator *indicator) const; void PostprocessEigen(int i, std::complex omega, double error1, double error2, int num_conv) const; + void PostprocessEPR(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, int i, std::complex omega, double Em) const; diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 6ac7e7861..d4697a806 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -8,7 +8,6 @@ #include "linalg/errorestimator.hpp" #include "linalg/ksp.hpp" #include "linalg/operator.hpp" -#include "linalg/vector.hpp" #include "models/laplaceoperator.hpp" #include "models/postoperator.hpp" #include "utils/communication.hpp" @@ -30,14 +29,6 @@ ElectrostaticSolver::Solve(const std::vector> &me auto K = laplaceop.GetStiffnessMatrix(); SaveMetadata(laplaceop.GetH1Spaces()); - // Construct the error estimator. - auto estimator = [&]() - { - BlockTimer bt(Timer::ESTCONSTRUCT); - return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), - laplaceop.GetH1Spaces()); - }(); - // Set up the linear solver. KspSolver ksp(iodata, laplaceop.GetH1Spaces()); ksp.SetOperators(*K, *K); @@ -51,7 +42,6 @@ ElectrostaticSolver::Solve(const std::vector> &me // Right-hand side term and solution vector storage. Vector RHS(K->Height()); std::vector V(nstep); - std::vector indicators(nstep); // Main loop over terminal boundaries. Mpi::Print("\nComputing electrostatic fields for {:d} terminal boundar{}\n", nstep, @@ -70,7 +60,6 @@ ElectrostaticSolver::Solve(const std::vector> &me BlockTimer bt1(Timer::SOLVE); ksp.Mult(RHS, V[step]); - indicators[step] = estimator.ComputeIndicators(V[step]); BlockTimer bt2(Timer::POSTPRO); Mpi::Print(" Sol. ||V|| = {:.6e} (||RHS|| = {:.6e})\n", @@ -84,15 +73,12 @@ ElectrostaticSolver::Solve(const std::vector> &me // Postprocess the capacitance matrix from the computed field solutions. BlockTimer bt1(Timer::POSTPRO); SaveMetadata(ksp); - Postprocess(laplaceop, postop, V, indicators); - PostprocessErrorIndicator( - ErrorIndicator(indicators).GetPostprocessData(laplaceop.GetComm())); - return indicators; + return Postprocess(laplaceop, postop, V); } -void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, - const std::vector &V, - const std::vector &indicators) const +ErrorIndicator ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, + PostOperator &postop, + const std::vector &V) const { // Postprocess the Maxwell capacitance matrix. See p. 97 of the COMSOL AC/DC Module manual // for the associated formulas based on the electric field energy based on a unit voltage @@ -110,6 +96,20 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & { Mpi::Print("\n"); } + + // Calculate and record the error indicators. + auto estimator = [&]() + { + BlockTimer bt(Timer::CONSTRUCTESTIMATE); + return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), + laplaceop.GetH1Spaces()); + }(); + ErrorIndicator indicator; + for (int i = 0; i < nstep; i++) + { + estimator.AddErrorIndicator(V[i], indicator); + } + int i = 0; for (const auto &[idx, data] : terminal_sources) { @@ -125,10 +125,13 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & PostprocessProbes(postop, "i", i, idx); if (i < iodata.solver.electrostatic.n_post) { - postop.SetIndicatorGridFunction(indicators[i].Local()); - PostprocessFields(postop, i, idx); + PostprocessFields(postop, i, idx, (i == 0) ? &indicator : nullptr); Mpi::Print(" Wrote fields to disk for terminal {:d}\n", idx); } + if (i == 0) + { + PostprocessErrorIndicator(postop, indicator); + } // Diagonal: C_ii = 2 U_e(V_i) / V_i². C(i, i) = Cm(i, i) = 2.0 * Ue; @@ -163,6 +166,7 @@ void ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, PostOperator & mfem::DenseMatrix Cinv(C); Cinv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(terminal_sources, C, Cinv, Cm); + return indicator; } void ElectrostaticSolver::PostprocessTerminals( diff --git a/palace/drivers/electrostaticsolver.hpp b/palace/drivers/electrostaticsolver.hpp index 61464e68c..0baecce63 100644 --- a/palace/drivers/electrostaticsolver.hpp +++ b/palace/drivers/electrostaticsolver.hpp @@ -35,9 +35,8 @@ class Timer; class ElectrostaticSolver : public BaseSolver { private: - void Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, - const std::vector &V, - const std::vector &indicators) const; + ErrorIndicator Postprocess(LaplaceOperator &laplaceop, PostOperator &postop, + const std::vector &V) const; void PostprocessTerminals(const std::map> &terminal_sources, const mfem::DenseMatrix &C, const mfem::DenseMatrix &Cinv, diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index d6fbc5821..288284546 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -29,14 +29,6 @@ MagnetostaticSolver::Solve(const std::vector> &me auto K = curlcurlop.GetStiffnessMatrix(); SaveMetadata(curlcurlop.GetNDSpaces()); - // Construct the error estimator. - auto estimator = [&]() - { - BlockTimer bt(Timer::ESTCONSTRUCT); - return CurlFluxErrorEstimator(iodata, curlcurlop.GetMaterialOp(), - curlcurlop.GetNDSpaces()); - }(); - // Set up the linear solver. KspSolver ksp(iodata, curlcurlop.GetNDSpaces(), &curlcurlop.GetH1Spaces()); ksp.SetOperators(*K, *K); @@ -47,10 +39,9 @@ MagnetostaticSolver::Solve(const std::vector> &me MFEM_VERIFY(nstep > 0, "No surface current boundaries specified for magnetostatic simulation!"); - // Source term, solution vector storage and error indicators storage. + // Source term and solution vector storage. Vector RHS(K->Height()); std::vector A(nstep); - std::vector indicators(nstep); // Main loop over current source boundaries. Mpi::Print("\nComputing magnetostatic fields for {:d} source boundar{}\n", nstep, @@ -70,7 +61,6 @@ MagnetostaticSolver::Solve(const std::vector> &me BlockTimer bt1(Timer::SOLVE); ksp.Mult(RHS, A[step]); - indicators[step] = estimator.ComputeIndicators(A[step]); BlockTimer bt2(Timer::POSTPRO); Mpi::Print(" Sol. ||A|| = {:.6e} (||RHS|| = {:.6e})\n", @@ -84,15 +74,12 @@ MagnetostaticSolver::Solve(const std::vector> &me // Postprocess the capacitance matrix from the computed field solutions. BlockTimer bt1(Timer::POSTPRO); SaveMetadata(ksp); - Postprocess(curlcurlop, postop, A, indicators); - PostprocessErrorIndicator( - ErrorIndicator(indicators).GetPostprocessData(curlcurlop.GetComm())); - return indicators; + return Postprocess(curlcurlop, postop, A); } -void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, - const std::vector &A, - const std::vector &indicators) const +ErrorIndicator MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, + PostOperator &postop, + const std::vector &A) const { // Postprocess the Maxwell inductance matrix. See p. 97 of the COMSOL AC/DC Module manual // for the associated formulas based on the magnetic field energy based on a current @@ -111,6 +98,20 @@ void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator { Mpi::Print("\n"); } + + // Calculate and record the error indicators. + auto estimator = [&]() + { + BlockTimer bt(Timer::CONSTRUCTESTIMATE); + return CurlFluxErrorEstimator(iodata, curlcurlop.GetMaterialOp(), + curlcurlop.GetNDSpaces()); + }(); + ErrorIndicator indicator; + for (int i = 0; i < nstep; i++) + { + estimator.AddErrorIndicator(A[i], indicator); + } + int i = 0; for (const auto &[idx, data] : surf_j_op) { @@ -130,10 +131,13 @@ void MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, PostOperator PostprocessProbes(postop, "i", i, idx); if (i < iodata.solver.magnetostatic.n_post) { - postop.SetIndicatorGridFunction(indicators[i].Local()); - PostprocessFields(postop, i, idx); + PostprocessFields(postop, i, idx, (i == 0) ? &indicator : nullptr); Mpi::Print(" Wrote fields to disk for terminal {:d}\n", idx); } + if (i == 0) + { + PostprocessErrorIndicator(postop, indicator); + } // Diagonal: M_ii = 2 U_m(A_i) / I_i². M(i, i) = Mm(i, i) = 2.0 * Um / (Iinc(i) * Iinc(i)); diff --git a/palace/drivers/magnetostaticsolver.hpp b/palace/drivers/magnetostaticsolver.hpp index 45cce6c23..d1fe3c48b 100644 --- a/palace/drivers/magnetostaticsolver.hpp +++ b/palace/drivers/magnetostaticsolver.hpp @@ -33,9 +33,8 @@ class Timer; class MagnetostaticSolver : public BaseSolver { private: - void Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, - const std::vector &A, - const std::vector &indicators) const; + ErrorIndicator Postprocess(CurlCurlOperator &curlcurlop, PostOperator &postop, + const std::vector &A) const; void PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, const mfem::DenseMatrix &M, const mfem::DenseMatrix &Minv, diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 2f3c7ec86..1a3bfd1df 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -80,7 +80,7 @@ TransientSolver::Solve(const std::vector> &mesh) // Initialize structures for storing and reducing the results of error estimation. auto estimator = [&]() { - BlockTimer bt(Timer::ESTCONSTRUCT); + BlockTimer bt(Timer::CONSTRUCTESTIMATE); return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); }(); ErrorIndicator indicator; @@ -127,19 +127,20 @@ TransientSolver::Solve(const std::vector> &mesh) } // Calculate and record the error indicators. - estimator.AddErrorIndicator(indicator, postop, E); + estimator.AddErrorIndicator(E, indicator); // Postprocess port voltages/currents and optionally write solution to disk. Postprocess(postop, spaceop.GetLumpedPortOp(), spaceop.GetSurfaceCurrentOp(), step, t, - J_coef(t), E_elec, E_mag, !iodata.solver.transient.only_port_post); + J_coef(t), E_elec, E_mag, !iodata.solver.transient.only_port_post, + (step == nstep - 1) ? &indicator : nullptr); // Increment time step. step++; } SaveMetadata(timeop.GetLinearSolver()); - PostprocessErrorIndicator(indicator.GetPostprocessData(spaceop.GetComm())); return indicator; } + std::function TransientSolver::GetTimeExcitation(bool dot) const { using namespace excitations; @@ -249,7 +250,7 @@ void TransientSolver::Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, const SurfaceCurrentOperator &surf_j_op, int step, double t, double J_coef, double E_elec, double E_mag, - bool full) const + bool full, const ErrorIndicator *indicator) const { // The internal GridFunctions for PostOperator have already been set from the E and B // solutions in the main time integration loop. @@ -269,9 +270,13 @@ void TransientSolver::Postprocess(const PostOperator &postop, step % iodata.solver.transient.delta_post == 0) { Mpi::Print("\n"); - PostprocessFields(postop, step / iodata.solver.transient.delta_post, ts); + PostprocessFields(postop, step / iodata.solver.transient.delta_post, ts, indicator); Mpi::Print(" Wrote fields to disk at step {:d}\n", step); } + if (indicator) + { + PostprocessErrorIndicator(postop, *indicator); + } } namespace diff --git a/palace/drivers/transientsolver.hpp b/palace/drivers/transientsolver.hpp index 19f0286df..660c985eb 100644 --- a/palace/drivers/transientsolver.hpp +++ b/palace/drivers/transientsolver.hpp @@ -38,11 +38,13 @@ class TransientSolver : public BaseSolver void Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, const SurfaceCurrentOperator &surf_j_op, int step, double t, - double J_coef, double E_elec, double E_mag, bool ful) const; + double J_coef, double E_elec, double E_mag, bool full, + const ErrorIndicator *indicator) const; void PostprocessCurrents(const PostOperator &postop, const SurfaceCurrentOperator &surf_j_op, int step, double t, double J_coef) const; + void PostprocessPorts(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, int step, double t, double J_coef) const; diff --git a/palace/utils/timer.hpp b/palace/utils/timer.hpp index 4c74325c7..dc6949d7c 100644 --- a/palace/utils/timer.hpp +++ b/palace/utils/timer.hpp @@ -30,13 +30,13 @@ class Timer CONSTRUCT, WAVEPORT, // Wave port solver SOLVE, - PRECONDITIONER, // Linear solver - COARSESOLVE, // Linear solver - ESTIMATION, // Estimation - ESTCONSTRUCT, // Construction of estimator - ESTSOLVE, // Evaluation of estimator - CONSTRUCTPROM, // Adaptive frequency sweep - SOLVEPROM, // Adaptive frequency sweep + PRECONDITIONER, // Linear solver + COARSESOLVE, // Linear solver + ESTIMATION, // Estimation + CONSTRUCTESTIMATE, // Construction of estimator + SOLVEESTIMATE, // Evaluation of estimator + CONSTRUCTPROM, // Adaptive frequency sweep + SOLVEPROM, // Adaptive frequency sweep POSTPRO, IO, TOTAL, From 20655eb9f06fb07b44048cedd7c7b861f873ded5 Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Tue, 10 Oct 2023 13:45:39 -0700 Subject: [PATCH 36/39] Rework errorestimator.cpp classes: Improve performance including avoiding legacy LinearForm assembly and simplify, removing need for custom Coefficients and improving ComplexVector organization --- palace/drivers/basesolver.cpp | 4 +- palace/drivers/drivensolver.cpp | 32 +- palace/drivers/eigensolver.cpp | 9 +- palace/drivers/electrostaticsolver.cpp | 10 +- palace/drivers/magnetostaticsolver.cpp | 11 +- palace/drivers/transientsolver.cpp | 9 +- palace/fem/coefficient.hpp | 63 ---- palace/linalg/errorestimator.cpp | 452 ++++++++++++------------- palace/linalg/errorestimator.hpp | 116 ++++--- palace/linalg/vector.cpp | 7 - palace/linalg/vector.hpp | 3 - palace/models/postoperator.cpp | 2 +- palace/models/postoperator.hpp | 2 +- palace/utils/timer.hpp | 14 +- 14 files changed, 337 insertions(+), 397 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index a2de02afa..9f0208ae4 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -578,7 +578,7 @@ void BaseSolver::PostprocessErrorIndicator(const PostOperator &postop, return; } MPI_Comm comm = postop.GetComm(); - std::array data = {indicator.Sum(comm), indicator.Min(comm), + std::array data = {indicator.Norml2(comm), indicator.Min(comm), indicator.Max(comm), indicator.Mean(comm), indicator.Normalization()}; if (root) @@ -587,7 +587,7 @@ void BaseSolver::PostprocessErrorIndicator(const PostOperator &postop, auto output = OutputFile(path, false); // clang-format off output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", - "Sum", table.w, + "Norm", table.w, "Min.", table.w, "Max.", table.w, "Mean", table.w, diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index 2b097790b..b3b005c57 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -98,9 +98,8 @@ DrivenSolver::Solve(const std::vector> &mesh) con Mpi::Print("\n"); // Main frequency sweep loop. - return adaptive - ? SweepAdaptive(spaceop, postop, estimator, nstep, step0, omega0, delta_omega) - : SweepUniform(spaceop, postop, estimator, nstep, step0, omega0, delta_omega); + return adaptive ? SweepAdaptive(spaceop, postop, nstep, step0, omega0, delta_omega) + : SweepUniform(spaceop, postop, nstep, step0, omega0, delta_omega); } ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator &postop, @@ -138,11 +137,10 @@ ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator & B = 0.0; // Initialize structures for storing and reducing the results of error estimation. - auto estimator = [&]() - { - BlockTimer bt(Timer::CONSTRUCTESTIMATE); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); - }(); + CurlFluxErrorEstimator estimator( + spaceop.GetMaterialOp(), spaceop.GetNDSpaces(), iodata.solver.linear.estimator_tol, + iodata.solver.linear.estimator_max_it, iodata.problem.verbose, + iodata.solver.pa_order_threshold); ErrorIndicator indicator; // Main frequency sweep loop. @@ -252,11 +250,10 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator B = 0.0; // Initialize structures for storing and reducing the results of error estimation. - auto estimator = [&]() - { - BlockTimer bt(Timer::CONSTRUCTESTIMATE); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); - }(); + CurlFluxErrorEstimator estimator( + spaceop.GetMaterialOp(), spaceop.GetNDSpaces(), iodata.solver.linear.estimator_tol, + iodata.solver.linear.estimator_max_it, iodata.problem.verbose, + iodata.solver.pa_order_threshold); ErrorIndicator indicator; // Configure the PROM operator which performs the parameter space sampling and basis @@ -382,7 +379,8 @@ void DrivenSolver::Postprocess(const PostOperator &postop, const LumpedPortOperator &lumped_port_op, const WavePortOperator &wave_port_op, const SurfaceCurrentOperator &surf_j_op, int step, - double omega, double E_elec, double E_mag, bool full) const + double omega, double E_elec, double E_mag, bool full, + const ErrorIndicator *indicator) const { // The internal GridFunctions for PostOperator have already been set from the E and B // solutions in the main frequency sweep loop. @@ -405,9 +403,13 @@ void DrivenSolver::Postprocess(const PostOperator &postop, if (iodata.solver.driven.delta_post > 0 && step % iodata.solver.driven.delta_post == 0) { Mpi::Print("\n"); - PostprocessFields(postop, step / iodata.solver.driven.delta_post, freq); + PostprocessFields(postop, step / iodata.solver.driven.delta_post, freq, indicator); Mpi::Print(" Wrote fields to disk at step {:d}\n", step + 1); } + if (indicator) + { + PostprocessErrorIndicator(postop, *indicator); + } } namespace diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 8dd97522f..9e0d3ef93 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -258,11 +258,10 @@ EigenSolver::Solve(const std::vector> &mesh) cons SaveMetadata(*ksp); // Calculate and record the error indicators. - auto estimator = [&]() - { - BlockTimer bt(Timer::CONSTRUCTESTIMATE); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); - }(); + CurlFluxErrorEstimator estimator( + spaceop.GetMaterialOp(), spaceop.GetNDSpaces(), iodata.solver.linear.estimator_tol, + iodata.solver.linear.estimator_max_it, iodata.problem.verbose, + iodata.solver.pa_order_threshold); ErrorIndicator indicator; for (int i = 0; i < iodata.solver.eigenmode.n; i++) { diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index d4697a806..31a85a989 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -98,12 +98,10 @@ ErrorIndicator ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, } // Calculate and record the error indicators. - auto estimator = [&]() - { - BlockTimer bt(Timer::CONSTRUCTESTIMATE); - return GradFluxErrorEstimator(iodata, laplaceop.GetMaterialOp(), - laplaceop.GetH1Spaces()); - }(); + GradFluxErrorEstimator estimator( + laplaceop.GetMaterialOp(), laplaceop.GetH1Spaces(), + iodata.solver.linear.estimator_tol, iodata.solver.linear.estimator_max_it, + iodata.problem.verbose, iodata.solver.pa_order_threshold); ErrorIndicator indicator; for (int i = 0; i < nstep; i++) { diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index 288284546..5f4606509 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -100,12 +100,10 @@ ErrorIndicator MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, } // Calculate and record the error indicators. - auto estimator = [&]() - { - BlockTimer bt(Timer::CONSTRUCTESTIMATE); - return CurlFluxErrorEstimator(iodata, curlcurlop.GetMaterialOp(), - curlcurlop.GetNDSpaces()); - }(); + CurlFluxErrorEstimator estimator( + curlcurlop.GetMaterialOp(), curlcurlop.GetNDSpaces(), + iodata.solver.linear.estimator_tol, iodata.solver.linear.estimator_max_it, + iodata.problem.verbose, iodata.solver.pa_order_threshold); ErrorIndicator indicator; for (int i = 0; i < nstep; i++) { @@ -172,6 +170,7 @@ ErrorIndicator MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, mfem::DenseMatrix Minv(M); Minv.Invert(); // In-place, uses LAPACK (when available) and should be cheap PostprocessTerminals(surf_j_op, M, Minv, Mm); + return indicator; } void MagnetostaticSolver::PostprocessTerminals(const SurfaceCurrentOperator &surf_j_op, diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 1a3bfd1df..13972388f 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -78,11 +78,10 @@ TransientSolver::Solve(const std::vector> &mesh) Mpi::Print("\n"); // Initialize structures for storing and reducing the results of error estimation. - auto estimator = [&]() - { - BlockTimer bt(Timer::CONSTRUCTESTIMATE); - return CurlFluxErrorEstimator(iodata, spaceop.GetMaterialOp(), spaceop.GetNDSpaces()); - }(); + CurlFluxErrorEstimator estimator( + spaceop.GetMaterialOp(), spaceop.GetNDSpaces(), iodata.solver.linear.estimator_tol, + iodata.solver.linear.estimator_max_it, iodata.problem.verbose, + iodata.solver.pa_order_threshold); ErrorIndicator indicator; // Main time integration loop. diff --git a/palace/fem/coefficient.hpp b/palace/fem/coefficient.hpp index c488a02e1..05c643a08 100644 --- a/palace/fem/coefficient.hpp +++ b/palace/fem/coefficient.hpp @@ -477,69 +477,6 @@ inline double DielectricInterfaceCoefficient:: return 0.5 * ts * epsilon * (V * V); } -// Computes the flux, μ⁻¹ ∇ × U, of a field, U, where U can be the electric field E, or the -// magnetic vector potential A. -class CurlFluxCoefficient : public mfem::VectorCoefficient -{ -private: - const mfem::ParGridFunction &U; - const MaterialOperator &mat_op; - mfem::Vector curl; - -public: - CurlFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) - : mfem::VectorCoefficient(gf.ParFESpace()->GetParMesh()->SpaceDimension()), U(gf), - mat_op(mat_op) - { - } - - void Eval(mfem::Vector &V, mfem::ElementTransformation &T, - const mfem::IntegrationPoint &ip) override - { - V.SetSize(vdim); - U.GetCurl(T, curl); - mat_op.GetInvPermeability(T.Attribute).Mult(curl, V); - } -}; - -// Computes the flux, ε ∇ϕ, of the electrostatic potential ϕ. -class GradFluxCoefficient : public mfem::Coefficient -{ -private: - const mfem::ParGridFunction Φ - const MaterialOperator &mat_op; - mfem::Vector grad, W; - int component; - -public: - GradFluxCoefficient(const mfem::ParGridFunction &gf, const MaterialOperator &mat_op) - : mfem::Coefficient(), Phi(gf), mat_op(mat_op), component(-1) - { - } - - void SetComponent(int i) - { - MFEM_ASSERT(i >= 0 && i < mat_op.SpaceDimension(), "Invalid component index!"); - component = i; - } - - double Eval(mfem::ElementTransformation &T, const mfem::IntegrationPoint &ip) override - { - MFEM_ASSERT(component >= 0 && component < mat_op.SpaceDimension(), - "Invalid component index, try calling SetComponent!"); - Eval(W, T, ip); - return W(component); - } - - void Eval(mfem::Vector &V, mfem::ElementTransformation &T, - const mfem::IntegrationPoint &ip) - { - V.SetSize(mat_op.SpaceDimension()); - Phi.GetGradient(T, grad); - mat_op.GetPermittivityReal(T.Attribute).Mult(grad, V); - } -}; - enum class EnergyDensityType { ELECTRIC, diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 57e07370f..826d2da65 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 #include "errorestimator.hpp" + #include #include "fem/coefficient.hpp" -#include "fem/errorindicator.hpp" #include "fem/integrator.hpp" #include "fem/multigrid.hpp" #include "linalg/amg.hpp" @@ -12,38 +12,29 @@ #include "linalg/iterative.hpp" #include "linalg/rap.hpp" #include "models/materialoperator.hpp" -#include "models/postoperator.hpp" #include "utils/communication.hpp" -#include "utils/iodata.hpp" #include "utils/timer.hpp" namespace palace { -using namespace fem; - -// Given a finite element space hierarchy, construct a vector of mass matrix -// operators corresponding to each level. -template -std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHierarchy &h, - int pa_order_threshold) +namespace { - constexpr int skip_zeros = 0; - constexpr bool ScalarFESpace = - std::is_same::value || - std::is_same::value; - - // Assemble the bilinear form operator - auto M = std::make_unique(h.GetNumLevels()); - for (int l = 0; l < h.GetNumLevels(); l++) +std::unique_ptr GetMassMatrix(mfem::ParFiniteElementSpaceHierarchy &fespaces, + int pa_order_threshold, int skip_zeros) +{ + const int dim = fespaces.GetFinestFESpace().GetParMesh()->Dimension(); + const auto type = fespaces.GetFinestFESpace().FEColl()->GetRangeType(dim); + auto M = std::make_unique(fespaces.GetNumLevels()); + for (int l = 0; l < fespaces.GetNumLevels(); l++) { - auto &h_l = h.GetFESpaceAtLevel(l); - auto m = std::make_unique(&h_l); - - if constexpr (ScalarFESpace) + // Force coarse level operator to be fully assembled always. + auto &fespace_l = fespaces.GetFESpaceAtLevel(l); + auto m = std::make_unique(&fespace_l); + if (type == mfem::FiniteElement::SCALAR) { - MFEM_VERIFY(h_l.GetVDim() == 1, + MFEM_VERIFY(fespace_l.GetVDim() == 1, "Scalar mass matrix hierarchy assumes a component-wise solve."); m->AddDomainIntegrator(new mfem::MassIntegrator); } @@ -54,20 +45,15 @@ std::unique_ptr BuildMassMatrixOperator(mfem::ParFiniteElementSpaceHie auto M_l = std::make_unique( fem::AssembleOperator(std::move(m), true, (l > 0) ? pa_order_threshold : 100, skip_zeros), - h_l); - - // Set the essential dofs (none). + fespace_l); M->AddOperator(std::move(M_l)); } return M; } -template -FluxProjector::FluxProjector( - mfem::ParFiniteElementSpaceHierarchy &fespaces, double tol, int max_it, int print, - int pa_order_threshold) - : M(BuildMassMatrixOperator(fespaces, - pa_order_threshold)) +std::unique_ptr +ConfigureLinearSolver(mfem::ParFiniteElementSpaceHierarchy &fespaces, double tol, + int max_it, int print, int pa_order_threshold) { // The system matrix for the projection is real and SPD. For the coarse-level AMG solve, // we don't use an exact solve on the coarsest level. @@ -91,260 +77,270 @@ FluxProjector::FluxProjector( pcg->SetAbsTol(std::numeric_limits::epsilon()); pcg->SetMaxIter(max_it); - ksp = std::make_unique(std::move(pcg), std::move(pc)); - ksp->SetOperators(*M, *M); + return std::make_unique(std::move(pcg), std::move(pc)); } -CurlFluxErrorEstimator::CurlFluxErrorEstimator( - const IoData &iodata, const MaterialOperator &mat_op, - mfem::ParFiniteElementSpaceHierarchy &nd_fespaces) - : mat_op(mat_op), nd_fespaces(nd_fespaces), - smooth_projector(nd_fespaces, iodata.solver.linear.estimator_tol, - iodata.solver.linear.estimator_max_it, iodata.problem.verbose, - iodata.solver.pa_order_threshold), - smooth_flux(nd_fespaces.GetFinestFESpace().GetTrueVSize()), - flux_rhs(nd_fespaces.GetFinestFESpace().GetTrueVSize()), - field_gf(&nd_fespaces.GetFinestFESpace()), - smooth_flux_gf(&nd_fespaces.GetFinestFESpace()) +} // namespace + +FluxProjector::FluxProjector(const MaterialOperator &mat_op, + mfem::ParFiniteElementSpaceHierarchy &nd_fespaces, double tol, + int max_it, int print, int pa_order_threshold) { + BlockTimer bt(Timer::CONSTRUCTESTIMATOR); + constexpr int skip_zeros = 0; + { + // XX TODO: No partial assembly is available yet for this operator. + constexpr auto MatType = MaterialPropertyType::INV_PERMEABILITY; + MaterialPropertyCoefficient muinv_func(mat_op); + auto flux = std::make_unique(&nd_fespaces.GetFinestFESpace(), + &nd_fespaces.GetFinestFESpace()); + flux->AddDomainIntegrator(new mfem::MixedVectorCurlIntegrator(muinv_func)); + flux->SetAssemblyLevel(mfem::AssemblyLevel::LEGACY); + flux->Assemble(skip_zeros); + flux->Finalize(skip_zeros); + Flux = std::make_unique(std::move(flux), nd_fespaces.GetFinestFESpace(), + nd_fespaces.GetFinestFESpace(), false); + } + M = GetMassMatrix(nd_fespaces, pa_order_threshold, skip_zeros); + + ksp = ConfigureLinearSolver(nd_fespaces, tol, max_it, print, pa_order_threshold); + ksp->SetOperators(*M, *M); + + rhs.SetSize(nd_fespaces.GetFinestFESpace().GetTrueVSize()); } -template <> -ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const ComplexVector &v) const +FluxProjector::FluxProjector(const MaterialOperator &mat_op, + mfem::ParFiniteElementSpaceHierarchy &h1_fespaces, + mfem::ParFiniteElementSpace &h1d_fespace, double tol, + int max_it, int print, int pa_order_threshold) { - BlockTimer bt_est(Timer::ESTIMATION); - auto &nd_fespace = nd_fespaces.GetFinestFESpace(); - const int nelem = nd_fespace.GetNE(); - - Vector smooth_vec, coarse_vec, estimates(nelem); - estimates = 0.0; - double normalization = 0.0; - for (bool real : {true, false}) + BlockTimer bt(Timer::CONSTRUCTESTIMATOR); + constexpr int skip_zeros = 0; { - field_gf.SetFromTrueDofs(real ? v.Real() : v.Imag()); + // XX TODO: Matrix coefficient support for GradientIntegrator. + // XX TODO: No partial assembly is available yet for this operator. + // constexpr auto MatType = MaterialPropertyType::PERMITTIVITY_REAL; + // MaterialPropertyCoefficient epsilon_func(mat_op); + auto flux = std::make_unique(&h1_fespaces.GetFinestFESpace(), + &h1d_fespace); + flux->AddDomainIntegrator(new mfem::GradientIntegrator); + flux->SetAssemblyLevel(mfem::AssemblyLevel::LEGACY); + flux->Assemble(skip_zeros); + flux->Finalize(skip_zeros); + Flux = std::make_unique(std::move(flux), h1_fespaces.GetFinestFESpace(), + h1d_fespace, false); + } + M = GetMassMatrix(h1_fespaces, pa_order_threshold, skip_zeros); + + ksp = ConfigureLinearSolver(h1_fespaces, tol, max_it, print, pa_order_threshold); + ksp->SetOperators(*M, *M); + + rhs.SetSize(h1d_fespace.GetTrueVSize()); +} - // Coefficients for computing the discontinuous flux component, i.e. (W, μ⁻¹∇ × V). - CurlFluxCoefficient coef(field_gf, mat_op); +template +void FluxProjector::Mult(const VecType &x, VecType &y) const +{ + BlockTimer bt(Timer::SOLVEESTIMATOR); + MFEM_ASSERT(y.Size() == rhs.Size(), "Invalid vector dimensions for FluxProjector::Mult!"); + MFEM_ASSERT( + y.Size() % x.Size() == 0, + "Invalid vector dimension for FluxProjector::Mult, does not yield even blocking!"); + auto MultImpl = [this](const Vector &x_, Vector &y_) + { + const int vdim = y_.Size() / x_.Size(); + Flux->Mult(x_, rhs); + if (vdim == 1) { - // Given the RHS vector of non-smooth flux, construct a flux projector and perform - // mass matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - Mpi::Print("Computing smooth flux approximation of {} component\n", - real ? "real" : "imaginary"); - BlockTimer bt(Timer::ESTSOLVE); - mfem::LinearForm rhs(&nd_fespace); - rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); - rhs.UseFastAssembly(false); - rhs.Assemble(); - nd_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); - smooth_projector.Mult(flux_rhs, smooth_flux); - } - smooth_flux_gf.SetFromTrueDofs(smooth_flux); - smooth_flux_gf.ExchangeFaceNbrData(); - // Loop over elements and accumulate the estimates from this component - for (int e = 0; e < nd_fespace.GetNE(); e++) + // XX TODO WIP PRINT STATEMENTS... + Mpi::Print("Computing smooth flux projection for error estimation\n"); + + ksp->Mult(rhs, y_); + } + else { - auto &T = *nd_fespace.GetElementTransformation(e); - // integration order 2p + q - const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), - 2 * nd_fespace.GetFE(e)->GetOrder() + T.Order()); - for (const auto &ip : ir) + for (int i = 0; i < vdim; i++) { - T.SetIntPoint(&ip); - - smooth_flux_gf.GetVectorValue(e, ip, smooth_vec); - coef.Eval(coarse_vec, T, ip); - double w_i = ip.weight * T.Weight(); - constexpr int vdim = 3; - for (int c = 0; c < vdim; c++) - { - estimates[e] += w_i * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); - normalization += w_i * coarse_vec[c] * coarse_vec[c]; - } + + // XX TODO WIP PRINT STATEMENTS... + Mpi::Print("Computing smooth flux projection of flux component {:d}/{:d} for error " + "estimation\n", + i + 1, vdim); + + const Vector rhsb(rhs, i * x_.Size(), x_.Size()); + Vector yb(y_, i * x_.Size(), x_.Size()); + ksp->Mult(rhsb, yb); } } + }; + if constexpr (std::is_same::value) + { + MultImpl(x.Real(), y.Real()); + MultImpl(x.Imag(), y.Imag()); } - linalg::Sqrt(estimates); - - Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm()); - normalization = std::sqrt(normalization); - if (normalization > 0) + else { - estimates /= normalization; + MultImpl(x, y); } - return ErrorIndicator(std::move(estimates), normalization); } -template <> -ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const Vector &v) const +template +CurlFluxErrorEstimator::CurlFluxErrorEstimator( + const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &nd_fespaces, + double tol, int max_it, int print_level, int pa_order_threshold) + : mat_op(mat_op), nd_fespaces(nd_fespaces), + projector(mat_op, nd_fespaces, tol, max_it, print_level, pa_order_threshold), + F(nd_fespaces.GetFinestFESpace().GetTrueVSize()), F_gf(&nd_fespaces.GetFinestFESpace()), + U_gf(&nd_fespaces.GetFinestFESpace()) { - BlockTimer bt_est(Timer::ESTIMATION); - auto &nd_fespace = nd_fespaces.GetFinestFESpace(); - field_gf.SetFromTrueDofs(v); - const int nelem = nd_fespace.GetNE(); +} - // Coefficients for computing the discontinuous flux., i.e. (W, μ⁻¹∇ × V). - CurlFluxCoefficient coef(field_gf, mat_op); +template +ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const VecType &U) const +{ + // Compute the projection of the discontinuous flux onto the smooth finite element space + // and populate the corresponding grid functions. + BlockTimer bt(Timer::ESTIMATION); + projector.Mult(U, F); + if constexpr (std::is_same::value) { - // Given the RHS vector of non-smooth flux, construct a flux projector and perform mass - // matrix inversion in the appropriate space, giving f = M⁻¹ f̂. - Mpi::Print("Computing smooth flux approximation\n"); - BlockTimer bt(Timer::ESTSOLVE); - mfem::LinearForm rhs(&nd_fespace); - rhs.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coef)); - rhs.UseFastAssembly(false); - rhs.Assemble(); - nd_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); - smooth_projector.Mult(flux_rhs, smooth_flux); + F_gf.real().SetFromTrueDofs(F.Real()); + F_gf.imag().SetFromTrueDofs(F.Imag()); + U_gf.real().SetFromTrueDofs(U.Real()); + U_gf.imag().SetFromTrueDofs(U.Imag()); + } + else + { + F_gf.SetFromTrueDofs(F); + U_gf.SetFromTrueDofs(U); } - // Given a complex solution represented with a ComplexVector, build a ComplexGridFunction - // for evaluation. - smooth_flux_gf.SetFromTrueDofs(smooth_flux); - smooth_flux_gf.ExchangeFaceNbrData(); - - Vector smooth_vec, coarse_vec, estimates(nelem); - estimates = 0.0; + // Loop over elements and accumulate the estimates from this component. The discontinuous + // flux is μ⁻¹ ∇ × U. + auto &nd_fespace = nd_fespaces.GetFinestFESpace(); + auto &mesh = *nd_fespace.GetParMesh(); + Vector estimates(mesh.GetNE()), V_smooth, V_ip; double normalization = 0.0; - for (int e = 0; e < nd_fespace.GetNE(); e++) + for (int e = 0; e < mesh.GetNE(); e++) { - auto &T = *nd_fespace.GetElementTransformation(e); - // integration order 2p + q - const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), - 2 * nd_fespace.GetFE(e)->GetOrder() + T.Order()); - for (const auto &ip : ir) + const mfem::FiniteElement &fe = *nd_fespace.GetFE(e); + mfem::ElementTransformation &T = *mesh.GetElementTransformation(e); + const int q_order = 2 * fe.GetOrder() + T.OrderW(); + const mfem::IntegrationRule &ir = mfem::IntRules.Get(T.GetGeometryType(), q_order); + for (int i = 0; i < ir.GetNPoints(); i++) { + const mfem::IntegrationPoint &ip = ir.IntPoint(i); T.SetIntPoint(&ip); - smooth_flux_gf.GetVectorValue(e, ip, smooth_vec); - coef.Eval(coarse_vec, T, ip); - const double w_i = ip.weight * T.Weight(); - constexpr int vdim = 3; - for (int c = 0; c < vdim; c++) + const double w = ip.weight * T.Weight(); + if constexpr (std::is_same::value) + { + // Real part + U_gf.real().GetCurl(T, V_smooth); + mat_op.GetInvPermeability(T.Attribute).Mult(V_smooth, V_ip); + F_gf.real().GetVectorValue(T, ip, V_smooth); + V_ip -= V_smooth; + estimates[e] = w * (V_ip * V_ip); + normalization += w * (V_smooth * V_smooth); + + // Imaginary part + U_gf.imag().GetCurl(T, V_smooth); + mat_op.GetInvPermeability(T.Attribute).Mult(V_smooth, V_ip); + F_gf.imag().GetVectorValue(T, ip, V_smooth); + V_ip -= V_smooth; + estimates[e] += w * (V_ip * V_ip); + normalization += w * (V_smooth * V_smooth); + } + else { - estimates[e] += w_i * std::pow(smooth_vec[c] - coarse_vec[c], 2.0); - normalization += w_i * coarse_vec[c] * coarse_vec[c]; + U_gf.GetCurl(T, V_smooth); + mat_op.GetInvPermeability(T.Attribute).Mult(V_smooth, V_ip); + F_gf.GetVectorValue(T, ip, V_smooth); + V_ip -= V_smooth; + estimates[e] = w * (V_ip * V_ip); + normalization += w * (V_smooth * V_smooth); } } estimates[e] = std::sqrt(estimates[e]); } - Mpi::GlobalSum(1, &normalization, nd_fespace.GetComm()); + // Finalize the element-wise error estimates. + Mpi::GlobalSum(1, &normalization, mesh.GetComm()); normalization = std::sqrt(normalization); - if (normalization > 0) + if (normalization > 0.0) { - estimates /= normalization; + estimates *= 1.0 / normalization; } return ErrorIndicator(std::move(estimates), normalization); } -template -void CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &indicator, - PostOperator &postop, - const VectorType &v) const -{ - auto i = ComputeIndicators(v); - BlockTimer bt(Timer::POSTPRO); - postop.SetIndicatorGridFunction(i.Local()); - indicator.AddIndicator(i); -} - -template -void CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &indicator, - const VectorType &v) const -{ - BlockTimer bt(Timer::POSTPRO); - indicator.AddIndicator(ComputeIndicators(v)); -} - -template void -CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &, PostOperator &, - const ComplexVector &) const; -template void CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &, - PostOperator &, - const Vector &) const; -template void -CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &, - const ComplexVector &) const; -template void CurlFluxErrorEstimator::AddErrorIndicator(ErrorIndicator &, - const Vector &) const; - GradFluxErrorEstimator::GradFluxErrorEstimator( - const IoData &iodata, const MaterialOperator &mat_op, - mfem::ParFiniteElementSpaceHierarchy &h1_fespaces) + const MaterialOperator &mat_op, mfem::ParFiniteElementSpaceHierarchy &h1_fespaces, + double tol, int max_it, int print_level, int pa_order_threshold) : mat_op(mat_op), h1_fespaces(h1_fespaces), - smooth_projector(h1_fespaces, iodata.solver.linear.estimator_tol, - iodata.solver.linear.estimator_max_it, iodata.problem.verbose, - iodata.solver.pa_order_threshold), - smooth_flux(h1_fespaces.GetFinestFESpace().GetTrueVSize()), - flux_rhs(h1_fespaces.GetFinestFESpace().GetTrueVSize()), - field_gf(&h1_fespaces.GetFinestFESpace()), - smooth_flux_gf{&h1_fespaces.GetFinestFESpace(), &h1_fespaces.GetFinestFESpace(), - &h1_fespaces.GetFinestFESpace()} + h1d_fespace(h1_fespaces.GetFinestFESpace().GetParMesh(), + h1_fespaces.GetFinestFESpace().FEColl(), + h1_fespaces.GetFinestFESpace().GetParMesh()->SpaceDimension(), + mfem::Ordering::byNODES), + projector(mat_op, h1_fespaces, h1d_fespace, tol, max_it, print_level, + pa_order_threshold), + F(h1d_fespace.GetTrueVSize()), F_gf(&h1d_fespace), U_gf(&h1_fespaces.GetFinestFESpace()) { } -ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &v) const +ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &U) const { - BlockTimer bt_est(Timer::ESTIMATION); - auto &h1_fespace = h1_fespaces.GetFinestFESpace(); - const int sdim = h1_fespace.GetMesh()->SpaceDimension(); - field_gf.SetFromTrueDofs(v); - const int nelem = h1_fespace.GetNE(); - - Vector estimates(nelem); - estimates = 0.0; + // Compute the projection of the discontinuous flux onto the smooth finite element space + // and populate the corresponding grid functions. + BlockTimer bt(Timer::ESTIMATION); + projector.Mult(U, F); + F_gf.SetFromTrueDofs(F); + U_gf.SetFromTrueDofs(U); + + // Loop over elements and accumulate the estimates from this component. The discontinuous + // flux is ε ∇U. + auto &mesh = *h1d_fespace.GetParMesh(); + Vector estimates(mesh.GetNE()), V_smooth, V_ip; double normalization = 0.0; - - // Coefficient for computing the discontinuous flux., i.e. (Vᵢ, (ϵ ∇ ϕ)ᵢ). - GradFluxCoefficient coef(field_gf, mat_op); - for (int c = 0; c < sdim; c++) - { - coef.SetComponent(c); - { - // Given the RHS vector of non-smooth flux component, compute fᵢ = M⁻¹ f̂ᵢ. - Mpi::Print("Computing smooth flux approximation of component {}\n", c); - BlockTimer bt0(Timer::ESTSOLVE); - mfem::LinearForm rhs(&h1_fespace); - rhs.AddDomainIntegrator(new mfem::DomainLFIntegrator(coef)); - rhs.UseFastAssembly(false); - rhs.Assemble(); - h1_fespace.GetProlongationMatrix()->MultTranspose(rhs, flux_rhs); - smooth_projector.Mult(flux_rhs, smooth_flux); - } - smooth_flux_gf[c].SetFromTrueDofs(smooth_flux); - smooth_flux_gf[c].ExchangeFaceNbrData(); - } - - Vector coef_eval(3); - for (int e = 0; e < h1_fespace.GetNE(); e++) + for (int e = 0; e < mesh.GetNE(); e++) { - auto &T = *h1_fespace.GetElementTransformation(e); - // integration order 2p + q - const auto &ir = mfem::IntRules.Get(T.GetGeometryType(), - 2 * h1_fespace.GetFE(e)->GetOrder() + T.Order()); - for (const auto &ip : ir) + const mfem::FiniteElement &fe = *h1d_fespace.GetFE(e); + mfem::ElementTransformation &T = *mesh.GetElementTransformation(e); + const int q_order = 2 * fe.GetOrder() + T.OrderW(); + const mfem::IntegrationRule &ir = mfem::IntRules.Get(T.GetGeometryType(), q_order); + for (int i = 0; i < ir.GetNPoints(); i++) { + // XX TODO: For now the flux is just ∇U since the coefficient support for + // matrix-valued + // permittivity is not yet there (coming soon). + const mfem::IntegrationPoint &ip = ir.IntPoint(i); T.SetIntPoint(&ip); - coef.Eval(coef_eval, T, ip); - for (int c = 0; c < sdim; c++) - { - coef.SetComponent(c); - double smooth_val = smooth_flux_gf[c].GetValue(e, ip); - const double w_i = ip.weight * T.Weight(); - estimates[e] += w_i * std::pow(smooth_val - coef_eval(c), 2.0); - normalization += w_i * std::pow(coef_eval(c), 2.0); - } + const double w = ip.weight * T.Weight(); + U_gf.GetGradient(T, V_smooth); + // mat_op.GetPermittivityReal(T.Attribute).Mult(V_smooth, V_ip); + V_ip = V_smooth; + F_gf.GetVectorValue(T, ip, V_smooth); + V_ip -= V_smooth; + estimates[e] = w * (V_ip * V_ip); + normalization += w * (V_smooth * V_smooth); } estimates[e] = std::sqrt(estimates[e]); } - Mpi::GlobalSum(1, &normalization, h1_fespace.GetComm()); + // Finalize the element-wise error estimates. + Mpi::GlobalSum(1, &normalization, mesh.GetComm()); normalization = std::sqrt(normalization); - if (normalization > 0) + if (normalization > 0.0) { - estimates /= normalization; + estimates *= 1.0 / normalization; } return ErrorIndicator(std::move(estimates), normalization); } +template void FluxProjector::Mult(const Vector &, Vector &) const; +template void FluxProjector::Mult(const ComplexVector &, ComplexVector &) const; + +template class CurlFluxErrorEstimator; +template class CurlFluxErrorEstimator; + } // namespace palace diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index c4f1b7c52..2f63d990e 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -14,93 +14,113 @@ namespace palace { -class IoData; class MaterialOperator; -class PostOperator; // +// Classes used in the estimation of element-wise solution errors via a global L2 projection +// of a discontinuous flux onto a smooth space. +// + // This solver computes a smooth reconstruction of a discontinuous flux. The difference // between this resulting smooth flux and the original non-smooth flux provides a // localizable error estimate. An instance of FluxProjector can be reused across solutions, // thus the construction of the operator is separated from the construction of the flux RHS. - -template class FluxProjector { private: // Operator for the mass matrix inversion. - std::unique_ptr M; + std::unique_ptr Flux, M; // Linear solver and preconditioner for the projected linear system M σ = σ̂. std::unique_ptr ksp; -public: - FluxProjector(mfem::ParFiniteElementSpaceHierarchy &smooth_flux_fes, double tol, - int max_it, int print_level, int pa_order_threshold); + // Workspace object for solver application. + mutable Vector rhs; - inline void Mult(const Vector &x, Vector &y) const { ksp->Mult(x, y); } - inline void Mult(const ComplexVector &x, ComplexVector &y) const - { - Mult(x.Real(), y.Real()); - Mult(x.Imag(), y.Imag()); - } +public: + FluxProjector(const MaterialOperator &mat_op, + mfem::ParFiniteElementSpaceHierarchy &nd_fespaces, double tol, int max_it, + int print_level, int pa_order_threshold); + FluxProjector(const MaterialOperator &mat_op, + mfem::ParFiniteElementSpaceHierarchy &h1_fespaces, + mfem::ParFiniteElementSpace &h1d_fespace, double tol, int max_it, + int print_level, int pa_order_threshold); + + template + void Mult(const VecType &x, VecType &y) const; }; -// Class used for computing curl flux error estimate, i.e. || μ⁻¹∇ × Vₕ - F ||_K where F -// denotes a smooth reconstruction of μ⁻¹∇ × Vₕ. +// Class used for computing curl flux error estimate, i.e. || μ⁻¹ ∇ × Uₕ - F ||_K where F +// denotes a smooth reconstruction of μ⁻¹ ∇ × Uₕ. +template class CurlFluxErrorEstimator { + using GridFunctionType = + typename std::conditional::value, + mfem::ParComplexGridFunction, mfem::ParGridFunction>::type; + + // Reference to input data (not owned). const MaterialOperator &mat_op; - // The finite element space used to represent V, and F. + + // Finite element space used to represent U and F. mfem::ParFiniteElementSpaceHierarchy &nd_fespaces; - FluxProjector smooth_projector; - mutable Vector smooth_flux, flux_rhs; - mutable mfem::ParGridFunction field_gf, smooth_flux_gf; + + // Global L2 projection solver. + FluxProjector projector; + + // Temporary vectors for error estimation. + mutable VecType F; + mutable GridFunctionType F_gf, U_gf; public: - // Constructor for using geometric and p multigrid. - CurlFluxErrorEstimator(const IoData &iodata, const MaterialOperator &mat_op, - mfem::ParFiniteElementSpaceHierarchy &nd_fespaces); + CurlFluxErrorEstimator(const MaterialOperator &mat_op, + mfem::ParFiniteElementSpaceHierarchy &nd_fespaces, double tol, + int max_it, int print_level, int pa_order_threshold); // Compute elemental error indicators given a vector of true DOF. - template - ErrorIndicator ComputeIndicators(const VectorType &v) const; - - // Compute elemental error indicators given a vector of true DOF, v, and fold into an - // existing indicator. Optionally set the error indicator field within a PostOperator. - template - void AddErrorIndicator(ErrorIndicator &indicator, PostOperator &postop, - const VectorType &v) const; - template - void AddErrorIndicator(ErrorIndicator &indicator, const VectorType &v) const; + ErrorIndicator ComputeIndicators(const VecType &U) const; + + // Compute elemental error indicators given a vector of true DOF and fold into an existing + // indicator. + void AddErrorIndicator(const VecType &U, ErrorIndicator &indicator) const + { + indicator.AddIndicator(ComputeIndicators(U)); + } }; -// Class used for computing grad flux error estimate, i.e. || ϵ ∇ ϕₕ - F ||_K where F -// denotes a smooth reconstruction of ϵ ∇ ϕₕ. +// Class used for computing gradient flux error estimate, i.e. || ε ∇Uₕ - F ||_K, where F +// denotes a smooth reconstruction of ε ∇Uₕ. class GradFluxErrorEstimator { + // Reference to input data (not owned). const MaterialOperator &mat_op; - // The finite element space used to represent ϕ, and components of F + + // Finite element space used to represent U. mfem::ParFiniteElementSpaceHierarchy &h1_fespaces; - FluxProjector smooth_projector; - mutable Vector smooth_flux, flux_rhs; - mutable mfem::ParGridFunction field_gf; - mutable std::array smooth_flux_gf; + // Vector H1 space used to represent the components of F, ordered by component. + mfem::ParFiniteElementSpace h1d_fespace; + + // Global L2 projection solver. + FluxProjector projector; + + // Temporary vectors for error estimation. + mutable Vector F; + mutable mfem::ParGridFunction F_gf, U_gf; public: - // Constructor for using geometric and p multigrid. - GradFluxErrorEstimator(const IoData &iodata, const MaterialOperator &mat_op, - mfem::ParFiniteElementSpaceHierarchy &h1_fespaces); + GradFluxErrorEstimator(const MaterialOperator &mat_op, + mfem::ParFiniteElementSpaceHierarchy &h1_fespaces, double tol, + int max_it, int print_level, int pa_order_threshold); // Compute elemental error indicators given a vector of true DOF. - ErrorIndicator ComputeIndicators(const Vector &v) const; + ErrorIndicator ComputeIndicators(const Vector &U) const; - // Compute elemental error indicators given a vector of true DOF, v, and fold into an - // existing indicator. - void AddErrorIndicator(ErrorIndicator &indicator, const Vector &v) const + // Compute elemental error indicators given a vector of true DOF and fold into an existing + // indicator. + void AddErrorIndicator(const Vector &U, ErrorIndicator &indicator) const { - indicator.AddIndicator(ComputeIndicators(v)); + indicator.AddIndicator(ComputeIndicators(U)); } }; diff --git a/palace/linalg/vector.cpp b/palace/linalg/vector.cpp index 62e5ac91f..5f4b57f34 100644 --- a/palace/linalg/vector.cpp +++ b/palace/linalg/vector.cpp @@ -356,13 +356,6 @@ void SetRandomSign(MPI_Comm comm, ComplexVector &x, int seed) { XI[i] = (XI[i] > 0.0) ? 1.0 : ((XI[i] < 0.0) ? -1.0 : 0.0); }); } -void Sqrt(Vector &x) -{ - const int N = x.Size(); - auto *X = x.ReadWrite(); - mfem::forall(N, [=] MFEM_HOST_DEVICE(int i) { X[i] = std::sqrt(X[i]); }); -} - template <> void SetSubVector(Vector &x, const mfem::Array &rows, double s) { diff --git a/palace/linalg/vector.hpp b/palace/linalg/vector.hpp index cc36296c1..8f25fb2d7 100644 --- a/palace/linalg/vector.hpp +++ b/palace/linalg/vector.hpp @@ -139,9 +139,6 @@ void SetRandomReal(MPI_Comm comm, VecType &x, int seed = 0); template void SetRandomSign(MPI_Comm comm, VecType &x, int seed = 0); -// Component-wise square root. -void Sqrt(Vector &x); - // Calculate the inner product yᴴ x or yᵀ x. template inline auto Dot(MPI_Comm comm, const VecType &x, const VecType &y) diff --git a/palace/models/postoperator.cpp b/palace/models/postoperator.cpp index 07fc9a444..bfa1c10c3 100644 --- a/palace/models/postoperator.cpp +++ b/palace/models/postoperator.cpp @@ -658,7 +658,7 @@ void PostOperator::WriteFields(int step, double time, const ErrorIndicator *indi if (indicator) { eta = std::make_unique(&pwconst_fespace); - MFEM_VERIFY(eta.Size() == indicator->Size(), + MFEM_VERIFY(eta->Size() == indicator->Local().Size(), "Size mismatch for provided ErrorIndicator for postprocessing!"); *eta = indicator->Local(); paraview.RegisterField("Indicator", eta.get()); diff --git a/palace/models/postoperator.hpp b/palace/models/postoperator.hpp index 8d49a2055..7ca5265e6 100644 --- a/palace/models/postoperator.hpp +++ b/palace/models/postoperator.hpp @@ -175,7 +175,7 @@ class PostOperator // Get the associated MPI communicator. MPI_Comm GetComm() const { - return (E) ? *E->ParFESpace()->GetComm() : *B->ParFESpace()->GetComm(); + return (E) ? E->ParFESpace()->GetComm() : B->ParFESpace()->GetComm(); } }; diff --git a/palace/utils/timer.hpp b/palace/utils/timer.hpp index dc6949d7c..0ad2e861a 100644 --- a/palace/utils/timer.hpp +++ b/palace/utils/timer.hpp @@ -30,13 +30,13 @@ class Timer CONSTRUCT, WAVEPORT, // Wave port solver SOLVE, - PRECONDITIONER, // Linear solver - COARSESOLVE, // Linear solver - ESTIMATION, // Estimation - CONSTRUCTESTIMATE, // Construction of estimator - SOLVEESTIMATE, // Evaluation of estimator - CONSTRUCTPROM, // Adaptive frequency sweep - SOLVEPROM, // Adaptive frequency sweep + PRECONDITIONER, // Linear solver + COARSESOLVE, // Linear solver + ESTIMATION, // Estimation + CONSTRUCTESTIMATOR, // Construction of estimator + SOLVEESTIMATOR, // Evaluation of estimator + CONSTRUCTPROM, // Adaptive frequency sweep + SOLVEPROM, // Adaptive frequency sweep POSTPRO, IO, TOTAL, From c6e51e19d2259053de0abedeea5cc869653e3e37 Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Tue, 10 Oct 2023 14:05:02 -0700 Subject: [PATCH 37/39] Final cleanup and bug fixes --- palace/drivers/basesolver.cpp | 4 +- palace/drivers/drivensolver.cpp | 6 +-- palace/drivers/eigensolver.cpp | 3 +- palace/drivers/electrostaticsolver.cpp | 9 ++--- palace/drivers/magnetostaticsolver.cpp | 4 +- palace/drivers/transientsolver.cpp | 3 +- palace/fem/errorindicator.cpp | 2 +- palace/linalg/errorestimator.cpp | 51 ++++++++++++-------------- palace/linalg/errorestimator.hpp | 5 +-- palace/utils/configfile.hpp | 2 +- test/runtests.jl | 16 ++++---- test/testcase.jl | 6 +-- 12 files changed, 51 insertions(+), 60 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index 9f0208ae4..e46337a3c 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -588,8 +588,8 @@ void BaseSolver::PostprocessErrorIndicator(const PostOperator &postop, // clang-format off output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", "Norm", table.w, - "Min.", table.w, - "Max.", table.w, + "Minimum", table.w, + "Maximum", table.w, "Mean", table.w, "Normalization", table.w); output.print("{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", diff --git a/palace/drivers/drivensolver.cpp b/palace/drivers/drivensolver.cpp index b3b005c57..93a7fa5a6 100644 --- a/palace/drivers/drivensolver.cpp +++ b/palace/drivers/drivensolver.cpp @@ -139,8 +139,7 @@ ErrorIndicator DrivenSolver::SweepUniform(SpaceOperator &spaceop, PostOperator & // Initialize structures for storing and reducing the results of error estimation. CurlFluxErrorEstimator estimator( spaceop.GetMaterialOp(), spaceop.GetNDSpaces(), iodata.solver.linear.estimator_tol, - iodata.solver.linear.estimator_max_it, iodata.problem.verbose, - iodata.solver.pa_order_threshold); + iodata.solver.linear.estimator_max_it, 0, iodata.solver.pa_order_threshold); ErrorIndicator indicator; // Main frequency sweep loop. @@ -252,8 +251,7 @@ ErrorIndicator DrivenSolver::SweepAdaptive(SpaceOperator &spaceop, PostOperator // Initialize structures for storing and reducing the results of error estimation. CurlFluxErrorEstimator estimator( spaceop.GetMaterialOp(), spaceop.GetNDSpaces(), iodata.solver.linear.estimator_tol, - iodata.solver.linear.estimator_max_it, iodata.problem.verbose, - iodata.solver.pa_order_threshold); + iodata.solver.linear.estimator_max_it, 0, iodata.solver.pa_order_threshold); ErrorIndicator indicator; // Configure the PROM operator which performs the parameter space sampling and basis diff --git a/palace/drivers/eigensolver.cpp b/palace/drivers/eigensolver.cpp index 9e0d3ef93..5de5b1ffb 100644 --- a/palace/drivers/eigensolver.cpp +++ b/palace/drivers/eigensolver.cpp @@ -260,8 +260,7 @@ EigenSolver::Solve(const std::vector> &mesh) cons // Calculate and record the error indicators. CurlFluxErrorEstimator estimator( spaceop.GetMaterialOp(), spaceop.GetNDSpaces(), iodata.solver.linear.estimator_tol, - iodata.solver.linear.estimator_max_it, iodata.problem.verbose, - iodata.solver.pa_order_threshold); + iodata.solver.linear.estimator_max_it, 0, iodata.solver.pa_order_threshold); ErrorIndicator indicator; for (int i = 0; i < iodata.solver.eigenmode.n; i++) { diff --git a/palace/drivers/electrostaticsolver.cpp b/palace/drivers/electrostaticsolver.cpp index 31a85a989..e47f2c105 100644 --- a/palace/drivers/electrostaticsolver.cpp +++ b/palace/drivers/electrostaticsolver.cpp @@ -86,7 +86,6 @@ ErrorIndicator ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, // charges from the prescribed voltage to get C directly as: // Q_i = ∫ ρ dV = ∫ ∇ ⋅ (ε E) dV = ∫ (ε E) ⋅ n dS // and C_ij = Q_i/V_j. The energy formulation avoids having to locally integrate E = -∇V. - // Additionally compute error estimates for each terminal. auto Grad = laplaceop.GetGradMatrix(); const std::map> &terminal_sources = laplaceop.GetSources(); int nstep = static_cast(terminal_sources.size()); @@ -98,10 +97,10 @@ ErrorIndicator ElectrostaticSolver::Postprocess(LaplaceOperator &laplaceop, } // Calculate and record the error indicators. - GradFluxErrorEstimator estimator( - laplaceop.GetMaterialOp(), laplaceop.GetH1Spaces(), - iodata.solver.linear.estimator_tol, iodata.solver.linear.estimator_max_it, - iodata.problem.verbose, iodata.solver.pa_order_threshold); + GradFluxErrorEstimator estimator(laplaceop.GetMaterialOp(), laplaceop.GetH1Spaces(), + iodata.solver.linear.estimator_tol, + iodata.solver.linear.estimator_max_it, 0, + iodata.solver.pa_order_threshold); ErrorIndicator indicator; for (int i = 0; i < nstep; i++) { diff --git a/palace/drivers/magnetostaticsolver.cpp b/palace/drivers/magnetostaticsolver.cpp index 5f4606509..bed9bc05e 100644 --- a/palace/drivers/magnetostaticsolver.cpp +++ b/palace/drivers/magnetostaticsolver.cpp @@ -102,8 +102,8 @@ ErrorIndicator MagnetostaticSolver::Postprocess(CurlCurlOperator &curlcurlop, // Calculate and record the error indicators. CurlFluxErrorEstimator estimator( curlcurlop.GetMaterialOp(), curlcurlop.GetNDSpaces(), - iodata.solver.linear.estimator_tol, iodata.solver.linear.estimator_max_it, - iodata.problem.verbose, iodata.solver.pa_order_threshold); + iodata.solver.linear.estimator_tol, iodata.solver.linear.estimator_max_it, 0, + iodata.solver.pa_order_threshold); ErrorIndicator indicator; for (int i = 0; i < nstep; i++) { diff --git a/palace/drivers/transientsolver.cpp b/palace/drivers/transientsolver.cpp index 13972388f..3c3a50a1a 100644 --- a/palace/drivers/transientsolver.cpp +++ b/palace/drivers/transientsolver.cpp @@ -80,8 +80,7 @@ TransientSolver::Solve(const std::vector> &mesh) // Initialize structures for storing and reducing the results of error estimation. CurlFluxErrorEstimator estimator( spaceop.GetMaterialOp(), spaceop.GetNDSpaces(), iodata.solver.linear.estimator_tol, - iodata.solver.linear.estimator_max_it, iodata.problem.verbose, - iodata.solver.pa_order_threshold); + iodata.solver.linear.estimator_max_it, 0, iodata.solver.pa_order_threshold); ErrorIndicator indicator; // Main time integration loop. diff --git a/palace/fem/errorindicator.cpp b/palace/fem/errorindicator.cpp index 40b234bfa..e4f9a1b52 100644 --- a/palace/fem/errorindicator.cpp +++ b/palace/fem/errorindicator.cpp @@ -46,7 +46,7 @@ void ErrorIndicator::AddIndicator(const ErrorIndicator &indicator) // More samples have been added, update for the running average. normalization = - (n * normalization + indicator.normalization * indicator.n) / (n + indicator.n); + (normalization * n + indicator.normalization * indicator.n) / (n + indicator.n); n += indicator.n; } diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index 826d2da65..bdbf32f3d 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -152,22 +152,16 @@ void FluxProjector::Mult(const VecType &x, VecType &y) const Flux->Mult(x_, rhs); if (vdim == 1) { - - // XX TODO WIP PRINT STATEMENTS... - Mpi::Print("Computing smooth flux projection for error estimation\n"); - + // Mpi::Print(" Computing smooth flux projection for error estimation\n"); ksp->Mult(rhs, y_); } else { for (int i = 0; i < vdim; i++) { - - // XX TODO WIP PRINT STATEMENTS... - Mpi::Print("Computing smooth flux projection of flux component {:d}/{:d} for error " - "estimation\n", - i + 1, vdim); - + // Mpi::Print(" Computing smooth flux projection of flux component {:d}/{:d} for " + // "error estimation\n", + // i + 1, vdim); const Vector rhsb(rhs, i * x_.Size(), x_.Size()); Vector yb(y_, i * x_.Size(), x_.Size()); ksp->Mult(rhsb, yb); @@ -220,7 +214,7 @@ ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const VecType // flux is μ⁻¹ ∇ × U. auto &nd_fespace = nd_fespaces.GetFinestFESpace(); auto &mesh = *nd_fespace.GetParMesh(); - Vector estimates(mesh.GetNE()), V_smooth, V_ip; + Vector estimates(mesh.GetNE()), V_ip(mesh.SpaceDimension()), V_smooth; double normalization = 0.0; for (int e = 0; e < mesh.GetNE(); e++) { @@ -228,6 +222,7 @@ ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const VecType mfem::ElementTransformation &T = *mesh.GetElementTransformation(e); const int q_order = 2 * fe.GetOrder() + T.OrderW(); const mfem::IntegrationRule &ir = mfem::IntRules.Get(T.GetGeometryType(), q_order); + double err = 0.0; for (int i = 0; i < ir.GetNPoints(); i++) { const mfem::IntegrationPoint &ip = ir.IntPoint(i); @@ -239,29 +234,29 @@ ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const VecType U_gf.real().GetCurl(T, V_smooth); mat_op.GetInvPermeability(T.Attribute).Mult(V_smooth, V_ip); F_gf.real().GetVectorValue(T, ip, V_smooth); - V_ip -= V_smooth; - estimates[e] = w * (V_ip * V_ip); - normalization += w * (V_smooth * V_smooth); + V_smooth -= V_ip; + err += w * (V_smooth * V_smooth); + normalization += w * (V_ip * V_ip); // Imaginary part U_gf.imag().GetCurl(T, V_smooth); mat_op.GetInvPermeability(T.Attribute).Mult(V_smooth, V_ip); F_gf.imag().GetVectorValue(T, ip, V_smooth); - V_ip -= V_smooth; - estimates[e] += w * (V_ip * V_ip); - normalization += w * (V_smooth * V_smooth); + V_smooth -= V_ip; + err += w * (V_smooth * V_smooth); + normalization += w * (V_ip * V_ip); } else { U_gf.GetCurl(T, V_smooth); mat_op.GetInvPermeability(T.Attribute).Mult(V_smooth, V_ip); F_gf.GetVectorValue(T, ip, V_smooth); - V_ip -= V_smooth; - estimates[e] = w * (V_ip * V_ip); - normalization += w * (V_smooth * V_smooth); + V_smooth -= V_ip; + err += w * (V_smooth * V_smooth); + normalization += w * (V_ip * V_ip); } } - estimates[e] = std::sqrt(estimates[e]); + estimates[e] = std::sqrt(err); } // Finalize the element-wise error estimates. @@ -299,8 +294,9 @@ ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &U) const // Loop over elements and accumulate the estimates from this component. The discontinuous // flux is ε ∇U. - auto &mesh = *h1d_fespace.GetParMesh(); - Vector estimates(mesh.GetNE()), V_smooth, V_ip; + auto &h1_fespace = h1_fespaces.GetFinestFESpace(); + auto &mesh = *h1_fespace.GetParMesh(); + Vector estimates(mesh.GetNE()), V_ip(mesh.SpaceDimension()), V_smooth; double normalization = 0.0; for (int e = 0; e < mesh.GetNE(); e++) { @@ -308,6 +304,7 @@ ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &U) const mfem::ElementTransformation &T = *mesh.GetElementTransformation(e); const int q_order = 2 * fe.GetOrder() + T.OrderW(); const mfem::IntegrationRule &ir = mfem::IntRules.Get(T.GetGeometryType(), q_order); + double err = 0.0; for (int i = 0; i < ir.GetNPoints(); i++) { // XX TODO: For now the flux is just ∇U since the coefficient support for @@ -320,11 +317,11 @@ ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &U) const // mat_op.GetPermittivityReal(T.Attribute).Mult(V_smooth, V_ip); V_ip = V_smooth; F_gf.GetVectorValue(T, ip, V_smooth); - V_ip -= V_smooth; - estimates[e] = w * (V_ip * V_ip); - normalization += w * (V_smooth * V_smooth); + V_smooth -= V_ip; + err += w * (V_smooth * V_smooth); + normalization += w * (V_ip * V_ip); } - estimates[e] = std::sqrt(estimates[e]); + estimates[e] = std::sqrt(err); } // Finalize the element-wise error estimates. diff --git a/palace/linalg/errorestimator.hpp b/palace/linalg/errorestimator.hpp index 2f63d990e..d38dec435 100644 --- a/palace/linalg/errorestimator.hpp +++ b/palace/linalg/errorestimator.hpp @@ -23,15 +23,14 @@ class MaterialOperator; // This solver computes a smooth reconstruction of a discontinuous flux. The difference // between this resulting smooth flux and the original non-smooth flux provides a -// localizable error estimate. An instance of FluxProjector can be reused across solutions, -// thus the construction of the operator is separated from the construction of the flux RHS. +// localizable error estimate. class FluxProjector { private: // Operator for the mass matrix inversion. std::unique_ptr Flux, M; - // Linear solver and preconditioner for the projected linear system M σ = σ̂. + // Linear solver and preconditioner for the projected linear system. std::unique_ptr ksp; // Workspace object for solver application. diff --git a/palace/utils/configfile.hpp b/palace/utils/configfile.hpp index 2b42fde30..a946a268c 100644 --- a/palace/utils/configfile.hpp +++ b/palace/utils/configfile.hpp @@ -820,7 +820,7 @@ struct LinearSolverData int divfree_max_it = 1000; // Relative tolerance for solving linear systems in the error estimator. - double estimator_tol = 1e-6; + double estimator_tol = 1.0e-6; // Maximum number of iterations for solving linear systems in the error estimator. int estimator_max_it = 100; diff --git a/test/runtests.jl b/test/runtests.jl index 41cb5f5ab..6304e1b15 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,7 +34,7 @@ abstol = 1.0e-16 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max", "Min"] + excluded_columns=["Maximum", "Minimum"] ) @info "Testing rings..." @@ -46,7 +46,7 @@ abstol = 1.0e-16 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max", "Min"] + excluded_columns=["Maximum", "Minimum"] ) @info "Testing cavity (PEC)..." @@ -58,7 +58,7 @@ abstol = 1.0e-16 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max", "Min"], + excluded_columns=["Maximum", "Minimum"], skip_rowcount=true ) @@ -71,7 +71,7 @@ abstol = 1.0e-16 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max", "Min"], + excluded_columns=["Maximum", "Minimum"], skip_rowcount=true ) @@ -88,7 +88,7 @@ abstol = 2.0e-12 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max", "Min"] + excluded_columns=["Maximum", "Minimum"] ) @info "Testing coaxial (matched)..." @@ -100,7 +100,7 @@ abstol = 2.0e-12 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max", "Min"] + excluded_columns=["Maximum", "Minimum"] ) @info "Testing CPW (lumped ports)" @@ -112,7 +112,7 @@ abstol = 2.0e-12 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max", "Min"] + excluded_columns=["Maximum", "Minimum"] ) @info "Testing CPW (wave ports)" @@ -124,7 +124,7 @@ abstol = 2.0e-12 np=numprocs, rtol=reltol, atol=abstol, - excluded_columns=["Max", "Min"] + excluded_columns=["Maximum", "Minimum"] ) # Don't check accuracy for adaptive frequency sweep simulations diff --git a/test/testcase.jl b/test/testcase.jl index 673209f65..ed97cea8f 100644 --- a/test/testcase.jl +++ b/test/testcase.jl @@ -69,11 +69,11 @@ function testcase( dataref = CSV.File(joinpath(refpostprodir, file); header=1) |> DataFrame data = CSV.File(joinpath(postprodir, file); header=1) |> DataFrame if !skip_rowcount - @test size(dataref, 1) <= size(data, 1) + @test nrow(data) == nrow(dataref) end - data = data[1:min(size(dataref, 1), size(data, 1)), :] + data = data[1:min(nrow(data), nrow(dataref)), :] - # Check the number of columns matches, before removing any excluded_columns + # Check the number of columns matches, before removing any excluded columns @test ncol(data) == ncol(dataref) for col ∈ excluded_columns select!(data, Not(Cols(contains(col)))) From 8be72f1e8d37e281cc55dbeae95591094aa0308e Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Tue, 10 Oct 2023 15:41:39 -0700 Subject: [PATCH 38/39] Remove normalization from ErrorIndicator class (not used, the stored indicators are already relative quantities) --- palace/drivers/basesolver.cpp | 15 ++++++--------- palace/fem/errorindicator.cpp | 3 --- palace/fem/errorindicator.hpp | 13 ++----------- palace/linalg/errorestimator.cpp | 4 ++-- 4 files changed, 10 insertions(+), 25 deletions(-) diff --git a/palace/drivers/basesolver.cpp b/palace/drivers/basesolver.cpp index e46337a3c..2b6719943 100644 --- a/palace/drivers/basesolver.cpp +++ b/palace/drivers/basesolver.cpp @@ -578,26 +578,23 @@ void BaseSolver::PostprocessErrorIndicator(const PostOperator &postop, return; } MPI_Comm comm = postop.GetComm(); - std::array data = {indicator.Norml2(comm), indicator.Min(comm), - indicator.Max(comm), indicator.Mean(comm), - indicator.Normalization()}; + std::array data = {indicator.Norml2(comm), indicator.Min(comm), + indicator.Max(comm), indicator.Mean(comm)}; if (root) { std::string path = post_dir + "error-indicators.csv"; auto output = OutputFile(path, false); // clang-format off - output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", + output.print("{:>{}s},{:>{}s},{:>{}s},{:>{}s}\n", "Norm", table.w, "Minimum", table.w, "Maximum", table.w, - "Mean", table.w, - "Normalization", table.w); - output.print("{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", + "Mean", table.w); + output.print("{:+{}.{}e},{:+{}.{}e},{:+{}.{}e},{:+{}.{}e}\n", data[0], table.w, table.p, data[1], table.w, table.p, data[2], table.w, table.p, - data[3], table.w, table.p, - data[4], table.w, table.p); + data[3], table.w, table.p); // clang-format on } } diff --git a/palace/fem/errorindicator.cpp b/palace/fem/errorindicator.cpp index e4f9a1b52..de6c293be 100644 --- a/palace/fem/errorindicator.cpp +++ b/palace/fem/errorindicator.cpp @@ -13,7 +13,6 @@ void ErrorIndicator::AddIndicator(const ErrorIndicator &indicator) if (n == 0) { local = indicator.local; - normalization = indicator.normalization; n = indicator.n; return; } @@ -45,8 +44,6 @@ void ErrorIndicator::AddIndicator(const ErrorIndicator &indicator) }); // More samples have been added, update for the running average. - normalization = - (normalization * n + indicator.normalization * indicator.n) / (n + indicator.n); n += indicator.n; } diff --git a/palace/fem/errorindicator.hpp b/palace/fem/errorindicator.hpp index 0487b4821..ae1213261 100644 --- a/palace/fem/errorindicator.hpp +++ b/palace/fem/errorindicator.hpp @@ -23,18 +23,12 @@ class ErrorIndicator // refinement and coarsening. Vector local; - // Normalization constant. - double normalization; - // Number of samples. int n; public: - ErrorIndicator(Vector &&local, double normalization) - : local(std::move(local)), normalization(normalization), n(1) - { - } - ErrorIndicator() : normalization(0.0), n(0) {} + ErrorIndicator(Vector &&local) : local(std::move(local)), n(1) {} + ErrorIndicator() : n(0) {} // Add an indicator to the running total. void AddIndicator(const ErrorIndicator &indicator); @@ -68,9 +62,6 @@ class ErrorIndicator Mpi::GlobalSum(1, &sum, comm); return sum / linalg::GlobalSize(comm, local); } - - // Return the normalization constant for the absolute error. - auto Normalization() const { return normalization; } }; } // namespace palace diff --git a/palace/linalg/errorestimator.cpp b/palace/linalg/errorestimator.cpp index bdbf32f3d..3f5e07929 100644 --- a/palace/linalg/errorestimator.cpp +++ b/palace/linalg/errorestimator.cpp @@ -266,7 +266,7 @@ ErrorIndicator CurlFluxErrorEstimator::ComputeIndicators(const VecType { estimates *= 1.0 / normalization; } - return ErrorIndicator(std::move(estimates), normalization); + return ErrorIndicator(std::move(estimates)); } GradFluxErrorEstimator::GradFluxErrorEstimator( @@ -331,7 +331,7 @@ ErrorIndicator GradFluxErrorEstimator::ComputeIndicators(const Vector &U) const { estimates *= 1.0 / normalization; } - return ErrorIndicator(std::move(estimates), normalization); + return ErrorIndicator(std::move(estimates)); } template void FluxProjector::Mult(const Vector &, Vector &) const; From 467a3aaf344b5a506a7dc4b93ed5294d04451d94 Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Tue, 10 Oct 2023 15:42:06 -0700 Subject: [PATCH 39/39] Rebaseline tests for changed column headings and fixed integration rule for error estimation --- test/ref/cavity/impedance/error-indicators.csv | 4 ++-- test/ref/cavity/pec/error-indicators.csv | 4 ++-- test/ref/coaxial/matched/error-indicators.csv | 4 ++-- test/ref/coaxial/open/error-indicators.csv | 4 ++-- test/ref/cpw/lumped_adaptive/error-indicators.csv | 4 ++-- test/ref/cpw/lumped_uniform/error-indicators.csv | 4 ++-- test/ref/cpw/wave_adaptive/error-indicators.csv | 4 ++-- test/ref/cpw/wave_uniform/error-indicators.csv | 4 ++-- test/ref/rings/error-indicators.csv | 4 ++-- test/ref/spheres/error-indicators.csv | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/test/ref/cavity/impedance/error-indicators.csv b/test/ref/cavity/impedance/error-indicators.csv index fb6548b0f..0e954bd0a 100644 --- a/test/ref/cavity/impedance/error-indicators.csv +++ b/test/ref/cavity/impedance/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +1.832232364e-02, +4.343456836e-04, +1.399358786e-03, +7.453986746e-04, +8.242436674e-01 + Norm, Minimum, Maximum, Mean + +1.832232122e-02, +4.343407680e-04, +1.399360704e-03, +7.453991764e-04 diff --git a/test/ref/cavity/pec/error-indicators.csv b/test/ref/cavity/pec/error-indicators.csv index 3e8f88db3..0366ce7b2 100644 --- a/test/ref/cavity/pec/error-indicators.csv +++ b/test/ref/cavity/pec/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +1.832231602e-02, +4.343391518e-04, +1.399358147e-03, +7.453990858e-04, +8.242435486e-01 + Norm, Minimum, Maximum, Mean + +1.832232013e-02, +4.343399227e-04, +1.399359231e-03, +7.453992522e-04 diff --git a/test/ref/coaxial/matched/error-indicators.csv b/test/ref/coaxial/matched/error-indicators.csv index 9cce1c0b0..07d9f1c32 100644 --- a/test/ref/coaxial/matched/error-indicators.csv +++ b/test/ref/coaxial/matched/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +2.096263303e-01, +1.115162750e-02, +2.559512195e-02, +1.801842180e-02, +1.099244137e+00 + Norm, Minimum, Maximum, Mean + +2.096263437e-01, +1.115162678e-02, +2.559512078e-02, +1.801842294e-02 diff --git a/test/ref/coaxial/open/error-indicators.csv b/test/ref/coaxial/open/error-indicators.csv index 8fc67cb44..79d93928e 100644 --- a/test/ref/coaxial/open/error-indicators.csv +++ b/test/ref/coaxial/open/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +1.022347200e-02, +2.024010727e-04, +2.416110589e-03, +5.982341447e-04, +1.742421290e+00 + Norm, Minimum, Maximum, Mean + +1.022362887e-02, +2.024179316e-04, +2.416129470e-03, +5.982596107e-04 diff --git a/test/ref/cpw/lumped_adaptive/error-indicators.csv b/test/ref/cpw/lumped_adaptive/error-indicators.csv index 1ac73b6ee..b017fdae0 100644 --- a/test/ref/cpw/lumped_adaptive/error-indicators.csv +++ b/test/ref/cpw/lumped_adaptive/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +7.058727272e-01, +1.275543633e-06, +1.731621567e-02, +8.748160640e-04, +2.521900498e+00 + Norm, Minimum, Maximum, Mean + +7.054779357e-01, +1.246930995e-06, +1.708706513e-02, +8.753657319e-04 diff --git a/test/ref/cpw/lumped_uniform/error-indicators.csv b/test/ref/cpw/lumped_uniform/error-indicators.csv index fb63b8082..95258be5a 100644 --- a/test/ref/cpw/lumped_uniform/error-indicators.csv +++ b/test/ref/cpw/lumped_uniform/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +7.056448946e-01, +1.148429809e-06, +1.689680158e-02, +8.751153409e-04, +2.287774332e+00 + Norm, Minimum, Maximum, Mean + +7.056448955e-01, +1.149476742e-06, +1.689680155e-02, +8.751155866e-04 diff --git a/test/ref/cpw/wave_adaptive/error-indicators.csv b/test/ref/cpw/wave_adaptive/error-indicators.csv index 213baba81..7caa2db96 100644 --- a/test/ref/cpw/wave_adaptive/error-indicators.csv +++ b/test/ref/cpw/wave_adaptive/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +6.809041632e-01, +6.042001464e-06, +1.659737114e-02, +1.093675108e-03, +3.034916068e+00 + Norm, Minimum, Maximum, Mean + +6.792700565e-01, +6.244398669e-06, +1.613828550e-02, +1.095287561e-03 diff --git a/test/ref/cpw/wave_uniform/error-indicators.csv b/test/ref/cpw/wave_uniform/error-indicators.csv index 27f9dafff..1c8d39bb2 100644 --- a/test/ref/cpw/wave_uniform/error-indicators.csv +++ b/test/ref/cpw/wave_uniform/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +6.978759848e-01, +4.868944903e-06, +1.714751896e-02, +9.637101290e-04, +2.154592309e+00 + Norm, Minimum, Maximum, Mean + +6.978759748e-01, +4.868999354e-06, +1.714751988e-02, +9.637098727e-04 diff --git a/test/ref/rings/error-indicators.csv b/test/ref/rings/error-indicators.csv index 006fd9db3..042669bb0 100644 --- a/test/ref/rings/error-indicators.csv +++ b/test/ref/rings/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +3.178909004e-01, +7.241787209e-08, +2.108657479e-02, +5.448306933e-04, +2.836557570e-01 + Norm, Minimum, Maximum, Mean + +3.178909044e-01, +7.241707644e-08, +2.108657480e-02, +5.448306955e-04 diff --git a/test/ref/spheres/error-indicators.csv b/test/ref/spheres/error-indicators.csv index 8f56cd272..25bc44ad2 100644 --- a/test/ref/spheres/error-indicators.csv +++ b/test/ref/spheres/error-indicators.csv @@ -1,2 +1,2 @@ - Sum, Min, Max, Mean, Normalization - +7.683970730e-03, +1.578064762e-06, +5.153233092e-04, +4.815651013e-05, +3.686180355e-01 + Norm, Minimum, Maximum, Mean + +7.683933839e-03, +1.569617836e-06, +5.151996544e-04, +4.815617253e-05