Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable p-refinement for ScalarWave executables #5573

Merged
merged 2 commits into from
Oct 23, 2023
Merged

Enable p-refinement for ScalarWave executables #5573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f3b41ad
Use NeighborInfo to initialize MortarMesh after AMR
kidder Oct 18, 2023
fd25de7
Enable AMR for ScalarWave
kidder Oct 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions src/Evolution/DiscontinuousGalerkin/Initialization/Mortars.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
#include "DataStructures/DataBox/DataBoxTag.hpp"
#include "DataStructures/DataBox/Prefixes.hpp"
#include "DataStructures/Variables.hpp"
#include "Domain/Amr/Info.hpp"
#include "Domain/Amr/Tags/NeighborFlags.hpp"
#include "Domain/Creators/Tags/InitialExtents.hpp"
#include "Domain/Structure/Direction.hpp"
#include "Domain/Structure/Element.hpp"
Expand All @@ -32,6 +34,7 @@
#include "ParallelAlgorithms/Amr/Protocols/Projector.hpp"
#include "ParallelAlgorithms/Initialization/MutateAssign.hpp"
#include "Time/TimeStepId.hpp"
#include "Utilities/Gsl.hpp"
#include "Utilities/TMPL.hpp"

/// \cond
Expand Down Expand Up @@ -183,7 +186,8 @@ struct ProjectMortars : tt::ConformsTo<amr::protocols::Projector> {
evolution::dg::Tags::NormalCovectorAndMagnitude<dim>,
Tags::MortarDataHistory<dim, typename dt_variables_tag::type>>;
using argument_tags =
tmpl::list<domain::Tags::Mesh<dim>, domain::Tags::Element<dim>>;
tmpl::list<domain::Tags::Mesh<dim>, domain::Tags::Element<dim>,
amr::Tags::NeighborInfo<dim>>;

static void apply(
const gsl::not_null<::dg::MortarMap<dim, evolution::dg::MortarData<dim>>*>
Expand All @@ -201,7 +205,8 @@ struct ProjectMortars : tt::ConformsTo<amr::protocols::Projector> {
normal_covector_and_magnitude,
const gsl::not_null<mortar_data_history_type*>
/*mortar_data_history*/,
const Mesh<dim>& /*new_mesh*/, const Element<dim>& new_element,
const Mesh<dim>& new_mesh, const Element<dim>& new_element,
const std::unordered_map<ElementId<dim>, amr::Info<dim>>& neighbor_info,
const std::pair<Mesh<dim>, Element<dim>>& /*old_mesh_and_element*/) {
static_assert(not Metavariables::local_time_stepping,
"AMR with local time-stepping is not yet supported");
Expand All @@ -216,7 +221,13 @@ struct ProjectMortars : tt::ConformsTo<amr::protocols::Projector> {
for (const auto& neighbor : neighbors) {
const auto mortar_id = std::make_pair(direction, neighbor);
mortar_data->emplace(mortar_id, MortarData<dim>{1});
nilsvu marked this conversation as resolved.
Show resolved Hide resolved
mortar_mesh->emplace(mortar_id, Mesh<dim - 1>{});
const auto new_neighbor_mesh = neighbors.orientation().inverse_map()(
neighbor_info.at(neighbor).new_mesh);
mortar_mesh->emplace(
mortar_id,
::dg::mortar_mesh(
new_mesh.slice_away(direction.dimension()),
new_neighbor_mesh.slice_away(direction.dimension())));
mortar_size->emplace(
mortar_id,
::dg::mortar_size(new_element.id(), neighbor, direction.dimension(),
Expand Down Expand Up @@ -246,6 +257,8 @@ struct ProjectMortars : tt::ConformsTo<amr::protocols::Projector> {
const gsl::not_null<mortar_data_history_type*>
/*mortar_data_history*/,
const Mesh<dim>& /*new_mesh*/, const Element<dim>& /*new_element*/,
const std::unordered_map<ElementId<dim>,
amr::Info<dim>>& /*neighbor_info*/,
const tuples::TaggedTuple<Tags...>& /*parent_items*/) {
ERROR("h-refinement not implemented yet");
}
Expand All @@ -268,6 +281,8 @@ struct ProjectMortars : tt::ConformsTo<amr::protocols::Projector> {
const gsl::not_null<mortar_data_history_type*>
/*mortar_data_history*/,
const Mesh<dim>& /*new_mesh*/, const Element<dim>& /*new_element*/,
const std::unordered_map<ElementId<dim>,
amr::Info<dim>>& /*neighbor_info*/,
const std::unordered_map<ElementId<dim>, tuples::TaggedTuple<Tags...>>&
/*children_items*/) {
ERROR("h-refinement not implemented yet");
Expand Down
3 changes: 3 additions & 0 deletions src/Evolution/Executables/ScalarWave/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@

set(LIBS_TO_LINK
Charmxx::main
Amr
AmrCriteria
AmrProjectors
CoordinateMaps
DiscontinuousGalerkin
DomainCreators
Expand Down
5 changes: 4 additions & 1 deletion src/Evolution/Executables/ScalarWave/EvolveScalarWave.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "Domain/FunctionsOfTime/RegisterDerivedWithCharm.hpp"
#include "Evolution/Systems/ScalarWave/BoundaryCorrections/RegisterDerived.hpp"
#include "Parallel/CharmMain.tpp"
#include "ParallelAlgorithms/Amr/Actions/RegisterCallbacks.hpp"
#include "Utilities/Serialization/RegisterDerivedClassesWithCharm.hpp"

// Chosen in CMakeLists.txt
Expand All @@ -22,6 +23,8 @@ extern "C" void CkRegisterMainModule() {
&domain::creators::time_dependence::register_derived_with_charm,
&domain::FunctionsOfTime::register_derived_with_charm,
&ScalarWave::BoundaryCorrections::register_derived_with_charm,
&register_factory_classes_with_charm<metavariables>},
&register_factory_classes_with_charm<metavariables>,
&amr::register_callbacks<metavariables,
typename metavariables::dg_element_array>},
{});
}
93 changes: 74 additions & 19 deletions src/Evolution/Executables/ScalarWave/EvolveScalarWave.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
#include "Evolution/ComputeTags.hpp"
#include "Evolution/DiscontinuousGalerkin/Actions/ApplyBoundaryCorrections.hpp"
#include "Evolution/DiscontinuousGalerkin/Actions/ComputeTimeDerivative.hpp"
#include "Evolution/DiscontinuousGalerkin/DgElementArray.hpp" // IWYU pragma: keep
#include "Evolution/DiscontinuousGalerkin/DgElementArray.hpp"
#include "Evolution/DiscontinuousGalerkin/Initialization/Mortars.hpp"
#include "Evolution/DiscontinuousGalerkin/Initialization/QuadratureTag.hpp"
#include "Evolution/EventsAndDenseTriggers/DenseTrigger.hpp"
Expand All @@ -35,12 +35,12 @@
#include "Evolution/Systems/ScalarWave/System.hpp"
#include "Evolution/Tags/Filter.hpp"
#include "IO/Observer/Actions/RegisterEvents.hpp"
#include "IO/Observer/Helpers.hpp" // IWYU pragma: keep
#include "IO/Observer/ObserverComponent.hpp" // IWYU pragma: keep
#include "IO/Observer/Helpers.hpp"
#include "IO/Observer/ObserverComponent.hpp"
#include "NumericalAlgorithms/DiscontinuousGalerkin/Formulation.hpp"
#include "NumericalAlgorithms/DiscontinuousGalerkin/Tags.hpp"
#include "NumericalAlgorithms/LinearOperators/ExponentialFilter.hpp"
#include "NumericalAlgorithms/LinearOperators/FilterAction.hpp" // IWYU pragma: keep
#include "NumericalAlgorithms/LinearOperators/FilterAction.hpp"
#include "Options/Protocols/FactoryCreation.hpp"
#include "Options/String.hpp"
#include "Parallel/Local.hpp"
Expand All @@ -56,26 +56,40 @@
#include "ParallelAlgorithms/Actions/InitializeItems.hpp"
#include "ParallelAlgorithms/Actions/MutateApply.hpp"
#include "ParallelAlgorithms/Actions/TerminatePhase.hpp"
#include "ParallelAlgorithms/Amr/Actions/CollectDataFromChildren.hpp"
#include "ParallelAlgorithms/Amr/Actions/Component.hpp"
#include "ParallelAlgorithms/Amr/Actions/CreateChild.hpp"
#include "ParallelAlgorithms/Amr/Actions/Initialize.hpp"
#include "ParallelAlgorithms/Amr/Actions/SendAmrDiagnostics.hpp"
#include "ParallelAlgorithms/Amr/Criteria/Criterion.hpp"
#include "ParallelAlgorithms/Amr/Criteria/DriveToTarget.hpp"
#include "ParallelAlgorithms/Amr/Criteria/Random.hpp"
#include "ParallelAlgorithms/Amr/Criteria/Tags/Criteria.hpp"
#include "ParallelAlgorithms/Amr/Criteria/TruncationError.hpp"
#include "ParallelAlgorithms/Amr/Projectors/DefaultInitialize.hpp"
#include "ParallelAlgorithms/Amr/Projectors/Tensors.hpp"
#include "ParallelAlgorithms/Amr/Projectors/Variables.hpp"
#include "ParallelAlgorithms/Amr/Protocols/AmrMetavariables.hpp"
#include "ParallelAlgorithms/Events/Factory.hpp" // IWYU pragma: keep
#include "ParallelAlgorithms/Events/Tags.hpp"
#include "ParallelAlgorithms/EventsAndTriggers/Completion.hpp"
#include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
#include "ParallelAlgorithms/EventsAndTriggers/EventsAndTriggers.hpp" // IWYU pragma: keep
#include "ParallelAlgorithms/EventsAndTriggers/EventsAndTriggers.hpp"
#include "ParallelAlgorithms/EventsAndTriggers/LogicalTriggers.hpp"
#include "ParallelAlgorithms/EventsAndTriggers/Trigger.hpp"
#include "PointwiseFunctions/AnalyticData/AnalyticData.hpp"
#include "PointwiseFunctions/AnalyticSolutions/AnalyticSolution.hpp"
#include "PointwiseFunctions/AnalyticSolutions/Tags.hpp"
#include "PointwiseFunctions/AnalyticSolutions/WaveEquation/Factory.hpp"
#include "PointwiseFunctions/AnalyticSolutions/WaveEquation/PlaneWave.hpp" // IWYU pragma: keep
#include "PointwiseFunctions/AnalyticSolutions/WaveEquation/RegularSphericalWave.hpp" // IWYU pragma: keep
#include "PointwiseFunctions/AnalyticSolutions/WaveEquation/PlaneWave.hpp"
#include "PointwiseFunctions/AnalyticSolutions/WaveEquation/RegularSphericalWave.hpp"
#include "PointwiseFunctions/MathFunctions/Factory.hpp"
#include "PointwiseFunctions/MathFunctions/MathFunction.hpp"
#include "Time/Actions/AdvanceTime.hpp" // IWYU pragma: keep
#include "Time/Actions/ChangeSlabSize.hpp" // IWYU pragma: keep
#include "Time/Actions/RecordTimeStepperData.hpp" // IWYU pragma: keep
#include "Time/Actions/SelfStartActions.hpp" // IWYU pragma: keep
#include "Time/Actions/UpdateU.hpp" // IWYU pragma: keep
#include "Time/Actions/AdvanceTime.hpp"
#include "Time/Actions/ChangeSlabSize.hpp"
#include "Time/Actions/RecordTimeStepperData.hpp"
#include "Time/Actions/SelfStartActions.hpp"
#include "Time/Actions/UpdateU.hpp"
#include "Time/StepChoosers/ByBlock.hpp"
#include "Time/StepChoosers/Factory.hpp"
#include "Time/StepChoosers/StepChooser.hpp"
Expand Down Expand Up @@ -115,7 +129,7 @@ struct EvolutionMetavars {
static constexpr dg::Formulation dg_formulation =
dg::Formulation::StrongInertial;
using temporal_id = Tags::TimeStepId;
static constexpr bool local_time_stepping = true;
static constexpr bool local_time_stepping = false;

using analytic_solution_fields = typename system::variables_tag::tags_list;
using deriv_compute = ::Tags::DerivCompute<
Expand Down Expand Up @@ -152,6 +166,11 @@ struct EvolutionMetavars {
struct factory_creation
: tt::ConformsTo<Options::protocols::FactoryCreation> {
using factory_classes = tmpl::map<
tmpl::pair<amr::Criterion,
tmpl::list<amr::Criteria::DriveToTarget<volume_dim>,
amr::Criteria::TruncationError<
volume_dim,
typename system::variables_tag::tags_list>>>,
tmpl::pair<DenseTrigger, DenseTriggers::standard_dense_triggers>,
tmpl::pair<DomainCreator<volume_dim>, domain_creators<volume_dim>>,
tmpl::pair<Event,
Expand Down Expand Up @@ -197,6 +216,32 @@ struct EvolutionMetavars {
// wave system, the user should determine whether this filter can be removed.
static constexpr bool use_filtering = (2 == volume_dim);

struct amr : tt::ConformsTo<::amr::protocols::AmrMetavariables> {
using projectors = tmpl::list<
Initialization::ProjectTimeStepping<volume_dim>,
evolution::dg::Initialization::ProjectDomain<volume_dim>,
Initialization::ProjectTimeStepperHistory<EvolutionMetavars>,
::amr::projectors::ProjectVariables<volume_dim,
typename system::variables_tag>,
::amr::projectors::ProjectTensors<volume_dim,
::ScalarWave::Tags::ConstraintGamma2>,
evolution::dg::Initialization::ProjectMortars<EvolutionMetavars>,
evolution::Actions::ProjectRunEventsAndDenseTriggers,
::amr::projectors::DefaultInitialize<
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you move some/all of these to the corresponding projectors? Would be easier if these are handled by the code that adds the tags, so you don't have to worry about them in the metavars.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can, but will defer doing so until I implement the h-refinement overloads for the projectors as what is default initialized may change...

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure I understand, the way these are handled might change with adding h-refinement, but moving the responsibility to the projectors (which live next to the initializer that adds the tag) already now seems easier than doing so later.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is psychological; replacing 5 lines of code with multiple commits adding dozens of lines now seems a waste of my time. On the other hand, I can easily make the changes across multiple PRs when implementing h-refinement.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok!

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have to add these to every executable though when we add support for AMR. If you prefer that fine with me, just checking.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Merging now, this can be done ina follow-up as well when adding AMR to more executables

Initialization::Tags::InitialTimeDelta,
Initialization::Tags::InitialSlabSize<local_time_stepping>,
::domain::Tags::InitialExtents<volume_dim>,
::domain::Tags::InitialRefinementLevels<volume_dim>,
evolution::dg::Tags::Quadrature,
evolution::dg::Tags::NeighborMesh<volume_dim>,
Tags::StepperError<typename system::variables_tag>,
Tags::PreviousStepperError<typename system::variables_tag>,
Tags::StepperErrorUpdated,
SelfStart::Tags::InitialValue<typename system::variables_tag>,
SelfStart::Tags::InitialValue<Tags::TimeStep>,
SelfStart::Tags::InitialValue<Tags::Next<Tags::TimeStep>>>>;
};

using step_actions = tmpl::flatten<tmpl::list<
evolution::dg::Actions::ComputeTimeDerivative<
volume_dim, system, AllStepChoosers, local_time_stepping>,
Expand All @@ -222,7 +267,8 @@ struct EvolutionMetavars {
tmpl::list<>>>>;

using const_global_cache_tags =
tmpl::list<evolution::initial_data::Tags::InitialData>;
tmpl::list<evolution::initial_data::Tags::InitialData,
::amr::Criteria::Tags::Criteria>;

using dg_registration_list =
tmpl::list<observers::Actions::RegisterEventsWithObservers>;
Expand All @@ -231,6 +277,7 @@ struct EvolutionMetavars {
Initialization::Actions::InitializeItems<
Initialization::TimeStepping<EvolutionMetavars, local_time_stepping>,
evolution::dg::Initialization::Domain<volume_dim>,
::amr::Initialization::Initialize<volume_dim>,
Initialization::TimeStepperHistory<EvolutionMetavars>>,
Initialization::Actions::NonconservativeSystem<system>,
evolution::Initialization::Actions::SetVariables<
Expand All @@ -257,6 +304,10 @@ struct EvolutionMetavars {
tmpl::list<dg_registration_list,
Parallel::Actions::TerminatePhase>>,

Parallel::PhaseActions<Parallel::Phase::CheckDomain,
tmpl::list<::amr::Actions::SendAmrDiagnostics,
Parallel::Actions::TerminatePhase>>,

Parallel::PhaseActions<
Parallel::Phase::Evolve,
tmpl::list<evolution::Actions::RunEventsAndTriggers,
Expand All @@ -271,18 +322,22 @@ struct EvolutionMetavars {
};

using component_list =
tmpl::list<observers::Observer<EvolutionMetavars>,
tmpl::list<::amr::Component<EvolutionMetavars>,
observers::Observer<EvolutionMetavars>,
observers::ObserverWriter<EvolutionMetavars>,
dg_element_array>;

static constexpr Options::String help{
"Evolve a Scalar Wave in Dim spatial dimension.\n\n"
"The numerical flux is: UpwindFlux\n"};

static constexpr std::array<Parallel::Phase, 5> default_phase_order{
{Parallel::Phase::Initialization,
Parallel::Phase::InitializeTimeStepperHistory, Parallel::Phase::Register,
Parallel::Phase::Evolve, Parallel::Phase::Exit}};
static constexpr auto default_phase_order =
std::array{Parallel::Phase::Initialization,
Parallel::Phase::InitializeTimeStepperHistory,
Parallel::Phase::Register,
Parallel::Phase::CheckDomain,
Parallel::Phase::Evolve,
Parallel::Phase::Exit};

// NOLINTNEXTLINE(google-runtime-references)
void pup(PUP::er& /*p*/) {}
Expand Down
5 changes: 4 additions & 1 deletion src/Parallel/PhaseControl/Factory.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@

namespace PhaseControl {
using factory_creatable_classes =
tmpl::list<VisitAndReturn<Parallel::Phase::LoadBalancing>,
tmpl::list<VisitAndReturn<Parallel::Phase::EvaluateAmrCriteria>,
VisitAndReturn<Parallel::Phase::AdjustDomain>,
VisitAndReturn<Parallel::Phase::CheckDomain>,
VisitAndReturn<Parallel::Phase::LoadBalancing>,
VisitAndReturn<Parallel::Phase::WriteCheckpoint>,
CheckpointAndExitAfterWallclock>;
}
29 changes: 18 additions & 11 deletions tests/InputFiles/ScalarWave/PlaneWave1D.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ Testing:

ResourceInfo:
AvoidGlobalProc0: false
Singletons: Auto

InitialData: &InitialData
PlaneWave:
Expand All @@ -21,25 +22,31 @@ InitialData: &InitialData
Wavenumber: 1.0
Phase: 0.0

Amr:
Criteria:
- TruncationError:
VariablesToMonitor: [Psi]
AbsoluteTarget: 1.e-6
RelativeTarget: 1.0
Verbosity: Verbose

PhaseChangeAndTriggers:
- Trigger:
Slabs:
EvenlySpaced:
Interval: 1
Offset: 0
PhaseChanges:
- VisitAndReturn(EvaluateAmrCriteria)
- VisitAndReturn(AdjustDomain)
- VisitAndReturn(CheckDomain)

Evolution:
InitialTime: 0.0
InitialTimeStep: 0.001
InitialSlabSize: 0.1
TimeStepper:
AdamsBashforth:
Order: 3
StepChoosers:
- PreventRapidIncrease
- Increase:
Factor: 2
- ErrorControl:
AbsoluteTolerance: 1.0e-5
RelativeTolerance: 1.0e-5
MaxFactor: 10000.0
MinFactor: 0.0
SafetyFactor: 0.9

DomainCreator:
RotatedIntervals:
Expand Down
21 changes: 18 additions & 3 deletions tests/InputFiles/ScalarWave/PlaneWave1DEventsAndTriggersExample.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ ExpectedOutput:

ResourceInfo:
AvoidGlobalProc0: false
Singletons: Auto

InitialData:
PlaneWave:
Expand All @@ -23,17 +24,31 @@ InitialData:
Wavenumber: 1.0
Phase: 0.0

Amr:
Criteria:
- TruncationError:
VariablesToMonitor: [Psi]
AbsoluteTarget: 1.e-6
RelativeTarget: 1.0
Verbosity: Verbose

PhaseChangeAndTriggers:
- Trigger:
Slabs:
EvenlySpaced:
Interval: 1
Offset: 0
PhaseChanges:
- VisitAndReturn(EvaluateAmrCriteria)
- VisitAndReturn(AdjustDomain)
- VisitAndReturn(CheckDomain)

Evolution:
InitialTime: 0.0
InitialTimeStep: 0.001
InitialSlabSize: 0.01
TimeStepper:
AdamsBashforth:
Order: 3
StepChoosers:
- Constant: 0.001

DomainCreator:
Interval:
Expand Down
Loading