From 62a732650b6db3c94684dde659356304c29bb0cc Mon Sep 17 00:00:00 2001 From: Kyle Nelli Date: Mon, 2 Oct 2023 16:41:57 -0700 Subject: [PATCH] Reinterpolate to all elements when interpolator receives new points This is believed to be what was causing a lot of deadlocks in BBH simulations. --- .../Callbacks/FindApparentHorizon.hpp | 4 +- .../Actions/InterpolatorReceivePoints.hpp | 56 ++- .../Actions/SendPointsToInterpolator.hpp | 11 +- .../Interpolation/InterpolatedVars.hpp | 4 + .../InterpolationTargetTestHelpers.hpp | 29 +- .../Test_InterpolatorReceivePoints.cpp | 457 +++++++++++++++--- 6 files changed, 468 insertions(+), 93 deletions(-) diff --git a/src/ParallelAlgorithms/ApparentHorizonFinder/Callbacks/FindApparentHorizon.hpp b/src/ParallelAlgorithms/ApparentHorizonFinder/Callbacks/FindApparentHorizon.hpp index 501c6f1482af..9064027eeae2 100644 --- a/src/ParallelAlgorithms/ApparentHorizonFinder/Callbacks/FindApparentHorizon.hpp +++ b/src/ParallelAlgorithms/ApparentHorizonFinder/Callbacks/FindApparentHorizon.hpp @@ -267,9 +267,11 @@ struct FindApparentHorizon auto& interpolation_target = Parallel::get_parallel_component< intrp::InterpolationTarget>( *cache); + // The iteration of these new coords is the fast flow iteration + 1 + // because the zeroth iteration was the initial guess Parallel::simple_action< Actions::SendPointsToInterpolator>( - interpolation_target, temporal_ids.front()); + interpolation_target, temporal_ids.front(), info.iteration + 1); // We return false because we don't want this iteration to clean // up the volume data, since we are using it for the next iteration // (i.e. the simple_action that we just called). diff --git a/src/ParallelAlgorithms/Interpolation/Actions/InterpolatorReceivePoints.hpp b/src/ParallelAlgorithms/Interpolation/Actions/InterpolatorReceivePoints.hpp index 38d136b6bbbc..9347c1764284 100644 --- a/src/ParallelAlgorithms/Interpolation/Actions/InterpolatorReceivePoints.hpp +++ b/src/ParallelAlgorithms/Interpolation/Actions/InterpolatorReceivePoints.hpp @@ -12,6 +12,7 @@ #include "DataStructures/Tensor/TypeAliases.hpp" #include "ParallelAlgorithms/Interpolation/Actions/TryToInterpolate.hpp" #include "ParallelAlgorithms/Interpolation/InterpolatedVars.hpp" +#include "Utilities/ErrorHandling/Error.hpp" #include "Utilities/Gsl.hpp" #include "Utilities/Requires.hpp" #include "Utilities/TaggedTuple.hpp" @@ -45,6 +46,17 @@ namespace Actions { /// After receiving the points, interpolates volume data onto them /// if it already has all the volume data. /// +/// The `iteration` parameter is used to order receives of +/// `block_logical_coords`. Because of the asynchronous nature of communication, +/// it is possible that a more recent set of points arrives before an older set. +/// It is assumed that if a more recent set arrives, then the old set is no +/// longer needed. This `iteration` parameter tags each communication as "more +/// recent" or "older" so if we receive an older set of points after a more +/// recent set, we don't overwrite the more recent set. +/// +/// \note If the interpolator receives points with the same iteration, an ERROR +/// will occur. +/// /// Uses: /// - Databox: /// - `Tags::NumberOfElements` @@ -69,9 +81,10 @@ struct ReceivePoints { std::vector>>>&& - block_logical_coords) { + block_logical_coords, + const size_t iteration = 0_st) { db::mutate>( - [&temporal_id, &block_logical_coords]( + [&temporal_id, &block_logical_coords, &iteration]( const gsl::not_null::type*> vars_holders) { @@ -80,12 +93,39 @@ struct ReceivePoints { Metavariables>>(*vars_holders) .infos; - // Add the target interpolation points at this temporal_id. - vars_infos.emplace(std::make_pair( - temporal_id, - intrp::Vars::Info{ - std::move(block_logical_coords)})); + // Add the new target interpolation points at this temporal_id. There + // are two conditions that allow us to overwrite the current target + // points. Either + // + // 1. There are no current target points at the temporal_id, OR + // 2. There are target points already at this temporal_id, but the + // iteration of the new target points is greater than the + // iteration of the current target points. + // + // If we already have target points and the iteration of the new + // points is less than or equal to the iteration of the current target + // points, then we ignore the new points. The new points are outdated + // and we definitely didn't have any of the new target points in our + // element by the fact that we have already received the next + // iteration of points. + // + // Whenever we overwrite the target points, we also empty the + // `interpolation_is_done_for_these_elements` (by virtue of a default + // constructed `intrp::Vars::Info`) so that we always check every + // element for this new set of target points. + if (vars_infos.count(temporal_id) == 0 or + vars_infos.at(temporal_id).iteration < iteration) { + vars_infos.insert_or_assign( + temporal_id, + intrp::Vars::Info{ + std::move(block_logical_coords), iteration}); + } else if (vars_infos.at(temporal_id).iteration == iteration) { + ERROR( + "Interpolator received target points at iteration " + << iteration + << " twice. Only one set of points per iteration is allowed."); + } }, make_not_null(&box)); diff --git a/src/ParallelAlgorithms/Interpolation/Actions/SendPointsToInterpolator.hpp b/src/ParallelAlgorithms/Interpolation/Actions/SendPointsToInterpolator.hpp index 806b47e6e2ac..abc6f856471d 100644 --- a/src/ParallelAlgorithms/Interpolation/Actions/SendPointsToInterpolator.hpp +++ b/src/ParallelAlgorithms/Interpolation/Actions/SendPointsToInterpolator.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include "DataStructures/DataBox/DataBox.hpp" @@ -18,6 +19,11 @@ namespace Actions { /// \brief Sets up points on an `InterpolationTarget` at a new `temporal_id` /// and sends these points to an `Interpolator`. /// +/// The `iteration` parameter tags each set of points so the `Interpolator` +/// knows which are newer points and which are older points. +/// +/// \see `intrp::Actions::ReceivePoints` +/// /// Uses: /// - DataBox: /// - `domain::Tags::Domain<3>` @@ -38,7 +44,8 @@ struct SendPointsToInterpolator { static void apply(db::DataBox& box, Parallel::GlobalCache& cache, const ArrayIndex& /*array_index*/, - const TemporalId& temporal_id) { + const TemporalId& temporal_id, + const size_t iteration = 0_st) { auto coords = InterpolationTarget_detail::block_logical_coords< InterpolationTargetTag>(box, cache, temporal_id); InterpolationTarget_detail::set_up_interpolation( @@ -46,7 +53,7 @@ struct SendPointsToInterpolator { auto& receiver_proxy = Parallel::get_parallel_component>(cache); Parallel::simple_action>( - receiver_proxy, temporal_id, std::move(coords)); + receiver_proxy, temporal_id, std::move(coords), iteration); } }; diff --git a/src/ParallelAlgorithms/Interpolation/InterpolatedVars.hpp b/src/ParallelAlgorithms/Interpolation/InterpolatedVars.hpp index d761bdd26c57..b1e0d1c29cca 100644 --- a/src/ParallelAlgorithms/Interpolation/InterpolatedVars.hpp +++ b/src/ParallelAlgorithms/Interpolation/InterpolatedVars.hpp @@ -45,6 +45,10 @@ struct Info { IdPair>>> block_coord_holders; + /// If a target needs to send points in a specific order, it should also send + /// along which iteration the `block_coord_holders` are for. That way they can + /// be properly ordered in the Interpolator. + size_t iteration{0_st}; /// `vars` holds the interpolated `Variables` on some subset of the /// points in `block_coord_holders`. The grid points inside vars /// are indexed according to `global_offsets` below. The size of diff --git a/tests/Unit/Helpers/ParallelAlgorithms/Interpolation/InterpolationTargetTestHelpers.hpp b/tests/Unit/Helpers/ParallelAlgorithms/Interpolation/InterpolationTargetTestHelpers.hpp index 529cc99b37f0..5b1371d36945 100644 --- a/tests/Unit/Helpers/ParallelAlgorithms/Interpolation/InterpolationTargetTestHelpers.hpp +++ b/tests/Unit/Helpers/ParallelAlgorithms/Interpolation/InterpolationTargetTestHelpers.hpp @@ -101,9 +101,10 @@ struct MockReceivePoints { std::vector>>>&& - block_coord_holders) { + block_coord_holders, + const size_t iteration = 0_st) { db::mutate>( - [&temporal_id, &block_coord_holders]( + [&temporal_id, &block_coord_holders, &iteration]( const gsl::not_null::type*> vars_holders) { @@ -117,7 +118,7 @@ struct MockReceivePoints { temporal_id, intrp::Vars::Info{ - std::move(block_coord_holders)})); + std::move(block_coord_holders), iteration})); }, make_not_null(&box)); } @@ -131,12 +132,11 @@ struct mock_interpolator { using phase_dependent_action_list = tmpl::list< Parallel::PhaseActions< Parallel::Phase::Initialization, - tmpl::list< - intrp::Actions::InitializeInterpolator< - intrp::Tags::VolumeVarsInfo< - Metavariables, typename Metavariables:: - InterpolationTargetA::temporal_id>, - intrp::Tags::InterpolatedVarsHolders>>>, + tmpl::list, + intrp::Tags::InterpolatedVarsHolders>>>, Parallel::PhaseActions>>; using component_being_mocked = intrp::Interpolator; @@ -235,9 +235,10 @@ void test_interpolation_target( const auto& info = vars_infos.at(temporal_id); const auto& block_coord_holders = info.block_coord_holders; - // Check number of points + // Check number of points and iteration const size_t number_of_points = expected_block_coord_holders.size(); CHECK(block_coord_holders.size() == number_of_points); + CHECK(info.iteration == 0_st); for (size_t i = 0; i < number_of_points; ++i) { CHECK(block_coord_holders[i].value().id == @@ -255,10 +256,12 @@ void test_interpolation_target( ActionTesting::invoke_queued_simple_action( make_not_null(&runner), 0); - // Should be two entries in the vars_infos + // Should be two entries in the vars_infos. Second should also have iteration + // 0 CHECK(vars_infos.size() == 2); - const auto& new_block_coord_holders = - vars_infos.at(new_temporal_id).block_coord_holders; + const auto& new_info = vars_infos.at(new_temporal_id); + const auto& new_block_coord_holders = new_info.block_coord_holders; + CHECK(new_info.iteration == 0_st); for (size_t i = 0; i < number_of_points; ++i) { CHECK(new_block_coord_holders[i].value().id == expected_block_coord_holders[i].value().id); diff --git a/tests/Unit/ParallelAlgorithms/Interpolation/Test_InterpolatorReceivePoints.cpp b/tests/Unit/ParallelAlgorithms/Interpolation/Test_InterpolatorReceivePoints.cpp index 0ed09eb727d5..f1df6e39d0b6 100644 --- a/tests/Unit/ParallelAlgorithms/Interpolation/Test_InterpolatorReceivePoints.cpp +++ b/tests/Unit/ParallelAlgorithms/Interpolation/Test_InterpolatorReceivePoints.cpp @@ -10,23 +10,26 @@ #include #include -#include "DataStructures/DataBox/DataBox.hpp" // IWYU pragma: keep +#include "DataStructures/DataBox/DataBox.hpp" #include "DataStructures/DataVector.hpp" #include "DataStructures/IdPair.hpp" #include "DataStructures/Tensor/Tensor.hpp" +#include "DataStructures/Variables.hpp" #include "Domain/BlockLogicalCoordinates.hpp" +#include "Domain/Creators/Brick.hpp" #include "Domain/Creators/RegisterDerivedWithCharm.hpp" -#include "Domain/Creators/Sphere.hpp" #include "Domain/Creators/Tags/Domain.hpp" #include "Domain/Domain.hpp" #include "Domain/Structure/BlockId.hpp" +#include "Domain/Structure/InitialElementIds.hpp" #include "Framework/ActionTesting.hpp" #include "Parallel/Phase.hpp" -#include "Parallel/PhaseDependentActionList.hpp" // IWYU pragma: keep +#include "Parallel/PhaseDependentActionList.hpp" #include "ParallelAlgorithms/Interpolation/Actions/InitializeInterpolationTarget.hpp" -#include "ParallelAlgorithms/Interpolation/Actions/InitializeInterpolator.hpp" // IWYU pragma: keep -#include "ParallelAlgorithms/Interpolation/Actions/InterpolatorReceivePoints.hpp" // IWYU pragma: keep -#include "ParallelAlgorithms/Interpolation/Actions/InterpolatorRegisterElement.hpp" // IWYU pragma: keep +#include "ParallelAlgorithms/Interpolation/Actions/InitializeInterpolator.hpp" +#include "ParallelAlgorithms/Interpolation/Actions/InterpolatorReceivePoints.hpp" +#include "ParallelAlgorithms/Interpolation/Actions/InterpolatorReceiveVolumeData.hpp" +#include "ParallelAlgorithms/Interpolation/Actions/InterpolatorRegisterElement.hpp" #include "ParallelAlgorithms/Interpolation/Actions/TryToInterpolate.hpp" #include "ParallelAlgorithms/Interpolation/Callbacks/ObserveTimeSeriesOnSurface.hpp" #include "ParallelAlgorithms/Interpolation/InterpolatedVars.hpp" @@ -45,8 +48,6 @@ #include "Utilities/TMPL.hpp" #include "Utilities/TaggedTuple.hpp" -// IWYU pragma: no_include - namespace intrp::Actions { template struct InterpolationTargetReceiveVars; @@ -71,6 +72,61 @@ struct Variables; } // namespace Tags namespace { +enum WhichElement { Left, Right }; + +struct PointsInWhichElement : db::SimpleTag { + using type = WhichElement; +}; + +// So there will be a point in every element in the y direction +constexpr size_t number_of_points = 20; +// Target points along line x=0.5, z=0.0 from y=0.0 to y=1.0 +constexpr std::array begin{0.5, 0.0, 0.0}; +constexpr std::array end{0.5, 1.0, 0.0}; + +template +struct SequentialLineSegment + : tt::ConformsTo { + using const_global_cache_tags = tmpl::list<>; + using is_sequential = std::true_type; + using frame = Frame; + using simple_tags = tmpl::list; + + template + static tnsr::I points( + const db::DataBox& box, + const tmpl::type_& /*meta*/) { + const auto& which_element = get(box); + + const double fractional_distance = 1.0 / (number_of_points - 1); + tnsr::I target_points(number_of_points); + for (size_t n = 0; n < number_of_points; ++n) { + for (size_t d = 0; d < 3; ++d) { + target_points.get(d)[n] = + gsl::at(begin, d) + static_cast(n) * fractional_distance * + (gsl::at(end, d) - gsl::at(begin, d)); + + // Move the points slightly into the left/right element depending on the + // options + if (d == 0) { + if (which_element == WhichElement::Left) { + target_points.get(d)[n] -= 0.1; + } else if (which_element == WhichElement::Right) { + target_points.get(d)[n] += 0.1; + } + } + } + } + return target_points; + } + + template + static tnsr::I points( + const db::DataBox& box, const tmpl::type_& meta, + const TemporalId& /*temporal_id*/) { + return points(box, meta); + } +}; size_t num_calls_of_target_receive_vars = 0; template @@ -117,8 +173,9 @@ struct mock_interpolation_target { using phase_dependent_action_list = tmpl::list< Parallel::PhaseActions< Parallel::Phase::Initialization, - tmpl::list>>, + tmpl::list>, + intrp::Actions::InitializeInterpolationTarget< + Metavariables, InterpolationTargetTag>>>, Parallel::PhaseActions>>; using replace_these_simple_actions = @@ -153,8 +210,7 @@ struct Metavariables { tmpl::list>; using compute_items_on_target = tmpl::list<>; using compute_target_points = - ::intrp::TargetPoints::LineSegment; + SequentialLineSegment; using post_interpolation_callback = intrp::callbacks::ObserveTimeSeriesOnSurface, InterpolationTargetA>; @@ -171,76 +227,339 @@ SPECTRE_TEST_CASE("Unit.NumericalAlgorithms.Interpolator.ReceivePoints", "[Unit]") { domain::creators::register_derived_with_charm(); using metavars = Metavariables; - using target_component = - mock_interpolation_target; + using target_tag = typename metavars::InterpolationTargetA; + using compute_points = typename target_tag::compute_target_points; + using target_component = mock_interpolation_target; using interp_component = mock_interpolator; - const auto domain_creator = domain::creators::Sphere( - 0.9, 4.9, domain::creators::Sphere::Excision{}, 1_st, 7_st, false); + // Eight elements + const auto domain_creator = domain::creators::Brick{ + std::array{0.0, 0.0, 0.0}, std::array{1.0, 1.0, 1.0}, + std::array{1_st, 2_st, 0_st}, std::array{2_st, 2_st, 2_st}, + std::array{false, false, false}}; + const Domain<3> domain = domain_creator.create_domain(); ActionTesting::MockRuntimeSystem runner{ - {domain_creator.create_domain()}}; + {domain_creator.create_domain()}, {}, std::vector{2_st}}; ActionTesting::set_phase(make_not_null(&runner), Parallel::Phase::Initialization); - ActionTesting::emplace_component(&runner, 0); - for (size_t i = 0; i < 2; ++i) { - ActionTesting::next_action(make_not_null(&runner), 0); - } - ActionTesting::emplace_component(&runner, 0); - for (size_t i = 0; i < 2; ++i) { - ActionTesting::next_action(make_not_null(&runner), 0); + // Two components of the interpolator + for (size_t array_index = 0; array_index < 2; array_index++) { + ActionTesting::emplace_component(&runner, array_index); + for (size_t i = 0; i < 2; ++i) { + ActionTesting::next_action(make_not_null(&runner), + array_index); + } } + ActionTesting::emplace_array_component_and_initialize( + &runner, ActionTesting::NodeId{0}, ActionTesting::LocalCoreId{0}, 0_st, + {}); + auto& box = + ActionTesting::get_databox(make_not_null(&runner), 0); + // Set the points to be in the x<0.5 elements first + db::mutate( + [](const gsl::not_null which_element) { + *which_element = WhichElement::Left; + }, + make_not_null(&box)); ActionTesting::set_phase(make_not_null(&runner), Parallel::Phase::Testing); - // Make sure that we have one Element registered, - // or else ReceivePoints will (correctly) do nothing because it - // thinks it will never have any Elements to interpolate onto. - runner.simple_action(0); - - const auto domain = domain_creator.create_domain(); - const auto block_logical_coords = [&domain]() { - const size_t n_pts = 15; - tnsr::I points(n_pts); - for (size_t d = 0; d < 3; ++d) { - for (size_t i = 0; i < n_pts; ++i) { - points.get(d)[i] = 1.0 + (0.1 + 0.02 * d) * i; // Chosen by hand. - } + // Make eight element ids and meshes + std::vector> element_ids{}; + element_ids.reserve(8); + std::vector> meshes{}; + meshes.reserve(8); + for (const auto& block : domain.blocks()) { + const auto initial_ref_levs = + domain_creator.initial_refinement_levels()[block.id()]; + auto elem_ids = initial_element_ids(block.id(), initial_ref_levs); + element_ids.insert(element_ids.end(), elem_ids.begin(), elem_ids.end()); + } + for (const auto& element_id : element_ids) { + meshes.emplace_back(domain_creator.initial_extents()[element_id.block_id()], + Spectral::Basis::Legendre, + Spectral::Quadrature::GaussLobatto); + } + // Same number of grid points in each element so just create one lapse.. + Variables>> lapse{ + meshes[0].number_of_grid_points(), 0.0}; + + // Make sure that we have four elements on each core registered, + for (size_t array_index = 0; array_index < 2; array_index++) { + for (size_t num_elements = 0; num_elements < 4; num_elements++) { + runner.simple_action( + array_index); } + } + + const auto create_block_logical_coords = [&domain, &box]() { + const tnsr::I points = + compute_points::points(box, tmpl::type_{}); return block_logical_coordinates(domain, points); - }(); + }; + + auto block_logical_coords = create_block_logical_coords(); Slab slab(0.0, 1.0); TimeStepId temporal_id(true, 0, Time(slab, Rational(11, 15))); + const auto& get_holders = [&runner](const size_t array_index) { + return ActionTesting::get_databox_tag< + interp_component, intrp::Tags::InterpolatedVarsHolders>( + runner, array_index); + }; + const auto& get_holder = [&get_holders](const size_t array_index) { + const auto& holders = get_holders(array_index); + return get< + intrp::Vars::HolderTag>( + holders); + }; + + // Send points to both interpolator cores. Not much should happen. We send at + // iteration 1 so we can test receiving points before, at, and after this + // iteration. + for (size_t array_index = 0; array_index < 2; array_index++) { + runner.simple_action>( + array_index, temporal_id, block_logical_coords, 1_st); + const auto& holder = get_holder(array_index); + + // Should now be a single info in holder, indexed by temporal_id. + CHECK(holder.infos.size() == 1); + const auto& info = holder.infos.at(temporal_id); + // We haven't done any interpolation because we haven't received + // volume data from any Elements, so these fields should be empty. + CHECK(holder.temporal_ids_when_data_has_been_interpolated.empty()); + CHECK(info.interpolation_is_done_for_these_elements.empty()); + CHECK(info.global_offsets.empty()); + CHECK(info.vars.empty()); + // But block_coord_holders should be filled and the iteration should be 1 + CHECK(info.block_coord_holders == block_logical_coords); + CHECK(info.iteration == 1_st); + // There should be no more queued actions; verify this. + CHECK(runner.is_simple_action_queue_empty(array_index)); + // Make sure that the action was not called. + CHECK(num_calls_of_target_receive_vars == 0); + } + + using vars_info_tag = + intrp::Tags::VolumeVarsInfo; + const auto& get_vars_info = [&runner](const size_t array_index) { + return ActionTesting::get_databox_tag( + runner, array_index); + }; + + // Send data to first interpolator core from the first three elements + for (size_t i = 0; i < 3; i++) { + INFO("Element " + get_output(i)); + runner.simple_action< + interp_component, + intrp::Actions::InterpolatorReceiveVolumeData>( + 0_st, temporal_id, element_ids[i], meshes[i], lapse); + + // Ensure the volume data got to the interpolator + const auto& vars_info = get_vars_info(0_st); + CHECK(vars_info.count(temporal_id) == 1); + CHECK(vars_info.at(temporal_id).count(element_ids[i]) == 1); + + // We should have interpolated some data but not called the + // target_receive_vars + const auto& holder = get_holder(0_st); + CHECK(holder.temporal_ids_when_data_has_been_interpolated.empty()); + CHECK(holder.infos.size() == 1); + const auto& info = holder.infos.at(temporal_id); + CHECK(info.interpolation_is_done_for_these_elements.count(element_ids[i]) == + 1); + CHECK(info.block_coord_holders == block_logical_coords); + CHECK(info.iteration == 1_st); + CHECK(info.global_offsets.size() == i + 1); + CHECK(info.vars.size() == i + 1); + + // There should be no queued actions and no calls to target_receive_vars + CHECK(runner.is_simple_action_queue_empty(0_st)); + CHECK(num_calls_of_target_receive_vars == 0); + } + + // Send data to first interpolator core from the fourth element + runner.simple_action< + interp_component, + intrp::Actions::InterpolatorReceiveVolumeData>( + 0_st, temporal_id, element_ids[3], meshes[3], lapse); + + { + INFO("Element 3"); + // Ensure the volume data got to the interpolator + const auto& vars_info = get_vars_info(0_st); + CHECK(vars_info.count(temporal_id) == 1); + CHECK(vars_info.at(temporal_id).count(element_ids[3]) == 1); + + // We should have interpolated some data and called the target_receive_vars. + // This should have cleaned things up + const auto& holder = get_holder(0_st); + CHECK(holder.temporal_ids_when_data_has_been_interpolated.empty()); + CHECK(holder.infos.empty()); + + // There should be one queued action; verify this, but not called yet + CHECK(runner.number_of_queued_simple_actions(0_st) == 1); + CHECK(num_calls_of_target_receive_vars == 0); + // Invoke the action and check that it's called + ActionTesting::invoke_queued_simple_action( + make_not_null(&runner), 0_st); + CHECK(num_calls_of_target_receive_vars == 1); + } + + // Send data to second interpolator core from fifth and sixth elements + for (size_t i = 4; i < 6; i++) { + INFO("Element " + get_output(i)); + runner.simple_action< + interp_component, + intrp::Actions::InterpolatorReceiveVolumeData>( + 1_st, temporal_id, element_ids[i], meshes[i], lapse); + + // Ensure the volume data got to the interpolator + const auto& vars_info = get_vars_info(1_st); + CHECK(vars_info.count(temporal_id) == 1); + CHECK(vars_info.at(temporal_id).count(element_ids[i]) == 1); + + // We should have a holder for this temporal id, but there shouldn't be any + // interpolated data. The block logical coords should be the new ones + const auto& holder = get_holder(1_st); + CHECK(holder.temporal_ids_when_data_has_been_interpolated.empty()); + CHECK(holder.infos.size() == 1); + const auto& info = holder.infos.at(temporal_id); + CHECK(info.interpolation_is_done_for_these_elements.count(element_ids[i]) == + 1); + CHECK(info.block_coord_holders == block_logical_coords); + CHECK(info.iteration == 1_st); + CHECK(info.global_offsets.empty()); + CHECK(info.vars.empty()); + + // There should be no queued actions and no extra calls to + // target_receive_vars + CHECK(runner.is_simple_action_queue_empty(0_st)); + CHECK(num_calls_of_target_receive_vars == 1); + } + + // Now send new points to the interpolator for x>0.5 + db::mutate( + [](const gsl::not_null which_element) { + *which_element = WhichElement::Right; + }, + make_not_null(&box)); + block_logical_coords = create_block_logical_coords(); + + // First send with iteration 0. These points should be ignored + for (size_t array_index = 0; array_index < 2; array_index++) { + runner.simple_action>( + array_index, temporal_id, block_logical_coords, 0_st); + } + + // Send data to second interpolator core from the seventh element. We should + // not have done an interpolation to these new points because of iteration 1 runner.simple_action< - mock_interpolator, - intrp::Actions::ReceivePoints>( - 0, temporal_id, block_logical_coords); - - const auto& holders = ActionTesting::get_databox_tag< - interp_component, intrp::Tags::InterpolatedVarsHolders>(runner, - 0); - const auto& holder = - get>( - holders); - - // Should now be a single info in holder, indexed by temporal_id. - CHECK(holder.infos.size() == 1); - const auto& vars_info = holder.infos.at(temporal_id); - - // We haven't done any interpolation because we never received - // volume data from any Elements, so these fields should be empty. - CHECK(holder.temporal_ids_when_data_has_been_interpolated.empty()); - CHECK(vars_info.interpolation_is_done_for_these_elements.empty()); - CHECK(vars_info.global_offsets.empty()); - CHECK(vars_info.vars.empty()); - - // But block_coord_holders should be filled. - CHECK(vars_info.block_coord_holders == block_logical_coords); - - // There should be no more queued actions; verify this. - CHECK(runner.is_simple_action_queue_empty>(0)); - - // Make sure that the action was not called. - CHECK(num_calls_of_target_receive_vars == 0); + interp_component, + intrp::Actions::InterpolatorReceiveVolumeData>( + 1_st, temporal_id, element_ids[6], meshes[6], lapse); + + { + INFO("Element 6 no interpolate"); + // Ensure the volume data got to the interpolator + const auto& vars_info = get_vars_info(1_st); + CHECK(vars_info.count(temporal_id) == 1); + CHECK(vars_info.at(temporal_id).count(element_ids[6]) == 1); + + // Now we should NOT have interpolated data on the 3 elements that have + // received volume data thus far. + const auto& holder = get_holder(1_st); + CHECK(holder.temporal_ids_when_data_has_been_interpolated.empty()); + CHECK(holder.infos.size() == 1); + const auto& info = holder.infos.at(temporal_id); + CHECK(info.interpolation_is_done_for_these_elements.count(element_ids[6]) == + 1); + // These should still be the old coords + CHECK_FALSE(info.block_coord_holders == block_logical_coords); + CHECK(info.iteration == 1_st); + CHECK(info.global_offsets.empty()); + CHECK(info.vars.empty()); + + // There should be no queued actions and no extra calls to + // target_receive_vars + CHECK(runner.is_simple_action_queue_empty(0_st)); + CHECK(num_calls_of_target_receive_vars == 1); + } + + // Now send with iteration 1. One core 0, this shouldn't error because we have + // cleaned up points. But on core 1 this should error + runner.simple_action>( + 0_st, temporal_id, block_logical_coords, 1_st); + CHECK_THROWS_WITH( + (runner.simple_action< + interp_component, + intrp::Actions::ReceivePoints>( + 1_st, temporal_id, block_logical_coords, 1_st)), + Catch::Matchers::ContainsSubstring( + "Interpolator received target points at iteration 1 twice.")); + + // Now send with iteration 2. These points should overwrite the previous ones + // and an interpolation should happen + for (size_t array_index = 0; array_index < 2; array_index++) { + runner.simple_action>( + array_index, temporal_id, block_logical_coords, 2_st); + } + + { + INFO("Element 6 interpolate"); + // Ensure the volume data got to the interpolator + const auto& vars_info = get_vars_info(1_st); + CHECK(vars_info.count(temporal_id) == 1); + CHECK(vars_info.at(temporal_id).count(element_ids[6]) == 1); + + // Now we should have interpolated data on the 3 elements that have received + // volume data thus far because our points are now in these elements + const auto& holder = get_holder(1_st); + CHECK(holder.temporal_ids_when_data_has_been_interpolated.empty()); + CHECK(holder.infos.size() == 1); + const auto& info = holder.infos.at(temporal_id); + CHECK(info.interpolation_is_done_for_these_elements.count(element_ids[6]) == + 1); + CHECK(info.block_coord_holders == block_logical_coords); + CHECK(info.iteration == 2_st); + CHECK(info.global_offsets.size() == 3); + CHECK(info.vars.size() == 3); + + // There should be no queued actions and no extra calls to + // target_receive_vars + CHECK(runner.is_simple_action_queue_empty(0_st)); + CHECK(num_calls_of_target_receive_vars == 1); + } + + // Send data to second interpolator core from the eighth and final element + runner.simple_action< + interp_component, + intrp::Actions::InterpolatorReceiveVolumeData>( + 1_st, temporal_id, element_ids[7], meshes[7], lapse); + + { + INFO("Element 7"); + // Ensure the volume data got to the interpolator + const auto& vars_info = get_vars_info(1_st); + CHECK(vars_info.count(temporal_id) == 1); + CHECK(vars_info.at(temporal_id).count(element_ids[6]) == 1); + + // Now we should have interpolated data on all 4 elements and called + // target_receive_vars. This should have cleaned things up + const auto& holder = get_holder(1_st); + CHECK(holder.temporal_ids_when_data_has_been_interpolated.empty()); + CHECK(holder.infos.empty()); + + // There should be one queued action; verify this, but not called yet + CHECK(runner.number_of_queued_simple_actions(0_st) == 1); + CHECK(num_calls_of_target_receive_vars == 1); + // Invoke the action and check that it's called + ActionTesting::invoke_queued_simple_action( + make_not_null(&runner), 0_st); + CHECK(num_calls_of_target_receive_vars == 2); + } } } // namespace