Merge pull request #5634 from kidder/amr_bookmark

Create a new element with the phase bookmarks of an existing element

src/Parallel/Algorithms/AlgorithmArray.ci

@@ -31,7 +31,8 @@ module AlgorithmArray {
 
     entry AlgorithmArray(
         Parallel::CProxy_GlobalCache<typename ParallelComponent::metavariables>,
-        Parallel::Phase, std::unique_ptr<Parallel::Callback>);
+        Parallel::Phase, std::unordered_map<Parallel::Phase, size_t>,
+        std::unique_ptr<Parallel::Callback>);
 
     // Ordinarily, indiscriminate use of [inline] has the danger of arbitrarily
     // deep recursive function calls, which then exceed the stack limits of the

src/Parallel/DistributedObject.hpp

@@ -209,6 +209,7 @@ class DistributedObject<ParallelComponent,
   DistributedObject(
       const Parallel::CProxy_GlobalCache<metavariables>& global_cache_proxy,
       Parallel::Phase current_phase,
+      std::unordered_map<Parallel::Phase, size_t> phase_bookmarks,
       const std::unique_ptr<Parallel::Callback>& callback);
 
   /// Charm++ migration constructor, used after a chare is migrated
@@ -322,6 +323,15 @@ class DistributedObject<ParallelComponent,
   /// Get the current phase
   Phase phase() const { return phase_; }
 
+  /// \brief Get the phase bookmarks
+  ///
+  /// \details These are used to allow a phase to be resumed at a specific
+  /// step in its iterable action list after PhaseControl is used to temporarily
+  /// switch to other phases.
+  const std::unordered_map<Parallel::Phase, size_t>& phase_bookmarks() const {
+    return phase_bookmarks_;
+  }
+
   /// Tell the Algorithm it should no longer execute the algorithm. This does
   /// not mean that the execution of the program is terminated, but only that
   /// the algorithm has terminated. An algorithm can be restarted by passing
@@ -525,6 +535,7 @@ DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
     DistributedObject(
         const Parallel::CProxy_GlobalCache<metavariables>& global_cache_proxy,
         Parallel::Phase current_phase,
+        std::unordered_map<Parallel::Phase, size_t> phase_bookmarks,
         const std::unique_ptr<Parallel::Callback>& callback)
     : DistributedObject() {
   static_assert(Parallel::is_array_proxy<cproxy_type>::value,
@@ -540,6 +551,7 @@ DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
     this->setMigratable(true);
     global_cache_proxy_ = global_cache_proxy;
     phase_ = current_phase;
+    phase_bookmarks_ = std::move(phase_bookmarks);
     ::Initialization::mutate_assign<distributed_object_tags>(
         make_not_null(&box_), metavariables{}, array_index_,
         global_cache_proxy_);

src/ParallelAlgorithms/Amr/Actions/AdjustDomain.hpp

@@ -111,6 +111,8 @@ struct AdjustDomain {
     const auto& my_amr_flags = my_amr_info.flags;
     auto& element_array =
         Parallel::get_parallel_component<ParallelComponent>(cache);
+    const auto& phase_bookmarks =
+        Parallel::local(element_array[element_id])->phase_bookmarks();
 
     // Check for h-refinement
     if (alg::any_of(my_amr_flags,
@@ -125,7 +127,8 @@ struct AdjustDomain {
           Parallel::get_parallel_component<amr::Component<Metavariables>>(
               cache);
       Parallel::simple_action<CreateChild>(amr_component, element_array,
-                                           element_id, children_ids, 0_st);
+                                           element_id, children_ids, 0_st,
+                                           phase_bookmarks);
 
     } else if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
                  return flag == amr::Flag::Join;
@@ -138,9 +141,9 @@ struct AdjustDomain {
         auto& amr_component =
             Parallel::get_parallel_component<amr::Component<Metavariables>>(
                 cache);
-        Parallel::simple_action<CreateParent>(amr_component, element_array,
-                                              std::move(parent_id), element_id,
-                                              std::move(ids_to_join));
+        Parallel::simple_action<CreateParent>(
+            amr_component, element_array, std::move(parent_id), element_id,
+            std::move(ids_to_join), phase_bookmarks);
       }
 
     } else {

src/ParallelAlgorithms/Amr/Actions/CreateChild.hpp

@@ -47,27 +47,33 @@ struct CreateChild {
       ElementProxy element_proxy,
       ElementId<Metavariables::volume_dim> parent_id,
       std::vector<ElementId<Metavariables::volume_dim>> children_ids,
-      const size_t index_of_child_id) {
+      const size_t index_of_child_id,
+      const std::unordered_map<Parallel::Phase, size_t>
+          parent_phase_bookmarks) {
     auto my_proxy = Parallel::get_parallel_component<ParallelComponent>(cache);
     const ElementId<Metavariables::volume_dim>& child_id =
         children_ids[index_of_child_id];
     if (index_of_child_id + 1 == children_ids.size()) {
       auto parent_proxy = element_proxy[parent_id];
       element_proxy[child_id].insert(
           cache.get_this_proxy(), Parallel::Phase::AdjustDomain,
+          parent_phase_bookmarks,
           std::make_unique<Parallel::SimpleActionCallback<
               SendDataToChildren, decltype(parent_proxy),
               std::vector<ElementId<Metavariables::volume_dim>>>>(
               parent_proxy, std::move(children_ids)));
     } else {
       element_proxy[child_id].insert(
           cache.get_this_proxy(), Parallel::Phase::AdjustDomain,
+          parent_phase_bookmarks,
           std::make_unique<Parallel::SimpleActionCallback<
               CreateChild, decltype(my_proxy), ElementProxy,
               ElementId<Metavariables::volume_dim>,
-              std::vector<ElementId<Metavariables::volume_dim>>, size_t>>(
+              std::vector<ElementId<Metavariables::volume_dim>>, size_t,
+              std::unordered_map<Parallel::Phase, size_t>>>(
               my_proxy, std::move(element_proxy), std::move(parent_id),
-              std::move(children_ids), index_of_child_id + 1));
+              std::move(children_ids), index_of_child_id + 1,
+              parent_phase_bookmarks));
     }
   }
 };

src/ParallelAlgorithms/Amr/Actions/CreateParent.hpp

@@ -40,10 +40,12 @@ struct CreateParent {
       ElementProxy element_proxy,
       ElementId<Metavariables::volume_dim> parent_id,
       const ElementId<Metavariables::volume_dim>& child_id,
-      std::deque<ElementId<Metavariables::volume_dim>> sibling_ids_to_collect) {
+      std::deque<ElementId<Metavariables::volume_dim>> sibling_ids_to_collect,
+      const std::unordered_map<Parallel::Phase, size_t> child_phase_bookmarks) {
     auto child_proxy = element_proxy[child_id];
     element_proxy[parent_id].insert(
         cache.get_this_proxy(), Parallel::Phase::AdjustDomain,
+        child_phase_bookmarks,
         std::make_unique<Parallel::SimpleActionCallback<
             CollectDataFromChildren, decltype(child_proxy),
             ElementId<Metavariables::volume_dim>,

src/ParallelAlgorithms/Amr/Actions/RegisterCallbacks.hpp

@@ -25,7 +25,8 @@ void register_callbacks() {
               amr::Actions::CreateChild,
               CProxy_AlgorithmSingleton<amr::Component<Metavariables>, int>,
               CProxy_AlgorithmArray<Component, ArrayIndex>, ArrayIndex,
-              std::vector<ArrayIndex>, size_t>,
+              std::vector<ArrayIndex>, size_t,
+              std::unordered_map<Parallel::Phase, size_t>>,
           Parallel::SimpleActionCallback<
               amr::Actions::SendDataToChildren,
               CProxyElement_AlgorithmArray<Component, ArrayIndex>,

tests/Unit/Framework/ActionTesting.hpp

@@ -525,9 +525,11 @@ class MockDistributedObjectProxy : public CProxyElement_ArrayElement {
   // creating/destroying MockDistributedObjects is not supported; it must be
   // called on an existing MockDistrubtedObject.
   template <typename CacheProxy>
-  void insert(const CacheProxy& /*global_cache_proxy*/,
-              Parallel::Phase /*current_phase*/,
-              const std::unique_ptr<Parallel::Callback>& callback) {
+  void insert(
+      const CacheProxy& /*global_cache_proxy*/,
+      Parallel::Phase /*current_phase*/,
+      const std::unordered_map<Parallel::Phase, size_t>& /*phase_bookmarks*/,
+      const std::unique_ptr<Parallel::Callback>& callback) {
     callback->invoke();
   }
 

tests/Unit/Framework/MockDistributedObject.hpp

@@ -380,6 +380,10 @@ class MockDistributedObject {
   void set_terminate(bool t) { terminate_ = t; }
   bool get_terminate() const { return terminate_; }
 
+  // There are no phase bookmarks in mock distributed objects, so we just
+  // return an empty map
+  std::unordered_map<Parallel::Phase, size_t> phase_bookmarks() { return {}; }
+
   // Actions may call this, but since tests step through actions manually it has
   // no effect.
   void perform_algorithm() {}

tests/Unit/Parallel/CMakeLists.txt

@@ -146,6 +146,9 @@ add_algorithm_test(
 add_algorithm_test(
   "AlgorithmPhaseControl"
   INPUT_FILE "Test_AlgorithmPhaseControl.yaml")
+add_algorithm_test(
+  "DynamicInsertionState"
+  INPUT_FILE "Test_DynamicInsertionState.yaml")
 
 add_algorithm_test_with_restart_from_checkpoint("CheckpointRestart")
 

tests/Unit/Parallel/Test_DynamicInsertion.cpp

@@ -174,12 +174,13 @@ struct CreateChild {
   template <typename ParallelComponent, typename DbTagList,
             typename Metavariables, typename ArrayIndex, typename ElementProxy,
             typename ElementIndex>
-  static void apply(db::DataBox<DbTagList>& /*box*/,
-                    Parallel::GlobalCache<Metavariables>& cache,
-                    const ArrayIndex& /*array_index*/,
-                    ElementProxy element_proxy, ElementIndex parent_id,
-                    ElementIndex child_id,
-                    std::vector<ElementIndex> children_ids) {
+  static void apply(
+      db::DataBox<DbTagList>& /*box*/,
+      Parallel::GlobalCache<Metavariables>& cache,
+      const ArrayIndex& /*array_index*/, ElementProxy element_proxy,
+      ElementIndex parent_id, ElementIndex child_id,
+      std::vector<ElementIndex> children_ids,
+      std::unordered_map<Parallel::Phase, size_t> phase_bookmarks) {
     auto my_proxy = Parallel::get_parallel_component<ParallelComponent>(cache);
     ASSERT(
         alg::count(children_ids, child_id) == 1,
@@ -188,7 +189,7 @@ struct CreateChild {
     if (*child_it == children_ids.back()) {
       auto parent_proxy = element_proxy(parent_id);
       element_proxy(child_id).insert(
-          cache.thisProxy, Parallel::Phase::AdjustDomain,
+          cache.thisProxy, Parallel::Phase::AdjustDomain, phase_bookmarks,
           std::make_unique<Parallel::SimpleActionCallback<
               SendDataToChildren, decltype(parent_proxy),
               std::vector<ElementIndex>>>(parent_proxy,
@@ -197,12 +198,13 @@ struct CreateChild {
       const auto next_child_it = std::next(child_it);
       auto next_child = *next_child_it;
       element_proxy(child_id).insert(
-          cache.thisProxy, Parallel::Phase::AdjustDomain,
+          cache.thisProxy, Parallel::Phase::AdjustDomain, phase_bookmarks,
           std::make_unique<Parallel::SimpleActionCallback<
               CreateChild, decltype(my_proxy), ElementProxy, ElementIndex,
-              ElementIndex, std::vector<ElementIndex>>>(
+              ElementIndex, std::vector<ElementIndex>,
+              std::unordered_map<Parallel::Phase, size_t>>>(
               my_proxy, std::move(element_proxy), std::move(parent_id),
-              std::move(next_child), std::move(children_ids)));
+              std::move(next_child), std::move(children_ids), phase_bookmarks));
     }
   }
 };
@@ -216,24 +218,27 @@ struct ChangeArray {
     auto& array_proxy =
         Parallel::get_parallel_component<ParallelComponent>(cache);
     auto my_proxy = array_proxy[array_index];
-    auto create_children = [&cache, &array_index,
-                            &array_proxy](const size_t number_of_new_elements) {
+    const auto& phase_bookmarks = Parallel::local(my_proxy)->phase_bookmarks();
+    auto create_children = [&cache, &array_index, &array_proxy,
+                            &phase_bookmarks](
+                               const size_t number_of_new_elements) {
       std::vector<int> new_ids(number_of_new_elements);
       std::iota(new_ids.begin(), new_ids.end(), 100 * array_index);
       auto& singleton_proxy =
           Parallel::get_parallel_component<amr::Component<Metavariables>>(
               cache);
-      Parallel::simple_action<CreateChild>(
-          singleton_proxy, array_proxy, array_index, new_ids.front(), new_ids);
+      Parallel::simple_action<CreateChild>(singleton_proxy, array_proxy,
+                                           array_index, new_ids.front(),
+                                           new_ids, phase_bookmarks);
     };
 
-    auto create_parent = [&cache, &array_proxy,
-                          &my_proxy](std::deque<int> ids_to_join) {
+    auto create_parent = [&cache, &array_proxy, &my_proxy,
+                          &phase_bookmarks](std::deque<int> ids_to_join) {
       int id_of_first_child = ids_to_join.front();
       ids_to_join.pop_front();
       int parent_id = 100 * id_of_first_child;
       array_proxy(parent_id).insert(
-          cache.thisProxy, Parallel::Phase::Execute,
+          cache.thisProxy, Parallel::Phase::Execute, phase_bookmarks,
           std::make_unique<Parallel::SimpleActionCallback<
               CollectDataFromChildren, decltype(my_proxy), int, std::deque<int>,
               double>>(my_proxy, std::move(parent_id), std::move(ids_to_join),
@@ -344,7 +349,8 @@ void register_callback() {
               CreateChild,
               CProxy_AlgorithmSingleton<amr::Component<TestMetavariables>, int>,
               CProxy_AlgorithmArray<TestArray<TestMetavariables>, int>, int,
-              int, std::vector<int>>,
+              int, std::vector<int>,
+              std::unordered_map<Parallel::Phase, size_t>>,
           Parallel::SimpleActionCallback<
               SendDataToChildren,
               CProxyElement_AlgorithmArray<TestArray<TestMetavariables>, int>,