Revamp some deadlock analysis for GH

src/ControlSystem/Actions/PrintCurrentMeasurement.hpp

@@ -20,13 +20,14 @@ struct PrintCurrentMeasurement {
             typename ArrayIndex>
   static void apply(db::DataBox<DbTags>& box,
                     const Parallel::GlobalCache<Metavariables>& /*cache*/,
-                    const ArrayIndex& /*array_index*/) {
+                    const ArrayIndex& /*array_index*/,
+                    const std::string& file_name) {
     const int current_measurement =
         db::get<control_system::Tags::CurrentNumberOfMeasurements>(box);
 
-    Parallel::printf("%s: Current measurement = %d\n",
-                     pretty_type::name<ParallelComponent>(),
-                     current_measurement);
+    Parallel::fprintf(file_name, "%s: Current measurement = %d\n",
+                      pretty_type::name<ParallelComponent>(),
+                      current_measurement);
   }
 };
 }  // namespace control_system::Actions

src/Evolution/Deadlock/CMakeLists.txt

@@ -10,13 +10,15 @@ spectre_target_headers(
   INCLUDE_DIRECTORY ${CMAKE_SOURCE_DIR}/src
   HEADERS
   PrintDgElementArray.hpp
+  PrintFunctionsOfTime.hpp
   )
 
 target_link_libraries(
   ${LIBRARY}
   INTERFACE
   DataStructures
   DiscontinuousGalerkin
+  FunctionsOfTime
   Parallel
   Printf
   Time

src/Evolution/Deadlock/PrintDgElementArray.hpp

@@ -52,20 +52,21 @@ struct PrintElementInfo {
             typename ArrayIndex>
   static void apply(db::DataBox<DbTags>& box,
                     Parallel::GlobalCache<Metavariables>& cache,
-                    const ArrayIndex& array_index) {
+                    const ArrayIndex& array_index,
+                    const std::string& file_name) {
     if constexpr (Parallel::is_dg_element_collection_v<ParallelComponent>) {
       auto& element =
           Parallel::local_synchronous_action<
               Parallel::Actions::GetItemFromDistributedOject<
                   typename ParallelComponent::element_collection_tag>>(
               Parallel::get_parallel_component<ParallelComponent>(cache))
               ->at(array_index);
-      apply<ParallelComponent>(box, cache, array_index, &element);
+      apply<ParallelComponent>(box, cache, array_index, &element, file_name);
     } else {
       apply<ParallelComponent>(
           box, cache, array_index,
           Parallel::local(Parallel::get_parallel_component<ParallelComponent>(
-              cache)[array_index]));
+              cache)[array_index]), file_name);
     }
   }
 
@@ -74,7 +75,8 @@ struct PrintElementInfo {
             typename ArrayIndex, typename T>
   static void apply(db::DataBox<DbTags>& box,
                     const Parallel::GlobalCache<Metavariables>& /*cache*/,
-                    const ArrayIndex& array_index, T* local_object_ptr) {
+                    const ArrayIndex& array_index, T* local_object_ptr,
+                    const std::string& file_name) {
     auto& local_object = *local_object_ptr;
 
     const bool terminated = local_object.get_terminate();
@@ -139,7 +141,7 @@ struct PrintElementInfo {
       ss << "\n";
     }
 
-    Parallel::printf("%s", ss.str());
+    Parallel::fprintf(file_name, "%s", ss.str());
   }
 };
 }  // namespace deadlock

src/Evolution/Deadlock/PrintFunctionsOfTime.hpp

@@ -0,0 +1,57 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#pragma once
+
+#include <cstddef>
+
+#include "DataStructures/DataBox/DataBox.hpp"
+#include "Domain/FunctionsOfTime/OutputTimeBounds.hpp"
+#include "Domain/FunctionsOfTime/Tags.hpp"
+#include "Parallel/GlobalCache.hpp"
+#include "Parallel/Info.hpp"
+#include "Parallel/ParallelComponentHelpers.hpp"
+#include "Parallel/Printf/Printf.hpp"
+
+namespace control_system::Tags {
+struct MeasurementTimescales;
+}  // namespace control_system::Tags
+
+namespace deadlock {
+/*!
+ * \brief Simple action that will print the `domain::Tags::FunctionsOfTime` and
+ * `control_system::Tags::MeasurementTimescales` (if it exists) time bounds for
+ * each node of a simulation.
+ */
+struct PrintFunctionsOfTime {
+  template <typename ParallelComponent, typename DbTags, typename Metavariables,
+            typename ArrayIndex>
+  static void apply(db::DataBox<DbTags>& /*box*/,
+                    Parallel::GlobalCache<Metavariables>& cache,
+                    const ArrayIndex& /*array_index*/,
+                    const std::string& file_name) {
+    const auto& functions_of_time =
+        Parallel::get<::domain::Tags::FunctionsOfTime>(cache);
+    const std::string time_bounds =
+        ::domain::FunctionsOfTime::output_time_bounds(functions_of_time);
+
+    if constexpr (Parallel::is_in_global_cache<
+                      Metavariables,
+                      control_system::Tags::MeasurementTimescales>) {
+      const auto& measurement_timescales =
+          Parallel::get<control_system::Tags::MeasurementTimescales>(cache);
+      const std::string measurement_time_bounds =
+          ::domain::FunctionsOfTime::output_time_bounds(measurement_timescales);
+
+      Parallel::fprintf(
+          file_name,
+          "Node %zu\nFunctionsOfTime:\n%s\n\nMeasurementTimescales:\n%s\n",
+          Parallel::my_node<size_t>(cache), time_bounds,
+          measurement_time_bounds);
+    } else {
+      Parallel::fprintf(file_name, "Node %zu\nFunctionsOfTime:%s\n",
+                        Parallel::my_node<size_t>(cache), time_bounds);
+    }
+  }
+};
+}  // namespace deadlock

src/Evolution/DiscontinuousGalerkin/DgElementArray.hpp

@@ -91,10 +91,11 @@ struct DgElementArray {
 
   static void execute_next_phase(
       const Parallel::Phase next_phase,
-      Parallel::CProxy_GlobalCache<Metavariables>& global_cache) {
+      Parallel::CProxy_GlobalCache<Metavariables>& global_cache,
+      const bool force = false) {
     auto& local_cache = *Parallel::local_branch(global_cache);
     Parallel::get_parallel_component<DgElementArray>(local_cache)
-        .start_phase(next_phase);
+        .start_phase(next_phase, force);
   }
 };
 

src/Evolution/Executables/GeneralizedHarmonic/Deadlock.hpp

@@ -0,0 +1,102 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "ControlSystem/Actions/PrintCurrentMeasurement.hpp"
+#include "Domain/FunctionsOfTime/Tags.hpp"
+#include "Evolution/Deadlock/PrintDgElementArray.hpp"
+#include "Evolution/Deadlock/PrintFunctionsOfTime.hpp"
+#include "Parallel/ArrayCollection/IsDgElementCollection.hpp"
+#include "Parallel/ArrayCollection/SimpleActionOnElement.hpp"
+#include "Parallel/GlobalCache.hpp"
+#include "Parallel/Invoke.hpp"
+#include "ParallelAlgorithms/Interpolation/Actions/PrintInterpolationTargetForDeadlock.hpp"
+#include "ParallelAlgorithms/Interpolation/Actions/PrintInterpolatorForDeadlock.hpp"
+#include "Utilities/FileSystem.hpp"
+#include "Utilities/PrettyType.hpp"
+#include "Utilities/TMPL.hpp"
+
+/// \cond
+namespace intrp {
+template <class Metavariables>
+struct Interpolator;
+template <class Metavariables, typename InterpolationTargetTag>
+struct InterpolationTarget;
+}  // namespace intrp
+namespace observers {
+template <class Metavariables>
+struct ObserverWriter;
+}  // namespace observers
+/// \endcond
+
+namespace gh::deadlock {
+template <typename DgElementArray, typename ControlComponents,
+          typename InterpolationTargetTags, typename Metavariables>
+void run_deadlock_analysis_simple_actions(
+    Parallel::GlobalCache<Metavariables>& cache,
+    const std::vector<std::string>& deadlocked_components) {
+  const std::string deadlock_dir{"deadlock"};
+  if (file_system::check_if_dir_exists(deadlock_dir)) {
+    file_system::rm(deadlock_dir, true);
+  }
+
+  file_system::create_directory(deadlock_dir);
+
+  Parallel::simple_action<::deadlock::PrintFunctionsOfTime>(
+      Parallel::get_parallel_component<
+          observers::ObserverWriter<Metavariables>>(cache),
+      deadlock_dir + "functions_of_time.out");
+
+  {
+    auto cache_proxy = cache.get_this_proxy();
+
+    cache_proxy.print_mutable_cache_callbacks(deadlock_dir +
+                                              "/mutable_cache_callbacks.out");
+  }
+
+  Parallel::simple_action<::deadlock::PrintInterpolator>(
+      Parallel::get_parallel_component<intrp::Interpolator<Metavariables>>(
+          cache),
+      deadlock_dir);
+
+  const std::string intrp_target_file =
+      deadlock_dir + "/interpolation_targets.out";
+  tmpl::for_each<InterpolationTargetTags>(
+      [&cache, &intrp_target_file](const auto tag_v) {
+        using TargetTag = tmpl::type_from<decltype(tag_v)>;
+
+        Parallel::simple_action<::deadlock::PrintInterpolationTarget>(
+            Parallel::get_parallel_component<
+                intrp::InterpolationTarget<Metavariables, TargetTag>>(cache),
+            intrp_target_file);
+      });
+
+  if (alg::count(deadlocked_components, pretty_type::name<DgElementArray>()) ==
+      1) {
+    tmpl::for_each<ControlComponents>([&cache,
+                                       &deadlock_dir](auto component_v) {
+      using component = tmpl::type_from<decltype(component_v)>;
+      Parallel::simple_action<control_system::Actions::PrintCurrentMeasurement>(
+          Parallel::get_parallel_component<component>(cache),
+          deadlock_dir + "/control_systems.out");
+    });
+
+    const std::string element_array_file =
+        deadlock_dir + "/dg_element_array.out";
+    if constexpr (Parallel::is_dg_element_collection_v<DgElementArray>) {
+      Parallel::threaded_action<Parallel::Actions::SimpleActionOnElement<
+          ::deadlock::PrintElementInfo, true>>(
+          Parallel::get_parallel_component<DgElementArray>(cache),
+          element_array_file);
+    } else {
+      Parallel::simple_action<::deadlock::PrintElementInfo>(
+          Parallel::get_parallel_component<DgElementArray>(cache),
+          element_array_file);
+    }
+  }
+}
+}  // namespace gh::deadlock

src/Evolution/Executables/GeneralizedHarmonic/EvolveGhBinaryBlackHole.hpp

@@ -8,7 +8,6 @@
 
 #include "ControlSystem/Actions/InitializeMeasurements.hpp"
 #include "ControlSystem/Actions/LimitTimeStep.hpp"
-#include "ControlSystem/Actions/PrintCurrentMeasurement.hpp"
 #include "ControlSystem/Component.hpp"
 #include "ControlSystem/ControlErrors/Size/Factory.hpp"
 #include "ControlSystem/ControlErrors/Size/State.hpp"
@@ -25,19 +24,17 @@
 #include "DataStructures/Tensor/EagerMath/RaiseOrLowerIndex.hpp"
 #include "Domain/Creators/BinaryCompactObject.hpp"
 #include "Domain/Creators/CylindricalBinaryCompactObject.hpp"
-#include "Domain/FunctionsOfTime/OutputTimeBounds.hpp"
-#include "Domain/FunctionsOfTime/Tags.hpp"
 #include "Domain/Tags.hpp"
 #include "Domain/TagsCharacteristicSpeeds.hpp"
 #include "Evolution/Actions/RunEventsAndDenseTriggers.hpp"
 #include "Evolution/Actions/RunEventsAndTriggers.hpp"
 #include "Evolution/ComputeTags.hpp"
-#include "Evolution/Deadlock/PrintDgElementArray.hpp"
 #include "Evolution/DiscontinuousGalerkin/Actions/ApplyBoundaryCorrections.hpp"
 #include "Evolution/DiscontinuousGalerkin/Actions/ComputeTimeDerivative.hpp"
 #include "Evolution/DiscontinuousGalerkin/DgElementArray.hpp"
 #include "Evolution/DiscontinuousGalerkin/InboxTags.hpp"
 #include "Evolution/DiscontinuousGalerkin/Initialization/Mortars.hpp"
+#include "Evolution/Executables/GeneralizedHarmonic/Deadlock.hpp"
 #include "Evolution/Initialization/DgDomain.hpp"
 #include "Evolution/Initialization/Evolution.hpp"
 #include "Evolution/Initialization/NonconservativeSystem.hpp"
@@ -75,7 +72,6 @@
 #include "Options/String.hpp"
 #include "Parallel/Algorithms/AlgorithmSingleton.hpp"
 #include "Parallel/ArrayCollection/DgElementCollection.hpp"
-#include "Parallel/ArrayCollection/IsDgElementCollection.hpp"
 #include "Parallel/ArrayCollection/SimpleActionOnElement.hpp"
 #include "Parallel/GlobalCache.hpp"
 #include "Parallel/Invoke.hpp"
@@ -87,7 +83,6 @@
 #include "Parallel/PhaseControl/Factory.hpp"
 #include "Parallel/PhaseControl/VisitAndReturn.hpp"
 #include "Parallel/PhaseDependentActionList.hpp"
-#include "Parallel/Printf/Printf.hpp"
 #include "Parallel/Protocols/RegistrationMetavariables.hpp"
 #include "Parallel/Reduction.hpp"
 #include "ParallelAlgorithms/Actions/AddComputeTags.hpp"
@@ -127,6 +122,7 @@
 #include "ParallelAlgorithms/Events/ObserveTimeStepVolume.hpp"
 #include "ParallelAlgorithms/EventsAndDenseTriggers/DenseTrigger.hpp"
 #include "ParallelAlgorithms/EventsAndDenseTriggers/DenseTriggers/Factory.hpp"
+#include "ParallelAlgorithms/EventsAndTriggers/Actions/RunEventsOnFailure.hpp"
 #include "ParallelAlgorithms/EventsAndTriggers/Completion.hpp"
 #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
 #include "ParallelAlgorithms/EventsAndTriggers/EventsAndTriggers.hpp"
@@ -618,7 +614,11 @@ struct EvolutionMetavars {
                   ::domain::Actions::CheckFunctionsOfTimeAreReady<volume_dim>,
                   evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
                   Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
-                  PhaseControl::Actions::ExecutePhaseChange>>>>>;
+                  PhaseControl::Actions::ExecutePhaseChange>>,
+          Parallel::PhaseActions<
+              Parallel::Phase::PostFailureCleanup,
+              tmpl::list<Actions::RunEventsOnFailure<::Tags::Time>,
+                         Parallel::Actions::TerminatePhase>>>>>;
 
   struct BondiSachs : tt::ConformsTo<intrp::protocols::InterpolationTargetTag> {
     static std::string name() { return "BondiSachsInterpolation"; }
@@ -652,32 +652,9 @@ struct EvolutionMetavars {
   static void run_deadlock_analysis_simple_actions(
       Parallel::GlobalCache<EvolutionMetavars>& cache,
       const std::vector<std::string>& deadlocked_components) {
-    const auto& functions_of_time =
-        Parallel::get<::domain::Tags::FunctionsOfTime>(cache);
-
-    const std::string time_bounds =
-        ::domain::FunctionsOfTime::output_time_bounds(functions_of_time);
-
-    Parallel::printf("%s\n", time_bounds);
-
-    if (alg::count(deadlocked_components,
-                   pretty_type::name<gh_dg_element_array>()) == 1) {
-      tmpl::for_each<control_components>([&cache](auto component_v) {
-        using component = tmpl::type_from<decltype(component_v)>;
-        Parallel::simple_action<
-            control_system::Actions::PrintCurrentMeasurement>(
-            Parallel::get_parallel_component<component>(cache));
-      });
-
-      if constexpr (Parallel::is_dg_element_collection_v<gh_dg_element_array>) {
-        Parallel::threaded_action<Parallel::Actions::SimpleActionOnElement<
-            deadlock::PrintElementInfo, true>>(
-            Parallel::get_parallel_component<gh_dg_element_array>(cache));
-      } else {
-        Parallel::simple_action<deadlock::PrintElementInfo>(
-            Parallel::get_parallel_component<gh_dg_element_array>(cache));
-      }
-    }
+    gh::deadlock::run_deadlock_analysis_simple_actions<
+        gh_dg_element_array, control_components, interpolation_target_tags>(
+        cache, deadlocked_components);
   }
 
   struct amr : tt::ConformsTo<::amr::protocols::AmrMetavariables> {