Validation now prints after each event, which could mitigate meory us…

…age in large runs. More cleanup needed
apt-sim · Oct 7, 2024 · 9567119 · 9567119
1 parent 1efd25d
commit 9567119
Show file tree

Hide file tree

Showing 8 changed files with 75 additions and 53 deletions.
diff --git a/examples/IntegrationBenchmark/include/EventAction.hh b/examples/IntegrationBenchmark/include/EventAction.hh
@@ -31,6 +31,8 @@
 #include "G4UserEventAction.hh"
 #include "G4Timer.hh"
 
+#include "RunAction.hh"
+
 class DetectorConstruction;
 class EventActionMessenger;
 
@@ -45,7 +47,7 @@ class EventActionMessenger;
 
 class EventAction : public G4UserEventAction {
 public:
-  EventAction(bool aDoValidation);
+  EventAction(RunAction *aRunAction);
   virtual ~EventAction();
 
   /// Timer is started
@@ -69,7 +71,7 @@ private:
   G4Timer fTimer;
   /// Messenger for this
   EventActionMessenger *fMessenger{nullptr};
-  bool fDoValidation{false};
+  RunAction *fRunAction{nullptr};
 };
 
 #endif /* EVENTACTION_HH */
diff --git a/examples/IntegrationBenchmark/include/Run.hh b/examples/IntegrationBenchmark/include/Run.hh
@@ -6,6 +6,8 @@
 
 #include "G4Run.hh"
 
+#include "RunAction.hh"
+
 #define TAG_TYPE int
 
 template <class TTag>
@@ -17,20 +19,20 @@ class TestManager;
 class Run : public G4Run {
 
 public:
-  Run();
+  Run(RunAction *aRunAction);
   ~Run();
 
   /** @brief Merge the results of the worker threads */
   void Merge(const G4Run *run) override;
 
   TestManager<TAG_TYPE> *GetTestManager() const { return fTestManager; }
-  void SetDoBenchmark(bool aDoBenchmark) { fDoBenchmark = aDoBenchmark; }
-  bool GetDoBenchmark() { return fDoBenchmark; }
-  void SetDoValidation(bool aDoValidation) { fDoValidation = aDoValidation; }
-  bool GetDoValidation() { return fDoValidation; }
+  // void SetDoBenchmark(bool aDoBenchmark) { fDoBenchmark = aDoBenchmark; }
+  // bool GetDoBenchmark() { return fDoBenchmark; }
+  // void SetDoValidation(bool aDoValidation) { fDoValidation = aDoValidation; }
+  // bool GetDoValidation() { return fDoValidation; }
 
   /** @brief Compute and display collected metrics */
-  void EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilename);
+  void EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilenam);
 
   /**
    * @brief Enum defining the timers that we can use for benchmarking
@@ -66,8 +68,11 @@ public:
 
 private:
   TestManager<TAG_TYPE> *fTestManager;
-  bool fDoBenchmark;
-  bool fDoValidation;
+  RunAction *fRunAction;
+  // bool fDoBenchmark;
+  // bool fDoValidation;
+  // G4String fOutputDirectory;
+  // G4String fOutputFilename;
 };
 
 #endif
diff --git a/examples/IntegrationBenchmark/include/RunAction.hh b/examples/IntegrationBenchmark/include/RunAction.hh
@@ -58,6 +58,11 @@ public:
 
   G4Run *GenerateRun() override;
 
+  bool &GetDoBenchmark(){return fDoBenchmark;};
+  bool &GetDoValidation(){return fDoValidation;};
+  const G4String &GetOutputDirectory(){return fOutputDirectory;};
+  const G4String &GetOutputFilename(){return fOutputFilename;};
+
 private:
   /// Pointer to detector construction to retrieve the detector dimensions to
   /// setup the histograms

diff --git a/examples/IntegrationBenchmark/src/ActionInitialisation.cc b/examples/IntegrationBenchmark/src/ActionInitialisation.cc
@@ -56,9 +56,9 @@ void ActionInitialisation::BuildForMaster() const
 void ActionInitialisation::Build() const
 {
   SetUserAction(new PrimaryGeneratorAction());
-  SetUserAction(new EventAction(fDoValidation));
   RunAction *aRunAction = new RunAction(fOutputDirectory, fOutputFilename, fDoBenchmark, fDoValidation);
   SetUserAction(aRunAction);
+  SetUserAction(new EventAction(aRunAction));
   TrackingAction *aTrackingAction = new TrackingAction();
   SetUserAction(aTrackingAction);
   SteppingAction *aSteppingAction = new SteppingAction();

diff --git a/examples/IntegrationBenchmark/src/EventAction.cc b/examples/IntegrationBenchmark/src/EventAction.cc
@@ -41,7 +41,8 @@
 #include <AdePT/benchmarking/TestManagerStore.h>
 #include "Run.hh"
 
-EventAction::EventAction(bool aDoValidation) : G4UserEventAction(), fHitCollectionID(-1), fTimer(), fDoValidation(aDoValidation)
+EventAction::EventAction(RunAction *aRunAction)
+    : G4UserEventAction(), fHitCollectionID(-1), fTimer(), fRunAction(aRunAction)
 {
   fMessenger = new EventActionMessenger(this);
 }
@@ -60,7 +61,7 @@ void EventAction::BeginOfEventAction(const G4Event *)
 
   // Get the Run object associated to this thread and start the timer for this event
   Run *currentRun = static_cast<Run *>(G4RunManager::GetRunManager()->GetNonConstCurrentRun());
-  if (currentRun->GetDoBenchmark()) {
+  if (fRunAction->GetDoBenchmark()) {
     currentRun->GetTestManager()->timerStart(Run::timers::EVENT);
   }
   // zero the counters
@@ -81,7 +82,7 @@ void EventAction::EndOfEventAction(const G4Event *aEvent)
   // Get the Run object associated to this thread and stop the timer for this event
   Run *currentRun   = static_cast<Run *>(G4RunManager::GetRunManager()->GetNonConstCurrentRun());
   auto aTestManager = currentRun->GetTestManager();
-  if (currentRun->GetDoBenchmark()) {
+  if (fRunAction->GetDoBenchmark()) {
     aTestManager->timerStop(Run::timers::EVENT);
   }
   aTestManager->addToAccumulator(Run::accumulators::NUM_PARTICLES, aEvent->GetPrimaryVertex()->GetNumberOfParticle());
@@ -99,24 +100,6 @@ void EventAction::EndOfEventAction(const G4Event *aEvent)
     G4Exception("EventAction::GetHitsCollection()", "MyCode0001", FatalException, msg);
   }
 
-  if(fDoValidation)
-  {
-    // Get test manager
-    Run *currentRun = static_cast<Run *>(G4RunManager::GetRunManager()->GetNonConstCurrentRun());
-    auto testManager = currentRun->GetTestManager();
-
-    // Fill test manager with PvolID : Edep
-    for(auto &hit: *hitsCollection->GetVector())
-    {
-      // Use IDs that won't overlap with other accumulators
-      auto id = hit->GetPhysicalVolumeId() + Run::accumulators::NUM_ACCUMULATORS;
-      testManager->addToAccumulator(id, hit->GetEdep());
-    }
-
-    // Store test manager
-    TestManagerStore<int>::GetInstance()->RecordState(testManager);
-  }
-
   SimpleHit *hit       = nullptr;
   G4double hitEn       = 0;
   G4double totalEnergy = 0;
@@ -149,7 +132,29 @@ void EventAction::EndOfEventAction(const G4Event *aEvent)
     G4cout << "EndOfEventAction " << eventId << "Total energy deposited: " << totalEnergy / MeV << " MeV" << G4endl;
   }
 
-  if (currentRun->GetDoBenchmark()) {
+  if (fRunAction->GetDoValidation()) {
+    // Get test manager
+    // Run *currentRun = static_cast<Run *>(G4RunManager::GetRunManager()->GetNonConstCurrentRun());
+    // auto testManager = currentRun->GetTestManager();
+
+    // Fill test manager with PvolID : Edep
+    for (auto &hit : *hitsCollection->GetVector()) {
+      // Use IDs that won't overlap with other accumulators
+      auto id = hit->GetPhysicalVolumeId() + Run::accumulators::NUM_ACCUMULATORS;
+      // Reset the accumulator from the last event
+      // aTestManager->setAccumulator(id, 0);
+      // Set the accumulator
+      aTestManager->setAccumulator(id, hit->GetEdep());
+      // aTestManager->addToAccumulator(id, hit->GetEdep());
+    }
+
+    // Write data to output file. Validation data can take a lot of memory, and we don't need to aggregate
+    // the results of multiple events at runtime, so for better performance it's easier to write the output here
+    aTestManager->exportCSV(false);
+
+    // Store test manager
+    // TestManagerStore<int>::GetInstance()->RecordState(aTestManager);
+  } else if (fRunAction->GetDoBenchmark()) {
     // Get the timings
     double eventTime = aTestManager->getDurationSeconds(Run::timers::EVENT);
     double nonEMTime = aTestManager->getAccumulator(Run::accumulators::NONEM_EVT);

diff --git a/examples/IntegrationBenchmark/src/Run.cc b/examples/IntegrationBenchmark/src/Run.cc
@@ -11,16 +11,19 @@
 
 #define STDEV(N, MEAN, SUM_SQUARES) N > 1 ? sqrt((SUM_SQUARES - N * MEAN * MEAN) / N) : 0
 
-Run::Run()
+Run::Run(RunAction *aRunAction) : fRunAction(aRunAction)
 {
   fTestManager = new TestManager<TAG_TYPE>();
+  // Set output directory and filename, needed for validation where this test manager is used for output
+  fTestManager->setOutputDirectory(fRunAction->GetOutputDirectory());
+  fTestManager->setOutputFilename(fRunAction->GetOutputFilename());
 }
 
 Run::~Run() {}
 
 void Run::Merge(const G4Run *run)
 {
-  if (fDoBenchmark) {
+  if (fRunAction->GetDoBenchmark()) {
     const Run *localRun = static_cast<const Run *>(run);
 
     TestManager<TAG_TYPE> *aTestManager = localRun->GetTestManager();
@@ -46,7 +49,7 @@ void Run::Merge(const G4Run *run)
 
 void Run::EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilename)
 {
-  if (fDoBenchmark && !fDoValidation) {
+  if (fRunAction->GetDoBenchmark() && !fRunAction->GetDoValidation()) {
     // Printout of global statistics
     double runTime    = fTestManager->getDurationSeconds(timers::TOTAL);
     double eventMean  = fTestManager->getAccumulator(accumulators::EVENT_SUM) / GetNumberOfEvent();
@@ -73,21 +76,21 @@ void Run::EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilename)
 
   // aBenchmarkStates->size() should correspond to the number of events
   for (int i = 0; i < aBenchmarkStates->size(); i++) {
-    if (fDoValidation) {
+    if (fRunAction->GetDoValidation()) {
       // If we are taking validation data, export it to the specified file
       // Each benchmark state contains one counter per LogicalVolume
       // Export one CSV containing a list of volume IDs and Edep per event
-      for (auto iter = (*aBenchmarkStates)[i].begin(); iter != (*aBenchmarkStates)[i].end(); ++iter) {
-        if(iter->first >= Run::accumulators::NUM_ACCUMULATORS)
-          aOutputTestManager.setAccumulator(std::to_string(iter->first - Run::accumulators::NUM_ACCUMULATORS), iter->second);
-      }
+      // for (auto iter = (*aBenchmarkStates)[i].begin(); iter != (*aBenchmarkStates)[i].end(); ++iter) {
+      //   if(iter->first >= Run::accumulators::NUM_ACCUMULATORS)
+      //     aOutputTestManager.setAccumulator(std::to_string(iter->first - Run::accumulators::NUM_ACCUMULATORS), iter->second);
+      // }
 
-      aOutputTestManager.setOutputDirectory(aOutputDirectory);
-      aOutputTestManager.setOutputFilename(aOutputFilename);
-      aOutputTestManager.exportCSV(false);
+      // aOutputTestManager.setOutputDirectory(aOutputDirectory);
+      // aOutputTestManager.setOutputFilename(aOutputFilename);
+      // aOutputTestManager.exportCSV(false);
 
-      aOutputTestManager.reset();
-    } else if (fDoBenchmark) {
+      // aOutputTestManager.reset();
+    } else if (fRunAction->GetDoBenchmark()) {
       // Recover the results from each event and output them to the specified file
       double eventTime = (*aBenchmarkStates)[i][Run::timers::EVENT];
       double nonEMTime = (*aBenchmarkStates)[i][Run::accumulators::NONEM_EVT];
@@ -96,8 +99,8 @@ void Run::EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilename)
       aOutputTestManager.setAccumulator("Non EM", nonEMTime);
       aOutputTestManager.setAccumulator("ECAL", ecalTime);
 
-      aOutputTestManager.setOutputDirectory(aOutputDirectory);
-      aOutputTestManager.setOutputFilename(aOutputFilename);
+      aOutputTestManager.setOutputDirectory(fRunAction->GetOutputDirectory());
+      aOutputTestManager.setOutputFilename(fRunAction->GetOutputFilename());
       aOutputTestManager.exportCSV();
 
       aOutputTestManager.reset();

diff --git a/examples/IntegrationBenchmark/src/RunAction.cc b/examples/IntegrationBenchmark/src/RunAction.cc
@@ -81,8 +81,6 @@ void RunAction::EndOfRunAction(const G4Run *)
 
 G4Run *RunAction::GenerateRun()
 {
-  fRun = new Run();
-  fRun->SetDoBenchmark(fDoBenchmark);
-  fRun->SetDoValidation(fDoValidation);
+  fRun = new Run(this);
   return fRun;
 }
diff --git a/include/AdePT/benchmarking/TestManager.h b/include/AdePT/benchmarking/TestManager.h
@@ -40,7 +40,6 @@ struct TimeInfo {
   bool counting;                                                        ///< Whether the timer is running
 };
 
-
 template <class TTag>
 class TestManager {
 
@@ -49,6 +48,7 @@ class TestManager {
   std::map<TTag, double> fAccumulators; ///< Maps a tag to a double accumulator
   std::string fOutputDir;               ///< Output directory
   std::string fOutputFilename;          ///< Output filename
+  inline static std::mutex fWriteMutex;        ///< Mutex for writing to file
 
 public:
   TestManager(){};
@@ -152,6 +152,10 @@ class TestManager {
     bool first_write = !std::filesystem::exists(path);
 
     std::ofstream output_file;
+
+    // Acquire Mutex
+    std::lock_guard<std::mutex> lock(fWriteMutex);
+
     output_file.open(path, overwrite ? std::ofstream::trunc : std::ofstream::app);
 
     // Write the header only the first time
@@ -218,4 +222,4 @@ class TestManager {
 #endif
 */
 
-#endif
+#endif