diff --git a/examples/IntegrationBenchmark/include/EventAction.hh b/examples/IntegrationBenchmark/include/EventAction.hh index 200b0da3..9808529c 100644 --- a/examples/IntegrationBenchmark/include/EventAction.hh +++ b/examples/IntegrationBenchmark/include/EventAction.hh @@ -31,6 +31,8 @@ #include "G4UserEventAction.hh" #include "G4Timer.hh" +#include "RunAction.hh" + class DetectorConstruction; class EventActionMessenger; @@ -45,7 +47,7 @@ class EventActionMessenger; class EventAction : public G4UserEventAction { public: - EventAction(bool aDoValidation); + EventAction(RunAction *aRunAction); virtual ~EventAction(); /// Timer is started @@ -69,7 +71,7 @@ private: G4Timer fTimer; /// Messenger for this EventActionMessenger *fMessenger{nullptr}; - bool fDoValidation{false}; + RunAction *fRunAction{nullptr}; }; #endif /* EVENTACTION_HH */ diff --git a/examples/IntegrationBenchmark/include/Run.hh b/examples/IntegrationBenchmark/include/Run.hh index 338dbb66..ed571668 100644 --- a/examples/IntegrationBenchmark/include/Run.hh +++ b/examples/IntegrationBenchmark/include/Run.hh @@ -6,6 +6,8 @@ #include "G4Run.hh" +#include "RunAction.hh" + #define TAG_TYPE int template @@ -17,20 +19,20 @@ class TestManager; class Run : public G4Run { public: - Run(); + Run(RunAction *aRunAction); ~Run(); /** @brief Merge the results of the worker threads */ void Merge(const G4Run *run) override; TestManager *GetTestManager() const { return fTestManager; } - void SetDoBenchmark(bool aDoBenchmark) { fDoBenchmark = aDoBenchmark; } - bool GetDoBenchmark() { return fDoBenchmark; } - void SetDoValidation(bool aDoValidation) { fDoValidation = aDoValidation; } - bool GetDoValidation() { return fDoValidation; } + // void SetDoBenchmark(bool aDoBenchmark) { fDoBenchmark = aDoBenchmark; } + // bool GetDoBenchmark() { return fDoBenchmark; } + // void SetDoValidation(bool aDoValidation) { fDoValidation = aDoValidation; } + // bool GetDoValidation() { return fDoValidation; } /** @brief Compute and display collected metrics */ - void EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilename); + void EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilenam); /** * @brief Enum defining the timers that we can use for benchmarking @@ -66,8 +68,11 @@ public: private: TestManager *fTestManager; - bool fDoBenchmark; - bool fDoValidation; + RunAction *fRunAction; + // bool fDoBenchmark; + // bool fDoValidation; + // G4String fOutputDirectory; + // G4String fOutputFilename; }; #endif \ No newline at end of file diff --git a/examples/IntegrationBenchmark/include/RunAction.hh b/examples/IntegrationBenchmark/include/RunAction.hh index 1edebdc5..890d989c 100644 --- a/examples/IntegrationBenchmark/include/RunAction.hh +++ b/examples/IntegrationBenchmark/include/RunAction.hh @@ -58,6 +58,11 @@ public: G4Run *GenerateRun() override; + bool &GetDoBenchmark(){return fDoBenchmark;}; + bool &GetDoValidation(){return fDoValidation;}; + const G4String &GetOutputDirectory(){return fOutputDirectory;}; + const G4String &GetOutputFilename(){return fOutputFilename;}; + private: /// Pointer to detector construction to retrieve the detector dimensions to /// setup the histograms diff --git a/examples/IntegrationBenchmark/src/ActionInitialisation.cc b/examples/IntegrationBenchmark/src/ActionInitialisation.cc index 81567235..5a5af80c 100644 --- a/examples/IntegrationBenchmark/src/ActionInitialisation.cc +++ b/examples/IntegrationBenchmark/src/ActionInitialisation.cc @@ -56,9 +56,9 @@ void ActionInitialisation::BuildForMaster() const void ActionInitialisation::Build() const { SetUserAction(new PrimaryGeneratorAction()); - SetUserAction(new EventAction(fDoValidation)); RunAction *aRunAction = new RunAction(fOutputDirectory, fOutputFilename, fDoBenchmark, fDoValidation); SetUserAction(aRunAction); + SetUserAction(new EventAction(aRunAction)); TrackingAction *aTrackingAction = new TrackingAction(); SetUserAction(aTrackingAction); SteppingAction *aSteppingAction = new SteppingAction(); diff --git a/examples/IntegrationBenchmark/src/EventAction.cc b/examples/IntegrationBenchmark/src/EventAction.cc index d6b74783..760b9bb4 100644 --- a/examples/IntegrationBenchmark/src/EventAction.cc +++ b/examples/IntegrationBenchmark/src/EventAction.cc @@ -41,7 +41,8 @@ #include #include "Run.hh" -EventAction::EventAction(bool aDoValidation) : G4UserEventAction(), fHitCollectionID(-1), fTimer(), fDoValidation(aDoValidation) +EventAction::EventAction(RunAction *aRunAction) + : G4UserEventAction(), fHitCollectionID(-1), fTimer(), fRunAction(aRunAction) { fMessenger = new EventActionMessenger(this); } @@ -60,7 +61,7 @@ void EventAction::BeginOfEventAction(const G4Event *) // Get the Run object associated to this thread and start the timer for this event Run *currentRun = static_cast(G4RunManager::GetRunManager()->GetNonConstCurrentRun()); - if (currentRun->GetDoBenchmark()) { + if (fRunAction->GetDoBenchmark()) { currentRun->GetTestManager()->timerStart(Run::timers::EVENT); } // zero the counters @@ -81,7 +82,7 @@ void EventAction::EndOfEventAction(const G4Event *aEvent) // Get the Run object associated to this thread and stop the timer for this event Run *currentRun = static_cast(G4RunManager::GetRunManager()->GetNonConstCurrentRun()); auto aTestManager = currentRun->GetTestManager(); - if (currentRun->GetDoBenchmark()) { + if (fRunAction->GetDoBenchmark()) { aTestManager->timerStop(Run::timers::EVENT); } aTestManager->addToAccumulator(Run::accumulators::NUM_PARTICLES, aEvent->GetPrimaryVertex()->GetNumberOfParticle()); @@ -99,24 +100,6 @@ void EventAction::EndOfEventAction(const G4Event *aEvent) G4Exception("EventAction::GetHitsCollection()", "MyCode0001", FatalException, msg); } - if(fDoValidation) - { - // Get test manager - Run *currentRun = static_cast(G4RunManager::GetRunManager()->GetNonConstCurrentRun()); - auto testManager = currentRun->GetTestManager(); - - // Fill test manager with PvolID : Edep - for(auto &hit: *hitsCollection->GetVector()) - { - // Use IDs that won't overlap with other accumulators - auto id = hit->GetPhysicalVolumeId() + Run::accumulators::NUM_ACCUMULATORS; - testManager->addToAccumulator(id, hit->GetEdep()); - } - - // Store test manager - TestManagerStore::GetInstance()->RecordState(testManager); - } - SimpleHit *hit = nullptr; G4double hitEn = 0; G4double totalEnergy = 0; @@ -149,7 +132,29 @@ void EventAction::EndOfEventAction(const G4Event *aEvent) G4cout << "EndOfEventAction " << eventId << "Total energy deposited: " << totalEnergy / MeV << " MeV" << G4endl; } - if (currentRun->GetDoBenchmark()) { + if (fRunAction->GetDoValidation()) { + // Get test manager + // Run *currentRun = static_cast(G4RunManager::GetRunManager()->GetNonConstCurrentRun()); + // auto testManager = currentRun->GetTestManager(); + + // Fill test manager with PvolID : Edep + for (auto &hit : *hitsCollection->GetVector()) { + // Use IDs that won't overlap with other accumulators + auto id = hit->GetPhysicalVolumeId() + Run::accumulators::NUM_ACCUMULATORS; + // Reset the accumulator from the last event + // aTestManager->setAccumulator(id, 0); + // Set the accumulator + aTestManager->setAccumulator(id, hit->GetEdep()); + // aTestManager->addToAccumulator(id, hit->GetEdep()); + } + + // Write data to output file. Validation data can take a lot of memory, and we don't need to aggregate + // the results of multiple events at runtime, so for better performance it's easier to write the output here + aTestManager->exportCSV(false); + + // Store test manager + // TestManagerStore::GetInstance()->RecordState(aTestManager); + } else if (fRunAction->GetDoBenchmark()) { // Get the timings double eventTime = aTestManager->getDurationSeconds(Run::timers::EVENT); double nonEMTime = aTestManager->getAccumulator(Run::accumulators::NONEM_EVT); diff --git a/examples/IntegrationBenchmark/src/Run.cc b/examples/IntegrationBenchmark/src/Run.cc index f668f05a..d19d5d03 100644 --- a/examples/IntegrationBenchmark/src/Run.cc +++ b/examples/IntegrationBenchmark/src/Run.cc @@ -11,16 +11,19 @@ #define STDEV(N, MEAN, SUM_SQUARES) N > 1 ? sqrt((SUM_SQUARES - N * MEAN * MEAN) / N) : 0 -Run::Run() +Run::Run(RunAction *aRunAction) : fRunAction(aRunAction) { fTestManager = new TestManager(); + // Set output directory and filename, needed for validation where this test manager is used for output + fTestManager->setOutputDirectory(fRunAction->GetOutputDirectory()); + fTestManager->setOutputFilename(fRunAction->GetOutputFilename()); } Run::~Run() {} void Run::Merge(const G4Run *run) { - if (fDoBenchmark) { + if (fRunAction->GetDoBenchmark()) { const Run *localRun = static_cast(run); TestManager *aTestManager = localRun->GetTestManager(); @@ -46,7 +49,7 @@ void Run::Merge(const G4Run *run) void Run::EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilename) { - if (fDoBenchmark && !fDoValidation) { + if (fRunAction->GetDoBenchmark() && !fRunAction->GetDoValidation()) { // Printout of global statistics double runTime = fTestManager->getDurationSeconds(timers::TOTAL); double eventMean = fTestManager->getAccumulator(accumulators::EVENT_SUM) / GetNumberOfEvent(); @@ -73,21 +76,21 @@ void Run::EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilename) // aBenchmarkStates->size() should correspond to the number of events for (int i = 0; i < aBenchmarkStates->size(); i++) { - if (fDoValidation) { + if (fRunAction->GetDoValidation()) { // If we are taking validation data, export it to the specified file // Each benchmark state contains one counter per LogicalVolume // Export one CSV containing a list of volume IDs and Edep per event - for (auto iter = (*aBenchmarkStates)[i].begin(); iter != (*aBenchmarkStates)[i].end(); ++iter) { - if(iter->first >= Run::accumulators::NUM_ACCUMULATORS) - aOutputTestManager.setAccumulator(std::to_string(iter->first - Run::accumulators::NUM_ACCUMULATORS), iter->second); - } + // for (auto iter = (*aBenchmarkStates)[i].begin(); iter != (*aBenchmarkStates)[i].end(); ++iter) { + // if(iter->first >= Run::accumulators::NUM_ACCUMULATORS) + // aOutputTestManager.setAccumulator(std::to_string(iter->first - Run::accumulators::NUM_ACCUMULATORS), iter->second); + // } - aOutputTestManager.setOutputDirectory(aOutputDirectory); - aOutputTestManager.setOutputFilename(aOutputFilename); - aOutputTestManager.exportCSV(false); + // aOutputTestManager.setOutputDirectory(aOutputDirectory); + // aOutputTestManager.setOutputFilename(aOutputFilename); + // aOutputTestManager.exportCSV(false); - aOutputTestManager.reset(); - } else if (fDoBenchmark) { + // aOutputTestManager.reset(); + } else if (fRunAction->GetDoBenchmark()) { // Recover the results from each event and output them to the specified file double eventTime = (*aBenchmarkStates)[i][Run::timers::EVENT]; double nonEMTime = (*aBenchmarkStates)[i][Run::accumulators::NONEM_EVT]; @@ -96,8 +99,8 @@ void Run::EndOfRunSummary(G4String aOutputDirectory, G4String aOutputFilename) aOutputTestManager.setAccumulator("Non EM", nonEMTime); aOutputTestManager.setAccumulator("ECAL", ecalTime); - aOutputTestManager.setOutputDirectory(aOutputDirectory); - aOutputTestManager.setOutputFilename(aOutputFilename); + aOutputTestManager.setOutputDirectory(fRunAction->GetOutputDirectory()); + aOutputTestManager.setOutputFilename(fRunAction->GetOutputFilename()); aOutputTestManager.exportCSV(); aOutputTestManager.reset(); diff --git a/examples/IntegrationBenchmark/src/RunAction.cc b/examples/IntegrationBenchmark/src/RunAction.cc index 0aa04d04..1e620970 100644 --- a/examples/IntegrationBenchmark/src/RunAction.cc +++ b/examples/IntegrationBenchmark/src/RunAction.cc @@ -81,8 +81,6 @@ void RunAction::EndOfRunAction(const G4Run *) G4Run *RunAction::GenerateRun() { - fRun = new Run(); - fRun->SetDoBenchmark(fDoBenchmark); - fRun->SetDoValidation(fDoValidation); + fRun = new Run(this); return fRun; } diff --git a/include/AdePT/benchmarking/TestManager.h b/include/AdePT/benchmarking/TestManager.h index e04c2d91..16f26f59 100644 --- a/include/AdePT/benchmarking/TestManager.h +++ b/include/AdePT/benchmarking/TestManager.h @@ -40,7 +40,6 @@ struct TimeInfo { bool counting; ///< Whether the timer is running }; - template class TestManager { @@ -49,6 +48,7 @@ class TestManager { std::map fAccumulators; ///< Maps a tag to a double accumulator std::string fOutputDir; ///< Output directory std::string fOutputFilename; ///< Output filename + inline static std::mutex fWriteMutex; ///< Mutex for writing to file public: TestManager(){}; @@ -152,6 +152,10 @@ class TestManager { bool first_write = !std::filesystem::exists(path); std::ofstream output_file; + + // Acquire Mutex + std::lock_guard lock(fWriteMutex); + output_file.open(path, overwrite ? std::ofstream::trunc : std::ofstream::app); // Write the header only the first time @@ -218,4 +222,4 @@ class TestManager { #endif */ -#endif \ No newline at end of file +#endif