From 8e131233f1970bf251de6136d719560250b00ba4 Mon Sep 17 00:00:00 2001 From: Mateusz Jakub Fila Date: Tue, 20 Aug 2024 08:58:40 +0200 Subject: [PATCH 1/4] add test for creating EventHeader with GaudiHive --- test/k4FWCoreTest/CMakeLists.txt | 3 + .../options/checkEventHeaderConcurrent.py | 48 +++++++++++++ .../options/createEventHeaderConcurrent.py | 59 ++++++++++++++++ .../components/ExampleEventHeaderCheck.cpp | 68 +++++++++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 test/k4FWCoreTest/options/checkEventHeaderConcurrent.py create mode 100644 test/k4FWCoreTest/options/createEventHeaderConcurrent.py create mode 100644 test/k4FWCoreTest/src/components/ExampleEventHeaderCheck.cpp diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index e8d9e22c..3e624734 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -137,6 +137,9 @@ add_test_with_env(FunctionalCollectionMerger options/ExampleFunctionalCollection add_test_with_env(FunctionalFilterFile options/ExampleFunctionalFilterFile.py) add_test_with_env(FunctionalMetadata options/ExampleFunctionalMetadata.py) add_test_with_env(FunctionalMetadataOldAlgorithm options/ExampleFunctionalMetadataOldAlgorithm.py) +add_test_with_env(createEventHeaderConcurrent options/createEventHeaderConcurrent.py) +add_test_with_env(checkEventHeaderConcurrent options/checkEventHeaderConcurrent.py PROPERTIES DEPENDS createEventHeaderConcurrent) + add_test_with_env(FunctionalWrongImport options/ExampleFunctionalWrongImport.py) set_tests_properties(FunctionalWrongImport PROPERTIES PASS_REGULAR_EXPRESSION "ImportError: Importing ApplicationMgr or IOSvc from Configurables is not allowed.") diff --git a/test/k4FWCoreTest/options/checkEventHeaderConcurrent.py b/test/k4FWCoreTest/options/checkEventHeaderConcurrent.py new file mode 100644 index 00000000..5c24269b --- /dev/null +++ b/test/k4FWCoreTest/options/checkEventHeaderConcurrent.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example checking EventHeader created in GaudiHive +# The checks include whether a correct range of EventNumber is created +# and if there are no duplicates + +from Gaudi.Configuration import * + +from Configurables import ExampleEventHeaderCheck +from Configurables import EventDataSvc +from k4FWCore import ApplicationMgr, IOSvc + + +eventHeaderCreator = ExampleEventHeaderCheck( + "eventHeaderCreator", + runNumber=42, + expectedEventNumbers=list(range(42, 42 + 10)), + OutputLevel=DEBUG, +) + +svc = IOSvc("IOSvc") +svc.input = "eventHeaderConcurrent.root" + +ApplicationMgr( + TopAlg=[eventHeaderCreator], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + MessageSvcType="InertMessageSvc", + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/createEventHeaderConcurrent.py b/test/k4FWCoreTest/options/createEventHeaderConcurrent.py new file mode 100644 index 00000000..10e8f4d0 --- /dev/null +++ b/test/k4FWCoreTest/options/createEventHeaderConcurrent.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example creating EventHeader created in GaudiHive + +from Gaudi.Configuration import * + +from Configurables import EventHeaderCreator +from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard, AvalancheSchedulerSvc +from k4FWCore import ApplicationMgr, IOSvc + +evtslots = 6 +threads = 6 + +whiteboard = HiveWhiteBoard( + "EventDataSvc", + EventSlots=evtslots, + ForceLeaves=True, +) + +slimeventloopmgr = HiveSlimEventLoopMgr( + "HiveSlimEventLoopMgr", SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING +) + +scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, ShowDataFlow=True, OutputLevel=WARNING) + + +eventHeaderCreator = EventHeaderCreator( + "eventHeaderCreator", runNumber=42, eventNumberOffset=42, OutputLevel=DEBUG +) + +svc = IOSvc("IOSvc") +svc.output = "eventHeaderConcurrent.root" + +ApplicationMgr( + TopAlg=[eventHeaderCreator], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[whiteboard], + EventLoop=slimeventloopmgr, + MessageSvcType="InertMessageSvc", + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderCheck.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderCheck.cpp new file mode 100644 index 00000000..c65abd8e --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleEventHeaderCheck.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "edm4hep/Constants.h" +#include "edm4hep/EventHeaderCollection.h" + +#include "k4FWCore/Consumer.h" + +#include + +#include + +#include +#include +#include +#include + +struct ExampleEventHeaderCheck final : k4FWCore::Consumer { + ExampleEventHeaderCheck(const std::string& name, ISvcLocator* svcLoc) + : Consumer(name, svcLoc, {KeyValues("EventHeaderName", {edm4hep::labels::EventHeader})}) {} + + void operator()(const edm4hep::EventHeaderCollection& evtHeaderColl) const override { + if (evtHeaderColl.empty()) { + throw std::runtime_error("EventHeader collection is empty"); + } + const auto evtHeader = evtHeaderColl[0]; + if (!evtHeader.isAvailable()) { + throw std::runtime_error("Cannot get a valid EventHeader"); + } + + if (evtHeader.getRunNumber() != m_runNumber) { + throw std::runtime_error(fmt::format("Run number is not set correctly (expected {}, actual {})", + m_runNumber.value(), evtHeader.getRunNumber())); + } + if (std::find(std::begin(m_expectedEventNumbers), std::end(m_expectedEventNumbers), evtHeader.getEventNumber()) == + std::end(m_expectedEventNumbers)) { + throw std::runtime_error(fmt::format("Event number {} is not in expected numbers", evtHeader.getEventNumber())); + } + auto lock = std::scoped_lock(m_mutex); + if (m_seenEventNumbers.contains(evtHeader.getEventNumber())) { + throw std::runtime_error(fmt::format("Event number {} is duplicated", evtHeader.getEventNumber())); + } + m_seenEventNumbers.insert(evtHeader.getEventNumber()); + } + + Gaudi::Property m_runNumber{this, "runNumber", 0, "The expected run number"}; + Gaudi::Property> m_expectedEventNumbers{this, "expectedEventNumbers", 0, "Expected event numbers"}; + mutable std::mutex m_mutex; + mutable std::set m_seenEventNumbers; +}; + +DECLARE_COMPONENT(ExampleEventHeaderCheck) From f1a2c95fb99c1637b24908c49cd6f9899c30ac8f Mon Sep 17 00:00:00 2001 From: Mateusz Jakub Fila Date: Tue, 20 Aug 2024 09:00:16 +0200 Subject: [PATCH 2/4] fix data race in EventHeaderCreator --- k4FWCore/components/EventHeaderCreator.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/k4FWCore/components/EventHeaderCreator.cpp b/k4FWCore/components/EventHeaderCreator.cpp index 1326e62a..182e1efc 100644 --- a/k4FWCore/components/EventHeaderCreator.cpp +++ b/k4FWCore/components/EventHeaderCreator.cpp @@ -26,13 +26,13 @@ EventHeaderCreator::EventHeaderCreator(const std::string& name, ISvcLocator* svc "Name of the EventHeaderCollection that will be stored in the output root file."); } -StatusCode EventHeaderCreator::execute(const EventContext&) const { - static int eventNumber = 0; +StatusCode EventHeaderCreator::execute(const EventContext& ctx) const { + auto eventNumber = ctx.evt(); debug() << "Filling EventHeader with runNumber " << int(m_runNumber) << " and eventNumber " << eventNumber + m_eventNumberOffset << endmsg; auto headers = m_headerCol.createAndPut(); auto header = headers->create(); header.setRunNumber(m_runNumber); - header.setEventNumber(eventNumber++ + m_eventNumberOffset); + header.setEventNumber(eventNumber + m_eventNumberOffset); return StatusCode::SUCCESS; } From 77f58b372dcb533b12ba608f5d3e3f443bab1658 Mon Sep 17 00:00:00 2001 From: Mateusz Jakub Fila Date: Tue, 20 Aug 2024 09:00:36 +0200 Subject: [PATCH 3/4] replace hardcoded EventHeaderCollection name with edm4hep label --- k4FWCore/components/EventHeaderCreator.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k4FWCore/components/EventHeaderCreator.h b/k4FWCore/components/EventHeaderCreator.h index a3b4dc40..c84ed458 100644 --- a/k4FWCore/components/EventHeaderCreator.h +++ b/k4FWCore/components/EventHeaderCreator.h @@ -19,6 +19,7 @@ #ifndef K4FWCORE_EVENTHEADERCREATOR #define K4FWCORE_EVENTHEADERCREATOR +#include #include "Gaudi/Algorithm.h" #include "k4FWCore/DataHandle.h" @@ -44,7 +45,8 @@ class EventHeaderCreator : public Gaudi::Algorithm { this, "eventNumberOffset", 0, "Event number offset, eventNumber will be filled with 'event_index + eventNumberOffset'"}; // datahandle for the EventHeader - mutable DataHandle m_headerCol{"EventHeader", Gaudi::DataHandle::Writer, this}; + mutable DataHandle m_headerCol{edm4hep::labels::EventHeader, + Gaudi::DataHandle::Writer, this}; }; #endif From 95161308fc1e7a676ca7b0cb7983b2762cbbb15f Mon Sep 17 00:00:00 2001 From: Mateusz Jakub Fila Date: Tue, 10 Sep 2024 12:56:29 +0200 Subject: [PATCH 4/4] test file contents with script rather than algo --- test/k4FWCoreTest/CMakeLists.txt | 4 +- test/k4FWCoreTest/options/CheckOutputFiles.py | 21 ++++++ .../options/checkEventHeaderConcurrent.py | 48 ------------- .../components/ExampleEventHeaderCheck.cpp | 68 ------------------- 4 files changed, 22 insertions(+), 119 deletions(-) delete mode 100644 test/k4FWCoreTest/options/checkEventHeaderConcurrent.py delete mode 100644 test/k4FWCoreTest/src/components/ExampleEventHeaderCheck.cpp diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index 3e624734..e65fc048 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -138,14 +138,12 @@ add_test_with_env(FunctionalFilterFile options/ExampleFunctionalFilterFile.py) add_test_with_env(FunctionalMetadata options/ExampleFunctionalMetadata.py) add_test_with_env(FunctionalMetadataOldAlgorithm options/ExampleFunctionalMetadataOldAlgorithm.py) add_test_with_env(createEventHeaderConcurrent options/createEventHeaderConcurrent.py) -add_test_with_env(checkEventHeaderConcurrent options/checkEventHeaderConcurrent.py PROPERTIES DEPENDS createEventHeaderConcurrent) - add_test_with_env(FunctionalWrongImport options/ExampleFunctionalWrongImport.py) set_tests_properties(FunctionalWrongImport PROPERTIES PASS_REGULAR_EXPRESSION "ImportError: Importing ApplicationMgr or IOSvc from Configurables is not allowed.") add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py) -set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc;FunctionalTransformerHist;FunctionalCollectionMerger;FunctionalFilterFile;FunctionalMetadata;FunctionalMetadataOldAlgorithm") +set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc;FunctionalTransformerHist;FunctionalCollectionMerger;FunctionalFilterFile;FunctionalMetadata;FunctionalMetadataOldAlgorithm;createEventHeaderConcurrent") # Do this after checking the files not to overwrite them add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py index 665c2562..90c8971d 100644 --- a/test/k4FWCoreTest/options/CheckOutputFiles.py +++ b/test/k4FWCoreTest/options/CheckOutputFiles.py @@ -188,3 +188,24 @@ def check_events(filename, number): raise RuntimeError( f"Metadata parameter {key} does not match the expected value, got {metadata.get_parameter(key)} but expected {value}" ) + +reader = podio.root_io.Reader("eventHeaderConcurrent.root") +events = reader.get("events") +expected_events_length = 10 +expected_run_number = 42 +expected_event_numbers = set(range(42, 42 + expected_events_length)) +seen_event_numbers = set() +if len(events) != expected_events_length: + raise RuntimeError("Number of events does not match expected number") +for frame in events: + event_header = frame.get("EventHeader")[0] + if (run_number := event_header.getRunNumber()) != expected_run_number: + raise RuntimeError( + f"Run number is not set correctly (expected {expected_run_number}, actual {run_number})" + ) + event_number = event_header.getEventNumber() + if event_number not in expected_event_numbers: + raise RuntimeError(f"Event number {event_number} is not in expected numbers") + if event_number in seen_event_numbers: + raise RuntimeError(f"Event number {event_number} is duplicated") + seen_event_numbers.add(event_number) diff --git a/test/k4FWCoreTest/options/checkEventHeaderConcurrent.py b/test/k4FWCoreTest/options/checkEventHeaderConcurrent.py deleted file mode 100644 index 5c24269b..00000000 --- a/test/k4FWCoreTest/options/checkEventHeaderConcurrent.py +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (c) 2014-2024 Key4hep-Project. -# -# This file is part of Key4hep. -# See https://key4hep.github.io/key4hep-doc/ for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This is an example checking EventHeader created in GaudiHive -# The checks include whether a correct range of EventNumber is created -# and if there are no duplicates - -from Gaudi.Configuration import * - -from Configurables import ExampleEventHeaderCheck -from Configurables import EventDataSvc -from k4FWCore import ApplicationMgr, IOSvc - - -eventHeaderCreator = ExampleEventHeaderCheck( - "eventHeaderCreator", - runNumber=42, - expectedEventNumbers=list(range(42, 42 + 10)), - OutputLevel=DEBUG, -) - -svc = IOSvc("IOSvc") -svc.input = "eventHeaderConcurrent.root" - -ApplicationMgr( - TopAlg=[eventHeaderCreator], - EvtSel="NONE", - EvtMax=10, - ExtSvc=[EventDataSvc("EventDataSvc")], - MessageSvcType="InertMessageSvc", - OutputLevel=INFO, -) diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderCheck.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderCheck.cpp deleted file mode 100644 index c65abd8e..00000000 --- a/test/k4FWCoreTest/src/components/ExampleEventHeaderCheck.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2014-2024 Key4hep-Project. - * - * This file is part of Key4hep. - * See https://key4hep.github.io/key4hep-doc/ for further info. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "edm4hep/Constants.h" -#include "edm4hep/EventHeaderCollection.h" - -#include "k4FWCore/Consumer.h" - -#include - -#include - -#include -#include -#include -#include - -struct ExampleEventHeaderCheck final : k4FWCore::Consumer { - ExampleEventHeaderCheck(const std::string& name, ISvcLocator* svcLoc) - : Consumer(name, svcLoc, {KeyValues("EventHeaderName", {edm4hep::labels::EventHeader})}) {} - - void operator()(const edm4hep::EventHeaderCollection& evtHeaderColl) const override { - if (evtHeaderColl.empty()) { - throw std::runtime_error("EventHeader collection is empty"); - } - const auto evtHeader = evtHeaderColl[0]; - if (!evtHeader.isAvailable()) { - throw std::runtime_error("Cannot get a valid EventHeader"); - } - - if (evtHeader.getRunNumber() != m_runNumber) { - throw std::runtime_error(fmt::format("Run number is not set correctly (expected {}, actual {})", - m_runNumber.value(), evtHeader.getRunNumber())); - } - if (std::find(std::begin(m_expectedEventNumbers), std::end(m_expectedEventNumbers), evtHeader.getEventNumber()) == - std::end(m_expectedEventNumbers)) { - throw std::runtime_error(fmt::format("Event number {} is not in expected numbers", evtHeader.getEventNumber())); - } - auto lock = std::scoped_lock(m_mutex); - if (m_seenEventNumbers.contains(evtHeader.getEventNumber())) { - throw std::runtime_error(fmt::format("Event number {} is duplicated", evtHeader.getEventNumber())); - } - m_seenEventNumbers.insert(evtHeader.getEventNumber()); - } - - Gaudi::Property m_runNumber{this, "runNumber", 0, "The expected run number"}; - Gaudi::Property> m_expectedEventNumbers{this, "expectedEventNumbers", 0, "Expected event numbers"}; - mutable std::mutex m_mutex; - mutable std::set m_seenEventNumbers; -}; - -DECLARE_COMPONENT(ExampleEventHeaderCheck)