Use podio::Reader and podio::Writer with IOSvc (#233)

k4FWCore/CMakeLists.txt

@@ -31,7 +31,7 @@ gaudi_install(PYTHON)
 gaudi_add_library(k4FWCore
 		  SOURCES src/PodioDataSvc.cpp
               src/KeepDropSwitch.cpp
-                  LINK Gaudi::GaudiKernel podio::podioRootIO ROOT::Core ROOT::RIO ROOT::Tree
+                  LINK Gaudi::GaudiKernel podio::podioIO ROOT::Core ROOT::RIO ROOT::Tree
                   )
 target_include_directories(k4FWCore PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>

k4FWCore/components/IIOSvc.h

@@ -22,7 +22,7 @@
 #include "GaudiKernel/IInterface.h"
 
 #include "podio/CollectionBase.h"
-#include "podio/ROOTWriter.h"
+#include "podio/Writer.h"
 
 #include <memory>
 #include <vector>
@@ -31,11 +31,6 @@
  * The interface implemented by any class making IO with functional algorithms
  */
 class IIOSvc : virtual public IInterface {
-public:
-  struct EndOfInput : std::logic_error {
-    EndOfInput() : logic_error("Reached end of input while more data were expected"){};
-  };
-
 public:
   /// InterfaceID
   DeclareInterfaceID(IIOSvc, 1, 0);
@@ -48,10 +43,10 @@ class IIOSvc : virtual public IInterface {
                                                     next()                     = 0;
   virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
 
-  virtual std::shared_ptr<podio::ROOTWriter> getWriter()                                         = 0;
-  virtual void                               deleteWriter()                                      = 0;
-  virtual void                               deleteReader()                                      = 0;
-  virtual bool                               checkIfWriteCollection(const std::string& collName) = 0;
+  virtual podio::Writer& getWriter()                                         = 0;
+  virtual void           deleteWriter()                                      = 0;
+  virtual void           deleteReader()                                      = 0;
+  virtual bool           checkIfWriteCollection(const std::string& collName) = 0;
 };
 
 #endif

k4FWCore/components/IOSvc.cpp

@@ -21,14 +21,14 @@
 
 #include "podio/Frame.h"
 #include "podio/FrameCategories.h"
+#include "podio/Reader.h"
 
 #include "k4FWCore/FunctionalUtils.h"
 #include "k4FWCore/KeepDropSwitch.h"
 
 #include "GaudiKernel/AnyDataWrapper.h"
 #include "GaudiKernel/IEventProcessor.h"
 
-#include <algorithm>
 #include <mutex>
 #include <tuple>
 
@@ -42,6 +42,9 @@ StatusCode IOSvc::initialize() {
   if (!m_importedFromk4FWCore) {
     error() << "Use 'from k4FWCore import IOSvc' instead of 'from Configurables import IOSvc' to access the service"
             << endmsg;
+  }
+  if (m_outputType != "default" && m_outputType != "ROOT" && m_outputType != "RNTuple") {
+    error() << "Unknown input type: " << m_outputType << ", expected ROOT, RNTuple or default" << endmsg;
     return StatusCode::FAILURE;
   }
 
@@ -50,15 +53,13 @@ StatusCode IOSvc::initialize() {
     m_readingFileNames = m_readingFileNamesDeprecated;
   }
   if (!m_readingFileNames.empty()) {
-    info() << m_readingFileNames.size() << " files to be read" << endmsg;
-    m_reader = std::make_unique<podio::ROOTReader>();
     try {
-      m_reader->openFiles(m_readingFileNames);
+      m_reader = podio::makeReader(m_readingFileNames.value());
     } catch (std::runtime_error& e) {
       error() << "Error when opening files: " << e.what() << endmsg;
       return StatusCode::FAILURE;
     }
-    m_entries = m_reader->getEntries(podio::Category::Event);
+    m_entries = m_reader->getEvents();
   }
 
   if ((m_entries && m_firstEventEntry >= m_entries) || m_firstEventEntry < 0) {
@@ -97,7 +98,7 @@ StatusCode IOSvc::initialize() {
     if (std::find(categories.begin(), categories.end(), podio::Category::Metadata) != categories.end() &&
         m_reader->getEntries(podio::Category::Metadata) > 0) {
       info() << "Setting metadata frame" << endmsg;
-      m_metadataSvc->setFrame(m_reader->readEntry(podio::Category::Metadata, 0));
+      m_metadataSvc->setFrame(m_reader->readFrame(podio::Category::Metadata, 0));
     }
   }
 
@@ -113,7 +114,7 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std:
   {
     std::scoped_lock<std::mutex> lock(m_changeBufferLock);
     if (m_nextEntry < m_entries) {
-      frame = podio::Frame(m_reader->readEntry(podio::Category::Event, m_nextEntry));
+      frame = podio::Frame(m_reader->readEvent(m_nextEntry));
     } else {
       return std::make_tuple(std::vector<std::shared_ptr<podio::CollectionBase>>(), std::vector<std::string>(),
                              std::move(frame));

k4FWCore/components/IOSvc.h

@@ -26,8 +26,8 @@
 #include "GaudiKernel/IIncidentSvc.h"
 #include "GaudiKernel/Service.h"
 
-#include "podio/ROOTReader.h"
-#include "podio/ROOTWriter.h"
+#include "podio/Reader.h"
+#include "podio/Writer.h"
 
 #include "IIOSvc.h"
 #include "k4FWCore/IMetadataSvc.h"
@@ -62,7 +62,8 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   Gaudi::Property<std::string> m_writingFileName{this, "Output", {}, "List of files to write output to"};
   Gaudi::Property<std::vector<std::string>> m_outputCommands{
       this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
-  Gaudi::Property<std::string> m_inputType{this, "IOType", "ROOT", "Type of input file (ROOT, RNTuple)"};
+  Gaudi::Property<std::string> m_outputType{this, "OutputType", "default",
+                                            "Type of the output file (ROOT or RNTuple, or default)"};
 
   Gaudi::Property<bool> m_importedFromk4FWCore{
       this, "ImportedFromk4FWCore", false,
@@ -73,14 +74,14 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
 
   KeepDropSwitch m_switch;
 
-  std::unique_ptr<podio::ROOTReader> m_reader{nullptr};
-  std::shared_ptr<podio::ROOTWriter> m_writer{nullptr};
+  std::optional<podio::Reader> m_reader;
+  std::optional<podio::Writer> m_writer;
 
-  std::shared_ptr<podio::ROOTWriter> getWriter() override {
+  podio::Writer& getWriter() override {
     if (!m_writer) {
-      m_writer = std::make_shared<podio::ROOTWriter>(m_writingFileName.value());
+      m_writer = podio::makeWriter(m_writingFileName.value(), m_outputType);
     }
-    return m_writer;
+    return *m_writer;
   }
 
   // Gaudi doesn't always run the destructor of the Services so we have to

k4FWCore/components/Writer.cpp

@@ -120,10 +120,10 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     if (const char* env_key4hep_stack = std::getenv("KEY4HEP_STACK")) {
       config_metadata_frame.putParameter("key4hepstack", env_key4hep_stack);
     }
-    iosvc->getWriter()->writeFrame(config_metadata_frame, "configuration_metadata");
+    iosvc->getWriter().writeFrame(config_metadata_frame, "configuration_metadata");
 
     if (const auto* metadata_frame = m_metadataSvc->getFrame(); metadata_frame) {
-      iosvc->getWriter()->writeFrame(*metadata_frame, podio::Category::Metadata);
+      iosvc->getWriter().writeFrame(*metadata_frame, podio::Category::Metadata);
     }
 
     iosvc->deleteWriter();
@@ -262,7 +262,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     }
 
     debug() << "Writing frame" << endmsg;
-    iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event, m_collectionsToSave);
+    iosvc->getWriter().writeFrame(ptr->getData(), podio::Category::Event, m_collectionsToSave);
   }
 };
 

test/k4FWCoreTest/CMakeLists.txt

@@ -21,7 +21,7 @@ file(GLOB k4fwcoretest_plugin_sources src/components/*.cpp)
 
 gaudi_add_module(k4FWCoreTestPlugins
                  SOURCES ${k4fwcoretest_plugin_sources}
-                 LINK Gaudi::GaudiKernel k4FWCore k4FWCore::k4Interface ROOT::Core ROOT::RIO ROOT::Tree EDM4HEP::edm4hepDict EDM4HEP::edm4hep)
+                 LINK Gaudi::GaudiKernel k4FWCore k4FWCore::k4Interface ROOT::Core ROOT::RIO ROOT::Tree podio::podioIO EDM4HEP::edm4hep)
 
 include(CTest)
 
@@ -172,6 +172,8 @@ add_test_with_env(FunctionalMetadataReadOldAlgorithm options/ExampleFunctionalMe
 add_test_with_env(FunctionalWrongImport options/ExampleFunctionalWrongImport.py)
 set_tests_properties(FunctionalWrongImport PROPERTIES PASS_REGULAR_EXPRESSION "ImportError: Importing ApplicationMgr or IOSvc from Configurables is not allowed.")
 add_test_with_env(FunctionalReadNthEvent options/ExampleFunctionalReadNthEvent.py ADD_TO_CHECK_FILES)
+add_test_with_env(FunctionalProducerRNTuple options/ExampleFunctionalProducerRNTuple.py ADD_TO_CHECK_FILES)
+add_test_with_env(FunctionalTTreeToRNTuple options/ExampleFunctionalTTreeToRNTuple.py ADD_TO_CHECK_FILES)
 
 # Do this after checking the files not to overwrite them
 add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION

test/k4FWCoreTest/options/ExampleFunctionalProducerRNTuple.py

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example using a producer with a single output and saving that to an RNTuple file
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducer
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
+
+iosvc = IOSvc("IOSvc")
+iosvc.Output = "functional_producer_rntuple.root"
+iosvc.OutputType = "RNTuple"
+
+
+producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
+
+ApplicationMgr(
+    TopAlg=[producer],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)

test/k4FWCoreTest/options/ExampleFunctionalTTreeToRNTuple.py

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example algorithm to modify a TTree into an RNTuple
+# Don't use! There is already a converter in podio called
+# podio-ttree-to-rntuple
+# Running with k4run will add some extra configuration metadata to the file
+
+from Gaudi.Configuration import INFO
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
+
+iosvc = IOSvc("IOSvc")
+iosvc.Input = "functional_producer.root"
+iosvc.Output = "functional_producer_rntuple_converted.root"
+iosvc.OutputType = "RNTuple"
+
+ApplicationMgr(
+    TopAlg=[],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)

test/k4FWCoreTest/options/ExampleFunctionalWrongImport.py

@@ -17,8 +17,7 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a transformer to create new
-# data
+# This is an example of imports that should fail with the checks from utils.py
 
 
 # fmt: off

test/k4FWCoreTest/scripts/CheckOutputFiles.py

@@ -183,6 +183,28 @@ def check_metadata(filename, expected_metadata):
     },
 )
 
+reader = podio.root_io.Reader("functional_metadata.root")
+metadata = reader.get("metadata")[0]
+for key, value in zip(
+    [
+        "NumberOfParticles",
+        "ParticleTime",
+        "PDGValues",
+        "MetadataString",
+        "FinalizeMetadataInt",
+    ],
+    [3, 1.5, [1, 2, 3, 4], "hello", 10],
+):
+    if metadata.get_parameter(key) != value:
+        raise RuntimeError(
+            f"Metadata parameter {key} does not match the expected value, got {metadata.get_parameter(key)} but expected {value}"
+        )
+for rntuple in ["functional_producer_rntuple.root", "functional_producer_rntuple_converted.root"]:
+    reader = podio.root_io.RNTupleReader(f"{rntuple}")
+    frames = podio_reader.get("events")
+    if len(frames) != 10:
+        raise RuntimeError(f"Expected 10 events but got {len(frames)}")
+
 check_metadata(
     "functional_metadata_propagate.root",
     {