Fix overwritting the input file for IOSvc from the command line (#243)

k4FWCore/scripts/k4run

@@ -24,7 +24,6 @@ FILTER_GAUDI_PROPS = [
     "NeededResources",
     "PropertiesPrint",
     "RegisterForContextService",
-    "RegularRowFormat",
     "RequireObjects",
     "RootInTES",
     "StatPrint",
@@ -87,6 +86,10 @@ def add_arguments(parser, app_mgr):
             if proptype is list:
                 if value:
                     proptype = type(value[0])
+                # If the list is empty, we can't determine the type
+                # Assume it's a string by default and hope it will work
+                else:
+                    proptype = str
                 args = "+"
 
             # add the argument twice, once as "--PodioOutput.filename"
@@ -224,6 +227,12 @@ def main():
     # Allow graceful exit with Ctrl + C
     ApplicationMgr().StopOnSignal = True
 
+    # Apply fixes from the k4FWCore wrapper
+
+    from k4FWCore import ApplicationMgr
+
+    ApplicationMgr().fix_properties()
+
     from Gaudi.Main import gaudimain
 
     gaudi = gaudimain()

python/k4FWCore/ApplicationMgr.py

@@ -16,9 +16,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from Configurables import ApplicationMgr as AppMgr
-from Configurables import Reader, Writer, IOSvc, MetadataSvc, Gaudi__Sequencer, EventLoopMgr
 import os
+from Configurables import ApplicationMgr as AppMgr
+from Configurables import Reader, Writer, IOSvc, Gaudi__Sequencer, EventLoopMgr
 from podio.root_io import Reader as PodioReader
 
 
@@ -37,89 +37,91 @@ class ApplicationMgr:
 
     def __init__(self, **kwargs):
         self._mgr = AppMgr(**kwargs)
+
+    def fix_properties(self):
         # If there isn't an EventLoopMgr then it's the default
         # This will suppress two warnings about not using external input
         try:
             self._mgr.EventLoop
         except AttributeError:
             self._mgr.EventLoop = EventLoopMgr(Warnings=False)
 
-        for conf in frozenset(self._mgr.allConfigurables.values()):
-            if isinstance(conf, MetadataSvc):
-                self._mgr.ExtSvc.append(conf)
-            if not isinstance(conf, IOSvc):
-                continue
-            props = conf.getPropertiesWithDescription()
-            reader = writer = None
-            add_reader = False
-            for alg in self._mgr.TopAlg:
-                if isinstance(alg, Reader):
-                    reader = alg
-                elif isinstance(alg, Writer):
-                    writer = alg
-            # Remove "input" when the corresponding property is removed from IOSvc
-            if reader is None and (props["input"][0] or props["Input"][0]):
-                reader = Reader("k4FWCore__Reader")
-                add_reader = True
-            # It seems for a single string the default without a value is '<no value>'
-            # while for a list it's an empty list
-            # Remove "output" when the corresponding property is removed from IOSvc
-            if (
-                writer is None
-                and (props["output"][0] and props["output"][0] != "<no value>")
-                or (props["Output"][0] and props["Output"][0] != "<no value>")
-            ):
-                writer = Writer("k4FWCore__Writer")
-            # Let's tell the Reader one of the input files so it can
-            # know which collections it's going to read
-            if reader is not None:
-                # Open the files and get the number of events. This is necessary to
-                # avoid errors when running multithreaded since if we have, for
-                # example, 10 events and we are running 9 at the same time, then
-                # (possibly) the first 9 complete and 9 more are scheduled, out of
-                # which only one will be finished without errors. If we know the
-                # number of events in advance then we can just schedule those.
-                inp = None
-                if props["input"][0]:
-                    inp = "input"
-                elif props["Input"][0]:
-                    inp = "Input"
-                if inp:
-                    if os.path.exists(props[inp][0][0]):
-                        path = props[inp][0][0]
-                    else:
-                        path = os.getcwd() + "/" + props[inp][0][0]
-                    podio_reader = PodioReader(path)
-                    if self._mgr.EvtMax == -1:
-                        self._mgr.EvtMax = podio_reader._reader.getEntries("events")
-                    try:
-                        frame = podio_reader.get("events")[0]
-                    except IndexError:
-                        print("Warning, the events category wasn't found in the input file")
-                        raise
-                    collections = list(frame.getAvailableCollections())
-                    reader.InputCollections = collections
-            self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg
-            # Assume the writer is at the end
-            # Algorithms are wrapped with Sequential=False so that they can run in parallel
-            # The algorithms and Writer are wrapped with Sequential=True so that they can not
-            # run in parallel
-            if writer:
-                self._mgr.TopAlg = [
-                    Gaudi__Sequencer(
-                        "k4FWCore__Sequencer",
-                        Sequential=True,
-                        Members=[
-                            Gaudi__Sequencer(
-                                "k4FWCore__Algs",
-                                Members=self._mgr.TopAlg,
-                                Sequential=False,
-                                ShortCircuit=False,
-                            ),
-                            writer,
-                        ],
-                    )
-                ]
+        if "MetadataSvc" in self._mgr.allConfigurables:
+            self._mgr.ExtSvc.append(self._mgr.allConfigurables["MetadataSvc"])
+
+        if "IOSvc" not in self._mgr.allConfigurables:
+            return
+        if not isinstance(self._mgr.allConfigurables["IOSvc"], IOSvc):
+            raise TypeError("The IOSvc is not an instance of IOSvc")
+        conf = self._mgr.allConfigurables["IOSvc"]
+
+        props = conf.getPropertiesWithDescription()
+        reader = writer = None
+        add_reader = False
+        for alg in self._mgr.TopAlg:
+            if isinstance(alg, Reader):
+                reader = alg
+            elif isinstance(alg, Writer):
+                writer = alg
+        # Remove "input" when the corresponding property is removed from IOSvc
+        if reader is None and (props["input"][0] or props["Input"][0]):
+            reader = Reader("k4FWCore__Reader")
+            add_reader = True
+        # It seems for a single string the default without a value is '<no value>'
+        # while for a list it's an empty list
+        # Remove "output" when the corresponding property is removed from IOSvc
+        if (
+            writer is None
+            and (props["output"][0] and props["output"][0] != "<no value>")
+            or (props["Output"][0] and props["Output"][0] != "<no value>")
+        ):
+            writer = Writer("k4FWCore__Writer")
+        # Let's tell the Reader one of the input files so it can
+        # know which collections it's going to read
+        if reader is not None:
+            # Open the files and get the number of events. This is necessary to
+            # avoid errors when running multithreaded since if we have, for
+            # example, 10 events and we are running 9 at the same time, then
+            # (possibly) the first 9 complete and 9 more are scheduled, out of
+            # which only one will be finished without errors. If we know the
+            # number of events in advance then we can just schedule those.
+            inp = None
+            if props["input"][0]:
+                inp = "input"
+            elif props["Input"][0]:
+                inp = "Input"
+            if inp:
+                podio_reader = PodioReader(props[inp][0])
+                if self._mgr.EvtMax == -1:
+                    self._mgr.EvtMax = podio_reader._reader.getEntries("events")
+                try:
+                    frame = podio_reader.get("events")[0]
+                except IndexError:
+                    print("Warning, the events category wasn't found in the input file")
+                    raise
+                collections = list(frame.getAvailableCollections())
+                reader.InputCollections = collections
+        self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg
+        # Assume the writer is at the end
+        # Algorithms are wrapped with Sequential=False so that they can run in parallel
+        # The algorithms and Writer are wrapped with Sequential=True so that they can not
+        # run in parallel
+        if writer:
+            self._mgr.TopAlg = [
+                Gaudi__Sequencer(
+                    "k4FWCore__Sequencer",
+                    Sequential=True,
+                    Members=[
+                        Gaudi__Sequencer(
+                            "k4FWCore__Algs",
+                            Members=self._mgr.TopAlg,
+                            Sequential=False,
+                            ShortCircuit=False,
+                        ),
+                        writer,
+                    ],
+                )
+            ]
 
     def __getattr__(self, name):
         return getattr(self._mgr, name)

test/k4FWCoreTest/CMakeLists.txt

@@ -168,16 +168,23 @@ add_test_with_env(FunctionalMetadataRead options/ExampleFunctionalMetadataRead.p
 add_test_with_env(FunctionalMetadataOldAlgorithm options/ExampleFunctionalMetadataOldAlgorithm.py ADD_TO_CHECK_FILES)
 add_test_with_env(createEventHeaderConcurrent options/createEventHeaderConcurrent.py ADD_TO_CHECK_FILES)
 add_test_with_env(FunctionalMetadataReadOldAlgorithm options/ExampleFunctionalMetadataReadOldAlgorithm.py PROPERTIES DEPENDS ExampleFunctionalMetadataOldAlgorithm ADD_TO_CHECK_FILES)
+add_test_with_env(FunctionalNonExistingFile options/ExampleFunctionalNonExistingFile.py)
+set_tests_properties(FunctionalNonExistingFile PROPERTIES PASS_REGULAR_EXPRESSION "File.*couldn't be found")
+add_test_with_env(FunctionalNonExistingFileOverwritten options/ExampleFunctionalNonExistingFile.py --IOSvc.Input functional_producer.root)
+add_test_with_env(FunctionalFileTooLong options/ExampleFunctionalFile.py -n 999 --IOSvc.Output toolong.root PROPERTIES PASS_REGULAR_EXPRESSION
+                  "Application Manager Terminated successfully with a user requested ScheduledStop")
+add_test_with_env(FunctionalFileTooShort options/ExampleFunctionalFile.py -n 2 --IOSvc.Output two_events.root ADD_TO_CHECK_FILES)
+add_test_with_env(FunctionalFileCLI options/ExampleFunctionalNoFile.py --IOSvc.Input functional_producer.root --IOSvc.Output functional_transformer_cli.root ADD_TO_CHECK_FILES)
+set_tests_properties(FunctionalFileCLI PROPERTIES DEPENDS FunctionalProducer)
+add_test_with_env(FunctionalFileCLIMultiple options/ExampleFunctionalNoFile.py --IOSvc.Input functional_producer.root functional_producer.root --IOSvc.Output functional_transformer_cli_multiple.root -n -1 ADD_TO_CHECK_FILES)
+set_tests_properties(FunctionalFileCLIMultiple PROPERTIES DEPENDS FunctionalProducer)
 
 add_test_with_env(FunctionalWrongImport options/ExampleFunctionalWrongImport.py)
 set_tests_properties(FunctionalWrongImport PROPERTIES PASS_REGULAR_EXPRESSION "ImportError: Importing ApplicationMgr or IOSvc from Configurables is not allowed.")
 add_test_with_env(FunctionalReadNthEvent options/ExampleFunctionalReadNthEvent.py ADD_TO_CHECK_FILES)
 add_test_with_env(FunctionalProducerRNTuple options/ExampleFunctionalProducerRNTuple.py ADD_TO_CHECK_FILES)
 add_test_with_env(FunctionalTTreeToRNTuple options/ExampleFunctionalTTreeToRNTuple.py ADD_TO_CHECK_FILES)
 
-# Do this after checking the files not to overwrite them
-add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION
-                  "Application Manager Terminated successfully with a user requested ScheduledStop")
 
 # The following is done to make the tests work without installing the files in
 # the installation directory. The k4FWCore in the build directory is populated by

test/k4FWCoreTest/options/ExampleFunctionalNoFile.py

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example when a file to be read is not provided, it will fail as it
+# is but the file can be provided through the command line
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalTransformer
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
+
+svc = IOSvc("IOSvc")
+
+transformer = ExampleFunctionalTransformer(
+    "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]
+)
+
+mgr = ApplicationMgr(
+    TopAlg=[transformer],
+    EvtSel="NONE",
+    EvtMax=1,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)

test/k4FWCoreTest/options/ExampleFunctionalNonExistingFile.py

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file that doesn't exist, it will fail
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalTransformer
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
+
+svc = IOSvc()
+svc.Input = "non_existing_file.root"
+
+transformer = ExampleFunctionalTransformer(
+    "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]
+)
+
+mgr = ApplicationMgr(
+    TopAlg=[transformer],
+    EvtSel="NONE",
+    EvtMax=-1,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)

test/k4FWCoreTest/scripts/CheckOutputFiles.py

@@ -70,6 +70,7 @@ def check_metadata(filename, expected_metadata):
 
 
 check_collections("functional_transformer.root", ["MCParticles", "NewMCParticles"])
+check_collections("functional_transformer_cli.root", ["MCParticles", "NewMCParticles"])
 check_collections(
     "functional_transformer_multiple.root",
     [
@@ -165,11 +166,6 @@ def check_metadata(filename, expected_metadata):
 if len(ev.get("NewMCParticles")) != 4:
     raise RuntimeError(f"Expected 4 NewMCParticles but got {len(ev.get('NewMCParticles'))}")
 
-check_events(
-    "functional_filter.root",
-    5,
-)
-
 check_collections("functional_metadata.root", ["MCParticles"])
 
 check_metadata(
@@ -251,7 +247,10 @@ def check_metadata(filename, expected_metadata):
         raise RuntimeError(f"Event number {event_number} is duplicated")
     seen_event_numbers.add(event_number)
 
-check_events(
-    "functional_nth_event.root",
-    3,
-)
+for name, events in {
+    "functional_filter.root": 5,
+    "functional_nth_event.root": 3,
+    "two_events.root": 2,
+    "functional_transformer_cli_multiple.root": 20,
+}.items():
+    check_events(name, events)