k4run: allow addition of additional parameters from steering files

README.md

@@ -1,8 +1,10 @@
 # k4FWCore (key4hep FrameWork Core)
 
-k4FWCore is a Gaudi package that provides the PodioDataService, that allows to
+k4FWCore is a Gaudi package that provides the PodioDataService, which allows to
 use podio-based event data models like EDM4hep in Gaudi workflows.
 
+k4FWCore also provides the `k4run` script used to run Gaudi steering files.
+
 ## Components
 
 ### Basic I/O
@@ -13,12 +15,39 @@ Component wrapping the PodioDataService to handle PODIO types and collections.
 
 #### PodioInput
 
-Algorithm to read data from input file(s) on disk.
+Algorithm to read data from one or multiple input file(s) on disk.
 
 #### PodioOutput
 
-Algorithm to write data to output file on disk.
+Algorithm to write data to an output file on disk.
+
+## k4run
+```bash
+$ k4run --help
+usage: k4run [-h] [--dry-run] [-v] [-n NUM_EVENTS] [-l] [--gdb] [--ncpus NCPUS] [config_files ...]
+
+Run job in the Key4HEP framework
 
+positional arguments:
+  config_files          Gaudi config (python) files describing the job
+
+options:
+  -h, --help            show this help message and exit
+  --dry-run             Do not actually run the job, just parse the config files
+  -v, --verbose         Run job with verbose output
+  -n NUM_EVENTS, --num-events NUM_EVENTS
+                        Number of events to run
+  -l, --list            Print all the configurable components available in the framework and exit
+  --gdb                 Attach gdb debugger
+  --ncpus NCPUS         Start Gaudi in parallel mode using NCPUS processes. 0 => serial mode (default), -1 => use all CPUs
+```
+When supplied with a Gaudi steering file `k4run --help file.py` also shows the settable properties of the Gaudi algorithms used in the file. Additionally, it is possible to add further arguments and use them in the steering file by using the Python `argparse.ArgumentParser` shared by `k4run`.
+```python
+from k4FWCore.parseArgs import parser
+parser.add_argument("-f", "--foo", type=int, help="hello world")
+my_opts = parser.parse_known_args()
+print(my_opts[0].foo)
+```
 
 ## Dependencies
 
@@ -30,7 +59,7 @@ Algorithm to write data to output file on disk.
 
 ## Installation and downstream usage.
 
-k4FWCore is a cmake project. After setting up the dependencies (use for example `source /cvmfs/sw.hsf.org/key4hep/setup.sh`)
+k4FWCore is a CMake project. After setting up the dependencies (use for example `source /cvmfs/sw.hsf.org/key4hep/setup.sh`)
 
 
 ```

k4FWCore/CMakeLists.txt

@@ -5,6 +5,7 @@
 find_package(podio 0.16.3 REQUIRED)
 
 gaudi_install(SCRIPTS)
+gaudi_install(PYTHON)
 
 
 gaudi_add_library(k4FWCore
@@ -31,3 +32,10 @@ install(TARGETS k4FWCore k4FWCorePlugins
   LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT shlib
   COMPONENT dev)
 
+# Copy python parsing file to genConfDir in Gaudi
+add_custom_command(
+        TARGET k4FWCore POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy
+                ${PROJECT_SOURCE_DIR}/k4FWCore/python/k4FWCore/parseArgs.py
+                ${CMAKE_CURRENT_BINARY_DIR}/genConfDir/k4FWCore/parseArgs.py)
+

k4FWCore/python/k4FWCore/parseArgs.py

@@ -0,0 +1,3 @@
+import argparse
+
+parser = argparse.ArgumentParser(description="Run job in the Key4HEP framework", add_help=False)

k4FWCore/scripts/k4run

@@ -22,65 +22,48 @@ FILTER_GAUDI_PROPS = [ "ContextService", "Cardinality", "Context", "CounterList"
 seen_files = set()
 option_db = {}
 
-
-# There is no way of knowing if parse_known_args() or parse_args() was called
-# so we'll track wether this is the first parsing or not
-first_run = True
-class LoadFromFile(argparse.Action):
-    def __call__(self, parser, namespace, values, option_string=None):
-      global first_run
-      if not values:
-        if not first_run:
-          print('Error: missing gaudi options file.\n'
-          'Usage: k4run <options_file.py>, use --help to get a complete list of arguments')
-          sys.exit(1)
-        first_run = False
-        return
-      first_run = False
-
-      for wrapper in values:
-        if wrapper.name in seen_files:
-            return
-        seen_files.add(wrapper.name)
-        exec(open(wrapper.name).read(), globals())
-      # loop over all components that were configured in the options file
-      # use dict.fromkeys so that the confs are sorted (Python >= 3.7)
-      for conf in dict.fromkeys(ApplicationMgr.allConfigurables.values()):
-        # skip public tools and the applicationmgr itself
-        if "ToolSvc" in conf.name() or "ApplicationMgr" in conf.name():
-          continue
-        props = conf.getPropertiesWithDescription() #dict propertyname: (propertyvalue, propertydescription)
-        for prop in props:
-          # only add arguments for relevant properties
-          if prop in FILTER_GAUDI_PROPS or "Audit" in prop or hasattr(props[prop][0], '__slots__'):
+def load_file(file):
+  exec(file.read())
+
+
+def add_arguments(parser, app_mgr):
+  configurables = app_mgr.allConfigurables.values()
+  for conf in configurables:
+    # skip public tools and the applicationmgr itself
+    if "ToolSvc" in conf.name() or "ApplicationMgr" in conf.name():
+      continue
+    props = conf.getPropertiesWithDescription() #dict propertyname: (propertyvalue, propertydescription)
+    for prop in props:
+      # only add arguments for relevant properties
+      if prop in FILTER_GAUDI_PROPS or "Audit" in prop or hasattr(props[prop][0], '__slots__'):
+        continue
+      propvalue = props[prop][0]
+
+      # if it is set to "no value" it hasn't been touched in the options file
+      if propvalue == conf.propertyNoValue:
+          propvalue = conf.getDefaultProperty(prop) # thus get the default value
+      proptype = type(props[prop][0])
+      # if the property is a list of something, we need to set argparse nargs to '+'
+      propnargs = "?"
+      if proptype == list:
+          # tricky edgecase: if the default is an empty list there is no way to get the type
+          if len(propvalue) == 0:
+          # just skip for now
+          #print("Warning: argparse cannot deduce type for property %s of %s. Needs to be set in options file." % (prop, conf.name()))
             continue
-          propvalue = props[prop][0]
-
-          # if it is set to "no value" it hasn't been touched in the options file
-          if propvalue == conf.propertyNoValue:
-              propvalue = conf.getDefaultProperty(prop) # thus get the default value
-          proptype = type(props[prop][0])
-          # if the property is a list of something, we need to set argparse nargs to '+'
-          propnargs = "?"
-          if proptype == list:
-              # tricky edgecase: if the default is an empty list there is no way to get the type
-              if len(propvalue) == 0:
-              # just skip for now
-              #print("Warning: argparse cannot deduce type for property %s of %s. Needs to be set in options file." % (prop, conf.name()))
-                continue
-              else:
-                # deduce type from first item of the list
-                proptype = type(propvalue[0])
-              propnargs = "+"
-
-          # add the argument twice, once as "--PodioOutput.filename"
-          # and once as "--filename.PodioOutput"
-          propName = conf.name() + '.' + prop
-          propNameReversed = prop + '.' + conf.name()
-          option_db[propName] = (conf, propName)
-          parser.add_argument( "--%s" % propName, "--%s" % propNameReversed, type=proptype, help=props[prop][1],
-              nargs=propnargs,
-              default=propvalue)
+          else:
+            # deduce type from first item of the list
+            proptype = type(propvalue[0])
+          propnargs = "+"
+
+      # add the argument twice, once as "--PodioOutput.filename"
+      # and once as "--filename.PodioOutput"
+      propName = conf.name() + '.' + prop
+      propNameReversed = prop + '.' + conf.name()
+      option_db[propName] = (conf, propName)
+      parser.add_argument( f"--{propName}", f"--{propNameReversed}", type=proptype, help=props[prop][1],
+          nargs=propnargs,
+          default=propvalue)
 
 
 if __name__ == "__main__":
@@ -95,8 +78,8 @@ if __name__ == "__main__":
     handler.setFormatter(formatter)
     logger.addHandler(handler)
 
-    parser = argparse.ArgumentParser(description="Run job in the Key4HEP framework")
-    parser.add_argument("config_files", type=open, action=LoadFromFile, nargs="*",
+    from k4FWCore.parseArgs import parser
+    parser.add_argument("config_files", type=open, action="store", nargs="*",
                         help="Gaudi config (python) files describing the job")
     parser.add_argument("--dry-run", action="store_true",
                         help="Do not actually run the job, just parse the config files")
@@ -139,6 +122,21 @@ if __name__ == "__main__":
                   print("  %s (from %s)" % (item, cfgDb[item]["lib"]))
         sys.exit()
 
+    if len(opts[0].config_files) == 0:
+      print('Error: missing gaudi options file.\n'
+      'Usage: k4run <options_file.py>, use --help to get a complete list of arguments')
+      sys.exit(1)
+
+    for file in opts[0].config_files:
+      load_file(file)
+
+    # ApplicationMgr is a singleton
+    from Configurables import ApplicationMgr
+    add_arguments(parser, ApplicationMgr())
+
+    # add help manually here, if it is added earlier the parser exits after the first parse_arguments call
+    parser.add_argument("-h", "--help", action="help", default=argparse.SUPPRESS, help="show this help message and exit")
+
     opts = parser.parse_args()
 
     # print a doc line showing the configured algorithms

test/k4FWCoreTest/CMakeLists.txt

@@ -136,4 +136,11 @@ add_test(NAME Testk4runNoArguments
          COMMAND ${K4RUN})
 set_test_env(Testk4runNoArguments)
 set_tests_properties(Testk4runNoArguments
-  PROPERTIES PASS_REGULAR_EXPRESSION "Usage: k4run <options_file.py>, use --help to get a complete list of arguments")
+  PROPERTIES PASS_REGULAR_EXPRESSION "Usage: k4run <options_file.py>, use --help to get a complete list of arguments")
+
+add_test(NAME Testk4runCustomArguments
+         WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+         COMMAND ${K4RUN} --foo=42 options/TestArgs.py)
+set_test_env(Testk4runCustomArguments)
+set_tests_properties(Testk4runCustomArguments
+  PROPERTIES PASS_REGULAR_EXPRESSION "The answer is 42")

test/k4FWCoreTest/options/TestArgs.py

@@ -0,0 +1,5 @@
+from k4FWCore.parseArgs import parser
+parser.add_argument("-f", "--foo", type=int, help="hello world")
+my_opts = parser.parse_known_args()
+
+print(f"The answer is {my_opts[0].foo}")