modifications for v11 reco (#142)

* v09 added as Detector Model choice

* VTXFinder, TOFs usw commented out cause bug with CentOS7

* convert Daniels MarlinReco for v09 to Gaudi

* SIM files

* hotfix, difference between not existing and empty type; or similar; dont remember

* modified ILDReco...py added

* Material plots from k4geo/utils added

* Modification to add some command line argument support similar to ILDReconstruction.py

* Converting Tutorial: Adapting the options file for EDM4hep

* formatting with black and isort added

* Tracking adapted for v11

* CLD VTX and Inner Tracker physics params added

* TrackingReco adaptations for v11

* steering v11; only tracking

* Changes requested by Thomas;
except for consts in .cfg file, not yet taken from CLD;
testing with v09 and v11 needed

* Parsing cleaned up;
store_true already sets false as default, see https://docs.python.org/3/library/argparse.html#action
removed one argument as the same dest was already used

* Minor Update from PR Conversation StandardConfig/production/Calibration/Calibration_ILD_l5_o1_v09.cfg

Co-authored-by: Thomas Madlener <[email protected]>

* Minor Update from PR Conversation StandardConfig/production/Calibration/Calibration_ILD_l5_v11.cfg

Co-authored-by: Thomas Madlener <[email protected]>

* Reverts removal of runBeamCalRepo argument

* Removed unnecessary defaults in argparse args;
default in runBeamCalRepo was even wrong

* add compatibility with calls specifying --compactFile

* v11 model added to gaudi-std-reco

* Update StandardConfig/production/ILDReconstruction.py

Co-authored-by: Thomas Madlener <[email protected]>

* Update StandardConfig/production/ILDReconstruction.py

Co-authored-by: Thomas Madlener <[email protected]>

* add FCC detector models as valid choices for detectorModel argument

* custom workflow for v11 added

* removed nightlies fom v11 smoke test workflow as this is not expected to work

* add ConformalTracking

* Use nightlies to run v11

---------

Co-authored-by: Victor Laurenz Schwan <[email protected]>

.github/workflows/key4hep.yml

@@ -56,7 +56,6 @@ jobs:
                    --constant.lcgeo_DIR=$lcgeo_DIR
             echo "::endgroup::"
 
-
   # Reconstruction using EDM4hep inputs and outputs
   gaudi-std-reco:
     runs-on: ubuntu-latest
@@ -87,3 +86,36 @@ jobs:
                   --inputFiles=bbudsc_3evt_SIM.edm4hep.root \
                   --outputFileBase=bbudsc_3evt_GaudiRec \
                   --detectorModel=${{ matrix.detector_model }}
+
+  # Reconstruction with the v11 model (FCCee) using EDM4hep inputs and outputs
+  gaudi-v11-reco:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        detector_model: [ILD_l5_v11]
+        key4hep_build: [sw-nightlies.hsf.org]
+        os: [ubuntu2204, el9]
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cvmfs-contrib/github-action-cvmfs@v4
+      - uses: aidasoft/run-lcg-view@v4
+        with:
+          container: ${{ matrix.os }}
+          view-path: /cvmfs/${{ matrix.key4hep_build }}/key4hep
+          run: |
+            echo "::group::Run simulation"
+            cd StandardConfig/production
+            ddsim --inputFiles Examples/bbudsc_3evt/bbudsc_3evt.stdhep \
+                  --outputFile bbudsc_3evt_SIM.edm4hep.root \
+                  --compactFile $lcgeo_DIR/ILD/compact/${{ matrix.detector_model }}/${{ matrix.detector_model }}.xml \
+                  --steeringFile ddsim_steer.py
+            echo "::endgroup::"
+            echo "::group::Run reconstruction"
+            k4run ILDReconstruction.py \
+                  --inputFiles=bbudsc_3evt_SIM.edm4hep.root \
+                  --outputFileBase=bbudsc_3evt_GaudiRec \
+                  --detectorModel=${{ matrix.detector_model }} \
+                  --noBeamCalRec \
+                  --trackingOnly

StandardConfig/production/Calibration/Calibration_ILD_l5_o1_v09.cfg

@@ -0,0 +1,95 @@
+# Copied from Calibration_ILD_l5_o1_v02.cfg
+CONSTANTS = {
+    "EcalBarrelMip": "0.0001575",
+    "EcalEndcapMip": "0.0001575",
+    "EcalRingMip": "0.0001575",
+    "HcalBarrelMip": "0.0004925",
+    "HcalEndcapMip": "0.0004725",
+    "HcalRingMip": "0.0004875",
+    "EcalBarrelEnergyFactors": ["0.0063520964756", "0.012902699188"],
+    "EcalEndcapEnergyFactors": ["0.0067218419842", "0.013653744940"],
+    "EcalRingEnergyFactors": ["0.0066536339", "0.0135151972"],
+    "HcalBarrelEnergyFactors": "0.0287783798145",
+    "HcalEndcapEnergyFactors": "0.0285819096797",
+    "HcalRingEnergyFactors": "0.0349940637704",
+    "MuonCalibration": "56.7",
+    "PandoraEcalToMip": "153.846",
+    "PandoraHcalToMip": "37.1747",
+    "PandoraMuonToMip": "10.5263",
+    "PandoraEcalToEMScale": "1.0",
+    "PandoraHcalToEMScale": "1.0",
+    "PandoraEcalToHadBarrelScale": "1.17344504717",
+    "PandoraEcalToHadEndcapScale": "1.17344504717",
+    "PandoraHcalToHadScale": "1.02821419758",
+    "PandoraSoftwareCompensationWeights": [
+        "1.59121",
+        "-0.0281982",
+        "0.000250616",
+        "-0.0424222",
+        "0.000335128",
+        "-2.06112e-05",
+        "0.148549",
+        "0.199618",
+        "-0.0697277",
+    ],
+    "ApplyPhotonPFOCorrections": "true",
+    "EcalTechnology": "SiWEcal",
+    "HcalTechnology": "AHcal",
+    "PandoraSettingsFile": "PandoraSettings/PandoraSettingsDefault.xml",
+    "DropCollectionsECal": [
+        "ECalBarrelScHitsEven",
+        "ECalBarrelScHitsOdd",
+        "ECalEndcapScHitsEven",
+        "ECalEndcapScHitsOdd",
+    ],
+    "DropCollectionsHCal": [
+        "HCalBarrelRPCHits",
+        "HCalEndcapRPCHits",
+        "HCalECRingRPCHits",
+    ],
+    "AdditionalDropCollectionsREC": [
+        "%(DropCollectionsECal)s",
+        "%(DropCollectionsHCal)s",
+    ],
+    "dEdXErrorFactor": "7.55",
+    "dEdXSmearingFactor": "0.029",
+    "PidPDFFile": "HighLevelReco/PIDFiles/LikelihoodPID_Standard_l5_v01.root",
+    "PidWeightFiles": [
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_02GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_03GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_04GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_05GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_06GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_07GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_08GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_09GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_10GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_11GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_12GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_13GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_14GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_15GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_16GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_17GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_18GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_19GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_20GeVP_clusterinfo.weights.xml",
+    ],
+    "ECalBarrelSimHitCollections": ["ECalBarrelSiHitsEven", "ECalBarrelSiHitsOdd"],
+    "ECalEndcapSimHitCollections": ["ECalEndcapSiHitsEven", "ECalEndcapSiHitsOdd"],
+    "ECalRingSimHitCollections": "EcalEndcapRingCollection",
+    "HCalBarrelSimHitCollections": "HcalBarrelRegCollection",
+    "HCalEndcapSimHitCollections": "HcalEndcapsCollection",
+    "HCalRingSimHitCollections": "HcalEndcapRingCollection",
+    "ECalSimHitCollections": [
+        "%(ECalBarrelSimHitCollections)s",
+        "%(ECalEndcapSimHitCollections)s",
+        "%(ECalRingSimHitCollections)s",
+    ],
+    "HCalSimHitCollections": [
+        "%(HCalBarrelSimHitCollections)s",
+        "%(HCalEndcapSimHitCollections)s",
+        "%(HCalRingSimHitCollections)s",
+    ],
+    "BeamCalBackgroundFile": "HighLevelReco/BeamCalBackground/BeamCalBackground-E%(CMSEnergy)s-B3.5-RealisticNominalAntiDid.root",
+}

StandardConfig/production/Calibration/Calibration_ILD_l5_v11.cfg

@@ -0,0 +1,95 @@
+# Copied from Calibration_ILD_l5_o1_v02.cfg
+CONSTANTS = {
+    "EcalBarrelMip": "0.0001575",
+    "EcalEndcapMip": "0.0001575",
+    "EcalRingMip": "0.0001575",
+    "HcalBarrelMip": "0.0004925",
+    "HcalEndcapMip": "0.0004725",
+    "HcalRingMip": "0.0004875",
+    "EcalBarrelEnergyFactors": ["0.0063520964756", "0.012902699188"],
+    "EcalEndcapEnergyFactors": ["0.0067218419842", "0.013653744940"],
+    "EcalRingEnergyFactors": ["0.0066536339", "0.0135151972"],
+    "HcalBarrelEnergyFactors": "0.0287783798145",
+    "HcalEndcapEnergyFactors": "0.0285819096797",
+    "HcalRingEnergyFactors": "0.0349940637704",
+    "MuonCalibration": "56.7",
+    "PandoraEcalToMip": "153.846",
+    "PandoraHcalToMip": "37.1747",
+    "PandoraMuonToMip": "10.5263",
+    "PandoraEcalToEMScale": "1.0",
+    "PandoraHcalToEMScale": "1.0",
+    "PandoraEcalToHadBarrelScale": "1.17344504717",
+    "PandoraEcalToHadEndcapScale": "1.17344504717",
+    "PandoraHcalToHadScale": "1.02821419758",
+    "PandoraSoftwareCompensationWeights": [
+        "1.59121",
+        "-0.0281982",
+        "0.000250616",
+        "-0.0424222",
+        "0.000335128",
+        "-2.06112e-05",
+        "0.148549",
+        "0.199618",
+        "-0.0697277",
+    ],
+    "ApplyPhotonPFOCorrections": "true",
+    "EcalTechnology": "SiWEcal",
+    "HcalTechnology": "AHcal",
+    "PandoraSettingsFile": "PandoraSettings/PandoraSettingsDefault.xml",
+    "DropCollectionsECal": [
+        "ECalBarrelScHitsEven",
+        "ECalBarrelScHitsOdd",
+        "ECalEndcapScHitsEven",
+        "ECalEndcapScHitsOdd",
+    ],
+    "DropCollectionsHCal": [
+        "HCalBarrelRPCHits",
+        "HCalEndcapRPCHits",
+        "HCalECRingRPCHits",
+    ],
+    "AdditionalDropCollectionsREC": [
+        "%(DropCollectionsECal)s",
+        "%(DropCollectionsHCal)s",
+    ],
+    "dEdXErrorFactor": "7.55",
+    "dEdXSmearingFactor": "0.029",
+    "PidPDFFile": "HighLevelReco/PIDFiles/LikelihoodPID_Standard_l5_v01.root",
+    "PidWeightFiles": [
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_02GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_03GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_04GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_05GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_06GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_07GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_08GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_09GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_10GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_11GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_12GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_13GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_14GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_15GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_16GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_17GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_18GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_19GeVP_clusterinfo.weights.xml",
+        "HighLevelReco/PIDFiles/LowMomMuPiSeparation/TMVAClassification_BDTG_l5_20GeVP_clusterinfo.weights.xml",
+    ],
+    "ECalBarrelSimHitCollections": ["ECalBarrelSiHitsEven", "ECalBarrelSiHitsOdd"],
+    "ECalEndcapSimHitCollections": ["ECalEndcapSiHitsEven", "ECalEndcapSiHitsOdd"],
+    "ECalRingSimHitCollections": "EcalEndcapRingCollection",
+    "HCalBarrelSimHitCollections": "HcalBarrelRegCollection",
+    "HCalEndcapSimHitCollections": "HcalEndcapsCollection",
+    "HCalRingSimHitCollections": "HcalEndcapRingCollection",
+    "ECalSimHitCollections": [
+        "%(ECalBarrelSimHitCollections)s",
+        "%(ECalEndcapSimHitCollections)s",
+        "%(ECalRingSimHitCollections)s",
+    ],
+    "HCalSimHitCollections": [
+        "%(HCalBarrelSimHitCollections)s",
+        "%(HCalEndcapSimHitCollections)s",
+        "%(HCalRingSimHitCollections)s",
+    ],
+    "BeamCalBackgroundFile": "HighLevelReco/BeamCalBackground/BeamCalBackground-E%(CMSEnergy)s-B3.5-RealisticNominalAntiDid.root",
+}

StandardConfig/production/HighLevelReco/HighLevelReco.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-from Gaudi.Configuration import INFO
 from Configurables import MarlinProcessorWrapper
+from Gaudi.Configuration import INFO
 
 MyAdd4MomCovMatrixCharged = MarlinProcessorWrapper("MyAdd4MomCovMatrixCharged")
 MyAdd4MomCovMatrixCharged.OutputLevel = INFO

StandardConfig/production/ILDReconstruction.py

@@ -1,28 +1,29 @@
 import os
 import sys
-
-from Gaudi.Configuration import INFO
+from pathlib import Path
 
 from Configurables import (
+    ApplicationMgr,
+    EDM4hep2LcioTool,
+    GeoSvc,
+    Lcio2EDM4hepTool,
     LcioEvent,
     MarlinProcessorWrapper,
-    k4DataSvc,
     PodioInput,
     PodioOutput,
-    EDM4hep2LcioTool,
-    Lcio2EDM4hepTool,
-    GeoSvc,
-    ApplicationMgr,
+    k4DataSvc,
 )
+from Gaudi.Configuration import INFO
 
 try:
-    from k4FWCore.utils import import_from, SequenceLoader
+    from k4FWCore.utils import SequenceLoader, import_from
 except ImportError:
     from py_utils import import_from, SequenceLoader
 
 from k4FWCore.parseArgs import parser
 from k4MarlinWrapper.parseConstants import parseConstants
 
+# only non-FCCMDI models
 DETECTOR_MODELS = (
     "ILD_l2_v02",
     "ILD_l4_o1_v02",
@@ -47,6 +48,11 @@
     "ILD_s5_o3_v02",
     "ILD_s5_o4_v02",
 )
+# only FCCMDI
+FCCeeMDI_DETECTOR_MODELS = (
+    "ILD_l5_o1_v09",
+    "ILD_l5_v11",
+)
 
 parser.add_argument(
     "--inputFiles",
@@ -80,21 +86,22 @@
 parser.add_argument(
     "--detectorModel",
     help="Which detector model to run reconstruction for",
-    choices=DETECTOR_MODELS,
+    choices=DETECTOR_MODELS + FCCeeMDI_DETECTOR_MODELS,
     type=str,
     default="ILD_l5_o1_v02",
 )
 parser.add_argument(
-    "--perfectPFA", help="Run perfect PandoraPFA", action="store_true", default=False
+    "--perfectPFA",
+    help="Run perfect PandoraPFA",
+    action="store_true",
 )
 parser.add_argument(
     "--runOverlay",
-    help="Run background overaly. NOTE: You have to make sure that the Overlay algorithms in "
+    help="Run background overlay. NOTE: You have to make sure that the Overlay algorithms in "
     " BgOverlay/BgOverlay.py are provided with the necessary overlay files",
     action="store_true",
-    default=False,
 )
-# BeamCal reco configuration
+
 parser.add_argument(
     "--runBeamCalReco",
     help="Run the BeamCal reco",
@@ -107,13 +114,17 @@
     action="store_false",
     dest="runBeamCalReco",
 )
-parser.set_defaults(runBeamCalReco=True)
 parser.add_argument(
     "--beamCalCalibFactor",
     help="The BeamCal calibration constant from sim hit energy to calibrated calo hit energy",
     type=float,
     default=79.6,
 )
+parser.add_argument(
+    "--trackingOnly",
+    help="Only Tracking is performed; built for reco testing purposes",
+    action="store_true",
+)
 
 reco_args = parser.parse_known_args()[0]
 
@@ -219,22 +230,34 @@ def create_reader(input_files):
 ecal_technology = CONSTANTS["EcalTechnology"]
 hcal_technology = CONSTANTS["HcalTechnology"]
 
-sequenceLoader.load("Tracking/TrackingDigi")
-sequenceLoader.load("Tracking/TrackingReco")
-sequenceLoader.load(f"CaloDigi/{ecal_technology}Digi")
-sequenceLoader.load(f"CaloDigi/{hcal_technology}Digi")
-sequenceLoader.load("CaloDigi/FcalDigi")
-sequenceLoader.load("CaloDigi/MuonDigi")
-
-if reco_args.perfectPFA:
-    sequenceLoader.load("ParticleFlow/PandoraPFAPerfect")
+# identify specified detector model
+if reco_args.compactFile:
+    det_model = Path(reco_args.compactFile).stem
 else:
-    sequenceLoader.load("ParticleFlow/PandoraPFA")
+    det_model = reco_args.detectorModel
+# load relevant tracking
+if det_model in FCCeeMDI_DETECTOR_MODELS:
+    sequenceLoader.load("Tracking/TrackingDigi_FCCeeMDI")
+    sequenceLoader.load("Tracking/TrackingReco_FCCeeMDI")
+elif det_model in DETECTOR_MODELS:
+    sequenceLoader.load("Tracking/TrackingDigi")
+    sequenceLoader.load("Tracking/TrackingReco")
+
+if not reco_args.trackingOnly:
+    sequenceLoader.load(f"CaloDigi/{ecal_technology}Digi")
+    sequenceLoader.load(f"CaloDigi/{hcal_technology}Digi")
+    sequenceLoader.load("CaloDigi/FcalDigi")
+    sequenceLoader.load("CaloDigi/MuonDigi")
+
+    if reco_args.perfectPFA:
+        sequenceLoader.load("ParticleFlow/PandoraPFAPerfect")
+    else:
+        sequenceLoader.load("ParticleFlow/PandoraPFA")
 
-if reco_args.runBeamCalReco:
-    sequenceLoader.load("HighLevelReco/BeamCalReco")
+    if reco_args.runBeamCalReco:
+        sequenceLoader.load("HighLevelReco/BeamCalReco")
 
-sequenceLoader.load("HighLevelReco/HighLevelReco")
+    sequenceLoader.load("HighLevelReco/HighLevelReco")
 
 
 MyPfoAnalysis = MarlinProcessorWrapper("MyPfoAnalysis")

StandardConfig/production/Tracking/TrackingDigi.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-from Gaudi.Configuration import INFO
 from Configurables import MarlinProcessorWrapper
+from Gaudi.Configuration import INFO
 
 MySplitCollectionByLayer = MarlinProcessorWrapper("MySplitCollectionByLayer")
 MySplitCollectionByLayer.OutputLevel = INFO