Jet Veto Map Selector

confirm_and_run.sh

@@ -0,0 +1,49 @@
+# Function to prompt user for confirmation if command is supposed to be skipped
+# if user types n or no then the command is executed
+# if user types anything else or nothing then the command is skipped
+# it is also possible to extend the command afte typing n or no
+
+# Define color variables
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+confirm_and_run() {
+    cmd="$1"
+    echo "________________________________________________________"
+    echo -e "${YELLOW}$cmd${NC}"
+    echo -e -n "    ${RED}Skip?${NC} (y/n): "
+    read response
+    case "$response" in
+        [nN][oO]|[nN])
+            echo -e "    ${RED}Any additional parameters?${NC} ('-ps' -> '--print-status 2,0', '-ro' -> '--remove-output 0,a,y' predefined, others possible)"
+            read -e -i "    " extra_params
+
+            # Add another case statement to handle the expansion
+            case "$extra_params" in
+                -ps)
+                    extra_params="--print-status 2,0"
+                    echo -e "    ${GREEN}Printing status...${NC}"
+                    ;;
+                -ro)
+                    extra_params="--remove-output 0,a,y"
+                    echo -e "    ${RED}Removing output...${NC}"
+                    ;;
+            esac
+            echo -e "    ${GREEN}Running...${NC}"
+            eval "$cmd $extra_params"
+            ;;
+        *)
+            echo -e "    ${GREEN}Skipped!${NC}"
+            ;;
+    esac
+}
+
+no_confirm() {
+    cmd="$1"
+    echo "________________________________________________________"
+    echo -e "${YELLOW}$cmd${NC}"
+    echo -e "    ${GREEN}Running...${NC}"
+    eval "$cmd"
+}

law.cfg

@@ -11,6 +11,17 @@ columnflow.tasks.cms.external
 topsf.tasks.plotting
 topsf.tasks.wp.efficiency
 topsf.tasks.inference
+topsf.tasks.inference_tasks.create_workspace
+topsf.tasks.inference_tasks.combine_task
+topsf.tasks.inference_tasks.postfitshapes
+topsf.tasks.inference_tasks.impacts
+topsf.tasks.inference_v2.workspace
+topsf.tasks.inference_v2.gen_toys
+topsf.tasks.inference_v2.multi_dim_fit
+topsf.tasks.inference_v2.post_fit_shapes
+topsf.tasks.inference_v2.impacts
+topsf.tasks.inference_v2.plot_impacts
+topsf.tasks.inference_v2.plot_shapes
 
 [logging]
 
@@ -22,19 +33,26 @@ columnflow.columnar_util-perf: INFO
 
 [analysis]
 
-default_analysis: topsf.config.run2.analysis_sf.analysis_sf
-default_config: run2_sf_2017_nano_v9_limited
+default_analysis: topsf.config.run3.analysis_sf.analysis_sf
+default_config: run3_sf_2022_postEE_nano_v12
 default_dataset: tt_fh_powheg
+run3_analysis: topsf.config.run3.analysis_sf.analysis_sf
+run3_config: run3_sf_2022_preEE_nano_v12
+
+default_keep_reduced_events: True
 
 production_modules: columnflow.production.{categories,normalization,mc_weight,pileup,processes,seeds}, columnflow.production.cms.{btag,electron,mc_weight,muon,pdf,pileup,scale,seeds}, topsf.production.{default,gen_top}
-calibration_modules: columnflow.calibration.cms.{jets,met}, topsf.calibration.default
+calibration_modules: columnflow.calibration.cms.{jets,met}, topsf.calibration.{default,skip_jec}
 selection_modules: columnflow.selection.cms.{json_filter,met_filters}, topsf.selection.{default,categories,jet,bjet,fatjet,lepton,wp}
 ml_modules: columnflow.ml
 inference_modules: columnflow.inference, topsf.inference.{default,uhh2}
 
 # namespace of all columnflow tasks
 cf_task_namespace: cf
 
+# sandbox for working with combine tasks
+combine_sandbox: bash::$TOPSF_BASE/sandboxes/combine_cmssw.sh
+
 # wether or not the ensure_proxy decorator should be skipped, even if used by task's run methods
 skip_ensure_proxy: False
 
@@ -47,26 +65,27 @@ slurm_partition: $CF_SLURM_PARTITION
 # ChunkedIOHandler defaults
 chunked_io_chunk_size: 100000
 chunked_io_pool_size: 2
-chunked_io_debug: False
+chunked_io_debug: True
 
 # csv list of task families that inherit from ChunkedReaderMixin and whose output arrays should be
 # checked for non-finite values before saving them to disk (right now, supported tasks are
 # cf.CalibrateEvents, cf.SelectEvents, cf.ProduceColumns, cf.PrepareMLEvents, cf.MLEvaluation,
 # cf.UniteColumns)
-check_finite_output: cf.CalibrateEvents, cf.SelectEvents, cf.ProduceColumns
+# check_finite_output: cf.CalibrateEvents, cf.SelectEvents, cf.ProduceColumns
+check_finite_output: None
 
 # whether to log runtimes of array functions by default
 log_array_function_runtime: False
 
 
-[outputs]
+[outputs]  
 
 # list of all used file systems
 wlcg_file_systems: wlcg_fs, wlcg_fs_infn_redirector, wlcg_fs_global_redirector
 
 # list of file systems used by columnflow.tasks.external.GetDatasetLFNs.iter_nano_files to
 # look for the correct fs per nano input file (in that order)
-lfn_sources: local_desy_dcache,wlcg_fs_infn_redirector, wlcg_fs_global_redirector
+lfn_sources: local_desy_dcache, wlcg_fs_infn_redirector, wlcg_fs_global_redirector
 
 # output locations per task family
 # for local targets : "local[, LOCAL_FS_NAME or STORE_PATH]"
@@ -78,12 +97,12 @@ cf.BundleBashSandbox: local
 cf.BundleCMSSWSandbox: local
 cf.BundleExternalFiles: local
 # GetDatasetLFNs requires a Grid certificate -> use a common space to store the output
-cf.GetDatasetLFNs: local, /nfs/dust/cms/user/dsavoiu/store/mttbar/data
+cf.GetDatasetLFNs: local
 cf.CalibrateEvents: wlcg
 cf.SelectEvents: wlcg
 cf.CreateCutflowHistograms: wlcg
-cf.PlotCutflow: wlcg
-cf.PlotCutflowVariables: wlcg
+cf.PlotCutflow: local
+cf.PlotCutflowVariables: local
 cf.ReduceEvents: wlcg
 cf.MergeReducedEvents: wlcg
 cf.ProduceColumns: wlcg
@@ -94,7 +113,7 @@ cf.MLEvaluation: wlcg
 cf.CreateHistograms: local
 cf.MergeHistograms: local
 cf.MergeShiftedHistograms: local
-cf.PlotVariables: local
+cf.PlotVariables1D: local
 cf.PlotShiftedVariables: local
 cf.CreateDatacards: local
 
@@ -131,7 +150,7 @@ cache_max_size: 50GB
 
 xrootd_base: root://dcache-cms-xrootd.desy.de:1094/pnfs/desy.de/cms/tier2/store/user/$CF_CERN_USER/$CF_STORE_NAME
 gsiftp_base: gsiftp://dcache-door-cms04.desy.de:2811/pnfs/desy.de/cms/tier2/store/user/$CF_CERN_USER/$CF_STORE_NAME
-base: &::gsiftp_base
+base: &::xrootd_base
 
 [wlcg_fs_infn_redirector]
 

sandboxes/_setup_combine.sh

@@ -252,6 +252,22 @@ setup_combine() {
                         return "3002"
                     }
 
+                    # clone the combine harvester repo
+                    cd ${CMSSW_BASE}/src
+                    git clone https://github.com/cms-analysis/CombineHarvester.git CombineHarvester || {
+                        >&2 echo "failed to clone CombineHarvester git repository from URL ${CF_COMBINE_HARVESTER_GIT_URL}"
+                        clear_pending
+                        return "3003"
+                    }
+
+                    # check out the specified combine harvester version
+                    cd CombineHarvester
+                    git checkout "${CF_COMBINE_HARVESTER_VERSION}" || {
+                        >&2 echo "failed to check out revision ${CF_COMBINE_HARVESTER_VERSION} from git repository"
+                        clear_pending
+                        return "3004"
+                    }
+
                     # compile
                     cd ${CMSSW_BASE}
                     scram b -j4

sandboxes/combine_cmssw.sh

@@ -11,9 +11,11 @@ action() {
     # set variables and source the combine setup
     export CF_SANDBOX_FILE="${CF_SANDBOX_FILE:-${this_file}}"
     export CF_COMBINE_GIT_URL="${CF_COMBINE_GIT_URL:-https://github.com/cms-analysis/HiggsAnalysis-CombinedLimit.git}"
-    export CF_COMBINE_SCRAM_ARCH="$( [ "${os_version}" = "8" ] && echo "el8" || echo "slc7" )_amd64_gcc10"
-    export CF_COMBINE_CMSSW_VERSION="CMSSW_12_6_2"
-    export CF_COMBINE_VERSION="${CF_COMBINE_VERSION:-v9.1.0}"
+    export CF_COMBINE_HARVESTER_GIT_URL="${CF_COMBINE_HARVESTER_GIT_URL:-https://github.com/cms-analysis/CombineHarvester.git}"
+    export CF_COMBINE_SCRAM_ARCH="el9_amd64_gcc12"
+    export CF_COMBINE_CMSSW_VERSION="CMSSW_14_1_0_pre5"  # from combine docu 23.07.24
+    export CF_COMBINE_HARVESTER_VERSION="${CF_COMBINE_HARVESTER_VERSION:-v3.0.0-pre1}"
+    export CF_COMBINE_VERSION="${CF_COMBINE_VERSION:-v10.0.1}"  # from combine docu 23.07.24
     export CF_COMBINE_ENV_NAME="$( basename "${this_file%.sh}" )"
     export CF_COMBINE_FLAG="1"  # increment when content changed
 

test_config.py

@@ -16,6 +16,12 @@
 
 analysis_inst = ana = AnalysisTask.get_analysis_inst(run3_analysis)
 config_inst = cfg = ana.get_config(run3_config)
+# ana = AnalysisTask.get_analysis_inst(default_analysis)
+# config_inst = cfg = ana.get_config(default_config)
+
+print(f"================= Analysis: {ana.name} ======================")
+print(f"ID: {ana.id}")
+print(f"Config: {cfg.name}")
 
 print(" ================ Processes ======================")
 process_insts = cfg.processes
@@ -33,7 +39,8 @@
 dataset_insts = cfg.datasets
 for data_inst in dataset_insts:
     print(f"{data_inst.name}; N_events: {data_inst.n_events:,}")
-print(f"Sum of all mc events: {sum([data_inst.n_events for data_inst in dataset_insts]):,}")
+    print(cfg.datasets.get(data_inst.name).processes.values()[0].xsecs)
+print(f"Sum of all mc events: {sum([data_inst.n_events for data_inst in dataset_insts if not data_inst.name.startswith('data')]):,}")
 print(f"Number of datasets: {len(dataset_insts)}")
 print(f"Number of dataset files: {sum([data_inst.n_files for data_inst in dataset_insts])}")
 
@@ -61,8 +68,10 @@
 print("================= Auxiliary ======================")
 aux = cfg.aux
 for key, value in aux.items():
-    print(key)
-print(cfg.tags)
+    print(f"aux entry {key}")
+print(f"tags: {cfg.tags}")
+if cfg.has_tag("is_top_sf"):
+    print("This is a top SF config")
 
 # print some features of an exemplary process inst
 proc_inst = cfg.get_process("tt")

topsf/columnflow_patches.py

@@ -8,11 +8,28 @@
 
 import law
 from columnflow.util import memoize
+import getpass
 
 
 logger = law.logger.get_logger(__name__)
 
 
+@memoize
+def patch_htcondor_workflow_naf_resources():
+    """
+    Patches the HTCondorWorkflow task to declare user-specific resources when running on the NAF.
+    """
+    from columnflow.tasks.framework.remote import HTCondorWorkflow
+
+    def htcondor_job_resources(self, job_num, branches):
+        # one "naf_<username>" resource per job, indendent of the number of branches in the job
+        return {f"naf_{getpass.getuser()}": 1}
+
+    HTCondorWorkflow.htcondor_job_resources = htcondor_job_resources
+
+    logger.debug(f"patched htcondor_job_resources of {HTCondorWorkflow.task_family}")
+
+
 @memoize
 def patch_bundle_repo_exclude_files():
     from columnflow.tasks.framework.remote import BundleRepo
@@ -37,3 +54,4 @@ def patch_bundle_repo_exclude_files():
 @memoize
 def patch_all():
     patch_bundle_repo_exclude_files()
+    patch_htcondor_workflow_naf_resources()

topsf/config/categories.py

@@ -219,7 +219,7 @@ def sel_tau32(
                 False,
             )
 
-        assert cat_idx < 10**cat_idx_ndigits - 1, "no space for category, ID reassignement necessary"
+        assert cat_idx < 10**cat_idx_ndigits - 1, "no space for category, ID reassignment necessary"
         cat = config.add_category(
             name=cat_name,
             id=int(10**cat_idx_lsd * ((cat_idx + 1) + 10 * 3)),
@@ -237,7 +237,7 @@ def sel_tau32(
             ("pass", "<", slice(None, cat_idx + 1)),
             ("fail", ">", slice(cat_idx + 1, None)),
         ]):
-            cat_label = rf"$\tau_{{3}}/\tau_{{2}}$ {comp_symbol} {tau32_val} ({pass_fail})"
+            cat_label = rf"{tau32_wp} wp: $\tau_{{3}}/\tau_{{2}}$ {comp_symbol} {tau32_val} ({pass_fail})"
 
             cat_name = f"tau32_wp_{tau32_wp}_{pass_fail}"
             sel_name = f"sel_{cat_name}"

topsf/config/categories_wp.py

@@ -41,7 +41,6 @@
 from columnflow.categorization import Categorizer, categorizer
 
 from topsf.config.util import create_category_combinations
-from topsf.production.probe_jet import probe_jet
 
 np = maybe_import("numpy")
 ak = maybe_import("awkward")
@@ -153,7 +152,6 @@ def sel_pt_init(self: Categorizer) -> None:
             column = self.cfg.get("column", "FatJet")
             self.uses.add(f"{column}.pt")
 
-
         assert cat_idx < 10**cat_idx_ndigits - 1, "no space for category, ID reassignment necessary"
         cat_id = int(10**cat_idx_lsd * ((cat_idx + 1) + 300))
         print(f"{cat_name = }, {cat_id = }, {cat_idx_lsd = }")