Merge branch 'master' into selection_without_weights

analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py

@@ -8,11 +8,11 @@
 import order as od
 from scinum import Number
 
-from columnflow.util import DotDict, maybe_import
-from columnflow.columnar_util import EMPTY_FLOAT, ColumnCollection
 from columnflow.config_util import (
     get_root_processes_from_campaign, add_shift_aliases, add_category, verify_config_processes,
 )
+from columnflow.columnar_util import EMPTY_FLOAT, ColumnCollection, skip_column
+from columnflow.util import DotDict, maybe_import
 
 ak = maybe_import("awkward")
 

columnflow/tasks/cms/inference.py

@@ -76,7 +76,7 @@ def workflow_requires(self):
 
             if cat_obj.config_data_datasets:
                 for dataset in cat_obj.config_data_datasets:
-                    data_dataset_params[dataset].add(cat_obj.config_variable)
+                    data_dataset_params[dataset]["variables"].add(cat_obj.config_variable)
 
         # set workflow requirements per mc dataset
         reqs["merged_hists"] = set(

columnflow/tasks/external.py

@@ -222,15 +222,16 @@ def iter_nano_files(
 
                 # measure the time required to perform the stat query
                 logger.debug(f"checking fs {selected_fs} for lfn {lfn}")
-                input_file = target_cls(lfn, fs=selected_fs)
+                input_file = target_cls(lfn.lstrip(os.sep) if is_local else lfn, fs=selected_fs)
                 t1 = time.perf_counter()
                 input_stat = input_file.exists(stat=True)
                 duration = time.perf_counter() - t1
                 i += 1
                 logger.info(f"file {lfn} does{'' if input_stat else ' not'} exist at fs {selected_fs}")
 
-                # when the stat query took longer than 2 seconds, eagerly try the next fs
+                # when the stat query took longer than some duration, eagerly try the next fs
                 # and check if it responds faster and if so, take it instead
+                latency = 4.0  # s
                 if input_stat and eager_lookup:
                     if (
                         isinstance(eager_lookup, int) and
@@ -239,9 +240,11 @@ def iter_nano_files(
                     ):
                         logger.debug(f"eager fs lookup skipped for fs {selected_fs} at index {i}")
                     else:
-                        if input_stat and not last_working and duration > 2.0 and i < len(fs):
+                        if input_stat and not last_working and duration > latency and i < len(fs):
                             last_working = selected_fs, input_file, input_stat, duration
-                            logger.debug("duration exceeded 2s, checking next fs for comparison")
+                            logger.debug(
+                                f"duration exceeded {latency}s, checking next fs for comparison",
+                            )
                             continue
                         if last_working and (not input_stat or last_working[3] < duration):
                             logger.debug("previously checked fs responded faster")