From 9dcbf9fe85d7a9f8c1a2514134e8543c5b97a654 Mon Sep 17 00:00:00 2001 From: "Marcel R." Date: Thu, 9 Nov 2023 09:52:10 +0100 Subject: [PATCH 01/43] Add named event weight sets. --- .../config/analysis___cf_short_name_lc__.py | 11 +-- columnflow/tasks/framework/mixins.py | 67 +++++++++++++++++-- columnflow/tasks/histograms.py | 14 +++- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py b/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py index c9eb07710..04acb0590 100644 --- a/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +++ b/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py @@ -228,11 +228,14 @@ }, }) -# event weight columns as keys in an OrderedDict, mapped to shift instances they depend on +# named sets of event weight columns as keys in an OrderedDict, mapped to shift instances they depend on get_shifts = functools.partial(get_shifts_from_sources, cfg) -cfg.x.event_weights = DotDict({ - "normalization_weight": [], - "muon_weight": get_shifts("mu"), +cfg.x.event_weights = DotDict.wrap({ + # the default set of event weights + "default": { + "normalization_weight": [], + "muon_weight": get_shifts("mu"), + }, }) # versions per task family, either referring to strings or to callables receving the invoking diff --git a/columnflow/tasks/framework/mixins.py b/columnflow/tasks/framework/mixins.py index d8e37ea27..e68487e6e 100644 --- a/columnflow/tasks/framework/mixins.py +++ b/columnflow/tasks/framework/mixins.py @@ -1349,13 +1349,73 @@ def shift_sources_repr(self): class EventWeightMixin(ConfigTask): + event_weights = law.CSVParameter( + default=("default",), + description="names of sets of event weights in the auxiliary data 'event_weights' of the " + "config to use (in the given order); when empty or NO_STR, no weights are applied; " + "default: 'default'", + brace_expand=True, + parse_empty=True, + ) + + @classmethod + def get_event_weights( + cls, + config_inst: od.Config, + names: str | Sequence[str], + ) -> dict[str, list[str]]: + """ + Returns event weights from the given configuration instance *config_inst* according to + *names*. Each name should correspond to one set of event weights in the auxiliary data field + ``event_weights`` of the config. Event weights of corresponding to latter names have + precedence over leading ones. + + :param config_inst: The configuration instance to get the event weights from. + :param names: The names of the set of event weights to get. + :return: A dictionary containing the requested event weights. + :raises ValueError: If the requested event weights are found to be unnamed in the config. + :raises KeyError: If the requested event weights are not found in the config. + """ + # first check if the weights are not stored in a nested way (legacy behavior) + all_weights = config_inst.x.event_weights + first_key = list(all_weights.keys())[0] + is_nested = isinstance(all_weights[first_key], dict) + + # collect weights, looping over names + event_weights = {} + for name in law.util.make_list(names): + # emtpy case + if name in ("", law.NO_STR): + continue + + # use weights as is if they are not nested and the "default" was requested + if not is_nested: + msg = ( + "event weights are found to be unnamed (not nested) in the config, consider " + "updating them accordingly (confit_inst.x.event_weights.some_name = {...})", + ) + if name == "default": + logger.warning(msg) + event_weights.update(all_weights) + raise ValueError(f"requested event weights '{name}' but {msg}") + + # check if they exist + if name not in all_weights: + raise KeyError(f"requested event weights '{name}' not found in the config") + + # add them + event_weights.update(all_weights[name]) + + return event_weights + @classmethod def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str], set[str]]: shifts, upstream_shifts = super().get_known_shifts(config_inst, params) # add shifts introduced by event weights if config_inst.has_aux("event_weights"): - for shift_insts in config_inst.x.event_weights.values(): + event_weights = cls.get_event_weights(config_inst, params.get("event_weights", ())) + for shift_insts in event_weights.values(): shifts |= {shift_inst.name for shift_inst in shift_insts} # optionally also for weights defined by a dataset @@ -1363,9 +1423,8 @@ def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str requested_dataset = params.get("dataset") if requested_dataset not in (None, law.NO_STR): dataset_inst = config_inst.get_dataset(requested_dataset) - if dataset_inst.has_aux("event_weights"): - for shift_insts in dataset_inst.x.event_weights.values(): - shifts |= {shift_inst.name for shift_inst in shift_insts} + for shift_insts in dataset_inst.x("event_weights", {}).values(): + shifts |= {shift_inst.name for shift_inst in shift_insts} return shifts, upstream_shifts diff --git a/columnflow/tasks/histograms.py b/columnflow/tasks/histograms.py index b8cebba98..c283153e2 100644 --- a/columnflow/tasks/histograms.py +++ b/columnflow/tasks/histograms.py @@ -117,6 +117,13 @@ def run(self): # get shift dependent aliases aliases = self.local_shift_inst.x("column_aliases", {}) + # prepare event weights + event_weights = ( + self.get_event_weights(self.config_inst, self.event_weights) + if self.dataset_inst.is_mc + else None + ) + # define columns that need to be read read_columns = {"process_id"} | set(self.category_id_columns) | set(aliases.values()) read_columns |= { @@ -132,8 +139,9 @@ def run(self): else variable_inst.x("inputs", []) ) } + if self.dataset_inst.is_mc: - read_columns |= {Route(column) for column in self.config_inst.x.event_weights} + read_columns |= {Route(column) for column in event_weights} read_columns |= {Route(column) for column in self.dataset_inst.x("event_weights", [])} read_columns = {Route(c) for c in read_columns} @@ -167,9 +175,9 @@ def run(self): ) # build the full event weight - weight = ak.Array(np.ones(len(events))) + weight = ak.Array(np.ones(len(events), dtype=np.float32)) if self.dataset_inst.is_mc and len(events): - for column in self.config_inst.x.event_weights: + for column in event_weights: weight = weight * Route(column).apply(events) for column in self.dataset_inst.x("event_weights", []): if has_ak_column(events, column): From b5ce64578891ee4fe47f7ede59147f66906913a8 Mon Sep 17 00:00:00 2001 From: "Marcel R." Date: Thu, 9 Nov 2023 10:02:46 +0100 Subject: [PATCH 02/43] Add logger. --- columnflow/tasks/framework/mixins.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/columnflow/tasks/framework/mixins.py b/columnflow/tasks/framework/mixins.py index e68487e6e..9d1cfe53a 100644 --- a/columnflow/tasks/framework/mixins.py +++ b/columnflow/tasks/framework/mixins.py @@ -27,6 +27,9 @@ ak = maybe_import("awkward") +logger = law.logger.get_logger(__name__) + + class CalibratorMixin(ConfigTask): calibrator = luigi.Parameter( From 1c4f36d84816505fce381407be5449df06114fd4 Mon Sep 17 00:00:00 2001 From: "Marcel R." Date: Mon, 13 Nov 2023 17:00:11 +0100 Subject: [PATCH 03/43] Add all new WeightProducer objects. --- .../config/analysis___cf_short_name_lc__.py | 11 +- .../__cf_module_name__/weight/__init__.py | 1 + .../__cf_module_name__/weight/example.py | 43 +++++ columnflow/__init__.py | 6 + columnflow/calibration/__init__.py | 22 +-- columnflow/categorization/__init__.py | 5 +- columnflow/columnar_util.py | 39 ++--- columnflow/production/__init__.py | 33 ++-- columnflow/selection/__init__.py | 28 ++-- columnflow/tasks/framework/base.py | 5 + columnflow/tasks/framework/mixins.py | 148 +++++++++--------- columnflow/tasks/histograms.py | 43 ++--- columnflow/tasks/plotting.py | 3 +- columnflow/weight/__init__.py | 106 +++++++++++++ columnflow/weight/empty.py | 17 ++ law.cfg | 1 + 16 files changed, 339 insertions(+), 172 deletions(-) create mode 100644 analysis_templates/cms_minimal/__cf_module_name__/weight/__init__.py create mode 100644 analysis_templates/cms_minimal/__cf_module_name__/weight/example.py create mode 100644 columnflow/weight/__init__.py create mode 100644 columnflow/weight/empty.py diff --git a/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py b/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py index 04acb0590..e778fed7c 100644 --- a/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +++ b/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py @@ -112,6 +112,7 @@ cfg.x.default_calibrator = "example" cfg.x.default_selector = "example" cfg.x.default_producer = "example" +cfg.x.default_weight_producer = "example" cfg.x.default_ml_model = None cfg.x.default_inference_model = "example" cfg.x.default_categories = ("incl",) @@ -228,16 +229,6 @@ }, }) -# named sets of event weight columns as keys in an OrderedDict, mapped to shift instances they depend on -get_shifts = functools.partial(get_shifts_from_sources, cfg) -cfg.x.event_weights = DotDict.wrap({ - # the default set of event weights - "default": { - "normalization_weight": [], - "muon_weight": get_shifts("mu"), - }, -}) - # versions per task family, either referring to strings or to callables receving the invoking # task instance and parameters to be passed to the task family cfg.x.versions = { diff --git a/analysis_templates/cms_minimal/__cf_module_name__/weight/__init__.py b/analysis_templates/cms_minimal/__cf_module_name__/weight/__init__.py new file mode 100644 index 000000000..57d631c3f --- /dev/null +++ b/analysis_templates/cms_minimal/__cf_module_name__/weight/__init__.py @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py b/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py new file mode 100644 index 000000000..6c657a58b --- /dev/null +++ b/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py @@ -0,0 +1,43 @@ +# coding: utf-8 + +""" +Example event weight producer. +""" + +from columnflow.weight import WeightProducer, weight_producer +from columnflow.util import maybe_import +from columnflow.config_util import get_shifts_from_sources +from columnflow.columnar_util import Route + +ak = maybe_import("awkward") +np = maybe_import("numpy") + + +@weight_producer( + # both produced columns and dependent shifts are defined in init below + # only run on mc + mc_only=True, +) +def example(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: + # build the full event weight + weight = ak.Array(np.ones(len(events), dtype=np.float32)) + for column in self.weight_columns: + weight = weight * Route(column).apply(events) + + return weight + + +@example.init +def example_init(self: WeightProducer) -> None: + # store column names referring to weights to multiply + self.weight_columns = [ + "normalization_weight", + "muon_weight", + ] + self.uses |= set(self.weight_columns) + + # declare shifts that the produced event weight depends on + shift_sources = [ + "mu", + ] + self.shifts |= set(get_shifts_from_sources(self.config_inst, *shift_sources)) diff --git a/columnflow/__init__.py b/columnflow/__init__.py index 632e73539..bce173341 100644 --- a/columnflow/__init__.py +++ b/columnflow/__init__.py @@ -56,6 +56,12 @@ logger.debug(f"loading production module '{m}'") maybe_import(m.strip()) +import columnflow.weight # noqa +if law.config.has_option("analysis", "weight_production_modules"): + for m in law.config.get_expanded("analysis", "weight_production_modules", [], split_csv=True): + logger.debug(f"loading weight production module '{m}'") + maybe_import(m.strip()) + import columnflow.calibration # noqa if law.config.has_option("analysis", "calibration_modules"): for m in law.config.get_expanded("analysis", "calibration_modules", [], split_csv=True): diff --git a/columnflow/calibration/__init__.py b/columnflow/calibration/__init__.py index 19c47ee2d..fe375292c 100644 --- a/columnflow/calibration/__init__.py +++ b/columnflow/calibration/__init__.py @@ -48,15 +48,17 @@ def calibrator( All additional *kwargs* are added as class members of the new subclasses. - :param func: Function to be wrapped and integrated into new :py:class:`Calibrator` - instance , defaults to None - :param bases: additional bases for new :py:class:`Calibrator`, defaults to () - :param mc_only: only run this :py:class:`Calibrator` on Monte Carlo simulation - , defaults to False - :param data_only: only run this :py:class:`Calibrator` on observed data, - defaults to False - :return: new :py:class:`Calibrator` instance with *func* as the `call_func` - or the decorator itself + :param func: Function to be wrapped and integrated into new :py:class:`Calibrator` class. + :param bases: Additional bases for the new :py:class:`Calibrator`. + :param mc_only: Boolean flag indicating that this :py:class:`Calibrator` should only run on + Monte Carlo simulation and skipped for real data. + :param data_only: Boolean flag indicating that this :py:class:`Calibrator` should only run + on real data and skipped for Monte Carlo simulation. + :param nominal_only: Boolean flag indicating that this :py:class:`Calibrator` should only + run on the nominal shift and skipped on any other shifts. + :param shifts_only: Shift names that this :py:class:`Calibrator` should only run on, + skipping all other shifts. + :return: New :py:class:`Calibrator` subclass. """ # prepare shifts_only if shifts_only: @@ -100,7 +102,7 @@ def skip_func(self): if data_only and not self.dataset_inst.is_data: return True - # check nominal_only + # check nominal_only and shifts_only if getattr(self, "global_shift_inst", None): if nominal_only and not self.global_shift_inst.is_nominal: return True diff --git a/columnflow/categorization/__init__.py b/columnflow/categorization/__init__.py index bab5025f5..354038018 100644 --- a/columnflow/categorization/__init__.py +++ b/columnflow/categorization/__init__.py @@ -33,10 +33,9 @@ def categorizer( All additional *kwargs* are added as class members of the new subclasses. - :param func: Function to be wrapped and integrated into new :py:class:`Categorizer` - instance. + :param func: Function to be wrapped and integrated into new :py:class:`Categorizer` class. :param bases: Additional base classes for new :py:class:`Categorizer`. - :return: The new :py:class:`Categorizer` instance. + :return: The new :py:class:`Categorizer` subclass. """ def decorator(func: Callable) -> DerivableMeta: # create the class dict diff --git a/columnflow/columnar_util.py b/columnflow/columnar_util.py index 4d6ae044b..80dc67e6b 100644 --- a/columnflow/columnar_util.py +++ b/columnflow/columnar_util.py @@ -1704,8 +1704,7 @@ def add_dep(cls_or_inst): instances.add(obj) else: - # here, obj must be anything that is accepted by route - instances.add(obj if isinstance(obj, Route) else Route(obj)) + instances.add(obj) # synchronize dependencies # this might remove deps that were present in self.deps already before this method is called @@ -1732,13 +1731,9 @@ def instantiate_dependency( return cls(**kwargs) - def get_dependencies( - self: ArrayFunction, - include_others: bool = False, - ) -> set[ArrayFunction | Any]: + def get_dependencies(self: ArrayFunction) -> set[ArrayFunction | Any]: """ - Returns a set of instances of all dependencies. When *include_others* is *True*, also - non-ArrayFunction types are returned. + Returns a set of instances of all dependencies. """ deps = set() @@ -1749,8 +1744,6 @@ def get_dependencies( obj = obj.wrapped if isinstance(obj, ArrayFunction): deps.add(obj) - elif include_others: - deps.add(obj) return deps @@ -1793,7 +1786,7 @@ def _get_columns( # add the columns columns |= flagged.wrapped._get_columns(flagged.io_flag, _cache=_cache) else: - columns.add(obj) + columns.add(Route(obj)) return columns @@ -2211,19 +2204,19 @@ def _get_all_shifts(self, _cache: set | None = None) -> set[str]: _cache = set() # add shifts and consider _this_ call cached - shifts = { - shift - for shift in self.shifts - if not isinstance(shift, (ArrayFunction, self.IOFlagged)) - } + shifts = set() + for shift in self.shifts: + if isinstance(shift, od.Shift): + shifts.add(shift.name) + elif isinstance(shift, str): + shifts.add(shift) _cache.add(self) # add shifts of all dependent objects - for obj in self.get_dependencies(include_others=False): - if isinstance(obj, TaskArrayFunction): - if obj not in _cache: - _cache.add(obj) - shifts |= obj._get_all_shifts(_cache=_cache) + for obj in self.get_dependencies(): + if isinstance(obj, TaskArrayFunction) and obj not in _cache: + _cache.add(obj) + shifts |= obj._get_all_shifts(_cache=_cache) return shifts @@ -2254,7 +2247,7 @@ def run_requires( # run the requirements of all dependent objects for dep in self.get_dependencies(): - if dep not in _cache: + if isinstance(dep, TaskArrayFunction) and dep not in _cache: _cache.add(dep) dep.run_requires(reqs=reqs, _cache=_cache) @@ -2287,7 +2280,7 @@ def run_setup( # run the setup of all dependent objects for dep in self.get_dependencies(): - if dep not in _cache: + if isinstance(dep, TaskArrayFunction) and dep not in _cache: _cache.add(dep) dep.run_setup(reqs, inputs, reader_targets, _cache=_cache) diff --git a/columnflow/production/__init__.py b/columnflow/production/__init__.py index d0bf8caeb..5c1bf7bd5 100644 --- a/columnflow/production/__init__.py +++ b/columnflow/production/__init__.py @@ -33,29 +33,32 @@ def producer( **kwargs, ) -> DerivableMeta | Callable: """ - Decorator for creating a new :py:class:`Producer` subclass with - additional, optional *bases* and attaching the decorated function to it - as :py:meth:`~Producer.call_func`. + Decorator for creating a new :py:class:`Producer` subclass with additional, optional *bases* + and attaching the decorated function to it as :py:meth:`~Producer.call_func`. - When *mc_only* (*data_only*) is *True*, the calibrator is skipped and not considered by - other calibrators, selectors and producers in case they are evalauted on a + When *mc_only* (*data_only*) is *True*, the producer is skipped and not considered by + other calibrators, selectors and producers in case they are evaluated on a :py:class:`order.Dataset` (using the :py:attr:`dataset_inst` attribute) whose ``is_mc`` (``is_data``) attribute is *False*. - When *nominal_only* is *True* or *shifts_only* is set, the calibrator is skipped and not - considered by other calibrators, selectors and producers in case they are evalauted on a + When *nominal_only* is *True* or *shifts_only* is set, the producer is skipped and not + considered by other calibrators, selectors and producers in case they are evaluated on a :py:class:`order.Shift` (using the :py:attr:`global_shift_inst` attribute) whose name does not match. All additional *kwargs* are added as class members of the new subclasses. - :param func: Callable function that produces new columns - :param bases: Additional bases for new Producer instance - :param mc_only: boolean flag indicating that this Producer instance - should only run on Monte Carlo simulation - :param data_only: boolean flag indicating that this Producer instance - should only run on observed data - :return: new Producer instance + :param func: Function to be wrapped and integrated into new :py:class:`Producer` class. + :param bases: Additional bases for the new :py:class:`Producer`. + :param mc_only: Boolean flag indicating that this :py:class:`Producer` should only run on + Monte Carlo simulation and skipped for real data. + :param data_only: Boolean flag indicating that this :py:class:`Producer` should only run on + real data and skipped for Monte Carlo simulation. + :param nominal_only: Boolean flag indicating that this :py:class:`Producer` should only run + on the nominal shift and skipped on any other shifts. + :param shifts_only: Shift names that this :py:class:`Producer` should only run on, skipping + all other shifts. + :return: New :py:class:`Producer` subclass. """ # prepare shifts_only if shifts_only: @@ -99,7 +102,7 @@ def skip_func(self): if data_only and not self.dataset_inst.is_data: return True - # check nominal_only + # check nominal_only and shifts_only if getattr(self, "global_shift_inst", None): if nominal_only and not self.global_shift_inst.is_nominal: return True diff --git a/columnflow/selection/__init__.py b/columnflow/selection/__init__.py index e557c3e32..da50b546f 100644 --- a/columnflow/selection/__init__.py +++ b/columnflow/selection/__init__.py @@ -50,25 +50,29 @@ def selector( Decorator for creating a new :py:class:`~.Selector` subclass with additional, optional *bases* and attaching the decorated function to it as ``call_func``. - When *mc_only* (*data_only*) is *True*, the calibrator is skipped and not considered by - other calibrators, selectors and producers in case they are evalauted on a + When *mc_only* (*data_only*) is *True*, the selector is skipped and not considered by + other calibrators, selectors and producers in case they are evaluated on a :py:class:`order.Dataset` (using the :py:attr:`dataset_inst` attribute) whose ``is_mc`` (``is_data``) attribute is *False*. - When *nominal_only* is *True* or *shifts_only* is set, the calibrator is skipped and not - considered by other calibrators, selectors and producers in case they are evalauted on a + When *nominal_only* is *True* or *shifts_only* is set, the selector is skipped and not + considered by other calibrators, selectors and producers in case they are evaluated on a :py:class:`order.Shift` (using the :py:attr:`global_shift_inst` attribute) whose name does not match. All additional *kwargs* are added as class members of the new subclasses. - :param func: Callable that is used to perform the selections - :param bases: Additional bases for new subclass - :param mc_only: Flag to indicate that this Selector should only run - on Monte Carlo Simulation - :param data_only: Flag to indicate that this Selector should only run - on observed data - :return: New Selector instance + :param func: Function to be wrapped and integrated into new :py:class:`Selector` class. + :param bases: Additional bases for the new :py:class:`Selector`. + :param mc_only: Boolean flag indicating that this :py:class:`Selector` should only run on + Monte Carlo simulation and skipped for real data. + :param data_only: Boolean flag indicating that this :py:class:`Selector` should only run on + real data and skipped for Monte Carlo simulation. + :param nominal_only: Boolean flag indicating that this :py:class:`Selector` should only run + on the nominal shift and skipped on any other shifts. + :param shifts_only: Shift names that this :py:class:`Selector` should only run on, + skipping all other shifts. + :return: New :py:class:`Selector` subclass. """ # prepare shifts_only if shifts_only: @@ -112,7 +116,7 @@ def skip_func(self): if data_only and not self.dataset_inst.is_data: return True - # check nominal_only + # check nominal_only and shifts_only if getattr(self, "global_shift_inst", None): if nominal_only and not self.global_shift_inst.is_nominal: return True diff --git a/columnflow/tasks/framework/base.py b/columnflow/tasks/framework/base.py index 16dcd4757..fac6bf825 100644 --- a/columnflow/tasks/framework/base.py +++ b/columnflow/tasks/framework/base.py @@ -216,6 +216,11 @@ def get_producer_kwargs(cls, *args, **kwargs) -> dict[str, Any]: # implemented here only for simplified mro control return cls.get_array_function_kwargs(*args, **kwargs) + @classmethod + def get_weight_producer_kwargs(cls, *args, **kwargs) -> dict[str, Any]: + # implemented here only for simplified mro control + return cls.get_array_function_kwargs(*args, **kwargs) + @classmethod def find_config_objects( cls, diff --git a/columnflow/tasks/framework/mixins.py b/columnflow/tasks/framework/mixins.py index 9d1cfe53a..4a893c470 100644 --- a/columnflow/tasks/framework/mixins.py +++ b/columnflow/tasks/framework/mixins.py @@ -20,6 +20,7 @@ from columnflow.calibration import Calibrator from columnflow.selection import Selector from columnflow.production import Producer +from columnflow.weight import WeightProducer from columnflow.ml import MLModel from columnflow.inference import InferenceModel from columnflow.util import maybe_import @@ -38,7 +39,7 @@ class CalibratorMixin(ConfigTask): "'default_calibrator' config", ) - # decibes whether the task itself runs the calibrator and implements its shifts + # decides whether the task itself runs the calibrator and implements its shifts register_calibrator_shifts = False @classmethod @@ -122,7 +123,7 @@ class CalibratorsMixin(ConfigTask): parse_empty=True, ) - # decibes whether the task itself runs the calibrators and implements their shifts + # decides whether the task itself runs the calibrators and implements their shifts register_calibrators_shifts = False @classmethod @@ -208,7 +209,7 @@ class SelectorMixin(ConfigTask): "'default_selector' config", ) - # decibes whether the task itself runs the selector and implements its shifts + # decides whether the task itself runs the selector and implements its shifts register_selector_shifts = False @classmethod @@ -344,7 +345,7 @@ class ProducerMixin(ConfigTask): "'default_producer' config", ) - # decibes whether the task itself runs the producer and implements its shifts + # decides whether the task itself runs the producer and implements its shifts register_producer_shifts = False @classmethod @@ -428,7 +429,7 @@ class ProducersMixin(ConfigTask): parse_empty=True, ) - # decibes whether the task itself runs the producers and implements their shifts + # decides whether the task itself runs the producers and implements their shifts register_producers_shifts = False @classmethod @@ -1350,87 +1351,92 @@ def shift_sources_repr(self): return f"{len(self.shift_sources)}_{law.util.create_hash(sorted(self.shift_sources))}" -class EventWeightMixin(ConfigTask): +class WeightProducerMixin(ConfigTask): - event_weights = law.CSVParameter( - default=("default",), - description="names of sets of event weights in the auxiliary data 'event_weights' of the " - "config to use (in the given order); when empty or NO_STR, no weights are applied; " - "default: 'default'", - brace_expand=True, - parse_empty=True, + weight_producer = luigi.Parameter( + default=RESOLVE_DEFAULT, + description="the name of the weight producer to be used; default: value of the " + "'default_weight_producer' config", ) + # decides whether the task itself runs the weight producer and implements its shifts + register_weight_producer_shifts = False + @classmethod - def get_event_weights( + def get_weight_producer_inst( cls, - config_inst: od.Config, - names: str | Sequence[str], - ) -> dict[str, list[str]]: - """ - Returns event weights from the given configuration instance *config_inst* according to - *names*. Each name should correspond to one set of event weights in the auxiliary data field - ``event_weights`` of the config. Event weights of corresponding to latter names have - precedence over leading ones. - - :param config_inst: The configuration instance to get the event weights from. - :param names: The names of the set of event weights to get. - :return: A dictionary containing the requested event weights. - :raises ValueError: If the requested event weights are found to be unnamed in the config. - :raises KeyError: If the requested event weights are not found in the config. - """ - # first check if the weights are not stored in a nested way (legacy behavior) - all_weights = config_inst.x.event_weights - first_key = list(all_weights.keys())[0] - is_nested = isinstance(all_weights[first_key], dict) - - # collect weights, looping over names - event_weights = {} - for name in law.util.make_list(names): - # emtpy case - if name in ("", law.NO_STR): - continue - - # use weights as is if they are not nested and the "default" was requested - if not is_nested: - msg = ( - "event weights are found to be unnamed (not nested) in the config, consider " - "updating them accordingly (confit_inst.x.event_weights.some_name = {...})", - ) - if name == "default": - logger.warning(msg) - event_weights.update(all_weights) - raise ValueError(f"requested event weights '{name}' but {msg}") + weight_producer: str, + kwargs: dict | None = None, + ) -> WeightProducer: + weight_producer_cls = WeightProducer.get_cls(weight_producer) + if not weight_producer_cls.exposed: + raise RuntimeError( + f"cannot use unexposed weight producer '{weight_producer}' in {cls.__name__}", + ) - # check if they exist - if name not in all_weights: - raise KeyError(f"requested event weights '{name}' not found in the config") + inst_dict = cls.get_weight_producer_kwargs(**kwargs) if kwargs else None + return weight_producer_cls(inst_dict=inst_dict) - # add them - event_weights.update(all_weights[name]) + @classmethod + def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: + params = super().resolve_param_values(params) + + config_inst = params.get("config_inst") + if config_inst: + # add the default weight producer when empty + params["weight_producer"] = cls.resolve_config_default( + params, + params.get("weight_producer"), + container=config_inst, + default_str="default_weight_producer", + multiple=False, + ) + params["weight_producer_inst"] = cls.get_weight_producer_inst( + params["weight_producer"], + params, + ) - return event_weights + return params @classmethod - def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str], set[str]]: + def get_known_shifts( + cls, + config_inst: od.Config, + params: dict[str, Any], + ) -> tuple[set[str], set[str]]: shifts, upstream_shifts = super().get_known_shifts(config_inst, params) - # add shifts introduced by event weights - if config_inst.has_aux("event_weights"): - event_weights = cls.get_event_weights(config_inst, params.get("event_weights", ())) - for shift_insts in event_weights.values(): - shifts |= {shift_inst.name for shift_inst in shift_insts} - - # optionally also for weights defined by a dataset - if "dataset" in params: - requested_dataset = params.get("dataset") - if requested_dataset not in (None, law.NO_STR): - dataset_inst = config_inst.get_dataset(requested_dataset) - for shift_insts in dataset_inst.x("event_weights", {}).values(): - shifts |= {shift_inst.name for shift_inst in shift_insts} + # get the weight producer, update it and add its shifts + weight_producer_inst = params.get("weight_producer_inst") + if weight_producer_inst: + if cls.register_weight_producer_shifts: + shifts |= weight_producer_inst.all_shifts + else: + upstream_shifts |= weight_producer_inst.all_shifts return shifts, upstream_shifts + def __init__(self: WeightProducerMixin, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + # cache for weight producer inst + self._weight_producer_inst = None + + @property + def weight_producer_inst(self: WeightProducerMixin) -> WeightProducer: + if self._weight_producer_inst is None: + self._weight_producer_inst = self.get_weight_producer_inst( + self.weight_producer, + {"task": self}, + ) + + return self._weight_producer_inst + + def store_parts(self: WeightProducerMixin) -> law.util.InsertableDict[str, str]: + parts = super().store_parts() + parts.insert_before("version", "weightprod", f"weight__{self.weight_producer}") + return parts + class ChunkedIOMixin(AnalysisTask): diff --git a/columnflow/tasks/histograms.py b/columnflow/tasks/histograms.py index c283153e2..ad9f66082 100644 --- a/columnflow/tasks/histograms.py +++ b/columnflow/tasks/histograms.py @@ -10,7 +10,7 @@ from columnflow.tasks.framework.base import Requirements, AnalysisTask, DatasetTask, wrapper_factory from columnflow.tasks.framework.mixins import ( CalibratorsMixin, SelectorStepsMixin, ProducersMixin, MLModelsMixin, VariablesMixin, - ShiftSourcesMixin, EventWeightMixin, ChunkedIOMixin, + ShiftSourcesMixin, WeightProducerMixin, ChunkedIOMixin, ) from columnflow.tasks.framework.remote import RemoteWorkflow from columnflow.tasks.reduction import MergeReducedEventsUser, MergeReducedEvents @@ -21,11 +21,11 @@ class CreateHistograms( VariablesMixin, + WeightProducerMixin, MLModelsMixin, ProducersMixin, SelectorStepsMixin, CalibratorsMixin, - EventWeightMixin, ChunkedIOMixin, MergeReducedEventsUser, law.LocalWorkflow, @@ -49,6 +49,9 @@ class CreateHistograms( # (might become a parameter at some point) category_id_columns = {"category_ids"} + # register shifts found in the chosen weight producer to this task + register_weight_producer_shifts = True + def workflow_requires(self): reqs = super().workflow_requires() @@ -117,15 +120,11 @@ def run(self): # get shift dependent aliases aliases = self.local_shift_inst.x("column_aliases", {}) - # prepare event weights - event_weights = ( - self.get_event_weights(self.config_inst, self.event_weights) - if self.dataset_inst.is_mc - else None - ) - # define columns that need to be read - read_columns = {"process_id"} | set(self.category_id_columns) | set(aliases.values()) + read_columns = {Route("process_id")} + read_columns |= set(map(Route, self.category_id_columns)) + read_columns |= set(self.weight_producer_inst.used_columns) + read_columns |= set(map(Route, aliases.values())) read_columns |= { Route(inp) for variable_inst in ( @@ -140,11 +139,6 @@ def run(self): ) } - if self.dataset_inst.is_mc: - read_columns |= {Route(column) for column in event_weights} - read_columns |= {Route(column) for column in self.dataset_inst.x("event_weights", [])} - read_columns = {Route(c) for c in read_columns} - # empty float array to use when input files have no entries empty_f32 = ak.Array(np.array([], dtype=np.float32)) @@ -175,18 +169,11 @@ def run(self): ) # build the full event weight - weight = ak.Array(np.ones(len(events), dtype=np.float32)) - if self.dataset_inst.is_mc and len(events): - for column in event_weights: - weight = weight * Route(column).apply(events) - for column in self.dataset_inst.x("event_weights", []): - if has_ak_column(events, column): - weight = weight * Route(column).apply(events) - else: - self.logger.warning_once( - f"missing_dataset_weight_{column}", - f"weight '{column}' for dataset {self.dataset_inst.name} not found", - ) + weight = ( + ak.Array(np.ones(len(events), dtype=np.float32)) + if self.weight_producer_inst.skip_func() + else self.weight_producer_inst(events) + ) # define and fill histograms, taking into account multiple axes for var_key, var_names in self.variable_tuples.items(): @@ -261,6 +248,7 @@ def expr(events, *args, **kwargs): class MergeHistograms( VariablesMixin, + WeightProducerMixin, MLModelsMixin, ProducersMixin, SelectorStepsMixin, @@ -361,6 +349,7 @@ def run(self): class MergeShiftedHistograms( VariablesMixin, ShiftSourcesMixin, + WeightProducerMixin, MLModelsMixin, ProducersMixin, SelectorStepsMixin, diff --git a/columnflow/tasks/plotting.py b/columnflow/tasks/plotting.py index 3cbceb5e5..daab3ebad 100644 --- a/columnflow/tasks/plotting.py +++ b/columnflow/tasks/plotting.py @@ -12,7 +12,7 @@ from columnflow.tasks.framework.base import Requirements, ShiftTask from columnflow.tasks.framework.mixins import ( - CalibratorsMixin, SelectorStepsMixin, ProducersMixin, MLModelsMixin, + CalibratorsMixin, SelectorStepsMixin, ProducersMixin, MLModelsMixin, WeightProducerMixin, CategoriesMixin, ShiftSourcesMixin, ) from columnflow.tasks.framework.plotting import ( @@ -29,6 +29,7 @@ class PlotVariablesBase( ProcessPlotSettingMixin, CategoriesMixin, MLModelsMixin, + WeightProducerMixin, ProducersMixin, SelectorStepsMixin, CalibratorsMixin, diff --git a/columnflow/weight/__init__.py b/columnflow/weight/__init__.py new file mode 100644 index 000000000..bce758ea9 --- /dev/null +++ b/columnflow/weight/__init__.py @@ -0,0 +1,106 @@ +# coding: utf-8 + +""" +Tools for producing new columns to be used as event or object weights. +""" + +from __future__ import annotations + +import inspect + +from columnflow.types import Callable +from columnflow.util import DerivableMeta +from columnflow.columnar_util import TaskArrayFunction + + +class WeightProducer(TaskArrayFunction): + """ + Base class for all weight producers, i.e., functions that produce and return a single column + that is meant to be used as a per-event or per-object weight. + """ + + exposed = True + + @classmethod + def weight_producer( + cls, + func: Callable | None = None, + bases: tuple = (), + mc_only: bool = False, + data_only: bool = False, + **kwargs, + ) -> DerivableMeta | Callable: + """ + Decorator for creating a new :py:class:`WeightProducer` subclass with additional, optional + *bases* and attaching the decorated function to it as :py:meth:`~WeightProducer.call_func`. + + When *mc_only* (*data_only*) is *True*, the weight producer is skipped and not considered by + other calibrators, selectors and producers in case they are evaluated on a + :py:class:`order.Dataset` (using the :py:attr:`dataset_inst` attribute) whose ``is_mc`` + (``is_data``) attribute is *False*. + + When *nominal_only* is *True* or *shifts_only* is set, the producer is skipped and not + considered by other calibrators, selectors and producers in case they are evaluated on a + :py:class:`order.Shift` (using the :py:attr:`global_shift_inst` attribute) whose name does + not match. + + All additional *kwargs* are added as class members of the new subclasses. + + :param func: Function to be wrapped and integrated into new :py:class:`WeightProducer` + class. + :param bases: Additional bases for the new :py:class:`WeightProducer`. + :param mc_only: Boolean flag indicating that this :py:class:`WeightProducer` should only run + on Monte Carlo simulation and skipped for real data. + :param data_only: Boolean flag indicating that this :py:class:`WeightProducer` should only + run on real data and skipped for Monte Carlo simulation. + :return: New :py:class:`WeightProducer` subclass. + """ + def decorator(func: Callable) -> DerivableMeta: + # create the class dict + cls_dict = { + "call_func": func, + "mc_only": mc_only, + "data_only": data_only, + } + cls_dict.update(kwargs) + + # get the module name + frame = inspect.stack()[1] + module = inspect.getmodule(frame[0]) + + # get the producer name + cls_name = cls_dict.pop("cls_name", func.__name__) + + # optionally add skip function + if mc_only and data_only: + raise Exception(f"weight producer {cls_name} received both mc_only and data_only") + if mc_only or data_only: + if cls_dict.get("skip_func"): + raise Exception( + f"weight producer {cls_name} received custom skip_func, but either mc_only " + "or data_only are set", + ) + + def skip_func(self): + # check mc_only and data_only + if getattr(self, "dataset_inst", None): + if mc_only and not self.dataset_inst.is_mc: + return True + if data_only and not self.dataset_inst.is_data: + return True + + # in all other cases, do not skip + return False + + cls_dict["skip_func"] = skip_func + + # create the subclass + subclass = cls.derive(cls_name, bases=bases, cls_dict=cls_dict, module=module) + + return subclass + + return decorator(func) if func else decorator + + +# shorthand +weight_producer = WeightProducer.weight_producer diff --git a/columnflow/weight/empty.py b/columnflow/weight/empty.py new file mode 100644 index 000000000..71e489148 --- /dev/null +++ b/columnflow/weight/empty.py @@ -0,0 +1,17 @@ +# coding: utf-8 + +""" +Empty event weight producer. +""" + +from columnflow.weight import WeightProducer, weight_producer +from columnflow.util import maybe_import + +np = maybe_import("numpy") +ak = maybe_import("awkward") + + +@weight_producer +def empty(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: + # simply return ones + return ak.Array(np.ones(len(events), dtype=np.float32)) diff --git a/law.cfg b/law.cfg index 559031a07..18bc72bbf 100644 --- a/law.cfg +++ b/law.cfg @@ -24,6 +24,7 @@ production_modules: columnflow.production.{categories,processes,normalization} calibration_modules: columnflow.calibration selection_modules: columnflow.selection categorization_modules: columnflow.categorization +weight_production_modules: columnflow.weight.{empty} ml_modules: columnflow.ml inference_modules: columnflow.inference From a880fb57f4eb66ba028beab5e4a489ee34dedb42 Mon Sep 17 00:00:00 2001 From: "Marcel R." Date: Mon, 13 Nov 2023 17:01:00 +0100 Subject: [PATCH 04/43] Linting. --- .../config/analysis___cf_short_name_lc__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py b/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py index e778fed7c..c88136264 100644 --- a/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +++ b/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py @@ -4,8 +4,6 @@ Configuration of the __cf_analysis_name__ analysis. """ -import functools - import law import order as od from scinum import Number @@ -13,8 +11,7 @@ from columnflow.util import DotDict, maybe_import from columnflow.columnar_util import EMPTY_FLOAT from columnflow.config_util import ( - get_root_processes_from_campaign, add_shift_aliases, get_shifts_from_sources, add_category, - verify_config_processes, + get_root_processes_from_campaign, add_shift_aliases, add_category, verify_config_processes, ) ak = maybe_import("awkward") From 8cc10e41bd29a226403847a600d4ba2505b71736 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 22 Feb 2024 11:34:49 +0100 Subject: [PATCH 05/43] fix wrong key in pileup variable map --- columnflow/production/cms/pileup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/columnflow/production/cms/pileup.py b/columnflow/production/cms/pileup.py index 2a7779b8f..cc194fddc 100644 --- a/columnflow/production/cms/pileup.py +++ b/columnflow/production/cms/pileup.py @@ -49,7 +49,7 @@ def pu_weight(self: Producer, events: ak.Array, **kwargs) -> ak.Array: ("pu_weight_minbias_xs_down", "down"), ): # get the inputs for this type of variation - variable_map_syst = {**variable_map, "systematic": syst} + variable_map_syst = {**variable_map, "weights": syst} inputs = [variable_map_syst[inp.name] for inp in self.pileup_corrector.inputs] # evaluate and store the produced column From de11ccfe104a2e6b233153a568d8e4c1bf7b38be Mon Sep 17 00:00:00 2001 From: Philip Keicher <26219567+pkausw@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:25:45 +0100 Subject: [PATCH 06/43] Update various sections of the docs (#397) * introduce dedicated section for tasks * move technical aspects of tasks to task overview section * update plotting user guide * include plotting tasks in documentation * adding new plotting tasks to top level tree of docs * include sphinx add-on for mermaid * example for automatic creating of inheritance tree with mermaid * create rst file to use mermaid extension more easily * added documentation basis for most abc functions and general usage * update plotting user guide * updated ml userguide by replacing dummy code with more physics related code * cleanup * add custom plotting function to template * start documentation Config class, small modification comments in analysis template as we are now using categorizers and not selectors * allow overwriting cms_label default with general_settings * add plot function example to docs * add some minor doc lines * add sphinx-design and sphinx-copybutton to docs requirements * update plotting user guide * added sphinx hook to import description of luigi parameters * add exemplary plots to plotting user guide * add plotting task graph and update plotting guide * add auxiliary part in config docu, put config docu in correct file, correct several small mistakes * Added Code example file to refere to in the documentation * Added cross reference to code blocks in documentation code file ml_code.py * finished ml guide for first review * add analysis and campaign sections, remove corresponding dataset TODO) * Apply suggestions from code review Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * review comments * remove autoclasstree * more review comments * outsource of the sandbox block within ml.md, into a separate tutorial (sandbox.ml) soley focusing on setting up sandbox environments * added section about uses. Increasing the readability by move models configuration on top and removing the previous section separation into steps. Separate derive and cls attribute definition and added more explanation about the whole init process of a model. * changes according to pull request comments mathis * add forgotten comment on x-sections * add missing - for required parameters * fixed hyperref to the uses function * Apply suggestions from code review add links Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review added links to functions Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review mostly typos Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review fixed mostly typos, add further details in explaining good practice for saving ml related files Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * implement comment pull request * Apply suggestions from code review Added explanation about cross validation, and rewrote some text to be less confusing * added sandbox.md into toc of user_guide. Correct code for trainings_calibrator/selector and added more clarification about how to use it * Apply suggestions from code review Added link to scikit for cross validation, fixed some typos, added some comments to the code of `produces` to be more clear about what to preserve. * made helper functions more explicit * fixed paths in sandboxes. fixed typo in uses and train in ml.md * Apply suggestions from code review added missing text about versioning * Apply suggestions from code review, these are mostly typos - fix typos - separate 1 code blocks to make copy easier - fixed link by remove 1 : Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review to sandbox file - fixed mostly typos - move ml_code.py into examples dir - changed CF_BASE into ANALYSIS_BASE, since users should not interact directly within CF Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * moved code exmaple into exmaples dir, and redirect all paths to this new one in ml.md * fixed some parsing errors of doc strings, started explicite typing of variables * switched to fully automatic summary of columnar_util * switched to fully automatic summary of config_util * added docs for categorization module * started docs for cms production module * added docstrings to ml __init__ api * Apply suggestions from code review added one word... * explain advantage of using general-settings * add description to general-settings resolving * add comment to task parameters for different plot tasks * implement comments up to custom retrieval datasets * first round corrections pull request ended * correct internal links * add dropdown, correct include statements * WIP implement categorization guide * update categorization guide * introduce dedicated section for tasks * move technical aspects of tasks to task overview section * update plotting user guide * include plotting tasks in documentation * adding new plotting tasks to top level tree of docs * include sphinx add-on for mermaid * example for automatic creating of inheritance tree with mermaid * create rst file to use mermaid extension more easily * update plotting user guide * cleanup * add custom plotting function to template * allow overwriting cms_label default with general_settings * add plot function example to docs * add some minor doc lines * add sphinx-design and sphinx-copybutton to docs requirements * update plotting user guide * added sphinx hook to import description of luigi parameters * add exemplary plots to plotting user guide * add plotting task graph and update plotting guide * Apply suggestions from code review Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * review comments * remove autoclasstree * more review comments * explain advantage of using general-settings * add description to general-settings resolving * add comment to task parameters for different plot tasks * added documentation basis for most abc functions and general usage * updated ml userguide by replacing dummy code with more physics related code * Added Code example file to refere to in the documentation * Added cross reference to code blocks in documentation code file ml_code.py * finished ml guide for first review * outsource of the sandbox block within ml.md, into a separate tutorial (sandbox.ml) soley focusing on setting up sandbox environments * added section about uses. Increasing the readability by move models configuration on top and removing the previous section separation into steps. Separate derive and cls attribute definition and added more explanation about the whole init process of a model. * fixed hyperref to the uses function * Apply suggestions from code review add links Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review added links to functions Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review mostly typos Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review fixed mostly typos, add further details in explaining good practice for saving ml related files Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review Added explanation about cross validation, and rewrote some text to be less confusing * added sandbox.md into toc of user_guide. Correct code for trainings_calibrator/selector and added more clarification about how to use it * Apply suggestions from code review Added link to scikit for cross validation, fixed some typos, added some comments to the code of `produces` to be more clear about what to preserve. * made helper functions more explicit * fixed paths in sandboxes. fixed typo in uses and train in ml.md * Apply suggestions from code review added missing text about versioning * Apply suggestions from code review, these are mostly typos - fix typos - separate 1 code blocks to make copy easier - fixed link by remove 1 : Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review to sandbox file - fixed mostly typos - move ml_code.py into examples dir - changed CF_BASE into ANALYSIS_BASE, since users should not interact directly within CF Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * moved code exmaple into exmaples dir, and redirect all paths to this new one in ml.md * added docstrings to ml __init__ api * Apply suggestions from code review added one word... * Apply suggestions from code review added one word... * fix typo in ml docs * remove duplicate definition of function which was introduced in previous merge * implementation requested changes pull request, add groups analysis template * add comments templates, add link to analysis template docu * correct variables linking, merge two variables examples * Apply suggestions from code review Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * review comments categorization guide * add minimal example for producer that runs category_ids * fix task call examples in category guide * remove duplicated line * Include rudimentary API docs for all modules (#386) * introduce dedicated section for tasks * move technical aspects of tasks to task overview section * update plotting user guide * include plotting tasks in documentation * adding new plotting tasks to top level tree of docs * include sphinx add-on for mermaid * example for automatic creating of inheritance tree with mermaid * create rst file to use mermaid extension more easily * update plotting user guide * cleanup * add custom plotting function to template * allow overwriting cms_label default with general_settings * add plot function example to docs * add some minor doc lines * add sphinx-design and sphinx-copybutton to docs requirements * update plotting user guide * added sphinx hook to import description of luigi parameters * add exemplary plots to plotting user guide * add plotting task graph and update plotting guide * Apply suggestions from code review Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * review comments * remove autoclasstree * more review comments * explain advantage of using general-settings * add description to general-settings resolving * add comment to task parameters for different plot tasks * added documentation basis for most abc functions and general usage * updated ml userguide by replacing dummy code with more physics related code * Added Code example file to refere to in the documentation * Added cross reference to code blocks in documentation code file ml_code.py * finished ml guide for first review * outsource of the sandbox block within ml.md, into a separate tutorial (sandbox.ml) soley focusing on setting up sandbox environments * added section about uses. Increasing the readability by move models configuration on top and removing the previous section separation into steps. Separate derive and cls attribute definition and added more explanation about the whole init process of a model. * fixed hyperref to the uses function * Apply suggestions from code review add links Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review added links to functions Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review mostly typos Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review fixed mostly typos, add further details in explaining good practice for saving ml related files Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review Added explanation about cross validation, and rewrote some text to be less confusing * added sandbox.md into toc of user_guide. Correct code for trainings_calibrator/selector and added more clarification about how to use it * Apply suggestions from code review Added link to scikit for cross validation, fixed some typos, added some comments to the code of `produces` to be more clear about what to preserve. * made helper functions more explicit * fixed paths in sandboxes. fixed typo in uses and train in ml.md * Apply suggestions from code review added missing text about versioning * Apply suggestions from code review, these are mostly typos - fix typos - separate 1 code blocks to make copy easier - fixed link by remove 1 : Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * Apply suggestions from code review to sandbox file - fixed mostly typos - move ml_code.py into examples dir - changed CF_BASE into ANALYSIS_BASE, since users should not interact directly within CF Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * moved code exmaple into exmaples dir, and redirect all paths to this new one in ml.md * added docstrings to ml __init__ api * Apply suggestions from code review added one word... * implemented doc strings for decorators * added more doc strings for mixins * generalized sub directory structure in existing api docs * added more doc strings to different mixin classes * added framework module to api docs * added cms-specific tasks to api docs * added all tasks to api docs * updated strucutre for calibration api * updated structure for selection api, added missing modules to api * generalized format in apis * added categorization module to api * added plotting modules to api * updated structure of production module api, added missing submodules * updated top level api structure * simple script to create rudimentary sphinx inputs for multiple files * added docs for types - doesn't define anything new, so currently empty. TBD * fix linting * updated docstring with proper mermaid graph * finalize rebase merge * add missing types * fix linting --------- Co-authored-by: Philip Daniel Keicher Co-authored-by: Mathis Frahm Co-authored-by: Mathis Frahm <49306645+mafrahm@users.noreply.github.com> Co-authored-by: Bogdan Wiederspan Co-authored-by: Bogdan-Wiederspan <79155113+Bogdan-Wiederspan@users.noreply.github.com> * Update docs/user_guide/building_blocks/categories.md Co-authored-by: Philip Keicher <26219567+pkausw@users.noreply.github.com> * use python classes as type hints where possible * lint --------- Co-authored-by: Philip Daniel Keicher Co-authored-by: Mathis Frahm Co-authored-by: Bogdan Wiederspan Co-authored-by: Nathan Prouvost Co-authored-by: Bogdan-Wiederspan <79155113+Bogdan-Wiederspan@users.noreply.github.com> Co-authored-by: Mathis Frahm <49306645+mafrahm@users.noreply.github.com> --- .../config/analysis___cf_short_name_lc__.py | 49 +- .../__cf_module_name__/plotting/__init__ | 1 + .../__cf_module_name__/plotting/example.py | 83 +++ columnflow/config_util.py | 129 +++-- columnflow/ml/__init__.py | 114 ++++ columnflow/tasks/external.py | 4 + columnflow/tasks/framework/decorators.py | 52 +- columnflow/tasks/framework/mixins.py | 294 ++++++++-- columnflow/tasks/framework/plotting.py | 8 +- columnflow/tasks/plotting.py | 7 + .../calibration/{cms.rst => cms/index.rst} | 4 +- docs/api/calibration/{ => cms}/jets.rst | 0 .../api/calibration/{ => cms}/jets_coffea.rst | 0 docs/api/calibration/{ => cms}/met.rst | 0 docs/api/calibration/index.rst | 13 +- docs/api/calibration/util.rst | 28 +- docs/api/categorization/index.rst | 8 + docs/api/columnar_util.rst | 158 +----- docs/api/config_util.rst | 39 +- docs/api/index.rst | 5 +- docs/api/plotting/index.rst | 2 + docs/api/plotting/plot_all.rst | 9 + docs/api/plotting/plot_functions_1d.rst | 9 + docs/api/plotting/plot_functions_2d.rst | 9 + docs/api/plotting/plot_util.rst | 9 + docs/api/production/categories.rst | 9 + docs/api/production/cms.rst | 7 + docs/api/production/cms/btag.rst | 9 + docs/api/production/cms/electron.rst | 9 + docs/api/production/cms/gen_top_decay.rst | 9 + docs/api/production/cms/index.rst | 21 + docs/api/production/cms/mc_weight.rst | 9 + docs/api/production/cms/muon.rst | 9 + docs/api/production/cms/pdf.rst | 9 + docs/api/production/cms/pileup.rst | 9 + docs/api/production/cms/scale.rst | 9 + docs/api/production/cms/seeds.rst | 9 + docs/api/production/index.rst | 22 +- docs/api/production/muon.rst | 8 + docs/api/production/normalization.rst | 9 + docs/api/production/processes.rst | 9 + docs/api/production/util.rst | 26 +- docs/api/selection/{cms.rst => cms/index.rst} | 3 +- docs/api/selection/{ => cms}/json_filter.rst | 6 +- docs/api/selection/{ => cms}/met_filters.rst | 7 +- docs/api/selection/empty.rst | 9 + docs/api/selection/index.rst | 28 +- docs/api/selection/matching.rst | 12 +- docs/api/selection/stats.rst | 9 + docs/api/selection/util.rst | 9 + docs/api/tasks/calibration.rst | 9 +- docs/api/tasks/cms.rst | 6 - docs/api/tasks/cms/base.rst | 8 + docs/api/tasks/cms/external.rst | 8 + docs/api/tasks/cms/index.rst | 15 + docs/api/tasks/cms/inference.rst | 8 + docs/api/tasks/cutflow.rst | 9 + docs/api/tasks/external.rst | 11 +- docs/api/tasks/framework/base.rst | 5 +- docs/api/tasks/framework/decorators.rst | 6 +- docs/api/tasks/framework/index.rst | 8 +- docs/api/tasks/framework/mixins.rst | 2 +- docs/api/tasks/framework/parameters.rst | 8 + docs/api/tasks/framework/plotting.rst | 8 + docs/api/tasks/framework/remote.rst | 8 + docs/api/tasks/histograms.rst | 9 + docs/api/tasks/index.rst | 30 +- docs/api/tasks/ml.rst | 9 + docs/api/tasks/plotting.rst | 9 + docs/api/tasks/production.rst | 9 + docs/api/tasks/reduction.rst | 11 +- docs/api/tasks/selection.rst | 11 +- docs/api/tasks/union.rst | 9 + docs/api/tasks/yields.rst | 9 + docs/api/types.rst | 9 + docs/build_simple_rst_docs.py | 133 +++++ docs/conf.py | 16 + docs/index.rst | 1 + ...lot__proc_st__cat_incl__var_cf_jet1_pt.pdf | Bin 0 -> 130 bytes ...lot__proc_tt__cat_incl__var_cf_jet1_pt.pdf | Bin 0 -> 130 bytes ...2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf | Bin 0 -> 130 bytes ...2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf | Bin 0 -> 130 bytes ...2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf | Bin 0 -> 130 bytes ..._analy__1__12a17bf79c__cutflow__cat_2j.pdf | Bin 0 -> 130 bytes ...naly__1__12a17bf79c__cutflow__cat_incl.pdf | Bin 0 -> 130 bytes ...11e799f__unc_mu__cat_incl__var_jet1_pt.pdf | Bin 0 -> 130 bytes ...2211e799f__unc_mu__cat_incl__var_n_jet.pdf | Bin 0 -> 130 bytes ..._a2211e799f__cat_incl__var_jet1_pt__c1.pdf | Bin 0 -> 130 bytes ..._2_a2211e799f__cat_incl__var_n_jet__c1.pdf | Bin 0 -> 130 bytes ...__proc_3_7727a49dc2__cat_2j__var_n_jet.pdf | Bin 0 -> 130 bytes ...proc_3_7727a49dc2__cat_incl__var_n_jet.pdf | Bin 0 -> 130 bytes ...proc_3_7727a49dc2__cat_incl__var_n_jet.pdf | Bin 0 -> 130 bytes ..._a2211e799f__cat_incl__var_jet1_pt__c3.pdf | Bin 0 -> 130 bytes ..._2_a2211e799f__cat_incl__var_n_jet__c3.pdf | Bin 0 -> 130 bytes ..._a2211e799f__cat_incl__var_jet1_pt__c2.pdf | Bin 0 -> 130 bytes ..._2_a2211e799f__cat_incl__var_n_jet__c2.pdf | Bin 0 -> 130 bytes ...2211e799f__cat_incl__var_jet1_pt-n_jet.pdf | Bin 0 -> 130 bytes ...2211e799f__cat_incl__var_n_jet-jet1_pt.pdf | Bin 0 -> 130 bytes docs/requirements.txt | 5 +- docs/task_overview/index.rst | 8 + docs/task_overview/introduction.md | 99 ++++ docs/task_overview/plotting.md | 106 ++++ docs/task_overview/plotting.rst | 2 + docs/user_guide/building_blocks/categories.md | 525 +++++++++++++++++- .../building_blocks/config_objects.md | 405 ++++++++++++-- docs/user_guide/building_blocks/producers.md | 2 +- docs/user_guide/building_blocks/selectors.md | 2 +- docs/user_guide/examples/ml_code.py | 289 ++++++++++ docs/user_guide/index.rst | 1 + docs/user_guide/law.md | 2 +- docs/user_guide/ml.md | 271 ++++++++- docs/user_guide/plotting.md | 394 ++++++++++++- docs/user_guide/sandbox.md | 59 ++ docs/user_guide/structure.md | 310 +++++------ modules/order | 2 +- 115 files changed, 3478 insertions(+), 697 deletions(-) create mode 100644 analysis_templates/cms_minimal/__cf_module_name__/plotting/__init__ create mode 100644 analysis_templates/cms_minimal/__cf_module_name__/plotting/example.py rename docs/api/calibration/{cms.rst => cms/index.rst} (75%) rename docs/api/calibration/{ => cms}/jets.rst (100%) rename docs/api/calibration/{ => cms}/jets_coffea.rst (100%) rename docs/api/calibration/{ => cms}/met.rst (100%) create mode 100644 docs/api/categorization/index.rst create mode 100644 docs/api/plotting/plot_all.rst create mode 100644 docs/api/plotting/plot_functions_1d.rst create mode 100644 docs/api/plotting/plot_functions_2d.rst create mode 100644 docs/api/plotting/plot_util.rst create mode 100644 docs/api/production/categories.rst create mode 100644 docs/api/production/cms.rst create mode 100644 docs/api/production/cms/btag.rst create mode 100644 docs/api/production/cms/electron.rst create mode 100644 docs/api/production/cms/gen_top_decay.rst create mode 100644 docs/api/production/cms/index.rst create mode 100644 docs/api/production/cms/mc_weight.rst create mode 100644 docs/api/production/cms/muon.rst create mode 100644 docs/api/production/cms/pdf.rst create mode 100644 docs/api/production/cms/pileup.rst create mode 100644 docs/api/production/cms/scale.rst create mode 100644 docs/api/production/cms/seeds.rst create mode 100644 docs/api/production/muon.rst create mode 100644 docs/api/production/normalization.rst create mode 100644 docs/api/production/processes.rst rename docs/api/selection/{cms.rst => cms/index.rst} (99%) rename docs/api/selection/{ => cms}/json_filter.rst (76%) rename docs/api/selection/{ => cms}/met_filters.rst (68%) create mode 100644 docs/api/selection/empty.rst create mode 100644 docs/api/selection/stats.rst create mode 100644 docs/api/selection/util.rst delete mode 100644 docs/api/tasks/cms.rst create mode 100644 docs/api/tasks/cms/base.rst create mode 100644 docs/api/tasks/cms/external.rst create mode 100644 docs/api/tasks/cms/index.rst create mode 100644 docs/api/tasks/cms/inference.rst create mode 100644 docs/api/tasks/cutflow.rst create mode 100644 docs/api/tasks/framework/parameters.rst create mode 100644 docs/api/tasks/framework/plotting.rst create mode 100644 docs/api/tasks/framework/remote.rst create mode 100644 docs/api/tasks/histograms.rst create mode 100644 docs/api/tasks/ml.rst create mode 100644 docs/api/tasks/plotting.rst create mode 100644 docs/api/tasks/production.rst create mode 100644 docs/api/tasks/union.rst create mode 100644 docs/api/tasks/yields.rst create mode 100644 docs/api/types.rst create mode 100644 docs/build_simple_rst_docs.py create mode 100644 docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__c3947accbb__plot__proc_st__cat_incl__var_cf_jet1_pt.pdf create mode 100644 docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__c3947accbb__plot__proc_tt__cat_incl__var_cf_jet1_pt.pdf create mode 100644 docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step0_Initial__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf create mode 100644 docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step1_jet__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf create mode 100644 docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step2_muon__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf create mode 100644 docs/plots/cf.PlotCutflow_tpl_config_analy__1__12a17bf79c__cutflow__cat_2j.pdf create mode 100644 docs/plots/cf.PlotCutflow_tpl_config_analy__1__12a17bf79c__cutflow__cat_incl.pdf create mode 100644 docs/plots/cf.PlotShiftedVariables1D_tpl_config_analy__1__42b45aba89__plot__proc_2_a2211e799f__unc_mu__cat_incl__var_jet1_pt.pdf create mode 100644 docs/plots/cf.PlotShiftedVariables1D_tpl_config_analy__1__42b45aba89__plot__proc_2_a2211e799f__unc_mu__cat_incl__var_n_jet.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__0191de868f__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt__c1.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__0191de868f__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c1.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__12dfac316a__plot__proc_3_7727a49dc2__cat_2j__var_n_jet.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__12dfac316a__plot__proc_3_7727a49dc2__cat_incl__var_n_jet.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__4601e8554b__plot__proc_3_7727a49dc2__cat_incl__var_n_jet.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__be60d3bca7__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt__c3.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__be60d3bca7__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c3.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__c80529af83__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt__c2.pdf create mode 100644 docs/plots/cf.PlotVariables1D_tpl_config_analy__1__c80529af83__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c2.pdf create mode 100644 docs/plots/cf.PlotVariables2D_tpl_config_analy__1__b27b994979__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt-n_jet.pdf create mode 100644 docs/plots/cf.PlotVariables2D_tpl_config_analy__1__b27b994979__plot__proc_2_a2211e799f__cat_incl__var_n_jet-jet1_pt.pdf create mode 100644 docs/task_overview/index.rst create mode 100644 docs/task_overview/introduction.md create mode 100644 docs/task_overview/plotting.md create mode 100644 docs/task_overview/plotting.rst create mode 100644 docs/user_guide/examples/ml_code.py create mode 100644 docs/user_guide/sandbox.md diff --git a/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py b/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py index a67f6a5ba..5767bfba6 100644 --- a/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +++ b/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py @@ -117,6 +117,7 @@ cfg.x.default_categories = ("incl",) cfg.x.default_variables = ("n_jet", "jet1_pt") + # process groups for conveniently looping over certain processs # (used in wrapper_factory and during plotting) cfg.x.process_groups = {} @@ -137,12 +138,41 @@ # (used during plotting) cfg.x.shift_groups = {} +# general_settings groups for conveniently looping over different values for the general-settings parameter +# (used during plotting) +cfg.x.general_settings_groups = {} + +# process_settings groups for conveniently looping over different values for the process-settings parameter +# (used during plotting) +cfg.x.process_settings_groups = {} + +# variable_settings groups for conveniently looping over different values for the variable-settings parameter +# (used during plotting) +cfg.x.variable_settings_groups = {} + +# custom_style_config groups for conveniently looping over certain style configs +# (used during plotting) +cfg.x.custom_style_config_groups = {} + # selector step groups for conveniently looping over certain steps # (used in cutflow tasks) cfg.x.selector_step_groups = { "default": ["muon", "jet"], } +# calibrator groups for conveniently looping over certain calibrators +# (used during calibration) +cfg.x.calibrator_groups = {} + +# producer groups for conveniently looping over certain producers +# (used during the ProduceColumns task) +cfg.x.producer_groups = {} + +# ml_model groups for conveniently looping over certain ml_models +# (used during the machine learning tasks) +cfg.x.ml_model_groups = {} + + # custom method and sandbox for determining dataset lfns cfg.x.get_dataset_lfns = None cfg.x.get_dataset_lfns_sandbox = None @@ -211,7 +241,7 @@ cfg.x.keep_columns = DotDict.wrap({ "cf.ReduceEvents": { # general event info, mandatory for reading files with coffea - ColumnCollection.MANDATORY_COFFEA, + ColumnCollection.MANDATORY_COFFEA, # additional columns can be added as strings, similar to object info # object info "Jet.pt", "Jet.eta", "Jet.phi", "Jet.mass", "Jet.btagDeepFlavB", "Jet.hadronFlavour", "Muon.pt", "Muon.eta", "Muon.phi", "Muon.mass", "Muon.pfRelIso04_all", @@ -248,7 +278,7 @@ cfg.add_channel(name="mutau", id=1) # add categories using the "add_category" tool which adds auto-generated ids -# the "selection" entries refer to names of selectors, e.g. in selection/example.py +# the "selection" entries refer to names of categorizers, e.g. in categorization/example.py # note: it is recommended to always add an inclusive category with id=1 or name="incl" which is used # in various places, e.g. for the inclusive cutflow plots and the "empty" selector add_category( @@ -296,6 +326,7 @@ x_title="Number of jets", discrete_x=True, ) +# pt of all jets in every event cfg.add_variable( name="jets_pt", expression="Jet.pt", @@ -303,14 +334,16 @@ unit="GeV", x_title=r"$p_{T} of all jets$", ) +# pt of the first jet in every event cfg.add_variable( - name="jet1_pt", - expression="Jet.pt[:,0]", - null_value=EMPTY_FLOAT, - binning=(40, 0.0, 400.0), - unit="GeV", - x_title=r"Jet 1 $p_{T}$", + name="jet1_pt", # variable name, to be given to the "--variables" argument for the plotting task + expression="Jet.pt[:,0]", # content of the variable + null_value=EMPTY_FLOAT, # value to be given if content not available for event + binning=(40, 0.0, 400.0), # (bins, lower edge, upper edge) + unit="GeV", # unit of the variable, if any + x_title=r"Jet 1 $p_{T}$", # x title of histogram when plotted ) +# eta of the first jet in every event cfg.add_variable( name="jet1_eta", expression="Jet.eta[:,0]", diff --git a/analysis_templates/cms_minimal/__cf_module_name__/plotting/__init__ b/analysis_templates/cms_minimal/__cf_module_name__/plotting/__init__ new file mode 100644 index 000000000..57d631c3f --- /dev/null +++ b/analysis_templates/cms_minimal/__cf_module_name__/plotting/__init__ @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/analysis_templates/cms_minimal/__cf_module_name__/plotting/example.py b/analysis_templates/cms_minimal/__cf_module_name__/plotting/example.py new file mode 100644 index 000000000..943d3ce33 --- /dev/null +++ b/analysis_templates/cms_minimal/__cf_module_name__/plotting/example.py @@ -0,0 +1,83 @@ +# coding: utf-8 + +""" +Examples for custom plot functions. +""" + +from __future__ import annotations + +from collections import OrderedDict + +from columnflow.util import maybe_import +from columnflow.plotting.plot_util import ( + remove_residual_axis, + apply_variable_settings, + apply_process_settings, +) + +hist = maybe_import("hist") +np = maybe_import("numpy") +mpl = maybe_import("matplotlib") +plt = maybe_import("matplotlib.pyplot") +mplhep = maybe_import("mplhep") +od = maybe_import("order") + + +def my_plot1d_func( + hists: OrderedDict[od.Process, hist.Hist], + config_inst: od.Config, + category_inst: od.Category, + variable_insts: list[od.Variable], + style_config: dict | None = None, + yscale: str | None = "", + process_settings: dict | None = None, + variable_settings: dict | None = None, + example_param: str | float | bool | None = None, + **kwargs, +) -> tuple(plt.Figure, tuple(plt.Axis,)): + """ + This is an exemplary custom plotting function. + + Exemplary task call: + + .. code-block:: bash + law run cf.PlotVariables1D --version v1 --processes st,tt --variables jet1_pt \ + --plot-function __cf_module_name__.plotting.example.my_plot1d_func \ + --general-settings example_param=some_text + """ + # we can add arbitrary parameters via the `general_settings` parameter to access them in the + # plotting function. They are automatically parsed either to a bool, float, or string + print(f"The example_param has been set to '{example_param}' (type: {type(example_param)})") + + # call helper function to remove shift axis from histogram + remove_residual_axis(hists, "shift") + + # call helper functions to apply the variable_settings and process_settings + variable_inst = variable_insts[0] + hists = apply_variable_settings(hists, variable_insts, variable_settings) + hists = apply_process_settings(hists, process_settings) + + # use the mplhep CMS stype + plt.style.use(mplhep.style.CMS) + + # create a figure and fill it with content + fig, ax = plt.subplots() + for proc_inst, h in hists.items(): + h.plot1d( + ax=ax, + label=proc_inst.label, + color=proc_inst.color1, + ) + + # styling and parameter implementation (e.g. `yscale`) + ax.set( + yscale=yscale, + ylabel=variable_inst.get_full_y_title(), + xlabel=variable_inst.get_full_x_title(), + xscale="log" if variable_inst.log_x else "linear", + ) + ax.legend() + mplhep.cms.label(ax=ax, fontsize=22, llabel="private work") + + # task expects a figure and a tuple of axes as output + return fig, (ax,) diff --git a/columnflow/config_util.py b/columnflow/config_util.py index 1047a3e9b..20da608d8 100644 --- a/columnflow/config_util.py +++ b/columnflow/config_util.py @@ -23,7 +23,8 @@ from collections import OrderedDict import law -import order as od +import order +od = order from columnflow.util import maybe_import from columnflow.types import Callable, Any, Sequence @@ -70,16 +71,20 @@ def get_events_from_categories( return events[mask] -def get_root_processes_from_campaign(campaign: od.Campaign) -> od.UniqueObjectIndex: - """ - Extracts all root process objects from datasets contained in an order *campaign* and returns +def get_root_processes_from_campaign(campaign: order.config.Campaign) -> order.unique.UniqueObjectIndex: + """Extracts all root process objects from datasets contained in an order *campaign* and returns them in a unique object index. + + :param campaign: :py:class:`~order.config.Campaign` object containing information + about relevant datasets + :return: Unique indices for :py:class:`~order.process.Process` instances of + root processes associated with these datasets """ # get all dataset processes - processes = set.union(*map(set, (dataset.processes for dataset in campaign.datasets))) + processes: set[od.Process] = set.union(*map(set, (dataset.processes for dataset in campaign.datasets))) # get their root processes - root_processes = set.union(*map(set, ( + root_processes: set[od.Process] = set.union(*map(set, ( (process.get_root_processes() or [process]) for process in processes ))) @@ -94,20 +99,19 @@ def get_root_processes_from_campaign(campaign: od.Campaign) -> od.UniqueObjectIn def get_datasets_from_process( - config: od.Config, - process: str | od.Process, + config: order.config.Config, + process: str | order.process.Process, strategy: str = "inclusive", only_first: bool = True, check_deep: bool = False, -) -> list[od.Dataset]: - r""" - Given a *process* and the *config* it belongs to, returns a list of order dataset objects that +) -> list[order.dataset.Dataset]: + r"""Given a *process* and the *config* it belongs to, returns a list of order dataset objects that contain matching processes. This is done by walking through *process* and its child processes and checking whether they are contained in known datasets. *strategy* controls how possible ambiguities are resolved: - ``"all"``: The full process tree is traversed and all matching datasets are considered. - Note that this might lead to a potential overrepresentation of the phase space. + Note that this might lead to a potential over-representation of the phase space. - ``"inclusive"``: If a dataset is found to match a process, its child processes are not checked further. - ``"exclusive"``: If **any** (deep) subprocess of *process* is found to be contained in a @@ -117,43 +121,74 @@ def get_datasets_from_process( As an example, consider the process tree - .. code-block:: none - --- single_top --- - / | \ - / | \ - s_channel t_channel tw_channel - / \ / \ / \ - / \ / \ / \ - t tbar t tbar t tbar + .. mermaid:: + :align: center + :zoom: + + flowchart BT + A[single top] + B{s channel} + C{t channel} + D{tw channel} + E(t) + F(tbar) + G(t) + H(tbar) + I(t) + J(tbar) + + B --> A + C --> A + D --> A + + E --> B + F --> B + + G --> C + H --> C + + I --> D + J --> D and datasets existing for - .. code-block:: none - 1. single_top__s_channel_t - 2. single_top__s_channel_tbar - 3. single_top__t_channel - 4. single_top__t_channel_t - 5. single_top__tw_channel - 6. single_top__tw_channel_t - 7. single_top__tw_channel_tbar - in the *config*. Depending on *strategy*, the returned datasets for process ``single_top``are: + 1. single top - s channel - t + 2. single top - s channel - tbar + 3. single top - t channel + 4. single top - t channel - t + 5. single top - tw channel + 6. single top - tw channel - t + 7. single top - tw channel - tbar + + in the *config*. Depending on *strategy*, the returned datasets for process ``single top``are: - ``"all"``: ``[1, 2, 3, 4, 5, 6, 7]``. Simply all datasets matching any subprocess. - - ``"inclusive"``: ``[1, 2, 3, 5]``. Skipping ``single_top__t_channel_t``, - ``single_top__tw_channel_t``, and ``single_top__tw_channel_tbar``, since more inclusive - datasets (``single_top__t_channel`` and ``single_top__tw_channel``) exist. - - ``"exclusive"``: ``[1, 2, 4, 6, 7]``. Skipping ``single_top__t_channel`` and - ``single_top__tw_channel`` since more exclusive datasets (``single_top__t_channel_t``, - ``single_top__tw_channel_t``, and ``single_top__tw_channel_tbar``) exist. + - ``"inclusive"``: ``[1, 2, 3, 5]``. Skipping ``single top - t channel - t``, + ``single top - tw channel - t``, and ``single top - tw channel - tbar``, since more inclusive + datasets (``single top - t channel`` and ``single top - tw channel``) exist. + - ``"exclusive"``: ``[1, 2, 4, 6, 7]``. Skipping ``single_top - t_channel`` and + ``single top - tw channel`` since more exclusive datasets (``single top - t channel - t``, + ``single top - tw channel - t``, and ``single top - tw channel - tbar``) exist. - ``"exclusive_strict"``: ``[1, 2, 3, 6, 7]``. Like ``"exclusive"``, but not skipping - ``single_top__t_channel`` since not all subprocesses of ``t_channel`` match a dataset - (there is no ``single_top__t_channel_tbar`` dataset). + ``single top - t channel`` since not all subprocesses of ``t channel`` match a dataset + (there is no ``single top - t channel - tbar`` dataset). In addition, two arguments configure how the check is performed whether a process is contained in a dataset. If *only_first* is *True*, only the first matching dataset is considered. Otherwise, all datasets matching a specific process are returned. For the check itself, *check_deep* is forwarded to :py:meth:`order.Dataset.has_process`. + + :param config: Config instance containing the information about known datasets. + :param process: Process instance or process name for which you want to obtain + list of datasets. + :param strategy: controls how possible ambiguities are resolved. Choices: + [``"all"``, ``"inclusive"``, ``"exclusive"``, ``"exclusive_strict"``] + :param only_first: If *True*, only the first matching dataset is considered. + :param check_deep: Forwarded to :py:meth:`order.Dataset.has_process` + :raises ValueError: If *strategy* is not in list of allowed choices + :return: List of datasets that correspond to *process*, depending on the + specifics of the query """ # check the strategy known_strategies = ["all", "inclusive", "exclusive", "exclusive_strict"] @@ -166,7 +201,7 @@ def get_datasets_from_process( # the tree traversal differs depending on the strategy, so distinguish cases if strategy in ["all", "inclusive"]: - dataset_insts = [] + dataset_insts: list[od.Dataset] = [] for process_inst, _, child_insts in root_inst.walk_processes(include_self=True, algo="bfs"): found_dataset = False @@ -187,36 +222,36 @@ def get_datasets_from_process( return law.util.make_unique(dataset_insts) # at this point, strategy is exclusive or exclusive_strict - dataset_insts = OrderedDict() + dataset_insts_dict: OrderedDict[str, od.Dataset] = OrderedDict() for process_inst, _, child_insts in root_inst.walk_processes(include_self=True, algo="dfs_post"): # check if child processes have matched datasets already if child_insts: - n_found = sum(int(child_inst in dataset_insts) for child_inst in child_insts) + n_found = sum(int(child_inst in dataset_insts_dict) for child_inst in child_insts) # potentially skip the current process if strategy == "exclusive" and n_found: continue if strategy == "exclusive_strict" and n_found == len(child_insts): # add a empty list to mark this is done - dataset_insts[process_inst] = [] + dataset_insts_dict[process_inst] = [] continue # at this point, the process itself must be checked, # so remove potentially found datasets of children - dataset_insts = { + dataset_insts_dict = OrderedDict({ child_inst: _dataset_insts - for child_inst, _dataset_insts in dataset_insts.items() + for child_inst, _dataset_insts in dataset_insts_dict.items() if child_inst not in child_insts - } + }) # check datasets for dataset_inst in config.datasets: if dataset_inst.has_process(process_inst, deep=check_deep): - dataset_insts.setdefault(process_inst, []).append(dataset_inst) + dataset_insts_dict.setdefault(process_inst, []).append(dataset_inst) # stop checking more datasets when only the first matters if only_first: break - return sum(dataset_insts.values(), []) + return sum(dataset_insts_dict.values(), []) def add_shift_aliases( @@ -250,7 +285,7 @@ def add_shift_aliases( shift.x.column_aliases = _aliases -def get_shifts_from_sources(config: od.Config, *shift_sources: str) -> list[od.Shift]: +def get_shifts_from_sources(config: od.Config, *shift_sources: Sequence[str]) -> list[od.Shift]: """ Takes a *config* object and returns a list of shift instances for both directions given a sequence *shift_sources*. diff --git a/columnflow/ml/__init__.py b/columnflow/ml/__init__.py index 2ff5fd765..d4a8e392d 100644 --- a/columnflow/ml/__init__.py +++ b/columnflow/ml/__init__.py @@ -270,6 +270,11 @@ def used_columns(self: MLModel) -> dict[od.Config, set[Route]]: @property def produced_columns(self: MLModel) -> dict[od.Config, set[Route]]: + """ + Helper function to resolve column names of produced with this MLModel instance. + + :returns: Set of column names + """ self._assert_configs("cannot determined produced columns") return { config_inst: set(map(Route, self.produces(config_inst))) @@ -320,6 +325,15 @@ def training_calibrators( and/or replace them to define a different set of calibrators for the preprocessing and training pipeline. This can be helpful in cases where training and evaluation phase spaces, as well as the required input columns are intended to diverge. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.training_calibrators + + :param config_inst: Config instance to extract the *requested_calibrators* from + :returns: Set with str of the *requested_calibrators* """ return list(requested_calibrators) @@ -333,6 +347,15 @@ def training_selector( different selector for the preprocessing and training pipeline. This can be helpful in cases where training and evaluation phase spaces, as well as the required input columns are intended to diverge. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.training_selector + + :param config_inst: Config instance to extract the *requested_selector* from + :returns: Set with str of the *requested_selector* """ return requested_selector @@ -346,6 +369,15 @@ def training_producers( replace them to define a different set of producers for the preprocessing and training pipeline. This can be helpful in cases where training and evaluation phase spaces, as well as the required input columns are intended to diverge. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.training_producers + + :param config_inst: Config instance to extract the *requested_producers* from + :returns: Set with str of the *requested_producers* """ return list(requested_producers) @@ -367,6 +399,16 @@ def sandbox(self: MLModel, task: law.Task) -> str: """ Given a *task*, returns the name of a sandbox that is needed to perform model training and evaluation. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.sandbox + + :param task: Task instance to extract the datasets from + :returns: path to the requested sandbox, optinally prefixed by the executing shell command + with trailing :: as separator """ return @@ -375,6 +417,15 @@ def datasets(self: MLModel, config_inst: od.Config) -> set[od.Dataset]: """ Returns a set of all required datasets for a certain *config_inst*. To be implemented in subclasses. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.datasets + + :param config_inst: Config instance to extract the datasets from + :returns: Set with :py:class:`~order.dataset.Dataset` instances """ return @@ -383,6 +434,15 @@ def uses(self: MLModel, config_inst: od.Config) -> set[Route]: """ Returns a set of all required columns for a certain *config_inst*. To be implemented in subclasses. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.uses + + :param config_inst: Config instance to extract the datasets from + :returns: Set with str of required columns """ return @@ -391,6 +451,15 @@ def produces(self: MLModel, config_inst: od.Config) -> set[Route]: """ Returns a set of all produced columns for a certain *config_inst*. To be implemented in subclasses. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.produces + + :param config_inst: Config instance to extract the datasets from + :returns: Set with str of produced columns """ return @@ -398,6 +467,15 @@ def produces(self: MLModel, config_inst: od.Config) -> set[Route]: def output(self: MLModel, task: law.Task) -> Any: """ Returns a structure of output targets. To be implemented in subclasses. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.output + + :param task: Task instance used extract task related information + :returns: Instance of :py:class:`~law.DirectoryTarget`, containing the path to directory. """ return @@ -406,6 +484,16 @@ def open_model(self: MLModel, target: Any) -> Any: """ Implemenents the opening of a trained model from *target* (corresponding to the structure returned by :py:meth:`output`). To be implemented in subclasses. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.open_model + + :param target: Instance of :py:class:`~law.DirectoryTarget`, + contains path to directory holding the machine learning model. + :returns: Machine learning model instance """ return @@ -419,6 +507,19 @@ def train( """ Performs the creation and training of a model, being passed a *task* and its *input* and *output*. To be implemented in subclasses. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.train + + :param task: Task instance used extract task related information + :param input: List of instances of :py:class:`~law.DirectoryTarget`, containing the paths + of all required *input* files + :param output: Instance of :py:class:`~law.DirectoryTarget`, contain path to *target* + directory of the task + :returns: None """ return @@ -436,5 +537,18 @@ def evaluate( of *models* corresponds to the number of folds generated by this model, and the already evaluated *fold_indices* for this event chunk that might used depending on *events_used_in_training*. To be implemented in subclasses. + + Example usage: + + .. literalinclude:: ../../user_guide/examples/ml_code.py + :language: python + :pyobject: TestModel.evaluate + + :param task: Task instance used to extract task related information + :param events: Awkward Array containing the events to evaluate + :param models: List containing trained models + :param fold_indices: Awkward Array containing the indices of the folds used for training + :param events_used_in_training: Boolean flag to indicate if events were used during training + :returns: Awkward array containing events with additional columns """ return diff --git a/columnflow/tasks/external.py b/columnflow/tasks/external.py index 9ae6cd8d4..a6a875400 100644 --- a/columnflow/tasks/external.py +++ b/columnflow/tasks/external.py @@ -32,6 +32,7 @@ class GetDatasetLFNs(DatasetTask, law.tasks.TransferLocalFile): default=5, description="number of replicas to generate; default: 5", ) + validate = law.OptionalBoolParameter( default=None, significant=False, @@ -39,7 +40,10 @@ class GetDatasetLFNs(DatasetTask, law.tasks.TransferLocalFile): "expected from the dataset info; default: obtained from 'validate_dataset_lfns' auxiliary " "entry in config", ) + version = None + """Version parameter - deactivated for :py:class:`~columnflow.tasks.external.GetDatasetLFNs` + """ @classmethod def resolve_param_values(cls, params: DotDict) -> DotDict: diff --git a/columnflow/tasks/framework/decorators.py b/columnflow/tasks/framework/decorators.py index 74f84eab9..8e6fc4ed0 100644 --- a/columnflow/tasks/framework/decorators.py +++ b/columnflow/tasks/framework/decorators.py @@ -1,21 +1,57 @@ -# coding: utf-8 - """ Custom law task method decorators. """ import law +from typing import Any, Callable @law.decorator.factory(accept_generator=True) -def view_output_plots(fn, opts, task, *args, **kwargs): - def before_call(): +def view_output_plots( + fn: Callable, + opts: Any, + task: law.Task, + *args: Any, + **kwargs: Any, +) -> tuple[Callable, Callable, Callable]: + """ + Decorator to view output plots. + + This decorator is used to view the output plots of a task. It checks if the task has a view command, + collects all the paths of the output files, and then opens each file using the view command. + + :param fn: The function to be decorated. + :param opts: Options for the decorator. + :param task: The task instance. + :param args: Variable length argument list. + :param kwargs: Arbitrary keyword arguments. + :return: A tuple containing the before_call, call, and after_call functions. + """ + + def before_call() -> None: + """ + Function to be called before the decorated function. + + :return: None + """ return None - def call(state): + def call(state: Any) -> Any: + """ + The decorated function. + + :param state: The state of the task. + :return: The result of the decorated function. + """ return fn(task, *args, **kwargs) - def after_call(state): + def after_call(state: Any) -> None: + """ + Function to be called after the decorated function. + + :param state: The state of the task. + :return: None + """ view_cmd = getattr(task, "view_cmd", None) if not view_cmd or view_cmd == law.NO_STR: return @@ -25,8 +61,8 @@ def after_call(state): view_cmd += " {}" # collect all paths to view - view_paths = [] - outputs = law.util.flatten(task.output()) + view_paths: list[str] = [] + outputs: list[Any] = law.util.flatten(task.output()) while outputs: output = outputs.pop(0) if isinstance(output, law.TargetCollection): diff --git a/columnflow/tasks/framework/mixins.py b/columnflow/tasks/framework/mixins.py index 67c68b2d1..82cefafc7 100644 --- a/columnflow/tasks/framework/mixins.py +++ b/columnflow/tasks/framework/mixins.py @@ -10,13 +10,12 @@ import time import itertools from collections import Counter -from typing import Iterable import luigi import law import order as od -from columnflow.types import Sequence, Any +from columnflow.types import Sequence, Any, Iterable, Union from columnflow.tasks.framework.base import AnalysisTask, ConfigTask, RESOLVE_DEFAULT from columnflow.calibration import Calibrator from columnflow.selection import Selector @@ -69,7 +68,7 @@ def get_calibrator_inst(cls, calibrator: str, kwargs=None) -> Calibrator: :return: The initialized :py:class:`~columnflow.calibration.Calibrator` instance. """ - calibrator_cls = Calibrator.get_cls(calibrator) + calibrator_cls: Calibrator = Calibrator.get_cls(calibrator) if not calibrator_cls.exposed: raise RuntimeError(f"cannot use unexposed calibrator '{calibrator}' in {cls.__name__}") @@ -77,7 +76,7 @@ def get_calibrator_inst(cls, calibrator: str, kwargs=None) -> Calibrator: return calibrator_cls(inst_dict=inst_dict) @classmethod - def resolve_param_values(cls, params: dict) -> dict: + def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: """Resolve parameter values *params* relevant for the :py:class:`CalibratorMixin` and all classes it inherits from. @@ -109,7 +108,7 @@ def resolve_param_values(cls, params: dict) -> dict: return params @classmethod - def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str], set[str]]: + def get_known_shifts(cls, config_inst: od.Config, params: dict[str, Any]) -> tuple[set[str], set[str]]: """Adds set of shifts that the current ``calibrator_inst`` registers to the set of known ``shifts`` and ``upstream_shifts``. @@ -140,7 +139,15 @@ def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str return shifts, upstream_shifts @classmethod - def req_params(cls, inst: law.Task, **kwargs) -> dict: + def req_params(cls, inst: law.Task, **kwargs) -> dict[str, Any]: + """ + Returns the required parameters for the task. + It prefers `--calibrator` set on task-level via command line. + + :param inst: The current task instance. + :param kwargs: Additional keyword arguments. + :return: Dictionary of required parameters. + """ # prefer --calibrator set on task-level via cli kwargs["_prefer_cli"] = law.util.make_set(kwargs.get("_prefer_cli", [])) | {"calibrator"} @@ -156,8 +163,8 @@ def __init__(self, *args, **kwargs): def calibrator_inst(self) -> Calibrator: """Access current :py:class:`~columnflow.calibration.Calibrator` instance. - Loads the current :py:class:`~columnflow.calibration.Calibrator` *calibrator_inst* from - the cache or initializes it. + This method loads the current :py:class:`~columnflow.calibration.Calibrator` + *calibrator_inst* from the cache or initializes it. If the calibrator requests a specific ``sandbox``, set this sandbox as the environment for the current :py:class:`~law.task.base.Task`. @@ -172,11 +179,11 @@ def calibrator_inst(self) -> Calibrator: return self._calibrator_inst - def store_parts(self) -> law.util.InsertableDict: + def store_parts(self) -> law.util.InsertableDict[str, str]: """Create parts to create the output path to store intermediary results for the current :py:class:`~law.task.base.Task`. - Calls :py:meth:`store_parts` of the ``super`` class and inserts + This method calls :py:meth:`store_parts` of the ``super`` class and inserts `{"calibrator": "calib__{self.calibrator}"}` before keyword ``version``. For more information, see e.g. :py:meth:`~columnflow.tasks.framework.base.ConfigTask.store_parts`. @@ -187,6 +194,14 @@ def store_parts(self) -> law.util.InsertableDict: return parts def find_keep_columns(self: ConfigTask, collection: ColumnCollection) -> set[Route]: + """ + Finds the columns to keep based on the *collection*. + + If the collection is `ALL_FROM_CALIBRATOR`, it includes the columns produced by the calibrator. + + :param collection: The collection of columns. + :return: Set of columns to keep. + """ columns = super().find_keep_columns(collection) if collection == ColumnCollection.ALL_FROM_CALIBRATOR: @@ -242,7 +257,10 @@ def get_calibrator_insts(cls, calibrators: Iterable[str], kwargs=None) -> list[C return insts @classmethod - def resolve_param_values(cls, params: law.util.InsertableDict) -> law.util.InsertableDict: + def resolve_param_values( + cls, + params: law.util.InsertableDict[str, Any], + ) -> law.util.InsertableDict[str, Any]: """Resolve values *params* and check against possible default values and calibrator groups. @@ -273,7 +291,11 @@ def resolve_param_values(cls, params: law.util.InsertableDict) -> law.util.Inser return params @classmethod - def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str], set[str]]: + def get_known_shifts( + cls, + config_inst: od.Config, + params: dict[str, Any], + ) -> tuple[set[str], set[str]]: """Adds set of all shifts that the list of ``calibrator_insts`` register to the set of known ``shifts`` and ``upstream_shifts``. @@ -304,7 +326,17 @@ def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str return shifts, upstream_shifts @classmethod - def req_params(cls, inst: law.Task, **kwargs) -> dict: + def req_params(cls, inst: law.Task, **kwargs) -> dict[str, Any]: + """ + Returns the required parameters for the task. + + It prefers ``--calibrators`` set on task-level via command line. + + :param inst: The current task instance. + :param kwargs: Additional keyword arguments. + :return: Dictionary of required parameters. + """ + # prefer --calibrators set on task-level via cli kwargs["_prefer_cli"] = law.util.make_set(kwargs.get("_prefer_cli", [])) | {"calibrators"} @@ -352,7 +384,15 @@ def store_parts(self): return parts def find_keep_columns(self: ConfigTask, collection: ColumnCollection) -> set[Route]: - columns = super().find_keep_columns(collection) + """ + Finds the columns to keep based on the *collection*. + + If the collection is ``ALL_FROM_CALIBRATORS``, it includes the columns produced by the calibrators. + + :param collection: The collection of columns. + :return: Set of columns to keep. + """ + columns: set[Route] = super().find_keep_columns(collection) if collection == ColumnCollection.ALL_FROM_CALIBRATORS: columns |= set.union(*( @@ -404,7 +444,7 @@ def get_selector_inst( return selector_cls(inst_dict=inst_dict) @classmethod - def resolve_param_values(cls, params: dict) -> dict: + def resolve_param_values(cls, params: dict[str, Any]) -> dict: """Resolve values *params* and check against possible default values and selector groups. @@ -437,7 +477,7 @@ def resolve_param_values(cls, params: dict) -> dict: def get_known_shifts( cls, config_inst: od.Config, - params: dict, + params: dict[str, Any], ) -> tuple[set[str], set[str]]: """Adds set of shifts that the current ``selector_inst`` registers to the set of known ``shifts`` and ``upstream_shifts``. @@ -469,7 +509,17 @@ def get_known_shifts( return shifts, upstream_shifts @classmethod - def req_params(cls, inst: law.Task, **kwargs) -> dict: + def req_params(cls, inst: law.Task, **kwargs) -> dict[str, Any]: + """Get the required parameters for the task, preferring the ``--selector`` set on task-level via CLI. + + This method first checks if the --selector parameter is set at the task-level via the command line. + If it is, this parameter is preferred and added to the '_prefer_cli' key in the kwargs dictionary. + The method then calls the 'req_params' method of the superclass with the updated kwargs. + + :param inst: The current task instance. + :param kwargs: Additional keyword arguments that may contain parameters for the task. + :return: A dictionary of parameters required for the task. + """ # prefer --selector set on task-level via cli kwargs["_prefer_cli"] = law.util.make_set(kwargs.get("_prefer_cli", [])) | {"selector"} @@ -580,7 +630,17 @@ def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: return params @classmethod - def req_params(cls, inst: law.Task, **kwargs) -> dict: + def req_params(cls, inst: law.Task, **kwargs) -> dict[str, Any]: + """Get the required parameters for the task, preferring the --selector-steps set on task-level via CLI. + + This method first checks if the --selector-steps parameter is set at the task-level via the command line. + If it is, this parameter is preferred and added to the '_prefer_cli' key in the kwargs dictionary. + The method then calls the 'req_params' method of the superclass with the updated kwargs. + + :param inst: The current task instance. + :param kwargs: Additional keyword arguments that may contain parameters for the task. + :return: A dictionary of parameters required for the task. + """ # prefer --selector-steps set on task-level via cli kwargs["_prefer_cli"] = law.util.make_set(kwargs.get("_prefer_cli", [])) | {"selector_steps"} @@ -648,7 +708,7 @@ def get_producer_inst(cls, producer: str, kwargs=None) -> Producer: :return: The initialized :py:class:`~columnflow.production.Producer` instance. """ - producer_cls = Producer.get_cls(producer) + producer_cls: Producer = Producer.get_cls(producer) if not producer_cls.exposed: raise RuntimeError(f"cannot use unexposed producer '{producer}' in {cls.__name__}") @@ -656,7 +716,7 @@ def get_producer_inst(cls, producer: str, kwargs=None) -> Producer: return producer_cls(inst_dict=inst_dict) @classmethod - def resolve_param_values(cls, params: dict) -> dict: + def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: """Resolve parameter values *params* relevant for the :py:class:`ProducerMixin` and all classes it inherits from. @@ -688,7 +748,7 @@ def resolve_param_values(cls, params: dict) -> dict: return params @classmethod - def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str], set[str]]: + def get_known_shifts(cls, config_inst: od.Config, params: dict[str, Any]) -> tuple[set[str], set[str]]: """Adds set of shifts that the current ``producer_inst`` registers to the set of known ``shifts`` and ``upstream_shifts``. @@ -719,7 +779,17 @@ def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str return shifts, upstream_shifts @classmethod - def req_params(cls, inst: law.Task, **kwargs) -> dict: + def req_params(cls, inst: law.Task, **kwargs) -> dict[str, Any]: + """Get the required parameters for the task, preferring the ``--producer`` set on task-level via CLI. + + This method first checks if the ``--producer`` parameter is set at the task-level via the command line. + If it is, this parameter is preferred and added to the '_prefer_cli' key in the kwargs dictionary. + The method then calls the 'req_params' method of the superclass with the updated kwargs. + + :param inst: The current task instance. + :param kwargs: Additional keyword arguments that may contain parameters for the task. + :return: A dictionary of parameters required for the task. + """ # prefer --producer set on task-level via cli kwargs["_prefer_cli"] = law.util.make_set(kwargs.get("_prefer_cli", [])) | {"producer"} @@ -751,7 +821,7 @@ def producer_inst(self) -> Producer: return self._producer_inst - def store_parts(self) -> law.util.InsertableDict: + def store_parts(self) -> law.util.InsertableDict[str, str]: """Create parts to create the output path to store intermediary results for the current :py:class:`~law.task.base.Task`. @@ -767,6 +837,15 @@ def store_parts(self) -> law.util.InsertableDict: return parts def find_keep_columns(self: ConfigTask, collection: ColumnCollection) -> set[Route]: + """Finds the columns to keep based on the *collection*. + + This method first calls the 'find_keep_columns' method of the superclass with the given *collection*. + If the *collection* is equal to ``ALL_FROM_PRODUCER``, it adds the + columns produced by the producer instance to the set of columns. + + :param collection: The collection of columns. + :return: A set of columns to keep. + """ columns = super().find_keep_columns(collection) if collection == ColumnCollection.ALL_FROM_PRODUCER: @@ -820,7 +899,10 @@ def get_producer_insts(cls, producers: Iterable[str], kwargs=None) -> list[Produ return insts @classmethod - def resolve_param_values(cls, params: law.util.InsertableDict) -> law.util.InsertableDict: + def resolve_param_values( + cls, + params: law.util.InsertableDict[str, Any], + ) -> law.util.InsertableDict[str, Any]: """Resolve values *params* and check against possible default values and producer groups. @@ -851,7 +933,7 @@ def resolve_param_values(cls, params: law.util.InsertableDict) -> law.util.Inser return params @classmethod - def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str], set[str]]: + def get_known_shifts(cls, config_inst: od.Config, params: dict[str, Any]) -> tuple[set[str], set[str]]: """Adds set of all shifts that the list of ``producer_insts`` register to the set of known ``shifts`` and ``upstream_shifts``. @@ -882,7 +964,17 @@ def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str return shifts, upstream_shifts @classmethod - def req_params(cls, inst: law.Task, **kwargs) -> dict: + def req_params(cls, inst: law.Task, **kwargs) -> dict[str, Any]: + """Get the required parameters for the task, preferring the --producers set on task-level via CLI. + + This method first checks if the --producers parameter is set at the task-level via the command line. + If it is, this parameter is preferred and added to the '_prefer_cli' key in the kwargs dictionary. + The method then calls the 'req_params' method of the superclass with the updated kwargs. + + :param inst: The current task instance. + :param kwargs: Additional keyword arguments that may contain parameters for the task. + :return: A dictionary of parameters required for the task. + """ # prefer --producers set on task-level via cli kwargs["_prefer_cli"] = law.util.make_set(kwargs.get("_prefer_cli", [])) | {"producers"} @@ -932,6 +1024,15 @@ def store_parts(self): return parts def find_keep_columns(self: ConfigTask, collection: ColumnCollection) -> set[Route]: + """Finds the columns to keep based on the *collection*. + + This method first calls the 'find_keep_columns' method of the superclass with the given *collection*. + If the *collection* is equal to ``ALL_FROM_PRODUCERS``, it adds the + columns produced by all producer instances to the set of columns. + + :param collection: The collection of columns. + :return: A set of columns to keep. + """ columns = super().find_keep_columns(collection) if collection == ColumnCollection.ALL_FROM_PRODUCERS: @@ -958,7 +1059,17 @@ class MLModelMixinBase(AnalysisTask): exclude_params_repr_empty = {"ml_model"} @classmethod - def req_params(cls, inst: law.Task, **kwargs) -> dict: + def req_params(cls, inst: law.Task, **kwargs) -> dict[str, Any]: + """Get the required parameters for the task, preferring the ``--ml-model`` set on task-level via CLI. + + This method first checks if the ``--ml-model`` parameter is set at the task-level via the command line. + If it is, this parameter is preferred and added to the '_prefer_cli' key in the kwargs dictionary. + The method then calls the 'req_params' method of the superclass with the updated kwargs. + + :param inst: The current task instance. + :param kwargs: Additional keyword arguments that may contain parameters for the task. + :return: A dictionary of parameters required for the task. + """ # prefer --ml-model set on task-level via cli kwargs["_prefer_cli"] = law.util.make_set(kwargs.get("_prefer_cli", [])) | {"ml_model"} @@ -969,26 +1080,22 @@ def get_ml_model_inst( cls, ml_model: str, analysis_inst: od.Analysis, - requested_configs: list[str] or None = None, + requested_configs: list[str] | None = None, **kwargs, ) -> MLModel: - """Get requested *ml_model*. - - :py:class:`~columnflow.ml.MLModel` instance is either initalized or - loaded from cache. - During the initialization, the *analysis_inst* as well as all *kwargs* - are forwarded to the init function of the *ml_model* class. - - :param ml_model: Name of :py:class:`~columnflow.ml.MLModel` to load - :param analysis_inst: Forward this analysis inst to the init function of - new MLModel sub class - :param requested_configs: Configs needed for the training of the ML - application - :param kwargs: Additional keyword arguments to forward to the - :py:class:`~columnflow.ml.MLModel` instance + """Get requested *ml_model* instance. + + This method retrieves the requested *ml_model* instance. + If *requested_configs* are provided, they are used for the training of + the ML application. + + :param ml_model: Name of :py:class:`~columnflow.ml.MLModel` to load. + :param analysis_inst: Forward this analysis inst to the init function of new MLModel sub class. + :param requested_configs: Configs needed for the training of the ML application. + :param kwargs: Additional keyword arguments to forward to the :py:class:`~columnflow.ml.MLModel` instance. :return: :py:class:`~columnflow.ml.MLModel` instance. """ - ml_model_inst = MLModel.get_cls(ml_model)(analysis_inst, **kwargs) + ml_model_inst: MLModel = MLModel.get_cls(ml_model)(analysis_inst, **kwargs) if requested_configs: configs = ml_model_inst.training_configs(list(requested_configs)) @@ -999,10 +1106,23 @@ def get_ml_model_inst( def events_used_in_training( self, - config_inst: od.Config, - dataset_inst: od.Dataset, - shift_inst: od.Shift, + config_inst: od.config.Config, + dataset_inst: od.dataset.Dataset, + shift_inst: od.shift.Shift, ) -> bool: + """Evaluate whether the events for the combination of *dataset_inst* and + *shift_inst* shall be used in the training. + + This method checks if the *dataset_inst* is in the set of datasets of + the current `ml_model_inst` based on the given *config_inst*. Additionally, + the function checks that the *shift_inst* does not have the tag + `"disjoint_from_nominal"`. + + :param config_inst: The configuration instance. + :param dataset_inst: The dataset instance. + :param shift_inst: The shift instance. + :return: True if the events shall be used in the training, False otherwise. + """ # evaluate whether the events for the combination of dataset_inst and shift_inst # shall be used in the training return ( @@ -1012,6 +1132,10 @@ def events_used_in_training( class MLModelTrainingMixin(MLModelMixinBase): + """A mixin class for training machine learning models. + + This class provides parameters for configuring the training of machine learning models. + """ configs = law.CSVParameter( default=(), @@ -1055,7 +1179,23 @@ def resolve_calibrators( ml_model_inst: MLModel, params: dict[str, Any], ) -> tuple[tuple[str]]: - calibrators = params.get("calibrators") or ((),) + """Resolve the calibrators for the given ML model instance. + + This method retrieves the calibrators from the parameters *params* and + broadcasts them to the configs if necessary. + It also resolves `calibrator_groups` and `default_calibrator` from the config(s) associated + with this ML model instance, and validates the number of sequences. + Finally, it checks the retrieved calibrators against + the training calibrators of the model using + :py:meth:`~columnflow.ml.MLModel.training_calibrators` and instantiates them if necessary. + + :param ml_model_inst: The ML model instance. + :param params: A dictionary of parameters that may contain the calibrators. + :return: A tuple of tuples containing the resolved calibrators. + :raises Exception: If the number of calibrator sequences does not match + the number of configs used by the ML model. + """ + calibrators: Union[tuple[str], tuple[tuple[str]]] = params.get("calibrators") or ((),) # broadcast to configs n_configs = len(ml_model_inst.config_insts) @@ -1101,6 +1241,22 @@ def resolve_selectors( ml_model_inst: MLModel, params: dict[str, Any], ) -> tuple[str]: + """Resolve the selectors for the given ML model instance. + + This method retrieves the selectors from the parameters *params* and + broadcasts them to the configs if necessary. + It also resolves `default_selector` from the config(s) associated + with this ML model instance, validates the number of sequences. + Finally, it checks the retrieved selectors against the training selectors + of the model, using + :py:meth:`~columnflow.ml.MLModel.training_selector`, and instantiates them. + + :param ml_model_inst: The ML model instance. + :param params: A dictionary of parameters that may contain the selectors. + :return: A tuple containing the resolved selectors. + :raises Exception: If the number of selector sequences does not match + the number of configs used by the ML model. + """ selectors = params.get("selectors") or (None,) # broadcast to configs @@ -1146,6 +1302,22 @@ def resolve_producers( ml_model_inst: MLModel, params: dict[str, Any], ) -> tuple[tuple[str]]: + """Resolve the producers for the given ML model instance. + + This method retrieves the producers from the parameters *params* and + broadcasts them to the configs if necessary. + It also resolves `producer_groups` and `default_producer` from the config(s) associated + with this ML model instance, validates the number of sequences. + Finally, it checks the retrieved producers against the training producers + of the model, using + :py:meth:`~columnflow.ml.MLModel.training_producers`, and instantiates them. + + :param ml_model_inst: The ML model instance. + :param params: A dictionary of parameters that may contain the producers. + :return: A tuple of tuples containing the resolved producers. + :raises Exception: If the number of producer sequences does not match + the number of configs used by the ML model. + """ producers = params.get("producers") or ((),) # broadcast to configs @@ -1188,6 +1360,16 @@ def resolve_producers( @classmethod def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: + """Resolve the parameter values for the given parameters. + + This method retrieves the parameters and resolves the ML model instance, configs, + calibrators, selectors, and producers. It also calls the model's setup hook. + + :param params: A dictionary of parameters that may contain the analysis instance and ML model. + :return: A dictionary containing the resolved parameters. + :raises Exception: If the ML model instance received configs to define training configs, + but did not define any. + """ params = super().resolve_param_values(params) if "analysis_inst" in params and "ml_model" in params: @@ -1231,7 +1413,21 @@ def __init__(self, *args, **kwargs): configs=list(self.configs), ) - def store_parts(self) -> law.util.InsertableDict: + def store_parts(self) -> law.util.InsertableDict[str, str]: + """Generate a dictionary of store parts for the current instance. + + This method extends the base method to include additional parts related to machine learning + model configurations, calibrators, selectors, producers (CSP), and the ML model instance itself. + If the list of either of the CSPs is empty, the corresponding part is set to ``"none"``, + otherwise, the first two elements of the list are joined with ``"__"``. + If the list of either of the CSPs contains more than two elements, the part is extended + with the number of elements and a hash of the remaining elements, which is + created with :py:meth:`law.util.create_hash`. + The parts are represented as strings and are used to create unique identifiers for the + instance's output. + + :return: An InsertableDict containing the store parts. + """ parts = super().store_parts() # since MLTraining is no CalibratorsMixin, SelectorMixin, ProducerMixin, ConfigTask, # all these parts are missing in the `store_parts` @@ -1833,7 +2029,7 @@ def shift_sources_repr(self): class EventWeightMixin(ConfigTask): @classmethod - def get_known_shifts(cls, config_inst: od.Config, params: dict) -> tuple[set[str], set[str]]: + def get_known_shifts(cls, config_inst: od.Config, params: dict[str, Any]) -> tuple[set[str], set[str]]: shifts, upstream_shifts = super().get_known_shifts(config_inst, params) # add shifts introduced by event weights diff --git a/columnflow/tasks/framework/plotting.py b/columnflow/tasks/framework/plotting.py index a0b843287..9ec714b19 100644 --- a/columnflow/tasks/framework/plotting.py +++ b/columnflow/tasks/framework/plotting.py @@ -60,11 +60,11 @@ class PlotBase(ConfigTask): description="when True, no legend is drawn; default: None", ) cms_label = luigi.Parameter( - default="pw", + default=law.NO_STR, significant=False, description="postfix to add behind the CMS logo; when 'skip', no CMS logo is shown at all; " "the following special values are expanded into the usual postfixes: wip, pre, pw, sim, " - "simwip, simpre, simpw, od, odwip, public; default: 'pw'", + "simwip, simpre, simpw, od, odwip, public; no default", ) @classmethod @@ -99,7 +99,7 @@ def get_plot_parameters(self) -> DotDict: # convert parameters to usable values during plotting params = DotDict() dict_add_strict(params, "skip_legend", self.skip_legend) - dict_add_strict(params, "cms_label", self.cms_label) + dict_add_strict(params, "cms_label", None if self.cms_label == law.NO_STR else self.cms_label) dict_add_strict(params, "general_settings", self.general_settings) dict_add_strict(params, "custom_style_config", self.custom_style_config) return params @@ -177,6 +177,8 @@ def update_plot_kwargs(self, kwargs: dict) -> dict: if isinstance(custom_style_config, dict) and isinstance(style_config, dict): style_config = law.util.merge_dicts(custom_style_config, style_config) kwargs["style_config"] = style_config + # update defaults after application of `general_settings` + kwargs.setdefault("cms_label", "pw") return kwargs diff --git a/columnflow/tasks/plotting.py b/columnflow/tasks/plotting.py index 3cbceb5e5..063aa15e2 100644 --- a/columnflow/tasks/plotting.py +++ b/columnflow/tasks/plotting.py @@ -36,6 +36,9 @@ class PlotVariablesBase( RemoteWorkflow, ): sandbox = dev_sandbox(law.config.get("analysis", "default_columnar_sandbox")) + """sandbox to use for this task. Defaults to *default_columnar_sandbox* from + analysis config. + """ exclude_index = True @@ -44,6 +47,8 @@ class PlotVariablesBase( RemoteWorkflow.reqs, MergeHistograms=MergeHistograms, ) + """Set upstream requirements, in this case :py:class:`~columnflow.tasks.histograms.MergeHistograms` + """ def store_parts(self): parts = super().store_parts() @@ -250,6 +255,8 @@ class PlotVariablesBaseMultiShifts( description="sets the title of the legend; when empty and only one process is present in " "the plot, the process_inst label is used; empty default", ) + """ + """ exclude_index = True diff --git a/docs/api/calibration/cms.rst b/docs/api/calibration/cms/index.rst similarity index 75% rename from docs/api/calibration/cms.rst rename to docs/api/calibration/cms/index.rst index 3c7aa658b..c91151f5e 100644 --- a/docs/api/calibration/cms.rst +++ b/docs/api/calibration/cms/index.rst @@ -1,5 +1,5 @@ -``columnflow.calibration.cms`` -=============================== +``cms`` +======= .. automodule:: columnflow.calibration.cms .. currentmodule:: columnflow.calibration.cms diff --git a/docs/api/calibration/jets.rst b/docs/api/calibration/cms/jets.rst similarity index 100% rename from docs/api/calibration/jets.rst rename to docs/api/calibration/cms/jets.rst diff --git a/docs/api/calibration/jets_coffea.rst b/docs/api/calibration/cms/jets_coffea.rst similarity index 100% rename from docs/api/calibration/jets_coffea.rst rename to docs/api/calibration/cms/jets_coffea.rst diff --git a/docs/api/calibration/met.rst b/docs/api/calibration/cms/met.rst similarity index 100% rename from docs/api/calibration/met.rst rename to docs/api/calibration/cms/met.rst diff --git a/docs/api/calibration/index.rst b/docs/api/calibration/index.rst index 72bcea387..f82c05aab 100644 --- a/docs/api/calibration/index.rst +++ b/docs/api/calibration/index.rst @@ -1,20 +1,15 @@ ``columnflow.calibration`` ========================== +.. currentmodule:: columnflow.calibration .. automodule:: columnflow.calibration :autosummary: :members: + :undoc-members: -.. currentmodule:: columnflow.calibration - -Summary -------- -.. :autoclass:: Calibrator - :members: - :inherited-members: .. toctree:: - :maxdepth: 2 + :maxdepth: 1 util - cms + cms/index diff --git a/docs/api/calibration/util.rst b/docs/api/calibration/util.rst index b126fe4ad..e83761555 100644 --- a/docs/api/calibration/util.rst +++ b/docs/api/calibration/util.rst @@ -1,23 +1,7 @@ -``columnflow.calibration.util`` -=============================== -.. automodule:: columnflow.calibration.util +``util`` +======== .. currentmodule:: columnflow.calibration.util - -Summary -------- - -.. autosummary:: - ak_random - propagate_met - - -Functions ---------- - -``ak_random`` -+++++++++++++ -.. autofunction:: ak_random - -``propagate_met`` -+++++++++++++++++ -.. autofunction:: propagate_met \ No newline at end of file +.. automodule:: columnflow.calibration.util + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/categorization/index.rst b/docs/api/categorization/index.rst new file mode 100644 index 000000000..2cd5cb51b --- /dev/null +++ b/docs/api/categorization/index.rst @@ -0,0 +1,8 @@ +``columnflow.categorization`` +============================= + +.. currentmodule:: columnflow.categorization +.. automodule:: columnflow.categorization + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/columnar_util.rst b/docs/api/columnar_util.rst index 566119401..e910490d9 100644 --- a/docs/api/columnar_util.rst +++ b/docs/api/columnar_util.rst @@ -2,161 +2,7 @@ ============================ .. currentmodule:: columnflow.columnar_util - .. automodule:: columnflow.columnar_util - -Summary -------- - -.. autosummary:: - - EMPTY_INT - EMPTY_FLOAT - mandatory_coffea_columns - eval_item - get_ak_routes - has_ak_column - set_ak_column - remove_ak_column - add_ak_alias - add_ak_aliases - update_ak_array - flatten_ak_array - sort_ak_fields - sorted_ak_to_parquet - attach_behavior - layout_ak_array - flat_np_view - Route - RouteFilter - ArrayFunction - TaskArrayFunction - ChunkedIOHandler - InsertableDict - -Attributes ----------- - -``EMPTY_INT`` -+++++++++++++ - -.. autoattribute:: columnflow.columnar_util.EMPTY_INT - -``EMPTY_FLOAT`` -+++++++++++++++ - -.. autoattribute:: columnflow.columnar_util.EMPTY_FLOAT - -``mandatory_coffea_columns`` -++++++++++++++++++++++++++++ - -.. autoattribute:: columnflow.columnar_util.mandatory_coffea_columns - -Functions ---------- - -``eval_item`` -+++++++++++++ - -.. autofunction:: eval_item - -``get_ak_routes`` -+++++++++++++++++ - -.. autofunction:: get_ak_routes - -``has_ak_column`` -+++++++++++++++++ - -.. autofunction:: has_ak_column - -``set_ak_column`` -+++++++++++++++++ - -.. autofunction:: set_ak_column - -``remove_ak_column`` -++++++++++++++++++++ - -.. autofunction:: remove_ak_column - -``add_ak_alias`` -++++++++++++++++ - -.. autofunction:: add_ak_alias - -``add_ak_aliases`` -++++++++++++++++++ - -.. autofunction:: add_ak_aliases - -``update_ak_array`` -+++++++++++++++++++ - -.. autofunction:: update_ak_array - -``flatten_ak_array`` -++++++++++++++++++++ - -.. autofunction:: flatten_ak_array - -``sort_ak_fields`` -++++++++++++++++++ - -.. autofunction:: sort_ak_fields - -``sorted_ak_to_parquet`` -++++++++++++++++++++++++ - -.. autofunction:: sorted_ak_to_parquet - -``attach_behavior`` -+++++++++++++++++++ - -.. autofunction:: attach_behavior - -``layout_ak_array`` -+++++++++++++++++++ - -.. autofunction:: layout_ak_array - -``flat_np_view`` -++++++++++++++++ - -.. autofunction:: flat_np_view - -Classes -------- - -``Route`` -+++++++++ - -.. autoclass:: Route - :members: - -``RouteFilter`` -+++++++++++++++ - -.. autoclass:: RouteFilter - :members: - -``ArrayFunction`` -+++++++++++++++++ - -.. autoclass:: ArrayFunction + :autosummary: :members: - -``TaskArrayFunction`` -+++++++++++++++++++++ - -.. autoclass:: TaskArrayFunction - :members: - -``ChunkedIOHandler`` -++++++++++++++++++++ - -.. autoclass:: ChunkedIOHandler - :members: - -.. autoclass:: InsertableDict - :members: \ No newline at end of file + :undoc-members: diff --git a/docs/api/config_util.rst b/docs/api/config_util.rst index 77b3db11c..dc19d18ad 100644 --- a/docs/api/config_util.rst +++ b/docs/api/config_util.rst @@ -1,39 +1,8 @@ ``columnflow.config_util`` ========================== -.. automodule:: columnflow.config_util - .. currentmodule:: columnflow.config_util - -Summary -------- - -.. autosummary:: - - get_root_processes_from_campaign - add_shift_aliases - get_shifts_from_sources - create_category_combinations - -Functions ---------- - -``get_root_processes_from_campaign`` -++++++++++++++++++++++++++++++++++++ - -.. autofunction:: get_root_processes_from_campaign - -``add_shift_aliases`` -+++++++++++++++++++++ - -.. autofunction:: add_shift_aliases - -``get_shifts_from_sources`` -+++++++++++++++++++++++++++ - -.. autofunction:: get_shifts_from_sources - -``create_category_combinations`` -++++++++++++++++++++++++++++++++ - -.. autofunction:: create_category_combinations +.. automodule:: columnflow.config_util + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/index.rst b/docs/api/index.rst index decccea77..9c085551a 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -5,12 +5,15 @@ API Reference :maxdepth: 3 calibration/index + categorization/index production/index selection/index - ml/index + categorization/index inference/index + ml/index plotting/index tasks/index util columnar_util config_util + types diff --git a/docs/api/plotting/index.rst b/docs/api/plotting/index.rst index 225126c63..07a4bb1e7 100644 --- a/docs/api/plotting/index.rst +++ b/docs/api/plotting/index.rst @@ -1,6 +1,8 @@ ``columnflow.plotting`` ======================= +.. currentmodule:: columnflow.plotting .. automodule:: columnflow.plotting :autosummary: :members: + :undoc-members: diff --git a/docs/api/plotting/plot_all.rst b/docs/api/plotting/plot_all.rst new file mode 100644 index 000000000..6144cbaf4 --- /dev/null +++ b/docs/api/plotting/plot_all.rst @@ -0,0 +1,9 @@ +``plot_all`` +============ + +.. currentmodule:: columnflow.plotting.plot_all +.. automodule:: columnflow.plotting.plot_all + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/plotting/plot_functions_1d.rst b/docs/api/plotting/plot_functions_1d.rst new file mode 100644 index 000000000..1aec5ea46 --- /dev/null +++ b/docs/api/plotting/plot_functions_1d.rst @@ -0,0 +1,9 @@ +``plot_functions_1d`` +===================== + +.. currentmodule:: columnflow.plotting.plot_functions_1d +.. automodule:: columnflow.plotting.plot_functions_1d + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/plotting/plot_functions_2d.rst b/docs/api/plotting/plot_functions_2d.rst new file mode 100644 index 000000000..8f41a4315 --- /dev/null +++ b/docs/api/plotting/plot_functions_2d.rst @@ -0,0 +1,9 @@ +``plot_functions_2d`` +===================== + +.. currentmodule:: columnflow.plotting.plot_functions_2d +.. automodule:: columnflow.plotting.plot_functions_2d + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/plotting/plot_util.rst b/docs/api/plotting/plot_util.rst new file mode 100644 index 000000000..098de5c77 --- /dev/null +++ b/docs/api/plotting/plot_util.rst @@ -0,0 +1,9 @@ +``plot_util`` +============= + +.. currentmodule:: columnflow.plotting.plot_util +.. automodule:: columnflow.plotting.plot_util + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/categories.rst b/docs/api/production/categories.rst new file mode 100644 index 000000000..0d504afc5 --- /dev/null +++ b/docs/api/production/categories.rst @@ -0,0 +1,9 @@ +``categories`` +============== + +.. currentmodule:: columnflow.production.categories +.. automodule:: columnflow.production.categories + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms.rst b/docs/api/production/cms.rst new file mode 100644 index 000000000..edb1ce0a7 --- /dev/null +++ b/docs/api/production/cms.rst @@ -0,0 +1,7 @@ +``production.cms`` +================== + +.. toctree:: + :maxdepth: 1 + + muon \ No newline at end of file diff --git a/docs/api/production/cms/btag.rst b/docs/api/production/cms/btag.rst new file mode 100644 index 000000000..04e8bbe26 --- /dev/null +++ b/docs/api/production/cms/btag.rst @@ -0,0 +1,9 @@ +``btag`` +======== + +.. currentmodule:: columnflow.production.cms.btag +.. automodule:: columnflow.production.cms.btag + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms/electron.rst b/docs/api/production/cms/electron.rst new file mode 100644 index 000000000..a6dd3370c --- /dev/null +++ b/docs/api/production/cms/electron.rst @@ -0,0 +1,9 @@ +``electron`` +============ + +.. currentmodule:: columnflow.production.cms.electron +.. automodule:: columnflow.production.cms.electron + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms/gen_top_decay.rst b/docs/api/production/cms/gen_top_decay.rst new file mode 100644 index 000000000..d79123a24 --- /dev/null +++ b/docs/api/production/cms/gen_top_decay.rst @@ -0,0 +1,9 @@ +``gen_top_decay`` +================= + +.. currentmodule:: columnflow.production.cms.gen_top_decay +.. automodule:: columnflow.production.cms.gen_top_decay + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms/index.rst b/docs/api/production/cms/index.rst new file mode 100644 index 000000000..05f5cc257 --- /dev/null +++ b/docs/api/production/cms/index.rst @@ -0,0 +1,21 @@ +``cms`` +========= + +.. currentmodule:: columnflow.production.cms +.. automodule:: columnflow.production.cms + :autosummary: + :members: + :undoc-members: + +.. toctree:: + :maxdepth: 1 + + btag + electron + gen_top_decay + mc_weight + muon + pdf + pileup + scale + seeds diff --git a/docs/api/production/cms/mc_weight.rst b/docs/api/production/cms/mc_weight.rst new file mode 100644 index 000000000..a719668a2 --- /dev/null +++ b/docs/api/production/cms/mc_weight.rst @@ -0,0 +1,9 @@ +``mc_weight`` +============= + +.. currentmodule:: columnflow.production.cms.mc_weight +.. automodule:: columnflow.production.cms.mc_weight + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms/muon.rst b/docs/api/production/cms/muon.rst new file mode 100644 index 000000000..85711059a --- /dev/null +++ b/docs/api/production/cms/muon.rst @@ -0,0 +1,9 @@ +``muon`` +======== + +.. currentmodule:: columnflow.production.cms.muon +.. automodule:: columnflow.production.cms.muon + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms/pdf.rst b/docs/api/production/cms/pdf.rst new file mode 100644 index 000000000..0571bba4e --- /dev/null +++ b/docs/api/production/cms/pdf.rst @@ -0,0 +1,9 @@ +``pdf`` +======= + +.. currentmodule:: columnflow.production.cms.pdf +.. automodule:: columnflow.production.cms.pdf + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms/pileup.rst b/docs/api/production/cms/pileup.rst new file mode 100644 index 000000000..77b54ff21 --- /dev/null +++ b/docs/api/production/cms/pileup.rst @@ -0,0 +1,9 @@ +``pileup`` +========== + +.. currentmodule:: columnflow.production.cms.pileup +.. automodule:: columnflow.production.cms.pileup + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms/scale.rst b/docs/api/production/cms/scale.rst new file mode 100644 index 000000000..94ce48801 --- /dev/null +++ b/docs/api/production/cms/scale.rst @@ -0,0 +1,9 @@ +``scale`` +========= + +.. currentmodule:: columnflow.production.cms.scale +.. automodule:: columnflow.production.cms.scale + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/cms/seeds.rst b/docs/api/production/cms/seeds.rst new file mode 100644 index 000000000..3e0dc86d7 --- /dev/null +++ b/docs/api/production/cms/seeds.rst @@ -0,0 +1,9 @@ +``seeds`` +========= + +.. currentmodule:: columnflow.production.cms.seeds +.. automodule:: columnflow.production.cms.seeds + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/index.rst b/docs/api/production/index.rst index 99b942d42..6938b513a 100644 --- a/docs/api/production/index.rst +++ b/docs/api/production/index.rst @@ -2,18 +2,16 @@ ========================= .. currentmodule:: columnflow.production - .. automodule:: columnflow.production - :autosummary: - -Classes -+++++++ + :autosummary: + :members: + :undoc-members: -.. autoclass:: Producer - :members: - :undoc-members: +.. toctree:: + :maxdepth: 1 -.. toctree:: - :maxdepth: 1 - - util + categories + normalization + processes + util + cms/index diff --git a/docs/api/production/muon.rst b/docs/api/production/muon.rst new file mode 100644 index 000000000..f81beecf4 --- /dev/null +++ b/docs/api/production/muon.rst @@ -0,0 +1,8 @@ +``muon`` +======== + +.. currentmodule:: columnflow.production.cms.muon +.. automodule:: columnflow.production.cms.muon + :autosummary: + :members: + :undoc-members: \ No newline at end of file diff --git a/docs/api/production/normalization.rst b/docs/api/production/normalization.rst new file mode 100644 index 000000000..c4839a3c8 --- /dev/null +++ b/docs/api/production/normalization.rst @@ -0,0 +1,9 @@ +``normalization`` +================= + +.. currentmodule:: columnflow.production.normalization +.. automodule:: columnflow.production.normalization + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/processes.rst b/docs/api/production/processes.rst new file mode 100644 index 000000000..3af781c8f --- /dev/null +++ b/docs/api/production/processes.rst @@ -0,0 +1,9 @@ +``processes`` +============= + +.. currentmodule:: columnflow.production.processes +.. automodule:: columnflow.production.processes + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/production/util.rst b/docs/api/production/util.rst index 017111599..bff11c4e1 100644 --- a/docs/api/production/util.rst +++ b/docs/api/production/util.rst @@ -1,25 +1,9 @@ ``util`` -=============================== -.. automodule:: columnflow.production.util -.. currentmodule:: columnflow.production.util - -Summary -------- - -.. autosummary:: - attach_coffea_behavior - -Attributes ----------- - -.. autoattribute:: columnflow.production.util.default_collections +======== -Producers ---------- - -``attach_coffea_behavior`` -++++++++++++++++++++++++++ -.. autoclass:: attach_coffea_behavior +.. currentmodule:: columnflow.production.util +.. automodule:: columnflow.production.util + :autosummary: :members: :undoc-members: - :exclude-members: skip_func \ No newline at end of file + diff --git a/docs/api/selection/cms.rst b/docs/api/selection/cms/index.rst similarity index 99% rename from docs/api/selection/cms.rst rename to docs/api/selection/cms/index.rst index 49885ded1..3cf4ef897 100644 --- a/docs/api/selection/cms.rst +++ b/docs/api/selection/cms/index.rst @@ -1,8 +1,7 @@ ``cms`` ======= - -.. automodule:: columnflow.selection.cms .. currentmodule:: columnflow.selection.cms +.. automodule:: columnflow.selection.cms .. toctree:: :maxdepth: 1 diff --git a/docs/api/selection/json_filter.rst b/docs/api/selection/cms/json_filter.rst similarity index 76% rename from docs/api/selection/json_filter.rst rename to docs/api/selection/cms/json_filter.rst index 85c04d3c7..377c63604 100644 --- a/docs/api/selection/json_filter.rst +++ b/docs/api/selection/cms/json_filter.rst @@ -4,11 +4,7 @@ .. currentmodule:: columnflow.selection.cms.json_filter .. automodule:: columnflow.selection.cms.json_filter :autosummary: - -``Selectors`` -------------- - -.. autoclass:: json_filter :members: :undoc-members: + diff --git a/docs/api/selection/met_filters.rst b/docs/api/selection/cms/met_filters.rst similarity index 68% rename from docs/api/selection/met_filters.rst rename to docs/api/selection/cms/met_filters.rst index e46773671..f8732e7b9 100644 --- a/docs/api/selection/met_filters.rst +++ b/docs/api/selection/cms/met_filters.rst @@ -4,10 +4,5 @@ .. currentmodule:: columnflow.selection.cms.met_filters .. automodule:: columnflow.selection.cms.met_filters :autosummary: - -``Selectors`` -------------- - -.. autoclass:: met_filters :members: - :undoc-members: + :undoc-members: \ No newline at end of file diff --git a/docs/api/selection/empty.rst b/docs/api/selection/empty.rst new file mode 100644 index 000000000..81625d54e --- /dev/null +++ b/docs/api/selection/empty.rst @@ -0,0 +1,9 @@ +``empty`` +========= + +.. currentmodule:: columnflow.selection.empty +.. automodule:: columnflow.selection.empty + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/selection/index.rst b/docs/api/selection/index.rst index acd9fd6eb..4c4be0f04 100644 --- a/docs/api/selection/index.rst +++ b/docs/api/selection/index.rst @@ -2,26 +2,16 @@ ======================== .. currentmodule:: columnflow.selection - .. automodule:: columnflow.selection - :autosummary: - -``Selector`` ------------- -.. autoclass:: Selector - :members: - :special-members: - -``SelectionResult`` -------------------- -.. autoclass:: SelectionResult - :members: - :special-members: + :autosummary: + :members: + :undoc-members: -``Submodules`` --------------- .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - matching - cms \ No newline at end of file + empty + matching + stats + util + cms/index \ No newline at end of file diff --git a/docs/api/selection/matching.rst b/docs/api/selection/matching.rst index c09873e85..30071fa7d 100644 --- a/docs/api/selection/matching.rst +++ b/docs/api/selection/matching.rst @@ -1,13 +1,9 @@ ``matching`` -================================= +============ .. currentmodule:: columnflow.selection.matching - -.. automodule::columnflow.selection.matching +.. automodule:: columnflow.selection.matching :autosummary: - -.. autofunction:: cleaning_factory - -.. autoclass:: jet_lepton_delta_r_cleaning :members: - :undoc-members: \ No newline at end of file + :undoc-members: + diff --git a/docs/api/selection/stats.rst b/docs/api/selection/stats.rst new file mode 100644 index 000000000..0d32d982c --- /dev/null +++ b/docs/api/selection/stats.rst @@ -0,0 +1,9 @@ +``stats`` +========= + +.. currentmodule:: columnflow.selection.stats +.. automodule:: columnflow.selection.stats + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/selection/util.rst b/docs/api/selection/util.rst new file mode 100644 index 000000000..12b1066c1 --- /dev/null +++ b/docs/api/selection/util.rst @@ -0,0 +1,9 @@ +``util`` +======== + +.. currentmodule:: columnflow.selection.util +.. automodule:: columnflow.selection.util + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/calibration.rst b/docs/api/tasks/calibration.rst index c0c896c40..1b60a80ea 100644 --- a/docs/api/tasks/calibration.rst +++ b/docs/api/tasks/calibration.rst @@ -1,14 +1,9 @@ ``calibration`` -============================== +=============== .. currentmodule:: columnflow.tasks.calibration .. automodule:: columnflow.tasks.calibration - :autosummary: - -.. autoclass:: CalibrateEvents + :autosummary: :members: :undoc-members: -.. autoclass:: CalibrateEventsWrapper - :members: - :undoc-members: \ No newline at end of file diff --git a/docs/api/tasks/cms.rst b/docs/api/tasks/cms.rst deleted file mode 100644 index b9442efed..000000000 --- a/docs/api/tasks/cms.rst +++ /dev/null @@ -1,6 +0,0 @@ -``columnflow.tasks.cms`` -============================== - -.. automodule:: columnflow.tasks.cms - :autosummary: - :members: diff --git a/docs/api/tasks/cms/base.rst b/docs/api/tasks/cms/base.rst new file mode 100644 index 000000000..42dbecedd --- /dev/null +++ b/docs/api/tasks/cms/base.rst @@ -0,0 +1,8 @@ +``base`` +======== + +.. currentmodule:: columnflow.tasks.cms.base +.. automodule:: columnflow.tasks.cms.base + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/tasks/cms/external.rst b/docs/api/tasks/cms/external.rst new file mode 100644 index 000000000..c361e696b --- /dev/null +++ b/docs/api/tasks/cms/external.rst @@ -0,0 +1,8 @@ +``external`` +============ + +.. currentmodule:: columnflow.tasks.cms.external +.. automodule:: columnflow.tasks.cms.external + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/tasks/cms/index.rst b/docs/api/tasks/cms/index.rst new file mode 100644 index 000000000..0987b8da3 --- /dev/null +++ b/docs/api/tasks/cms/index.rst @@ -0,0 +1,15 @@ +``cms`` +============= + +.. currentmodule:: columnflow.tasks.cms +.. automodule:: columnflow.tasks.cms + :autosummary: + :members: + :undoc-members: + +.. toctree:: + :maxdepth: 1 + + base + external + inference diff --git a/docs/api/tasks/cms/inference.rst b/docs/api/tasks/cms/inference.rst new file mode 100644 index 000000000..b74dceffd --- /dev/null +++ b/docs/api/tasks/cms/inference.rst @@ -0,0 +1,8 @@ +``inference`` +============= + +.. currentmodule:: columnflow.tasks.cms.inference +.. automodule:: columnflow.tasks.cms.inference + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/tasks/cutflow.rst b/docs/api/tasks/cutflow.rst new file mode 100644 index 000000000..6d144b2fe --- /dev/null +++ b/docs/api/tasks/cutflow.rst @@ -0,0 +1,9 @@ +``cutflow`` +=========== + +.. currentmodule:: columnflow.tasks.cutflow +.. automodule:: columnflow.tasks.cutflow + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/external.rst b/docs/api/tasks/external.rst index 5025efcfc..6db4e361c 100644 --- a/docs/api/tasks/external.rst +++ b/docs/api/tasks/external.rst @@ -1,6 +1,9 @@ -``columnflow.tasks.external`` -============================== +``external`` +============ +.. currentmodule:: columnflow.tasks.external .. automodule:: columnflow.tasks.external - :autosummary: - :members: + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/framework/base.rst b/docs/api/tasks/framework/base.rst index 3d979c66e..cf5fea3ca 100644 --- a/docs/api/tasks/framework/base.rst +++ b/docs/api/tasks/framework/base.rst @@ -1,6 +1,7 @@ -``columnflow.tasks.framework.base`` -=================================== +``base`` +======== .. automodule:: columnflow.tasks.framework.base :autosummary: :members: + :undoc-members: \ No newline at end of file diff --git a/docs/api/tasks/framework/decorators.rst b/docs/api/tasks/framework/decorators.rst index 33f43ceef..85a4048a9 100644 --- a/docs/api/tasks/framework/decorators.rst +++ b/docs/api/tasks/framework/decorators.rst @@ -1,6 +1,8 @@ -``columnflow.tasks.framework.decorators`` -=================================== +``decorators`` +============== +.. currentmodule:: columnflow.tasks.framework.decorators .. automodule:: columnflow.tasks.framework.decorators :autosummary: :members: + :undoc-members: diff --git a/docs/api/tasks/framework/index.rst b/docs/api/tasks/framework/index.rst index 01e9c5f7c..ee10d96cf 100644 --- a/docs/api/tasks/framework/index.rst +++ b/docs/api/tasks/framework/index.rst @@ -1,14 +1,18 @@ ``framework`` -============================== +============= .. currentmodule:: columnflow.tasks.framework .. automodule:: columnflow.tasks.framework :autosummary: :members: + :undoc-members: .. toctree:: :maxdepth: 1 base decorators - mixins \ No newline at end of file + mixins + parameters + plotting + remote \ No newline at end of file diff --git a/docs/api/tasks/framework/mixins.rst b/docs/api/tasks/framework/mixins.rst index 4ee59e0f3..9e6da8447 100644 --- a/docs/api/tasks/framework/mixins.rst +++ b/docs/api/tasks/framework/mixins.rst @@ -1,5 +1,5 @@ ``mixins`` -=================================== +========== .. currentmodule:: columnflow.tasks.framework.mixins .. automodule:: columnflow.tasks.framework.mixins diff --git a/docs/api/tasks/framework/parameters.rst b/docs/api/tasks/framework/parameters.rst new file mode 100644 index 000000000..ee5de7fe1 --- /dev/null +++ b/docs/api/tasks/framework/parameters.rst @@ -0,0 +1,8 @@ +``parameters`` +============== + +.. currentmodule:: columnflow.tasks.framework.parameters +.. automodule:: columnflow.tasks.framework.parameters + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/tasks/framework/plotting.rst b/docs/api/tasks/framework/plotting.rst new file mode 100644 index 000000000..631345ec4 --- /dev/null +++ b/docs/api/tasks/framework/plotting.rst @@ -0,0 +1,8 @@ +``plotting`` +============ + +.. currentmodule:: columnflow.tasks.framework.plotting +.. automodule:: columnflow.tasks.framework.plotting + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/tasks/framework/remote.rst b/docs/api/tasks/framework/remote.rst new file mode 100644 index 000000000..a7cee691a --- /dev/null +++ b/docs/api/tasks/framework/remote.rst @@ -0,0 +1,8 @@ +``remote`` +========== + +.. currentmodule:: columnflow.tasks.framework.remote +.. automodule:: columnflow.tasks.framework.remote + :autosummary: + :members: + :undoc-members: diff --git a/docs/api/tasks/histograms.rst b/docs/api/tasks/histograms.rst new file mode 100644 index 000000000..cb4a3065e --- /dev/null +++ b/docs/api/tasks/histograms.rst @@ -0,0 +1,9 @@ +``histograms`` +============== + +.. currentmodule:: columnflow.tasks.histograms +.. automodule:: columnflow.tasks.histograms + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/index.rst b/docs/api/tasks/index.rst index 9d8f4688d..8e670a99a 100644 --- a/docs/api/tasks/index.rst +++ b/docs/api/tasks/index.rst @@ -1,19 +1,29 @@ ``columnflow.tasks`` -==================== +====================== .. include:: introduction.md :parser: myst_parser.sphinx_ + +.. currentmodule:: columnflow.tasks .. automodule:: columnflow.tasks - :autosummary: - :members: + :autosummary: + :members: + :undoc-members: .. toctree:: - :maxdepth: 1 + :maxdepth: 1 - selection - calibration - external - reduction - framework/index - cms \ No newline at end of file + calibration + cutflow + external + histograms + ml + plotting + production + reduction + selection + union + yields + cms/index + framework/index diff --git a/docs/api/tasks/ml.rst b/docs/api/tasks/ml.rst new file mode 100644 index 000000000..003e01865 --- /dev/null +++ b/docs/api/tasks/ml.rst @@ -0,0 +1,9 @@ +``ml`` +====== + +.. currentmodule:: columnflow.tasks.ml +.. automodule:: columnflow.tasks.ml + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/plotting.rst b/docs/api/tasks/plotting.rst new file mode 100644 index 000000000..bcceb3dfe --- /dev/null +++ b/docs/api/tasks/plotting.rst @@ -0,0 +1,9 @@ +``plotting`` +============ + +.. currentmodule:: columnflow.tasks.plotting +.. automodule:: columnflow.tasks.plotting + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/production.rst b/docs/api/tasks/production.rst new file mode 100644 index 000000000..f0f1f8785 --- /dev/null +++ b/docs/api/tasks/production.rst @@ -0,0 +1,9 @@ +``production`` +============== + +.. currentmodule:: columnflow.tasks.production +.. automodule:: columnflow.tasks.production + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/reduction.rst b/docs/api/tasks/reduction.rst index ba1bb50e8..ecd80f75c 100644 --- a/docs/api/tasks/reduction.rst +++ b/docs/api/tasks/reduction.rst @@ -1,6 +1,9 @@ -``columnflow.tasks.reduction`` -============================== +``reduction`` +============= +.. currentmodule:: columnflow.tasks.reduction .. automodule:: columnflow.tasks.reduction - :autosummary: - :members: + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/selection.rst b/docs/api/tasks/selection.rst index e6676b4b6..af98ef917 100644 --- a/docs/api/tasks/selection.rst +++ b/docs/api/tasks/selection.rst @@ -1,6 +1,9 @@ -``columnflow.tasks.selection`` -============================== +``selection`` +============= +.. currentmodule:: columnflow.tasks.selection .. automodule:: columnflow.tasks.selection - :autosummary: - :members: + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/union.rst b/docs/api/tasks/union.rst new file mode 100644 index 000000000..b3b4294be --- /dev/null +++ b/docs/api/tasks/union.rst @@ -0,0 +1,9 @@ +``union`` +========= + +.. currentmodule:: columnflow.tasks.union +.. automodule:: columnflow.tasks.union + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/tasks/yields.rst b/docs/api/tasks/yields.rst new file mode 100644 index 000000000..50b7fd46d --- /dev/null +++ b/docs/api/tasks/yields.rst @@ -0,0 +1,9 @@ +``yields`` +========== + +.. currentmodule:: columnflow.tasks.yields +.. automodule:: columnflow.tasks.yields + :autosummary: + :members: + :undoc-members: + diff --git a/docs/api/types.rst b/docs/api/types.rst new file mode 100644 index 000000000..e2c680ef4 --- /dev/null +++ b/docs/api/types.rst @@ -0,0 +1,9 @@ +``columnflow.types`` +========================== + +.. currentmodule:: columnflow.types +.. automodule:: columnflow.types + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/build_simple_rst_docs.py b/docs/build_simple_rst_docs.py new file mode 100644 index 000000000..6f9234693 --- /dev/null +++ b/docs/build_simple_rst_docs.py @@ -0,0 +1,133 @@ +from __future__ import annotations +import os +import sys + +rst_template = """{title} +{underline} + +.. currentmodule:: {modulename} +.. automodule:: {modulename} + :autosummary: + :members: + :undoc-members: + +""" + +rst_toplevel_template = """{title} +{underline} + +.. currentmodule:: {modulename} +.. automodule:: {modulename} + :autosummary: + :members: + :undoc-members: + +.. toctree:: + :maxdepth: 1 + + {rst_files} +""" + + +def build_rst_file( + output_path: str, + file_path: str, + rst_path: str | None = None, +) -> bool: + """Build a rst file for a given python file. + + :param output_path: Output path for the rst file. + :param file_path: Path of the python file to document. + :return: Path of the generated rst file. + """ + + if "." in file_path: + file_path = ".".join(file_path.split(".")[:-1]) + fname: str = os.path.basename(file_path) + module_name: str = file_path.replace(os.sep, ".") + title: str = F"``{fname}``" + underline: str = "=" * len(title) + rst: str = rst_template.format( + title=title, + underline=underline, + modulename=module_name, + ) + + if not rst_path: + rst_path = os.path.join(output_path, fname + ".rst") + print(f"Writing rst file: {rst_path}") + with open(rst_path, "w") as f: + f.write(rst) + + return os.path.exists(rst_path) + + +def create_toplevel_rst_file( + output_path: str, + dirname: str, + documented_modules: list[str], +) -> bool: + """Create a toplevel rst file for a given directory. + + :param output_path: Output path for the rst file. + :param dirname: Name of the directory to document. + :param documented_modules: List of documented modules. + :return: Path of the generated rst file. + """ + + title: str = "``{dirname}``".format(dirname=dirname.replace(os.sep, ".")) + underline: str = "=" * len(title) + rst_files: str = "\n ".join(documented_modules) + rst: str = rst_toplevel_template.format( + title=title, + underline=underline, + modulename=dirname.replace(os.sep, "."), + rst_files=rst_files, + ) + rst_path: str = os.path.join(output_path, "index.rst") + with open(rst_path, "w") as f: + f.write(rst) + + return os.path.exists(rst_path) + + +def main(output_path, *files): + output_path = os.path.abspath(output_path) + if not os.path.exists(output_path): + print(f"Creating output path: {output_path}") + os.makedirs(output_path) + dirname = None + documented_modules = list() + documented_submodules = list() + if len(files) == 1 and os.path.basename(files[0]) == "__init__.py": + build_rst_file( + output_path, + file_path=os.path.dirname(files[0]), + rst_path=os.path.join(output_path, "index.rst"), + ) + for file in files: + if os.path.isdir(file): + subdir = os.path.basename(file) + main( + os.path.join(output_path, subdir), + *os.listdir(file), + ) + documented_submodules.append(f"{subdir}/index") + elif os.path.basename(file).startswith("__"): + continue + else: + this_dirname = os.path.dirname(file) + if dirname is None: + dirname = this_dirname + elif dirname != this_dirname: + raise ValueError(f"Files must have the same basename: {dirname} != {this_dirname}") + + if build_rst_file(output_path, file): + documented_modules.append(os.path.basename(file).split(".")[0]) + documented_modules.extend(documented_submodules) + if len(documented_modules) > 0: + create_toplevel_rst_file(output_path, dirname, documented_modules) + + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/docs/conf.py b/docs/conf.py index 8b4b939ad..99d139078 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -66,17 +66,22 @@ }) extensions = [ + "sphinx_design", + "sphinx_copybutton", "sphinx.ext.intersphinx", "sphinx.ext.autodoc", "sphinx_autodoc_typehints", "sphinx.ext.viewcode", "sphinx.ext.autosectionlabel", + "sphinxcontrib.mermaid", "sphinx_lfs_content", "autodocsumm", "myst_parser", "pydomain_patch", ] +myst_enable_extensions = ["colon_fence"] + typehints_defaults = "comma" autodoc_default_options = { @@ -103,8 +108,17 @@ "scipy": ("https://docs.scipy.org/doc/scipy/", None), "uproot": ("https://uproot.readthedocs.io/en/latest/", None), "numpy": ("https://numpy.org/doc/stable/", None), + "scinum": ("https://scinum.readthedocs.io/en/stable/", None), } +import luigi + + +def process_docstring(app, what, name, obj, options, lines): + if isinstance(obj, luigi.parameter.Parameter): + if obj.description: + lines.append(f"Description: {obj.description}") + def add_intersphinx_aliases_to_inv(app): from sphinx.ext.intersphinx import InventoryAdapter @@ -131,3 +145,5 @@ def setup(app): app.add_css_file("styles_common.css") if html_theme in ("sphinx_rtd_theme", "alabaster", "sphinx_book_theme"): app.add_css_file("styles_{}.css".format(html_theme)) + + app.connect("autodoc-process-docstring", process_docstring) diff --git a/docs/index.rst b/docs/index.rst index c951bc5b0..353cc5bb6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ start.md user_guide/index + task_overview/index api/index .. include:: ../README.md diff --git a/docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__c3947accbb__plot__proc_st__cat_incl__var_cf_jet1_pt.pdf b/docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__c3947accbb__plot__proc_st__cat_incl__var_cf_jet1_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..86f59bc86f8db27fa47b863507ebdc3b3cc51345 GIT binary patch literal 130 zcmWN?NfN>!5CFhCuiyiQWyk>e4ciD+Dk%qZ@b%i4zS^6|e96An$%j(+v2G9I?SKE| zt<0z5vlVq2F-Mi$lil%vgr0y}bIAed;x$EIg9TH_r<`;pA9iaIL1T{9$AXZX0}K!e NK=AlBX@fRV`~c@#Cl>$! literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__c3947accbb__plot__proc_tt__cat_incl__var_cf_jet1_pt.pdf b/docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__c3947accbb__plot__proc_tt__cat_incl__var_cf_jet1_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..daa13b60457ac44c640cfe780b0686f34c9fafd7 GIT binary patch literal 130 zcmWN{K@!3s3;@78uiyg~2&BROhLRx6sB{E-@b&hzmp$b(T7T<0=P@>89&O&9Wh|HV zyDz!F%s3>Q)2ZHAkD372O_D?B8B*~Gks?qgkCtsj5@41U6QKu787Sn4!3G~O@jZn- Nl%huai3Ylk`UAelCaeGe literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step0_Initial__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf b/docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step0_Initial__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8d3e2ecde13de9c0a971aad525e23224b41f7b33 GIT binary patch literal 130 zcmWN?K@!3s3;@7;U%>|~CYZ?dHzWaJMx`UP2Vbvy*)xAe>u*=*+{fnqqs_~+jP-v# zWYzxiW6$bOSb8%zYC_^ILXmO>CSw+ZTapyWK%|I-NNVxU!~u{AP=j(24VVpbv1kdG L(Z1OsN5JI=1h)dTRo}CZmuPF9^N{l1*X_Y`|KC2i zm+_SItVLdW8=b^#8Js_?i3u=QHe{?);GIVZ(FfM0P))Xgzb%p+;6{498*$ MZ?k`^03Vgxe(R_x00000 literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step2_muon__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf b/docs/plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step2_muon__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0dedf8909cdc725c1b870aeffa1edcde2f14573c GIT binary patch literal 130 zcmWm2K@P$o5CFhCuiyg~c7X-@8(652sBMJy;Oo`XBy-jG==)2yIgha_b?@``*v9g+ zoq6H?sm4)RT_E;e=13;@tQr{Dq=!zZY>ArJ~PZFL%S(bLz}TfB?E()W*UV;r&>^X%jCsxsZS zlcv1CjU1fK1#0g_OG`H3Mw}cip%7$DXsI~P86tT?#!?|c1YpbLAh@OYIa8n&k=9D= NLsZg#tnGZK`U56!C~*J) literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotCutflow_tpl_config_analy__1__12a17bf79c__cutflow__cat_incl.pdf b/docs/plots/cf.PlotCutflow_tpl_config_analy__1__12a17bf79c__cutflow__cat_incl.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b618df158ed0ccc293bac9445ca1f572cf067247 GIT binary patch literal 130 zcmWN@Q4+%t5CG7Br*MJB5{u+EmQ`lj%9P}zr*FRAyZAd=|HwM$F?OY%Z9ZOQEVuQf zCH1!&2PbuD(OZ_IMuTp5B}Q0*%BF)3839wyK%jXG#X=%RcRzZ|J`y3?2-%thM&n9M MVzeJi1KwKk2l*-|&;S4c literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotShiftedVariables1D_tpl_config_analy__1__42b45aba89__plot__proc_2_a2211e799f__unc_mu__cat_incl__var_jet1_pt.pdf b/docs/plots/cf.PlotShiftedVariables1D_tpl_config_analy__1__42b45aba89__plot__proc_2_a2211e799f__unc_mu__cat_incl__var_jet1_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ae685ed7e0e5f9a5517093dead552a7a689f1c5c GIT binary patch literal 130 zcmWN|xe>!45CFiODrmss=Rn=?9EKU&XnaV4tGA+!_QEfX`Qd%7gLWaFV?ADtw%dN@ zt<1Ne1BIsk`c_o>h)dT#qZh2OSHAlc}RKh>-MN~|KC1& zr}32YWRjQKMlWJE)XpDd7*%pu75HEfbB!5CFh?Ucm^UHN=h&XU$1@XtGs6)FVWUI=ON|2uiK-}{eSz! zJ&mWFXG!vcZS*W=3ux?Nm3J{4#=^EZ7eRvoRjYww2?8pITp@a!(4bD41CiC5DMX0L N>uvUL5lD=*{Q$Q0CUgJ* literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__0191de868f__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c1.pdf b/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__0191de868f__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7d1cf19c224f4ea8a0bc7f7a64590a6fecd1050c GIT binary patch literal 130 zcmWN?NfN>!5CFh?UcmPZ~q@0WCA-6w%vERheu`0St;Tfz3vNRWUjQRgjsHK}Sed Nxo!3@32k&R+YjZ6Ck6lj literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__12dfac316a__plot__proc_3_7727a49dc2__cat_2j__var_n_jet.pdf b/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__12dfac316a__plot__proc_3_7727a49dc2__cat_2j__var_n_jet.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7b28550608c31eded8b7d229d646e70a47ead15b GIT binary patch literal 130 zcmWN?OA^8$5Cy<}PQe8XJbod$4Ge=&rIHfN!qaPS`r`L!{Uz$0=QyOiw|RS%vHq{0 zv{HY{aWcsZOK(|>8nyF>MsONreMy^z%E}Z+o|&Q{3xGip-kY)~gbWI1(<#~#0(%Wr MmeIZ}w1eB1AJauA9{>OV literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__12dfac316a__plot__proc_3_7727a49dc2__cat_incl__var_n_jet.pdf b/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__12dfac316a__plot__proc_3_7727a49dc2__cat_incl__var_n_jet.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8a4bf6cb79c16fd4ff66facfa11fed7912c4584d GIT binary patch literal 130 zcmWN?%MrpL5CG6SRnUNeWmyR6hR-6*sAL3luzG!$ckz4n@se$=a~?|F`?@{q-2S&u z+VXg+d2&`4h|!CjEgEu1-T_>yem9;7lBNg+wez-M&yem9;7lBNg+wez-M&|~5=tiYHv|%4Mx`UP2Vbvy*-Jj7^|z^W9%Hxm(dO-`WBp%G zUUPrhaY(k8TY4*I)C2){C00-s&*UM5AQTB9amhYnLUJXrdJmrAzGc89NZFxFMI#mA MWwfstfX&6_2LPQY0RR91 literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__be60d3bca7__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c3.pdf b/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__be60d3bca7__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c3.pdf new file mode 100644 index 0000000000000000000000000000000000000000..be94a2adae687647afb9d5aba99ed18796dce392 GIT binary patch literal 130 zcmWN?OA^8$3;@tQr{DsXUsFhL10)DDDjie1@bvmN@8Y+N`O^J757~{mkMs6uvi$F# zvefxBa&R@5Q*)4(o;_@PN(_}EgpzARAZ*#K901#zYf8nlwZ%H#5E&dh)dTRo=6YmuzdD^HA#E*X>c~_P>2- z%j2o$S+csc7@g&8XnZ)B5ip?1o&seb+-?&lQ_=e33P-)`NR>H8O*#QvrX2~qa|B>A Mh}pkI8VlBo9}3+k(f|Me literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__c80529af83__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c2.pdf b/docs/plots/cf.PlotVariables1D_tpl_config_analy__1__c80529af83__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c2.pdf new file mode 100644 index 0000000000000000000000000000000000000000..259ca72dc6bbd5411ee9190d2ebe90356c713b01 GIT binary patch literal 130 zcmWN?Q4WJ33;@u7Pr(Hy0$Pfo@LV#F8TERMNif07OhB;*R NZj$!P%GzLH`UBB}CnNv> literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotVariables2D_tpl_config_analy__1__b27b994979__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt-n_jet.pdf b/docs/plots/cf.PlotVariables2D_tpl_config_analy__1__b27b994979__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt-n_jet.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fa06f1b0e01dd80ca9109caa2b34494ebba65372 GIT binary patch literal 130 zcmWN?%MrpL5CG6SRnUNeg@r)6;j;)cDj64ZuzG!$chQ^2e96An$%j(+v2KsX+yDN_ zTbWPAClz&pn4?PW$vA&RjX-3JX%wT`0Wp-QbHHLXXt2(r%ZYZX*?Y{`N~_Un3Q)a+ Nlm(A3NvqKZ@dM2LCo%v4 literal 0 HcmV?d00001 diff --git a/docs/plots/cf.PlotVariables2D_tpl_config_analy__1__b27b994979__plot__proc_2_a2211e799f__cat_incl__var_n_jet-jet1_pt.pdf b/docs/plots/cf.PlotVariables2D_tpl_config_analy__1__b27b994979__plot__proc_2_a2211e799f__cat_incl__var_n_jet-jet1_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..944f2f97f77768897c594a54ecd591640d325f64 GIT binary patch literal 130 zcmWl~!4ZQX5CzaXRnS1hMO?vdWcdVUlHdgOkm{3M-n;PbwY_vXj=itO-0N|BoRs1rCm?3`xksvtE%--eX0%L+~U=Vx~!jh8pLKY)!h;d|A Ly?&sl(R-`DDZ(hx literal 0 HcmV?d00001 diff --git a/docs/requirements.txt b/docs/requirements.txt index 7604d8812..3716b2297 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,12 +1,15 @@ -# version 5 +# version 7 # documentation packages sphinx~=6.2.1 +sphinx-design~=0.5.0 +sphinx-copybutton~=0.5.2 sphinx-autodoc-typehints~=1.22,<1.23 sphinx-book-theme~=1.0.1 sphinx-lfs-content~=1.1.3,!=1.1.5 autodocsumm~=0.2.11 myst-parser~=2.0.0 +sphinxcontrib-mermaid==0.9.2 # prod packages -r ../sandboxes/cf.txt diff --git a/docs/task_overview/index.rst b/docs/task_overview/index.rst new file mode 100644 index 000000000..4afe5ebd8 --- /dev/null +++ b/docs/task_overview/index.rst @@ -0,0 +1,8 @@ +Task Overview +============= + +.. toctree:: + :maxdepth: 3 + + introduction.md + plotting.rst \ No newline at end of file diff --git a/docs/task_overview/introduction.md b/docs/task_overview/introduction.md new file mode 100644 index 000000000..9b265f198 --- /dev/null +++ b/docs/task_overview/introduction.md @@ -0,0 +1,99 @@ +# Task Overview + +This page gives a small overview of the current tasks available by default in columnflow. + +In general (at least for people working in the CMS experiment), +using columnflow requires a grid proxy, as the +dataset files are accessed through it. The grid proxy should be activated after the setup of the +default environment. It is however possible to create a custom law config file to be used by people +without a grid certificate, although someone with a grid proxy must run the tasks +{py:class}`~columnflow.tasks.external.GetDatasetLFNs` (and potentially the cms-specific +{py:class}`~columnflow.tasks.cms.external.CreatePileUpWeights`) for them. Once these +tasks are done, the local task outputs can be used without grid certificate by other users if +they are able to access them with the storage location declared in the custom law config file. +An example for such a custom config file can be found in the {doc}`examples` section of this +documentation. + +While the name of each task is fairly descriptive of its purpose, a short introduction of the most +important facts and parameters about each task group is provided below. As some tasks require +others to run, the arguments for a task higher in the tree will also be required for tasks below +in the tree (sometimes in a slightly different version, e.g. with an "s" if the task allows several +instances of the parameter to be given at once (e.g. several dataset**s**)). + +It should be mentioned that in your analysis, the command line argument for running the columnflow +tasks described below will contain an additional "cf."-prefix +before the name of the task, as these are columnflow tasks and not new tasks created explicitely +for your analysis. For example, running the {py:class}`~columnflow.tasks.selection.SelectEvents` +task will require the following syntax: + +```shell +law run cf.SelectEvents --your-parameters parameter_value +``` + +- {py:class}`~columnflow.tasks.external.GetDatasetLFNs`: This task looks for the logical file names +of the datasets to be used and saves them in a json file. The argument ```--dataset``` followed by +the name of the dataset to be searched for, as defined in the analysis config is needed for this +task to run. + +- {py:class}`~columnflow.tasks.calibration.CalibrateEvents`: Task to implement corrections to be +applied on the datasets, e.g. jet-energy corrections. This task uses objects of the +{py:class}`~columnflow.calibration.Calibrator` class to apply the calibration. The argument +```--calibrator``` followed by the name of the Calibrator +object to be run is needed for this task to run. A default value for this argument can be set in +the analysis config. Similarly, the ```--shift``` argument can be given, in order to choose which +corrections are to be used, e.g. which variation (up, down, nominal) of the jet-energy corrections +are to be used. + +- {py:class}`~columnflow.tasks.selection.SelectEvents`: Task to implement selections to be applied +on the datssets. This task uses objects of the {py:class}`~columnflow.selection.Selector` class to +apply the selection. The output are masks for the events and objects to be selected, saved in a +parquet file, and some additional parameters stored in a dictionary format, like the statistics of +the selection (which are needed for the plotting tasks further down the task tree), saved in a json +file. The mask are not applied to the columns during this task. +The argument ```--selector``` followed by the name of the +Selector object to be run is needed for this task to run. +A default value for this argument can be set in the analysis config. From this task on, the +```--calibrator``` argument is replaced by ```--calibrators```. + +- {py:class}`~columnflow.tasks.reduction.ReduceEvents`: Task to apply the masks created in +{py:class}`~columnflow.tasks.selection.SelectEvents` on the datasets. All +tasks below ReduceEvents in the task graph use the parquet +file resulting from ReduceEvents to work on, not the +original dataset. The columns to be conserved after +ReduceEvents are to be given in the analysis config under +the ```config.x.keep_columns``` argument in a ```DotDict``` structure +(from {py:mod}`~columnflow.util`). + +- {py:class}`~columnflow.tasks.production.ProduceColumns`: Task to produce additional columns for +the reduced datasets, e.g. for new high level variables. This task uses objects of the +{py:class}`~columnflow.production.Producer` class to create the new columns. The new columns are +saved in a parquet file that can be used by the task below on the task graph. The argument +```--producer``` followed by the name of the Producer object +to be run is needed for this task to run. A default value for this argument can be set in the +analysis config. + +- {py:class}`~columnflow.tasks.ml.PrepareMLEvents`, {py:class}`~columnflow.tasks.ml.MLTraining`, +{py:class}`~columnflow.tasks.ml.MLEvaluation`: Tasks to +train, evaluate neural networks and plot (to be implemented) their results. + +- {py:class}`~columnflow.tasks.histograms.CreateHistograms`: Task to create histograms with the +python package [Hist](https://hist.readthedocs.io/en/latest/) which can be used by the tasks below +in the task graph. From this task on, the ```--producer``` argument is replaced by +```--producers```. The histograms are saved in a pickle file. + +- {py:class}`~columnflow.tasks.cms.inference.CreateDatacards`: TODO + +- ```Merge``` tasks (e.g. {py:class}`~columnflow.tasks.reduction.MergeReducedEvents`, +{py:class}`~columnflow.tasks.histograms.MergeHistograms`): Tasks to merge the local outputs from +the various occurences of the corresponding tasks. + +There are also CMS-specialized tasks, like +{py:class}`~columnflow.tasks.cms.external.CreatePileUpWeights`, which are described in the +{ref}`CMS specializations section `. As a note, the CreatePileUpWeights task is +interesting from a workflow point of view as it is an example of a task required through an object +of the {py:class}`~columnflow.production.Producer` class. This behaviour can be observed in the +{py:meth}`~columnflow.production.cms.pileup.pu_weight_requires` method. + +TODO: maybe interesting to have examples e.g. for the usage of the +parameters for the 2d plots. Maybe in the example section, or after the next subsection, such that +all parameters are explained? If so, at least to be mentioned here. diff --git a/docs/task_overview/plotting.md b/docs/task_overview/plotting.md new file mode 100644 index 000000000..4c82c0dc3 --- /dev/null +++ b/docs/task_overview/plotting.md @@ -0,0 +1,106 @@ +```{mermaid} +:name: plotting-tasks +:theme: forest +:zoom: +:align: center +:caption: Class diagram of all plotting tasks +classDiagram + PlotBase <|-- PlotBase1D + PlotBase <|-- PlotBase2D + PlotBase <|-- VariablePlotSettingMixin + PlotBase <|-- ProcessPlotSettingMixin + + PlotBase1D <|-- PlotVariables1D + PlotBase1D <|-- PlotShiftedVariables1D + + VariablePlotSettingMixin <|-- PlotVariablesBase + ProcessPlotSettingMixin <|-- PlotVariablesBase + PlotVariablesBase <|-- PlotVariablesBaseSingleShift + PlotVariablesBaseSingleShift <|-- PlotVariables1D + PlotVariablesBaseSingleShift <|-- PlotVariables2D + + PlotBase2D <|-- PlotVariables2D + PlotVariables2D <|-- PlotVariablesPerProcess2D + PlotVariablesBase <|-- PlotVariablesBaseMultiShifts + PlotVariablesBaseMultiShifts <|-- PlotShiftedVariables1D + PlotShiftedVariables1D <|-- PlotShiftedVariablesPerProcess1D + + class PlotBase{ + plot_function: luigi.Parameter + file_types: law.CSVParameter + plot_suffix: luigi.Parameter + view_cmd: luigi.Parameter + general_settings: SettingParameter + skip_legend: law.OptionalBoolParameter + cms_label: luigi.Parameter + } + + class PlotBase1D{ + skip_ratio: law.OptionalBoolParameter + density: law.OptionalBoolParameter + yscale: luigi.ChoiceParameter + shape_norm: law.OptionalBoolParameter + hide_errors: law.OptionalBoolParameter + } + + class PlotBase2D{ + zscale: luigi.ChoiceParameter + density: law.OptionalBoolParameter + shape_norm: law.OptionalBoolParameter + colormap: luigi.Parameter + zlim: law.CSVParameter + extremes: luigi.ChoiceParameter + extreme_colors: law.CSVParameter + } + + class ProcessPlotSettingMixin{ + process_settings: MultiSettingsParameter + } + + class VariablePlotSettingMixin{ + variable_settings: MultiSettingsParameter + } + + class PlotVariablesBaseMultiShifts{ + legend: luigi.Parameter + } +``` + +# Plotting tasks + +The following tasks are dedicated to plotting. +For more information, check out the [plotting user guide](../user_guide/plotting.md) + + {doc}`Plotting `. + +(PlotVariablesTasks)= +- ```PlotVariables*```, ```PlotShiftedVariables*``` (e.g. +{py:class}`~columnflow.tasks.plotting.PlotVariables1D`, +{py:class}`~columnflow.tasks.plotting.PlotVariables2D`, +{py:class}`~columnflow.tasks.plotting.PlotShiftedVariables1D`): Tasks to plot the histograms created by +{py:class}`~columnflow.tasks.histograms.CreateHistograms` using the python package +[matplotlib](https://matplotlib.org/) with [mplhep](https://mplhep.readthedocs.io/en/latest/) style. +Several plot types are possible, including +plots of variables for different physical processes or plots of variables for a single physical +process but different shifts (e.g. jet-energy correction variations). The argument ```--variables``` +followed by the name of the variables defined in the analysis config, separated by a comma, is +needed for these tasks to run. It is also possible to replace the ```--datasets``` argument +for these tasks by the ```--processes``` argument followed by the name of the physical processes to +be plotted, as defined in the analysis config. For the ```PlotShiftedVariables*``` plots, the +argument ```shift-sources``` is needed and replaces the argument ```shift```. The output format for +these plots can be given with the ```--file-types``` argument. It is possible to set a default for the +variables in the analysis config. + + +- ```PlotCutflow*``` (e.g. {py:class}`~columnflow.tasks.cutflow.PlotCutflow`, +{py:class}`~columnflow.tasks.cutflow.PlotCutflowVariables1D`): Tasks to plot the histograms created +by {py:class}`~columnflow.tasks.cutflow.CreateCutflowHistograms`. The +{py:class}`~columnflow.tasks.cutflow.PlotCutflowVariables1D` are plotted in a similar way to the +["PlotVariables*"](PlotVariablesTasks) tasks. The difference is that these plots show the selection +yields of the different selection steps defined in +{py:class}`~columnflow.tasks.selection.SelectEvents` instead of only after the +{py:class}`~columnflow.tasks.reduction.ReduceEvents` procedure. The selection steps to be shown +can be chosen with the ```--selector-steps``` argument. Without further argument, the outputs are +as much plots as the number of selector steps given. On the other hand, the +PlotCutflow task gives a single histograms containing only +the total event yields for each selection step given. diff --git a/docs/task_overview/plotting.rst b/docs/task_overview/plotting.rst new file mode 100644 index 000000000..1e759d2e9 --- /dev/null +++ b/docs/task_overview/plotting.rst @@ -0,0 +1,2 @@ +.. include:: plotting.md + :parser: myst_parser.sphinx_ diff --git a/docs/user_guide/building_blocks/categories.md b/docs/user_guide/building_blocks/categories.md index 12f694f16..a3e8ba9ef 100644 --- a/docs/user_guide/building_blocks/categories.md +++ b/docs/user_guide/building_blocks/categories.md @@ -1,3 +1,526 @@ +Work in Progess! This guide has not been finalized and the presented code has not yet been tested. + +TODO: make the code testable + +(categories)= # Categories -TODO +In columnflow, there are many tools to create a complex and flexible categorization of all analysed +events. +Generally, this categorization can be layered. +We refer to the smallest building block of these layers as **leaf categories**, which can subsequently +be either run individually or combined into more complex categories. +This guide presents how to implement a set of categories in columnflow and shows how to use the +resulting categories via the {py:class}`~columnflow.tasks.yields.CreateYieldTable` task. + + + +## Create a category + +If you used the analysis template to setup your analysis, there are already two categories +included, named ```incl``` and ```2j```. +To test that the categories are properly implemented, +we can use the CreateYieldTable task: +```shell +law run cf.CreateYieldTable --version v1 \ + --calibrators example --selector example --producers example \ + --processes tt,st --categories incl,2j +``` + +When all tasks are finished, this should create a table presenting the event yield of each process +for all categories. To create new categories, we essentially need three ingredients: + +1. We need to define our categories as {external+order:py:class}`order.category.Category` instances in the config +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: add_category( +:end-at: ) +``` +2. We need to write a {py:class}`~columnflow.categorization.Categorizer` that defines which events +to select for each category. The name of the Categorizer needs to be the same as the "selection" +attribute of the category inst. +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/categorization/example.py +:language: python +``` +Keep in mind that the module in which your define your Categorizer needs to be included in the law config + +3. We need to run the {py:class}`~columnflow.production.categories.category_ids` Producer to create +the ```category_ids``` column. This is directly included via the ```--producers category_ids```, +but you could also run the ```category_ids``` Producer as part of another Selector or Producer. +```python +@producer( + uses={category_ids}, + produces={category_ids}, +) +def my_producer(self: Producer, events: ak.Array, **kwargs) -> ak.Array: + # produce your category ids + events = self[category_ids](events, **kwargs) + + # do whatever else your producer needs to do + ... + + return events +``` + +:::{dropdown} But how does this actually work? +The `category_ids` Producer loads all category instances from your config. For each leaf category inst +(which is the "smallest unit" of categories), +it maps the `category_inst.selection` string to a `Categorizer` and adds it the to the `uses` and +`produces`, meaning that columns required (produced) by the `Categorizer` will automatically be +loaded (stored) when running the `category_ids` Producer. + +During the event processing, the `Categorizer` of each leaf category is evaluated to generate +a mask, which defines, whether the event is part of this category or not. +The mask is then transformed to an array of ids (either the `category_inst.id` if True, and +`None` for False entries).t + +In the end, we return a jagged array of category ids, which allows us to categorize one event +into multiple different types of categories. + +You can also store a list of strings in the `category_inst.selection` field. In that case, the logical +`and` of all masks obtained by the `Categorizer` is used to define the mask corresponding to this +category. +::: + + +:::{dropdown} And where are the ```category_ids``` actually used? +The `category_ids` column is primarily used when creating our histograms (e.g. in the +{py:class}`~columnflow.tasks.histograms.CreateHistograms` task). +The created histograms always contain one axis for categories, using the values from the `category_ids` +column. Since this column is a jagged array, it is possible to fill events either never or multiple +times in a histogram. + +Other tasks such as {py:class}`~columnflow.tasks.plotting.PlotVariables1D` are then using this +axis with categories to obtain all entries from the histogram corresponding to the category +that is requested via the `--categories` parameter. When the given category is not a leaf category +but contains categories itself, the ids of all its leaf categories combined are used for the category. +::: + + +## Creation of nested categories + +:::{note} +The following section is mostly included for didactic purposes. To implement a set of nested +categories, it is recommended to follow the section +{ref}`Categorization with multiple layers `. +::: + +Often times, there are multiple layers of categorization. For example, we might want to categorize +based on the number of leptons and the number of jets. + +```python +# do 0 lepton vs >= 1 lepton instead? +cat_1e = config.add_category( + name="1e", + id=10, + selection="cat_1e", + label="1 Electron, 0 Muons", +) +cat_1mu = config.add_category( + name="1mu", + id=20, + selection="cat_1mu", + label="1 Muon, 0 Electrons", +) +cat_2jet = config.add_category( + name="2jet", + id=200, + selection="cat_2jet", + label=r"$N_{jets} \leq 2$", +) +cat_3jet = config.add_category( + name="3jet", + id=300, + selection="cat_3jet", + label=r"$N_{jets} \geq 3$", +) +``` + +```python +@categorizer(uses={"Jet.pt"}) +def cat_2jet(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + return events, ak.num(events.Jet.pt, axis=1) <= 2 + +@categorizer(uses={"Jet.pt"}) +def cat_3jet(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + return events, ak.num(events.Jet.pt, axis=1) >= 3 + +@categorizer(uses={"Muon.pt", "Electron.pt"}) +def cat_1e(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + return events, (ak.num(events.Muon.pt, axis=1) == 0 & ak.num(events.Electron.pt, axis=1) == 1) + +@categorizer(uses={"Muon.pt", "Electron.pt"}) +def cat_1mu(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + return events, (ak.num(events.Muon.pt, axis=1) == 1 & ak.num(events.Electron.pt, axis=1) == 0) +``` + +This code snippet produces 4 new categories, which (after rerunning the ```category_ids``` +producer and recreating all necessary histograms) can already be used: + +```shell +law run cf.CreateYieldTable --version v1 \ + --calibrators example --selector example --producers example \ + --processes tt,st --categories "*" +``` + + +As a next step, one might want to combine these categories to categorize based on the number of +jets and leptons simutaneously. You could simply create all combinations by hand. The following code +snippet shows how to combine the "2jet" and the "1lep" category: + +```python +cat_2jet = config.get_category("2jet") +cat_3jet = config.get_category("3jet") +cat_1e = config.get_category("1e") +cat_1mu = config.get_category("1mu") + +# add combined category as child to the existing categories +cat_1e__2jet = cat_2jet.add_category( + name="1e__2jet", + id=cat_2jet.id + cat_1e.id, + # our `category_ids` producer can automatically combine multiple selections via logical 'and' + selection=[cat_1e.selection, cat_2jet.selection], + label="1 electron, 0 muons, >2 jets", +) +cat_1mu__2jet = cat_2jet.add_category( + name="1e__2jet", + id=cat_2jet.id + cat_1mu.id, + selection=[cat_1mu.selection, cat_2jet.selection], + label="1 muon, 0 electrons, 2 jets", +) +cat_1e__3jet = cat_3jet.add_category( + name="1e__3jet", + id=cat_3jet.id + cat_1e.id, + selection=[cat_1e.selection, cat_3jet.selection], + label="1 electron, 0 muons, 3 jets", +) +cat_1mu__3jet = cat_3jet.add_category( + name="1e__3jet", + id=cat_3jet.id + cat_1mu.id, + selection=[cat_1mu.selection, cat_3jet.selection], + label="1 muon, 0 electrons, 3 jets", +) + +# add children also to lepton categories +cat_1e.add_category(cat_1e__2jet) +cat_1e.add_category(cat_1e__3jet) +cat_1mu.add_category(cat_1mu__2jet) +cat_1mu.add_category(cat_1mu__3jet) +``` + +We now created categories with dependencies between each other. In columnflow, we always use the +smallest units of categories (commonly called "leaf categories") to build our parent categories. For +example, the `2jet` category consists of the leaf categories `1e__2jet` and `1mu__2jet`. To select +events corresponding to `2jet`, we will add all events with category ids corresponding to either +`1e__2jet` or `1mu__2jet`. + + +:::{note} We might add unexpected selections by combining categorization. + +In this example, the `cat_1e` and `cat_1mu` Categorizer only select events with exactly one lepton +(either electron or muon). This means, that events with zero or multiple leptons will not be considered +in our categorization, even when building e.g. the `2jet` category. +::: + +Let's test if our leaf categories are working as intended: + +```shell +law run cf.CreateYieldTable --version v1 \ + --calibrators example --selector example --producers example \ + --processes tt,st --categories 1e,2jet,1e__2jet,1e__3jet,1mu__2jet,1mu__3jet +``` + +We should see that the yield of the `2jet` category is the same as the `1e__2jet` and `1mu__2jet` +categories combined. This is to be expected, since the `2jet` category will be built by combining +histograms from all of their leaf categories. + +:::{note} Take care to define your categories orthogonal! + +There is no mechanism that automatically prevents double counting. It is therefore essential to define +categories such that there is no overlap between leaf categories of any defined category. + +:::::{dropdown} Example of what NOT to do +We might have two categories selecting either exactly two or at least two jets. + +```python +cat_2jet = config.add_category( + name="2jet", + id=200, + selection="cat_2jet", + label=r"$N_{jets} = 2$", +) +cat_geq2jet = config.add_category( + name="geq2jet", + id=300, + selection="cat_geq2jet", + label=r"$N_{jets} \geq 2$", +) + +@categorizer(uses={"Jet.pt"}) +def cat_2jet(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + return events, ak.num(events.Jet.pt, axis=1) == 2 + +@categorizer(uses={"Jet.pt"}) +def cat_geq2jet(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + return events, ak.num(events.Jet.pt, axis=1) >= 2 +``` + +This can be done without any issues since these two categories are independent of each other +(they will each produce their own category id). +However, if we now add a parent category that contains both `2jet` and `geq2jet`, this will result +in couting all events twice that contain exactly two jets. + +```python +non_orthogonal_cat = config.add_category( + name="non_orthogonal_cat", + id=100, + label="Exactly two and at least two jets, resulting in double-counting. You should not do this", +) +non_orthogonal_cat.add_category(cat_2jet) +non_orthogonal_cat.add_category(cat_geq2jet) +``` +::::: +::: + +(categorization_multiple_layers)= +## Categorization with multiple layers + +But what should you do when the number of category combinations is getting large? For example, you might +want to define categories based on the number of leptons, the number of jets, and based on a simple +obervable such as ```HT```: + +```python +jet_categories = (2, 3, 4, 5, 6) +lep_categories = (1, 2, 3) +ht_categories = ((0, 200), (200, 400), (400, 600)) +``` + +If you want to build all combinations of this set of 5+3+3 categories, you end up with 5 x 3 x 3 = 45 +leaf categories. To build all of them by hand is very tedious, unflexible, and error prone. +Luckily, columnflow provides all necessary tools to build combinations of categories automatically. +To use these tools, we first need to create our base categories and their corresponding Categorizers. + +```python +# +# add categories to config +# + +for n_lep in lep_categories: + config.add_category( + id=10 * n_lep, + name=f"{n_lep}lep", + selection=f"cat_{n_lep}lep", + label=f"{n_lep} leptons", + ) +for n_jet in jet_categories: + config.add_category( + id=100 * n_jet, + name=f"{n_jet}jet", + selection=f"cat_{n_jet}jet", + label=f"{n_jet} jets", + ) +for i, (ht_down, ht_up) in enumerate(ht_categories): + config.add_category( + id=1000 * i, + name=f"ht_{ht_down}to{ht_up}", + selection=f"cat_ht{ht_down}to{ht_up}", + label=f"HT in ({ht_down}, {ht_up}]", + ) + +# +# define Categorizer modules +# + +for n_jet in jet_categories: + @categorizer(name=f"cat_{n_jet}jet", uses={"Jet.pt"}) + def cat_n_jet(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + return events, ak.num(events.Jet.pt, axis=1) == n_jet + +for n_lep in (1, 2, 3): + @categorizer(name=f"cat_{n_lep}lep", uses={"Electron.pt", "Muon.pt"}) + def cat_n_lep(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + return events, (ak.num(events.Electron.pt, axis=1) + ak.num(events.Muon.pt, axis=1)) == n_lep + +for ht_down, ht_up in ht_categories: + @categorizer(name=f"cat_ht{ht_down}to{ht_up}", uses={"Jet.pt"}) + def cat_ht(self: Categorizer, events: ak.Array, **kwargs) -> tuple[ak.Array, ak.Array]: + ht = ak.sum(events.Jet.pt, axis=1) + return events, ((ht > ht_down) & (ht <= ht_up)) +``` + +This will only create the (5+3+3) parent categories. To create all possible combinations of +these categories, we can use the {py:func}`~columnflow.config_util.create_category_combinations` +function that will do most of the work for us: + +```python +def name_fn(root_cats): + """ define how to combine names for category combinations """ + cat_name = "__".join(cat.name for cat in root_cats.values()) + return cat_name + + +def kwargs_fn(root_cats): + """ define how to combine category attributes for category combinations """ + kwargs = { + "id": sum([c.id for c in root_cats.values()]), + "label": ", ".join([c.name for c in root_cats.values()]), + # optional: store the information on how the combined category has been created + "aux": { + "root_cats": {key: value.name for key, value in root_cats.items()}, + }, + } + return kwargs + +# +# define how to combine categories +# + + +category_blocks = OrderedDict({ + "lep": [config.get_category(f"{n_jet}lep") for n_lep in lep_categories], + "jet": [config.get_category(f"{n_jet}jet") for n_jet in jet_categories], + "ht": [config.get_category(f"ht{ht_down}to{ht_up}") for ht_down, ht_up in ht_categories], +}) + +from columnflow.config_util import create_category_combinations + +# this function creates all possible category combinations without producing duplicates +# (e.g. category '1lep__2jet__ht200to400' will be produced, but '2jet__1lep__ht200to400' will be skipped) +n_cats = create_category_combinations( + config, + category_blocks, + name_fn=name_fn, + kwargs_fn=kwargs_fn, + skip_existing=False, +) + +# let's check what categories have been produced +print(f"{n_cats} have been created by the create_category_combinations function") +all_cats = [cat.name for cat, _, _ in config.walk_categories()] +print(f"List of all cateogries in our config: \n{all_cats}") +``` + +To test our final set of categories, we can call our `CreateYieldTable` task again. + +```shell +law run cf.CreateYieldTable --version v1 \ + --calibrators example --selector example --producers example \ + --processes tt,st --categories "*" +``` + + +:::{note} This categorization is not inclusive! + +Since we only store leaf category ids, all the events that are not considered by one of the category +blocks will not be considered at all. For this set of categories it means that we implicitly added cuts +on the number of jets (>= 2 and <= 6), the number of leptons (>= 1 and <= 3), and HT (<= 600) as soon +as we use one of these categories. + +TODO: it might be more instructive to build this example such that our categorization is inclusive. +::: + + +## Extend your categorization as part of a Producer + +Some of your categories might need some computationally expensive reconstructed observables +and can therefore only be called after certain requirements are met. +In these cases, it might be beneficial to first produce columns and then load these columns when necessary. +To streamline this, you can set these dependencies as part of the Producer instance that creates the category ids (i.e. via the `category_ids` producer). + +To properly set this up, add the category instances as part of the `Producer.init`. Columns from +custom requirements can be added to a Producer via the `Producer.requires` in combination +with the `Producer.setup`. + +```python +@producer( + uses={category_ids}, + produces={category_ids}, + ml_model_name="my_ml_model", +) +def my_categorization_producer(self: Producer, events: ak.Array, **kwargs) -> ak.Array: + # do whatever else this Producer needs to to + ... + + # reproduce category ids + events = self[category_ids](events, **kwargs) + + return events + + +# The *requires* function can be used to add custom requirements to your Producer task call. +# In this example, we add the MLEvaluation output to our requirements, e.g. to +# categorize events based on a DNN output score +@my_categorization_producer.requires +def my_categorization_producer_reqs(self: Producer, reqs: dict) -> None: + if "ml" in reqs: + return + + from columnflow.tasks.ml import MLEvaluation + reqs["ml"] = MLEvaluation.req(self.task, ml_model=self.ml_model_name) + + +# To also load columns from our custom requirements, we need to tell our Producer, how to access +# columns from the inputs. This is achieved via this *setup* function +@my_categorization_producer.setup +def my_categorization_producer_setup(self: Producer, reqs: dict, inputs: dict, reader_targets: InsertableDict) -> None: + reader_targets["mlcolumns"] = inputs["ml"]["mlcolumns"] + + +# This *init* function is used to add categories as part of the Producer init. This means that +# your config will always contain this category as long as you're running some task where this +# particular Producer has been requested. +@my_categorization_producer.init +def my_categorization_producer_init(self: Producer) -> None: + # ensure that categories are only added to the config once + tag = f"{my_categorization_producer_init.__name__}_called" + if self.config_inst.has_tag(tag): + return + + # add categories to config inst + self.config_inst.add_category( + name="my_category", + id=12345, + selection="my_categorizer", + ) +``` + + +## Helper functions for categories + +### get_events_from_categories + +To obtain the events of a certain category (or categories) of an akward array `events` that contains the +`category_ids` column, you can use the {py:func}`~columnflow.util.get_events_from_categories` function. + + +```python +from columnflow.util import get_events_from_cateogries + +# you can pass a string of a category in combination with the config inst +selected_events = get_events_from_cateogries(events, "my_category", config_inst) + +# it is also possible to directly pass a category. In that case, no config inst is needed +selected_events = get_events_from_cateogries(events, config_inst.get_category("my_category")) + +# it is also possible to pass a list of categories. In that case, all events +# that belong to one of the requested categories are selected +selected_events = get_events_from_cateogries(events, ["cat1", "cat2"], config_inst) +``` + +This function automatically consideres category ids of all leaf categories from the requested +categories. This is especially helpful when your category contains multiple leaf categories. + + +## Key points (TL; DR) + +- Categories are {external+order:py:class}`order.category.Category` instances defined in the config. +- We can add categories to each other; the "smallest unit" of category is referred to as "leaf category". +- We build each category by combining all of it's leaf categories (both in columns and in histograms). +- Leaf categories need to define one or multiple {py:class}`~columnflow.categorization.Categorizer`, +which is a function that defines whether or not an event belongs in this category. +- The `category_ids` column is produced via the +{py:class}`~columnflow.production.categories.category_ids` Producer. +- Make sure that for each category, all of its leaf categories are defined orthogonal to prevent double counting. +- Groups of categories can be combined via {py:func}`~columnflow.config_util.create_category_combinations`. +- Combining categories can impact your parent categories; this can be prevented by defining +your categories such that each group of categories is inclusive. diff --git a/docs/user_guide/building_blocks/config_objects.md b/docs/user_guide/building_blocks/config_objects.md index 737905a9c..0d6be01d0 100644 --- a/docs/user_guide/building_blocks/config_objects.md +++ b/docs/user_guide/building_blocks/config_objects.md @@ -1,61 +1,374 @@ # Config Objects -Columnflow uses a config file for each analysis, saving the variables specific to the analysis. The -[order](https://github.com/riga/order) library is used for the conservation and use of the -metavariables. In this section, the most important config objects to be defined are presented. +## Generalities -## Datasets +The [order](https://github.com/riga/order) package defines several classes to implement the metavariables of an Analysis. +The order documentation and its {external+order:doc}`quickstart` section provide an introduction to these different classes. +In this section we will concentrate on the use of the order classes to define your analysis. -TODO +The three main classes needed to define your analysis are {external+order:py:class}`order.analysis.Analysis`, {external+order:py:class}`order.config.Campaign` and {external+order:py:class}`order.config.Config`. +Their purpose and definition can be found in [the Analysis, Campaign and Config section](https://python-order.readthedocs.io/en/latest/quickstart.html#analysis-campaign-and-config) of the Quickstart section of the order documentation. -## Processes +After defining your Analysis object and your Campaign object(s), you can use the command +```python +cfg = analysis.add_config(campaign, name=your_config_name, id=your_config_id) +``` +to create the new Config object `cfg`, which will be associated to both the Analysis object and the Campaign object needed for its creation. +As the Config object should contain the analysis-dependent information related to a certain campaign, it should contain most of the information needed for running your analysis. +Therefore, in this section, the Config parameters required by Columnflow and some convenience parameters will be presented. + +To start your analysis, do not forget to use the already existing analysis template in the +`analysis_templates/cms_minimal` Git directory and its +[config](https://github.com/columnflow/columnflow/blob/master/analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py). + + +The Config saves information under two general formats: the objects from the order package, which are necessary for your analysis to run in columnflow, and the additional parameters, which are saved under the auxiliary key, accessible through the "x" key. +In principle, the auxiliary field can contain any parameter the user wants to save and reuse for parts of the analysis. +However, several names in the auxiliary field do already have a meaning in columnflow and their values should respect the format used in columnflow. +These two general formats are presented below. + +Additionally, please note that some columnflow objects, like some Calibrators and Producers, require specific information that is needed to be accessible with predefined keywords. +As explained in the {ref}`object-specific variables section `, please check the documentation of these objects before using them. + + +It is generally advised to use functions to set up Config objects. +This enables easy and reliable reusage of parts of your analysis that are the same or similar between Campaigns (e.g. parts of the uncertainty model). +Additionally, other parts of the analysis that might be changed quite often, e.g. the definition of variables, can be defined separately, thus improving the overall organization and readability of your code. +An example of such a separation can be found in the existing [hh2bbtautau analysis](https://github.com/uhh-cms/hh2bbtautau). -TODO -## Shifts +## Parameters from the order package (required) -TODO +### Processes -## "Variable" creation +The physical processes to be included in the analysis. +These should be saved as objects of the {external+order:py:class}`order.process.Process` class and added to the Config object using its {external+order:py:meth}`order.config.Config.add_process()` method. +An example is given in the columnflow analysis template: -In order to create histograms out of the processed datasets, columnflow uses -{external+order:py:class}`order.variable.Variable`s. These -Variables need to be added to the config using the -function {external+order:py:meth}`order.config.Config.add_variable`. The standard syntax is as -follows: +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: process_names = [ +:end-at: proc = cfg.add_process(procs.get(process_name)) +``` + +Additionally, these processes must have corresponding datasets that are to be added to the Config as well (see {ref}`next section `). +It is possible to get all root processes from a specific campaign using the {py:func}`~columnflow.config_util.get_root_processes_from_campaign()` function from columnflow. +Examples of information carried by a process could be the cross section of the process, registered under the {external+order:py:attr}`order.process.Process.xsecs` attribute and further used for {py:class}`~columnflow.production.normalization.normalization_weights` in columnflow, and a color for the plotting scripts, which can be set using the {external+order:py:attr}`order.mixins.ColorMixin.color1` attribute of the process. +An example of a Process definition is given in the {ref}`Analysis, Campaign and Config ` section of the columnflow documentation. +More information about processes can be found in the {external+order:py:class}`order.process.Process` and the {external+order:doc}`quickstart` sections of the order documentation. + + +(datasets_docu_section)= +### Datasets + +The actual datasets to be processed in the analysis. +These should be saved as objects of the {external+order:py:class}`order.dataset.Dataset` class and added to the Config object using its {external+order:py:meth}`order.config.Config.add_dataset()` method. +The datasets added to the Config object must correspond to the datasets added to the Campaign object associated to the Config object. +They are accessible through the {external+order:py:meth}`order.config.Campaign.get_dataset()` method of the Campaign class. +An example is given in the columnflow analysis template: + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: dataset_names = [ +:end-at: dataset = cfg.add_dataset(campaign.get_dataset(dataset_name)) +``` +The Dataset objects should contain for example information about the number of files and number of events present in a Dataset as well as its keys (= the identifiers or origins of a dataset, used by the `cfg.x.get_dataset_lfns` parameter presented below in {ref}`the section on custom retrieval of datasets `) and wether it contains observed or simulated data. +It is also possible to change information of a dataset in the config script. +An example would be reducing the number of files to process for test purposes in a specific test config. +This could be done with the following lines of code: +e.g. ```python -config.add_variable( - name=variable_name, # this is to be given to the "--variables" argument for the plotting task - expression=content_of_the_variable, - null_value=value_to_be_given_if_content_not_available_for_event, - binning=(bins, lower_edge, upper_edge), - unit=unit_of_the_variable_if_any, - x_title=x_title_of_histogram_when_plotted, -) +n_files_max = 5 +for info in dataset.info.values(): + info.n_files = min(info.n_files, n_files_max) +``` + +Once the processes and datasets have both been added to the config, one can check that the root process of all datasets is part of any of the registered processes, using the columnflow function {py:func}`~columnflow.config_util.verify_config_processes`. + +An example of a Dataset definition is given in the {ref}`Analysis, Campaign and Config ` section of the columnflow documentation. + + +### Variables + +In order to create histograms out of the processed datasets, columnflow uses {external+order:py:class}`order.variable.Variable`s. +These Variables need to be added to the config using the function {external+order:py:meth}`order.config.Config.add_variable`. +An example of the standard syntax for the Config object `cfg` would be as follows for the transverse momentum of the first jet: + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: "# pt of the first jet in every event" +:end-before: "# eta of the first jet in every event" +``` + +It is worth mentioning, that you do not need to select a specific jet per event in the `expression` argument (here with `Jet.pt[:,0]`), you can get a flattened histogram for all jets in all events with `expression="Jet.pt"`. + +In histogramming tasks such as {py:class}`~columnflow.tasks.histograms.CreateHistograms`, one histogram is created per Variable given via the ```--variables``` argument, accessing information from columns based on the `expression` of the Variable and storing them in histograms with binning defined via the `binning` argument of the Variable. + +The list of possible keyword arguments can be found in the order documentation for the class {external+order:py:class}`order.variable.Variable`. +The values in the ```expression``` argument can be either a one-dimensional or a more dimensional array. +In this second case the information is flattened before plotting. +It is to be mentioned that {py:attr}`~columnflow.columnar_util.EMPTY_FLOAT` is a columnflow internal null value. + +### Category + +Categories built to investigate specific parts of the phase-space, for example for plotting. +These objects are described in [the Channel and Category](https://python-order.readthedocs.io/en/latest/quickstart.html#channel-and-category) part of the Quickstart section of the order documentation. +You can add such a category with the {py:func}`~columnflow.config_util.add_category()` method. +When adding this object to your Config instance, the `selection` argument is expected to take the name of an object of the {py:class}`~columnflow.categorization.Categorizer` class instead of a boolean expression in a string format. +An example for an inclusive category with the Categorizer `cat_incl` defined in the cms_minimal analysis template is given below: + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: add_category( +:end-at: ) ``` -An example with the transverse momentum of the first jet would be: +It is recommended to always add an inclusive category with ``id=1`` or ``name="incl"`` which is used in various places, e.g. for the inclusive cutflow plots and the "empty" selector. + +A more detailed description of the usage of categories in columnflow is given in the {ref}`Categories ` section of this documentation. + +### Channel + +Similarly to categories, Channels are built to investigate specific parts of the phase space and are described in the [Channel and Category](https://python-order.readthedocs.io/en/latest/quickstart.html#channel-and-category) part of the Quickstart section of the order documentation. +They can be added to the Config object using {external+order:py:meth}`order.config.Config.add_channel()`. + +### Shift + +In order to implement systematic variations in the Config object, the {external+order:py:class}`order.shift.Shift` class can be used. +Implementing systematic variations using shifts can take different forms depending on the kind of systematic variation involved, therefore a complete section specialized in the description of these implementations is to be found in (TODO: add link Shift section). +Adding a Shift object to the Config object happens through the {external+order:py:meth}`order.config.Config.add_shift()` function. +An example is given in the columnflow analysis template: + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: cfg.add_shift(name="nominal", id=0) +:end-at: cfg.add_shift(name="nominal", id=0) +``` + + +Often, shifts are related to auxiliary parameters of the Config, like the name of the scale factors involved, or the paths of source files in case the shift requires external information. + + + +## Auxiliary Parameters (optional) + +In principle, the auxiliaries of the Config may contain any kind of variables. +However, there are several keys with a special meaning for columnflow, for which you would need to respect the expected format. +These are presented below at first, followed by a few examples of the kind of information you might want to save in the auxiliary part of the config on top of these. +If you would like to use modules that ship with Columnflow, it is generally a good idea to first check their documentation to understand what kind of information you need to specify in the auxiliaries of your Config object for a successful run. + +### Keep_columns + +During the Task {py:class}`~columnflow.tasks.reduction.ReduceEvents` new files containing all remaining events and objects after the selections are created in parquet format. +If the auxiliary argument `keep_columns`, accessible through `cfg.x.keep_columns`, exists in the Config object, only the columns declared explicitely will be kept after the reduction. +Actually, several tasks can make use of such an argument in the Config object for the reduction of their output. +Therefore, the `keep_columns` argument expects a {py:class}`~columnflow.util.DotDict` containing the name of the tasks (with the `cf.` prefix) for which such a reduction should be applied as keys and the set of columns to be kept in the output of this task as values. + +For easier handling of the list of columns, the class {py:class}`~columnflow.columnar_util.ColumnCollection` was created. +It defines several enumerations containing columns to be kept according to a certain category. +For example, it is possible to keep all the columns created during the SelectEvents task with the enum `ALL_FROM_SELECTOR`. +An example is given below: + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-after: "# columns to keep after certain steps" +:end-before: "# event weight columns as keys in an OrderedDict, mapped to shift instances they depend on" +``` + +(custom_retrieval_of_dataset_files_section)= +### Custom retrieval of dataset files + +The Columnflow task {py:class}`~columnflow.tasks.external.GetDatasetLFNs` obtains by default the logical file names of the datasets based on the `keys` argument of the corresponding order Dataset. +By default, the function {py:meth}`~columnflow.external.GetDatasetLFNs.get_dataset_lfns_dasgoclient` is used, which obtains the information through the [CMS DAS](https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookLocatingDataSamples). +However, this default behaviour can be changed using the auxiliary parameter `cfg.x.get_dataset_lfns`. +You can set this to a custom function with the same keyword arguments as the default. +For more information, please consider the documentation of {py:meth}`~columnflow.external.GetDatasetLFNs.get_dataset_lfns_dasgoclient`. +Based on these parameters, the custom function should implement a way to create the list of paths corresponding to this dataset (the paths should not include the path to the remote file system) and return this list. +Two other auxiliary parameters can be changed: + +- `get_dataset_lfns_sandbox` provides the sandbox in which the task GetDatasetLFNS will be run and expects therefore a {external+law:py:class}`law.sandbox.base.Sandbox` object, which can be for example obtained through the {py:func}`~columnflow.util.dev_sandbox` function. + +- `get_dataset_lfns_remote_fs` provides the remote file system on which the LFNs for the specific dataset can be found. It expects a function with the `dataset_inst` as a parameter and returning the name of the file system as defined in the law config file. + +An example of such a function and the definition of the corresponding config parameters for a campaign where all datasets have been custom processed and stored on a single remote file system is given below. + ```python -config.add_variable( - name="jet1_pt", - expression="Jet.pt[:,0]", - null_value=EMPTY_FLOAT, - binning=(40, 0.0, 400.0), - unit="GeV", - x_title=r"Jet 1 $p_{T}$", -) -``` - -In histogramming tasks such as -{py:class}`~columnflow.tasks.histograms.CreateHistograms`, one histogram is created per Variable -given via the ```--variables``` argument, accessing information from columns based on -the `expression` of the Variable and storing them in histograms with binning defined -via the `binning` argument of the Variable. - -The list of possible keyword arguments can be found in -Variable. The values in the ```expression``` argument can -be either a one-dimensional or a more dimensional array. In this second case the information is -flattened before plotting. It is to be mentioned that -{py:attr}`~columnflow.columnar_util.EMPTY_FLOAT` is a columnflow internal null value and -corresponds to the value ```-9999.0```. +# custom lfn retrieval method in case the underlying campaign is custom uhh +if cfg.campaign.x("custom", {}).get("creator") == "uhh": + def get_dataset_lfns( + dataset_inst: od.Dataset, + shift_inst: od.Shift, + dataset_key: str, + ) -> list[str]: + # destructure dataset_key into parts and create the lfn base directory + dataset_id, full_campaign, tier = dataset_key.split("/")[1:] + main_campaign, sub_campaign = full_campaign.split("-", 1) + lfn_base = law.wlcg.WLCGDirectoryTarget( + f"/store/{dataset_inst.data_source}/{main_campaign}/{dataset_id}/{tier}/{sub_campaign}/0", + fs=f"wlcg_fs_{cfg.campaign.x.custom['name']}", + ) + + # loop though files and interpret paths as lfns + return [ + lfn_base.child(basename, type="f").path + for basename in lfn_base.listdir(pattern="*.root") + ] + + # define the lfn retrieval function + cfg.x.get_dataset_lfns = get_dataset_lfns + + # define a custom sandbox + cfg.x.get_dataset_lfns_sandbox = dev_sandbox("bash::$CF_BASE/sandboxes/cf.sh") + + # define custom remote fs's to look at + cfg.x.get_dataset_lfns_remote_fs = lambda dataset_inst: f"wlcg_fs_{cfg.campaign.x.custom['name']}" +``` + +### External_files + +If some files from outside columnflow are needed for an analysis, be them local files or online (and accessible through wget), these can be indicated in the `cfg.x.external_files` auxiliary parameter. +These can then be copied to the columnflow outputs using the {py:class}`~columnflow.tasks.external.BundleExternalFiles` task and used by being required by the object needing them. +The `cfg.x.external_files` parameter expects a (possibly nested) {py:class}`~columnflow.util.DotDict` with a user-defined key to retrieve the target in columnflow and the link/path as value. +It is also possible to give a tuple as value, with the link/path as the first entry of the tuple and a version as a second entry. +As an example, the `cfg.x.external_files` parameter might look like this, where `json_mirror` is a local path to a mirror directory of a specific commit of the [jsonPOG-integration Gitlab](https://gitlab.cern.ch/cms-nanoAOD/jsonpog-integration/-/tree/master) (CMS-specific): + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: "cfg.x.external_files = DotDict.wrap({" +:end-at: "})" +``` + +An example of usage of the `muon_sf`, including the requirement of the BundleExternalFiles task is given in the {py:class}`~columnflow.production.cms.muon.muon_weights` Producer. + +:::{dropdown} How to require a task in a Producer? + +Showing how to require the BundleExternalFiles task to have run in the example of the muon weights Producer linked above: + +```{literalinclude} ../../../columnflow/production/cms/muon.py +:language: python +:start-at: "@muon_weights.requires" +:end-at: reqs["external_files"] = BundleExternalFiles.req(self.task) +``` +::: + +### Luminosity + +The luminosity, needed for some normalizations and for the labels in the standard columnflow plots, needs to be given in the auxiliary arguments `cfg.x.luminosity` as an object of the {external+scinum:py:class}`scinum.Number` class, such that for example the `nominal` parameter exists. +An example for a CMS luminosity of 2017 with uncertainty sources given as relative errors is given below. + +```python +from scinum import Number +cfg.x.luminosity = Number(41480, { + "lumi_13TeV_2017": 0.02j, + "lumi_13TeV_1718": 0.006j, + "lumi_13TeV_correlated": 0.009j, +}) +``` + + +### Defaults + +Default values can be given for several command line parameters in columnflow, using the `cfg.x.default_{parameter}` entry in the Config object. +The expected format is either: + +- a single string containing the name of the object to be used as default for parameters accepting only one argument or +- a tuple for parameters accepting several arguments. + +The command-line arguments supporting a default value from the Config object are given in the cms_minimal example of the analysis_templates and shown again below: + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: cfg.x.default_calibrator = "example" +:end-at: cfg.x.default_variables = ("n_jet", "jet1_pt") +``` + +### Groups + +It is also possible to create groups, which allow to conveniently loop over certain command-line parameters. +This is done with the `cfg.x.{parameter}_group` arguments. +The expected format of the group is a dictionary containing the custom name of the groups as keys and the list of the parameter values as values. +The name of the group can then be given as command-line argument instead of the single values. +An example with a selector_steps group is given below. + + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: "# selector step groups for conveniently looping over certain steps" +:end-at: "}" +``` + + + +With this group defined in the Config object, running over the "muon" and "jet" selector_steps in this order in a cutflow task can done with the argument `--selector-steps default`. + +All parameters for which groups are possible are given below: + +```{literalinclude} ../../../analysis_templates/cms_minimal/__cf_module_name__/config/analysis___cf_short_name_lc__.py +:language: python +:start-at: "# process groups for conveniently looping over certain processs" +:end-at: "cfg.x.ml_model_groups = {}" +``` + +### Reduced File size + +The target size of the files in MB after the {py:class}`~columnflow.tasks.reduction.MergeReducedEvents` task can be set in the Config object with the `cfg.x.reduced_file_size` argument. +A float number corresponding to the size in MB is expected. +This value can also be changed with the `merged_size` argument when running the task. +If nothing is set, the default value implemented in columnflow will be used (defined in the {py:meth}`~columnflow.tasks.reduction.MergeReducedEvents.resolve_param_values` method). +An example is given below. + +```python +# target file size after MergeReducedEvents in MB +cfg.x.reduced_file_size = 512.0 +``` + +(object-specific_variables_section)= +### Object-specific variables + +Other than the variables mentioned above, several might be needed for specific Producers for example. +These won't be discussed here as they are not general parameters. +Hence, we invite the users to check which Config entries are needed for each {py:class}`~columnflow.calibration.Calibrator`, +{py:class}`~columnflow.selection.Selector` and {py:class}`~columnflow.production.Producer` and in general each CMS-specific object (=objects in the cms-subfolders) they want to use. +Since the {py:class}`~columnflow.production.cms.muon.muon_weights` Producer was already mentioned above, we will remind users here that the `cfg.x.muon_sf_names` Config entry is needed for this Producer to run, as indicated in the docstring of the Producer. + +As for the CMS-specific objects, an example could be the task {py:class}`~columnflow.tasks.cms.external.CreatePileupWeights`, which requires for example the minimum bias cross sections `cfg.x.minbias_xs` and the `pu` entry in the external files. + +Examples of these entries in the Config objects can be found in already existing CMS-analyses working with columnflow, for example the [hh2bbtautau analysis](https://github.com/uhh-cms/hh2bbtautau) or the [hh2bbww analysis](https://github.com/uhh-cms/hh2bbww) from UHH. + + +### Other examples of auxiliaries in the Config object + +As mentioned above, any kind of python variables can be stored in the auxiliary of the Config object. +To give an idea of the kind of variables an analysis might want to include in the Config object additionally to the ones needed by columnflow, a few examples of variables which do not receive any specific treatment in native columnflow are given. + +- Triggers + +- b-tag working points + +- MET filters + +For applications of these examples, you can look at already existing columnflow analyses, for example the [hh2bbtautau analysis](https://github.com/uhh-cms/hh2bbtautau) from UHH. + + + + diff --git a/docs/user_guide/building_blocks/producers.md b/docs/user_guide/building_blocks/producers.md index a6852a26a..f52452384 100644 --- a/docs/user_guide/building_blocks/producers.md +++ b/docs/user_guide/building_blocks/producers.md @@ -136,7 +136,7 @@ can be found in the {ref}`Law config section `. columns only after the reduction, using the ProduceColumns task. - Other useful functions (e.g. for easier handling of columns) can be found in the -{doc}`best_practices` section of this documentation. +{doc}`../best_practices` section of this documentation. ## ProduceColumns task diff --git a/docs/user_guide/building_blocks/selectors.md b/docs/user_guide/building_blocks/selectors.md index f76f473c5..173370c4f 100644 --- a/docs/user_guide/building_blocks/selectors.md +++ b/docs/user_guide/building_blocks/selectors.md @@ -613,7 +613,7 @@ done in the {py:class}`~columnflow.tasks.production.ProduceColumns` task, using created in this task if needed. - Other useful functions (e.g. for easier handling of columns) can be found in the -{doc}`best_practices` section of this documentation. +{doc}`../best_practices` section of this documentation. diff --git a/docs/user_guide/examples/ml_code.py b/docs/user_guide/examples/ml_code.py new file mode 100644 index 000000000..75b91e3d9 --- /dev/null +++ b/docs/user_guide/examples/ml_code.py @@ -0,0 +1,289 @@ +# coding: utf-8 + +""" +Test model definition. +""" + +from __future__ import annotations + +from typing import Any, Sequence + +import law +import order as od + +from columnflow.ml import MLModel +from columnflow.util import maybe_import +from columnflow.columnar_util import Route, set_ak_column, remove_ak_column + + +ak = maybe_import("awkward") +tf = maybe_import("tensorflow") +np = maybe_import("numpy") + + +law.contrib.load("tensorflow") + + +class TestModel(MLModel): + # shared between all model instances + datasets: dict = { + "datasets_name": [ + "hh_ggf_bbtautau_madgraph", + "tt_sl_powheg", + ], + } + + def __init__( + self, + *args, + folds: int | None = None, + **kwargs, + ): + super().__init__(*args, **kwargs) + # your instance variables + # these are exclusive to your model instance + + def setup(self): + # dynamically add variables for the quantities produced by this model + if f"{self.cls_name}.n_muon" not in self.config_inst.variables: + self.config_inst.add_variable( + name=f"{self.cls_name}.n_muon", + null_value=-1, + binning=(4, -1.5, 2.5), + x_title="Predicted number of muons", + ) + self.config_inst.add_variable( + name=f"{self.cls_name}.n_electron", + null_value=-1, + binning=(4, -1.5, 2.5), + x_title="Predicted number of electrons", + ) + return + + def sandbox(self, task: law.Task) -> str: + return "bash::$HBT_BASE/sandboxes/venv_columnar_tf.sh" + + def datasets(self, config_inst: od.Config) -> set[od.Dataset]: + # normally you would pass this to the model via config and loop through these names ... + all_datasets_names = self.datasets_name + + dataset_inst = [] + for dataset_name in all_datasets_names: + dataset_inst.append(config_inst.get_dataset(dataset_name)) + + # ... but you can also add one dataset by using its name + dataset_inst.append(config_inst.get_dataset("tt_sl_powheg")) + + return set(dataset_inst) + + def uses(self, config_inst: od.Config) -> set[Route | str]: + columns = set(self.input_features) | set(self.target_features) | {"normalization_weight"} + return columns + + def produces(self, config_inst: od.Config) -> set[Route | str]: + # mark columns that you don't want to be filtered out + # preserve the networks prediction of a specific feature for each fold + # cls_name would be the name of your model + ml_predictions = {f"{self.cls_name}.fold{fold}.{feature}" + for fold in range(self.folds) + for feature in self.target_columns} + + # save indices used to create the folds + util_columns = {f"{self.cls_name}.fold_indices"} + + # combine all columns to a unique set + preserved_columns = ml_predictions | util_columns + return preserved_columns + + def output(self, task: law.Task) -> law.FileSystemDirectoryTarget: + # needs to be given via config + max_folds = self.folds + current_fold = task.fold + + # create directory at task.target, if it does not exist + target = task.target(f"mlmodel_f{current_fold}of{max_folds}", dir=True) + return target + + def open_model(self, target: law.FileSystemDirectoryTarget): + # if a formatter exists use formatter + # e.g. for keras models: target.load(formatter="tf_keras_model") + loaded_model = tf.keras.models.load_model(target.path) + return loaded_model + + def open_input_files(self, inputs): + # contains files from all datasets + events_of_datasets = inputs["events"][self.config_inst.name] + + # get datasets names + # datasets = [dataset.label for dataset in self.datasets(self.config_inst)] + + # extract all columns from parquet files for all datasets and stack them + all_events = [] + for dataset, parquet_file_targets in events_of_datasets.items(): + for parquet_file_target in parquet_file_targets: + parquet_file_path = parquet_file_target["mlevents"].path + events = ak.from_parquet(parquet_file_path) + all_events.append(events) + + all_events = ak.concatenate(all_events) + return all_events + + def prepare_events(self, events): + # helper function to extract events and prepare them for training + + column_names = set(events.fields) + input_features = set(self.input_features) + target_features = set(self.target_features) + + # remove columns not used in training + to_remove_columns = list(column_names - (input_features | target_features)) + + for to_remove_column in to_remove_columns: + print(f"removing column {to_remove_column}") + events = remove_ak_column(events, to_remove_column) + + # ml model can't work with awkward arrays + # we need to convert them to tf.tensors + # this is done by following step chain: + # ak.array -> change type to uniform type -> np.recarray -> np.array -> tf.tensor + + # change dtype to uniform type + events = ak.values_astype(events, "float32") + # split data in inputs and target + input_columns = [events[input_column] for input_column in self.input_features] + target_columns = [events[target_column] for target_column in self.target_features] + + # convert ak.array -> np.array -> bring in correct shape + input_data = ak.concatenate(input_columns).to_numpy().reshape( + len(self.input_features), -1).transpose() + target_data = ak.concatenate(target_columns).to_numpy().reshape( + len(self.target_features), -1).transpose() + return tf.convert_to_tensor(input_data), tf.convert_to_tensor(target_data) + + def build_model(self): + # helper function to handle model building + x = tf.keras.Input(shape=(2,)) + a1 = tf.keras.layers.Dense(10, activation="elu")(x) + y = tf.keras.layers.Dense(2, activation="softmax")(a1) + model = tf.keras.Model(inputs=x, outputs=y) + return model + + def train( + self, + task: law.Task, + input: dict[str, list[law.FileSystemFileTarget]], + output: law.FileSystemDirectoryTarget, + ) -> None: + + # use helper functions to define model, open input parquet files and prepare events + # init a model structure + model = self.build_model() + + # get data tensors + events = self.open_input_files(input) + input_tensor, target_tensor = self.prepare_events(events) + + # setup everything needed for training + optimizer = tf.keras.optimizers.SGD() + model.compile( + optimizer, + loss="mse", + steps_per_execution=10, + ) + + # train, throw model_history away + _ = model.fit( + input_tensor, + target_tensor, + epochs=5, + steps_per_epoch=10, + validation_split=0.25, + ) + + # save your model and everything you want to keep + output.dump(model, formatter="tf_keras_model") + return + + def evaluate( + self, + task: law.Task, + events: ak.Array, + models: list[Any], + fold_indices: ak.Array, + events_used_in_training: bool = False, + ) -> ak.Array: + # prepare ml_models input features, target is not important + inputs_tensor, _ = self.prepare_events(events) + + # do evaluation on all data + # one can get test_set of fold using: events[fold_indices == fold] + for fold, model in enumerate(models): + # convert tf.tensor -> np.array -> ak.array + # to_regular is necessary to make array contigous (otherwise error) + prediction = ak.to_regular(model(inputs_tensor).numpy()) + + # update events with predictions, sliced by feature, and fold_indices for identification purpose + for index_feature, target_feature in enumerate(self.target_features): + events = set_ak_column( + events, + f"{self.cls_name}.fold{fold}.{target_feature}", + prediction[:, index_feature], + ) + + events = set_ak_column( + events, + f"{self.cls_name}.fold_indices", + fold_indices, + ) + + return events + + def training_selector( + self, + config_inst: od.Config, + requested_selector: str, + ) -> str: + # training uses the default selector + return "default" + + def training_producers( + self, + config_inst: od.Config, + requested_producers: Sequence[str], + ) -> list[str]: + # training uses the default producer + return ["default"] + + def training_calibrators( + self, + config_inst: od.Config, + requested_calibrators: Sequence[str], + ) -> list[str]: + # training uses a calibrator named "skip_jecunc" + return ["skip_jecunc"] + + +# usable derivations +# create configuration dictionaries +hyperparameters = { + "folds": 3, + "epochs": 5, +} + +# input and target features +configuration_dict = { + "input_features": ( + "n_jet", + "ht", + ), + "target_features": ( + "n_electron", + "n_muon", + ), +} + +# combine configuration dictionary +configuration_dict.update(hyperparameters) + +# init model instance with config dictionary +test_model = TestModel.derive(cls_name="test_model", cls_dict=configuration_dict) diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 4b2ce4fcf..106057ec8 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -6,6 +6,7 @@ User Guide structure.md building_blocks/index + sandbox.md ml.md plotting.md examples.md diff --git a/docs/user_guide/law.md b/docs/user_guide/law.md index 2993a552a..b67d3dafd 100644 --- a/docs/user_guide/law.md +++ b/docs/user_guide/law.md @@ -3,7 +3,7 @@ This analysis tool uses [law](https://github.com/riga/law) for the workflow orchestration. Therefore, a short introduction to the most essential functions of law you should be aware of when using this tool are provided here. More informations are available for example in the -"[Examples]((https://github.com/riga/law#examples))" section of this +"[Examples](https://github.com/riga/law#examples)" section of this [Github repository](https://github.com/riga/law). This section can be ignored if you are already familiar with law. diff --git a/docs/user_guide/ml.md b/docs/user_guide/ml.md index 1044c6360..903b1289a 100644 --- a/docs/user_guide/ml.md +++ b/docs/user_guide/ml.md @@ -1,8 +1,271 @@ # Machine Learning -In this section, the users will learn how to implement machine learning in their analysis with -columnflow. +In this section, the users will learn how to implement machine learning in their analysis with columnflow. -Note: which tasks, how to trigger them, how to train, test and plot networks and outputs +# How training happens in Columnflow: K-fold cross validation +Machine learning in columnflow is implemented in a way that k-fold cross validation is enabled by default. +In k-fold cross validation the dataset is split in k-parts of equal size. +For each training, k-1 parts are used for the actual training and the remaining part is used to test the model. +This process is repeated k-times, resulting in the training of k-model instances. +In the end of the training columnflow will save all k-models, which are then usable for evaluation. +An overview and further details about possible variations of k-fold cross validation can be found in the [sci-kit documentation](https://scikit-learn.org/stable/modules/cross_validation.html). -TODO +# Configure your custom machine learning class: +To create a custom machine learning (ML) class in columnflow, it is imperative to inherit from the {py:class}`~columnflow.ml.MLModel` class. +This inheritance ensures the availability of functions to manage and access config and model instances, as well as the necessary producers. +The name of your custom ML class can be arbitrary, since `law` accesses your machine learning model using a `cls_name` in {py:meth}`~columnflow.util.DerivableMeta.derive`, e.g. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: test_model +:end-at: test_model +``` +The second argument in {py:meth}`~columnflow.util.DerivableMeta.derive` is a `cls_dict` which configures your subclass. +The `cls_dict` needs to be flat. +The keys of the dictionary are set as class attributes and are therefore also accessible by using `self`. +The configuration with `derive` has two main advantages: +- **manageability**, since the dictionary can come from loading a config file and these can be changed fairly easy +- **flexibility**, multiple settings require only different configuration files + +A possible configuration and model initialization for such a model could look like this: +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: hyperparameters = +:end-at: test_model = TestModel.derive( +``` +One can also simply define class variables within the model. +This is can be useful for attributes that don't change often, for example to define the datasets your model uses. +Since these are also class attributes, they are accessible by using `self` and are also shared between all instances of this model. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: class TestModel( +:end-at: your instance variables +``` +If you have settings that should not be shared between all instances, define them within `__init__`. + +# After the configuration +After the configuration of your ML model, `law` needs to be informed in the `law.cfg` about the existence of the new machine learning model. +Add in your `law.cfg`, under the sections `analysis`, a `ml_modules` keyword, where you point to the Python module where the model's definition and derivation happens. +These import structures are relative to the analysis root directory. +```bash +[analysis] +# any other config entries + +# if you want to include multiple things from the same parent module, you can use a comma-separated list in {} +ml_modules: hbt.ml.{test} +inference_modules: hbt.inference.test +``` +# ABC functions +In the following we will go through several abstract functions that you must overwrite, in order to be able to use your custom ML class with columnflow. + +## sandbox: +In {py:meth}`~columnflow.ml.MLModel.sandbox`, you specify which sandbox setup file should be sourced to setup the environment for ML usage. +The return value of {py:meth}`~columnflow.ml.MLModel.sandbox` is the path to your shell (sh) file, e.g: +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def sandbox +:end-at: return "bash::$HBT_BASE/sandboxes/venv_columnar_tf.sh" +``` +It is recommended to start the path with `bash::`, to indicate that you want to source the {py:meth}`~columnflow.ml.MLModel.sandbox` with `bash`. +How to actually write the setup and requirement files can be found in the section about [setting up a sandbox](building_blocks/sandbox). + +## datasets: +In the {py:meth}`~columnflow.ml.MLModel.datasets` function, you specify which datasets are important for your machine learning model and which dataset instance(s) should be extracted from your config. +To use this function your datasets needs to be added to your campaign, as defined by the [Order](https://python-order.readthedocs.io/en/latest/) Module. +An example can be found [here](https://github.com/uhh-cms/cmsdb/blob/d83fb085d6e43fe9fc362df75bbc12816b68d413/cmsdb/campaigns/run2_2018_nano_uhh_v11/top.py). +It is recommended to return this as `set`, to prevent double counting. +In the following example all datasets given by the {ref}`external config ` are taken, but also an additional dataset is given. +Note how the name of each dataset is used to get a `dataset instance` from your `config instance`. +This ensures that you properly use the correct dataset. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def datasets +:end-at: return set(dataset_inst) +``` + + +## produces: +By default, intermediate columns are not saved within the columnflow framework but are filtered out afterwards. +If you want to prevent certain columns from being filtered out, you need to tell columnflow. +You can tell columnflow, by let {py:meth}`~columnflow.ml.MLModel.produces` return the names of all columns that should be preserved, do this by define the names as strings within an interable. +More information can be found in the official documentation about [producers](building_blocks/producers). + +In the following example, I want tell columnflow to preserve the output of the neural network, but also the fold indices, that are used to create the trainings and test fold. +I do not store the input and target columns of that fold to save disk space, since they are already stored in the training set parquet file. +To avoid confusion, we are not producing the columns in this function, we only tell columnflow to not throwing them away. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def produces +:end-at: return preserved_columns +``` +Thus, the choice to include these columns is your choice. + +## uses: +In `uses` you define the columns that are needed by your machine learning model, and are forwarded to the ML model during the execution of the various tasks +In this case we want to request the input and target features, as well as some weights: +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def uses( +:end-at: return columns +``` + +## output: +In the {py:meth}`~columnflow.ml.MLModel.output` function, you define your local target directory that your current model instance will have access to. + +Since machine learning in columnflow uses k-fold cross validation by default, it is a good idea to have a separate directory for each fold, and this should be reflected in the {py:meth}`~columnflow.ml.MLModel.output` path. +It is of good practice to store your "machine-learning-instance" files within the directory of the models instance. To get the path to this directory use `task.target`. +In this example we want to save each fold separately e.g: +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def output +:end-at: return target +``` + + +## open_model: +In the {py:meth}`~columnflow.ml.MLModel.open_model` function, you implement the loading of the trained model and if necessary its configuration, so it is ready to use for {py:meth}`~columnflow.ml.MLModel.evaluate`. +This does not define how the model is build for {py:meth}`~columnflow.ml.MLModel.train`. + +The `target` parameter represents the local path to the models directory. +In the the following example a TensorFlow model saved with Keras API is loaded and returned, and no further configuration happens: +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def open_model +:end-at: return loaded_model +``` + +## train: +In the {py:meth}`~columnflow.ml.MLModel.train` function, you implement the initialization of your models and the training loop. +The `task` corresponding to the models training is {py:class}`~columnflow.tasks.ml.MLTraining`. +By default {py:meth}`~columnflow.ml.MLModel.train` has access to the location of the models `inputs` and `outputs`. + +In columnflow, k-fold cross validation is enabled by default. +The `self` argument in {py:meth}`~columnflow.ml.MLModel.train` referes to the instance of the fold. +Using `self`, you have also access to the entire `analysis_inst`ance the `config_inst`ance of the current fold, and to all the derived parameters of your model. + +With this information, you can call and prepare the columns to be used by the model for training. +In the following example a very simple dummy training loop is performed using the Keras fit function. +Within this function some helper functions are used that are further explained in the following chapter about good practices. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def train +:end-at: return +``` +### Good practice for training: +It is considered to be a good practice to define helper functions for model building and column preprocessing and call them in {py:meth}`~columnflow.ml.MLModel.train`. +The reasoning behind this design decision is that in {py:meth}`~columnflow.ml.MLModel.train`, you want to focus more on the definition of the actual trainings loop. +Another reason is that especially the preparation of events can take a large amount of code lines, making it messy to debug. +In the following example it is shown how to define these helper functions. +First of all one needs a function to handle the opening and combination of all parquet files. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def open_input_files +:end-at: return all_events +``` + +In `prepare_events` we use the combined awkward array and filter out all columns we are not interested in during the training, to stay lightweight. +Next we split the remaining columns into input and target column and bring these columns into the correct shape, data type, and also use certain preprocessing transformation. +At the end we transform the awkward array into a Tensorflow tensor, something that our machine learning model can handle. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def prepare_events +:end-at: return tf.convert_to_tensor +``` +The actual building of the model is also handled by a separate function. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def build_model +:end-at: return model +``` +With this little example one can see why it is of good practice to separate this process into small chunks. + +## evaluate: +Within {py:meth}`~columnflow.ml.MLModel.train` one defined the trainings process of the ML model, while in {py:meth}`~columnflow.ml.MLModel.evaluate` its evaluation is defined. +The corresponding `task` is {py:class}`~columnflow.tasks.ml.MLEvaluation`, which depends on {py:class}`~columnflow.tasks.ml.MLTraining` and will therefore trigger a training if no training was performed before. + +For each fold of the k-folds a neural network model is trained and can be accessed by `models`. +The actual loading, of the trained model stored in the list, is defined in {py:meth}`~columnflow.ml.MLModel.open_model` function. + +The awkward array `events` is loaded in chunks and contains the merged events of all folds. +To filter out the test set, create a mask with `fold_indices`. + +If you want to preserve columns and write them out into a parquet file, append the columns to `events` using {py:func}`~columnflow.columnar_util.set_ak_column` and return `events`. +All columns not present in {py:meth}`~columnflow.ml.MLModel.produces` are then filtered out. + +In the following example, the models prediction as well as the number of muons and electrons are saved, all the other columns in events are thrown away, since they are not present in `produce`: + +Don't confuse this behavior with the parameter `events_used_in_training`. +This flag determines if a certain `dataset` and shift combination can be used by the task. + +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def evaluate +:end-at: return events +``` + +The evaluations output is saved as parquet file with the name of the model as field name. +To get the files path afterwards, rerun the same law command again, but with `--print-output 0` at the end. + +# Commands to start training and evaluation +To start the machine learning training `law run cf.MLTraining`, while for evaluation use `law run cf.MLEvaluation`. +When not using the `config`, `calibrators`, `selector`s or `dataset` +```bash +law run cf.MLTraining \ + --version your_current_version \ + --ml-model test_model_name \ + --config run2_2017_nano_uhh_v11_limited \ + --calibrators calibrator_name \ + --selector selector_name \ + --dataset datasets_1,dataset_2,dataset_3,... +``` +```bash +law run cf.MLEvaluation \ + --version your_current_version \ + --ml-model test_model_name \ + --config run2_2017_nano_uhh_v11_limited \ + --calibrators calibrator_name +``` +Most of these settings should sound familiar, if not look into the corresponding tutorial. +`version` defines a setup configuration of your ML task, think more of a label than of an actual `version`. +If you change the version label, columnflow will rerun all dependencies that are unique for this label, this typically just means you will retrain a new model. You can then switch freely between both models version. + +# Optional useful functions: +## Separate training and evaluation configuration for configs, calibrators, selector and producers: +By default chosen configs, calibrators, selector and producer are used for both training and evaluation. +Sometimes one does not want to share the same environment or does not need all the columns in evaluation as in training. +Another possible scenario is the usage of different selectors or datasets, to be able to explore different phase spaces. +This is where a separation of both comes in handy. + +To separate this behavior one need to define the training_{configs,calibrators, selector,producers}. +These functions take always the `config_inst` as first, and the requested_{configs,calibrators,selector,producers} as second parameter. +If this function is defined the evaluation will use the externally-defined `config`,`calibrator`, `selector` or `producer`, while the training will use one defined in the function. + +In the following case, training will use a fixed selector and producer called `default`, and a custom-defined calibrator called `skip_jecunc`: +the calibrator for the evaluation is provided by the used command. +Take special note on the numerus of the functions name and of course of the type hint. +The selector expects only a string, since we typically apply only 1 selection, while the calibrator or producers expect a sequence of strings. +In this special case we use an own defined calibrator called "skip_jecunc", which is of course defined within the law.cfg. +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def training_selector +:end-at: return "default" +``` +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def training_producers +:end-at: return ["default"] +``` +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def training_calibrators +:end-at: return ["skip_jecunc"] +``` +## setup +Setup is called at the end of `__init__` of your ML model. +Within this function you can prepare operations that should stay open during the whole life time of your instance. +A typical example for this would be the opening of a file or dynamically appending variables that are only relevant to your ML model, and no where else. +In the following example definitions needed to plot variables that are created in ML model are added to the `config_inst`: +```{literalinclude} ./examples/ml_code.py +:language: python +:start-at: def setup +:end-at: return +``` diff --git a/docs/user_guide/plotting.md b/docs/user_guide/plotting.md index 9896e2b48..51eb25ac2 100644 --- a/docs/user_guide/plotting.md +++ b/docs/user_guide/plotting.md @@ -1,11 +1,395 @@ # Plotting -This section showcases how to use the various plotting tasks in columnflow. +In columnflow, there are multiple tasks to create plots. This section showcases how to create and +customize a plot based on the {py:class}`~columnflow.tasks.plotting.PlotVariables1D` task. +The usage of other plotting tasks is mostly analogous to the ```PlotVariables1D``` task. The most +important differences compared to ```PlotVariables1D``` are presented in a separate section +for each of the other plotting tasks. +An overview of all plotting tasks is given in the [Plotting tasks](../task_overview/plotting.md) section. -Note: which tasks, how to trigger them, ALL POSSIBLE PARAMETERS!!, variables +## Creating your first plot -TODO +Assuming you used the analysis template to setup your analysis, you can create a first plot by running +```shell +law run cf.PlotVariables1D --version v1 \ + --calibrators example --selector example --producer example \ + --processes data,tt,st --variables n_jet --categories incl,2j +``` +This will run the full analysis chain for the given processes (data, tt, st) and should create +plots looking like this: -## Variables creation -see the Variables creation section in the Config Objects section of the documentation +::::{grid} 1 1 2 2 +:::{figure} ../plots/cf.PlotVariables1D_tpl_config_analy__1__12dfac316a__plot__proc_3_7727a49dc2__cat_incl__var_n_jet.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotVariables1D_tpl_config_analy__1__12dfac316a__plot__proc_3_7727a49dc2__cat_2j__var_n_jet.pdf +:width: 100% +::: +:::: + + +:::{dropdown} Where do I find that plot? +You can add ```--print-output 0``` to every task call, which will print the full filename of all +outputs of the requested task. Alternatively, you can add ```--fetch-output 0,a``` to directly +copy all outputs of this task into the directory you are currently in. +Finally, there is the ```--view-cmd``` parameter you can add to directly display the plot during +the runtime of the task, e.g. via ```--view-cmd evince-previewer```. +::: + + +The ```PlotVariables1D``` task is located at the bottom of our +[task graph](https://github.com/columnflow/columnflow/wiki#default-task-graph), which means that +all tasks leading to ```PlotVariables1D``` will be run for all datasets corresponding to the +```--processes``` we requested using the +{py:class}`~columnflow.calibration.Calibrator`s, {py:class}`~columnflow.selection.Selector`, and +{py:class}`~columnflow.production.Producer`s +(often referred to as CSPs) as requested. In the following examples, we will skip the +```--calibrators```, ```--selector``` and ```--producers``` parameters, which means that the defaults +defined in the config will be used automatically. Examples on how to +implement your own CSPs can be found in the [calibrators](building_blocks/calibrators), +[selectors](building_blocks/selectors), and [producers](building_blocks/producers) sections of the +user guide. The ```--variables``` parameter defines, for which variables we want to create histograms +and plots. Variables are [order](https://github.com/riga/order) objects that need to be defined +in the config as shown in the +[config objects](building_blocks/config_objects) section. The column corresponding to the expression +statement needs to be stored either after the {py:class}`~columnflow.tasks.reduction.ReduceEvents` +task or as part of a ```Producer``` used in the {py:class}`~columnflow.tasks.production.ProduceColumns` +task. +For each of the category given with the ```--categories``` parameter, one plot will be produced. +A detailed guide on how to implement categories in Columnflow is given in the +[categories](building_blocks/categories) section. + +To define which processes and datasets to consider when plotting, you can use the ```--processes``` +and ```--datasets``` parameter. When only processes are given, all datasets corresponding to the +requested processes will be considered. When only datasets are given, all processes in the config +will be considered. +The ```--processes``` parameter can be used to change the order of processes in the stack and the legend +(try for example ```--processes st,tt``` instead) and to further distinguish between sub-processes +(e.g. via ```--processses tt_sl,tt_dl,tt_fh```). + +:::{dropdown} IMPORTANT! Do not add the same dataset via multiple processes! +At the time of writing this documentation, there is still an issue present that histograms corresponding +to a dataset can accidentally be used multiple times. For example, when adding ```--processes tt,tt_sl```, +the events corresponding to the dataset ```tt_sl_powheg``` will be displayed twice in the resulting +plot. +::: + + +## Customization of plots + +There are many different parameters implemented that allow customizing the style of a plot. A short +overview to all plotting parameters is given in the [Plotting tasks](../task_overview/plotting.md). +In the following, a few exemplary task calls are given to present the usage of our plotting parameters, +using the {py:class}`~columnflow.tasks.plotting.PlotVariables1D` task. +Most paramaters are shared between the different plotting tasks. The most important changes regarding +the task parameters are discussed in separate sections for each type of plotting task. + +Per default, the ```PlotVariables1D``` task creates one plot +per variable with all Monte Carlo processes being included +in a stack and data being shown as separate points. The bottom subplot shows the ratio between signal +and all processes included in the stack and can be disabled via the ```--skip_ratio``` parameter. +To change the text next to the label, you can add the ```--cms-label``` parameter. +:::{dropdown} What are the ```cms-label``` options? +In general, this parameter accepts all types of strings, but there is a set of shortcuts for commonly +used labels that will automatically be resolved: +```{literalinclude} ../../columnflow/plotting/plot_all.py +:language: python +:start-at: label_options +:end-at: "}" +``` +::: + +To compare shapes of multiple processes, you might want to plot each process separately as one line. +To achieve this, you can use the ```unstack``` option of the ```--process-settings``` parameter. This +parameter can also be used to change other attributes of your process instances, such as color, label, +and the scale. To better compare shapes of processes, we can normalize each line with the +```--shape-norm``` parameter. Combining all the previously discussed parameters might lead to a task +call such as + +```shell +law run cf.PlotVariables1D --version v1 --processes tt,st --variables n_jet,jet1_pt \ + --skip-ratio --shape-norm --cms-label simpw \ + --process-settings "tt,unstack,color=#e41a1c:st,unstack,label=Single Top" +``` + +to produce the following plot: + +::::{grid} 1 1 2 2 +:::{figure} ../plots/cf.PlotVariables1D_tpl_config_analy__1__0191de868f__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt__c1.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotVariables1D_tpl_config_analy__1__0191de868f__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c1.pdf +:width: 100% +::: +:::: + +Parameters that only contain a single value can also be passed via the ```--general-settings```, which +is a single comma-separated list of parameters, where the name and the value are separated via a `=`. +The value of each parameter is automatically resolved to either a float, bool, or a string. When no `=` +is present, the parameter is automatically set to True. +:::{dropdown} What is the advantage of setting parameters via the ```--general-settings``` parameter? +While there is no direct advantage of setting parameters via the ```--general-settings```, this +parameter provides some convenience by allowing you to define defaults and groups in the config +(will be discussed later in the guide). + +Additionally, this parameter allows you to set parameters on the command line that are not directly +implemented as task parameters. This is especially helpful when you want to parametrize +{ref}`custom plotting functions `. +::: + +We can also change the y-scale of the plot to a log scale by adding ```--yscale log``` and change some +properties of specific variables via the ```variable-settings``` parameter. For example, we might +want to create the plots of our two obserables in one call, but would like to try out a +rebinned version of `jet1_pt` that merges bins by a factor of 10. A corresponding task call +might be + +```shell +law run cf.PlotVariables1D --version v1 --processes tt,st --variables n_jet,jet1_pt \ + --general-settings "skip_ratio,shape_norm,yscale=log,cms-label=simpw" \ + --variable-settings "n_jet,y_title=Events,x_title=N jets:jet1_pt,rebin=10,x_title=Leading jet \$p_{T}\$" +``` + +::::{grid} 1 1 2 2 +:::{figure} ../plots/cf.PlotVariables1D_tpl_config_analy__1__c80529af83__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt__c2.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotVariables1D_tpl_config_analy__1__c80529af83__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c2.pdf +:width: 100% +::: +:::: + + +:::{dropdown} Limitations of the ```variable_settings``` +While in theory we can change anything inside the variable and process instances via the +```variable_settings``` parameter, there are certain attributes that are already used during the creation +of the histograms (e.g. the ```expression``` and the ```binning```). Since our ```variable_settings``` +parameter only modifies these attributes during the runtime of our plotting task, this will not +impact our final results. +::: + + +For the ```general_settings```, ```process_settings```, and ```variable_settings``` you can define +defaults and groups in the config, e.g. via + +```python +config_inst.x.default_variable_settings = {"jet1_pt": {"rebin": 4, "x_title": r"Leading jet $p_{T}$"}} +config_inst.x.process_settings_groups = { + "unstack_processes": {proc: {"unstack": True} for proc in ("tt", "st")}, +} +config_inst.x.general_settings_groups = { + "compare_shapes": {"skip_ratio": True, "shape_norm": True, "yscale": "log", "cms_label": "simpw"}, +} +``` +The default is automatically used when no parameter is given in the task call, and the groups can +be used directly on the command line and will be resolved automatically. Our previously defined +defaults and groups will be used e.g. by the following task call: + +```shell +law run cf.PlotVariables1D --version v1 --processes tt,st --variables n_jet,jet1_pt \ + --process-settings unstack_processes --general-settings compare_shapes +``` + +::::{grid} 1 1 2 2 +:::{figure} ../plots/cf.PlotVariables1D_tpl_config_analy__1__be60d3bca7__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt__c3.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotVariables1D_tpl_config_analy__1__be60d3bca7__plot__proc_2_a2211e799f__cat_incl__var_n_jet__c3.pdf +:width: 100% +::: +:::: + + +## Creating 2D plots + +Columnflow also provides the {py:class}`~columnflow.tasks.plotting.PlotVariables2D` task to create +two-dimensional plots. Two-dimensional histograms are created by passing two variables to the +```--variables``` parameter, separated by a ```-```. Here is an exemplary task call and their +outputs. +```shell +law run cf.PlotVariables2D --version v1 \ + --processes tt,st --variables n_jet-jet1_pt,jet1_pt-n_jet +``` + +::::{grid} 1 1 2 2 +:::{figure} ../plots/cf.PlotVariables2D_tpl_config_analy__1__b27b994979__plot__proc_2_a2211e799f__cat_incl__var_jet1_pt-n_jet.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotVariables2D_tpl_config_analy__1__b27b994979__plot__proc_2_a2211e799f__cat_incl__var_n_jet-jet1_pt.pdf +:width: 100% +::: +:::: + +While most of the plotting parameters used in the ```PlotVariables1D``` task can be reused for this +task, there are also some additional parameters only available for 2D plotting tasks. +For more information on the task parameters of the ```PlotVariables2D``` task, take a look into the +[plotting task overview](../task_overview/plotting.md). + +## Creating cutflow plots + +The previously discussed plotting functions only create plots after applying the full event selection. +To allow inspecting and optimizing an event and object selection, Columnflow also includes plotting +tasks that can produce plots after each individual selection step. + +To create a simple cutflow plot, displaying event yields after each individual selection step, +you can use the {py:class}`~columnflow.tasks.cutflow.PlotCutflow` task, e.g. via calling +```shell +law run cf.PlotCutflow --version v1 \ + --calibrators example --selector example --categories incl,2j \ + --shape-norm --process-settings tt,unstack:st,unstack \ + --processes tt,st --selector-steps jet,muon +``` + +::::{grid} 1 1 2 2 +:::{figure} ../plots/cf.PlotCutflow_tpl_config_analy__1__12a17bf79c__cutflow__cat_incl.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotCutflow_tpl_config_analy__1__12a17bf79c__cutflow__cat_2j.pdf +:width: 100% +::: +:::: + +This will produce a plot with three bins, containing the event yield before applying any selection +and after each selector step, where we always apply the logical ```and``` of all previous selector steps. + +:::{dropdown} What are the options for the ```--selector-steps```? Can I customize the step labels? +The steps listed in the ```--selector-steps``` parameter need to be defined by the +{py:class}`~columnflow.selection.Selector` that has been used. +A detailed guide on how to implement your own selector can be found in the +[Selections](building_blocks/selectors) guide. + +Per default, the name of the selector step is used on the x-axis, but you can also provide custom +step labels via the config: +```python +config_inst.x.selector_step_labels = { + "muon": r"$N_{muon} = 1$", + "jet": r"$N_{jets}^{AK4} \geq 1$", +} +``` +::: + + +To create plots of variables as part of the cutflow, we also provide the +{py:class}`~columnflow.tasks.cutflow.PlotCutflowVariables1D`, which mostly behaves the same as the +{py:class}`~columnflow.tasks.plotting.PlotVariables1D` task. + +```shell +law run cf.PlotCutflowVariables1D --version v1 \ + --calibrators example --selector example \ + --processes tt,st --variables cf_jet1_pt --categories incl \ + --selector-steps jet,muon --per-plot processes +``` + +::::{grid} 1 1 3 3 +:::{figure} ../plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step0_Initial__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step1_jet__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__d8a37d3da9__plot__step2_muon__proc_2_a2211e799f__cat_incl__var_cf_jet1_pt.pdf +:width: 100% +::: +:::: + +The ```per-plot``` parameter defines whether to produce one plot per selector step +(```per-plot processes```) or one plot per process (```per-plot steps```). +For the ```per-plot steps``` option, try the following task call: + +```shell +law run cf.PlotCutflowVariables1D --version v1 \ + --calibrators example --selector example \ + --processes tt,st --variables cf_jet1_pt --categories incl \ + --selector-steps jet,muon --per-plot steps +``` + +::::{grid} 1 1 2 2 +:::{figure} ../plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__c3947accbb__plot__proc_st__cat_incl__var_cf_jet1_pt.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotCutflowVariables1D_tpl_config_analy__1__c3947accbb__plot__proc_tt__cat_incl__var_cf_jet1_pt.pdf +:width: 100% +::: +:::: + + +## Creating plots for different shifts + +Like most tasks, our plotting tasks also contain the ```--shift``` parameter that allows requesting +the outputs for a certain type of systematic variation. Per default, the ```shift``` parameter is set +to "nominal", but you could also produce your plot with a certain systematic uncertainty varied +up or down, e.g. via running +```shell +law run cf.PlotVariables1D --version v1 \ + --processes tt,st --variables n_jet --shift mu_up +``` + +If you already ran the same task call with ```--shift nominal``` before, this will only require to +produce new histograms and plots, as a shift such as the ```mu_up``` is typically implemented as an +event weight and therefore does not require to reproduce any columns. Other shifts such as ```jec_up``` +also impact our event selection and therefore also need to re-run anything starting from +{py:class}`~columnflow.tasks.selection.SelectEvents`. A detailed overview on how to implement different +types of systematic uncertainties is given in the [systematics](systematics) +section (TODO: not existing). + +For directly comparing differences introduced by one shift source, we provide the +{py:class}`~columnflow.tasks.plotting.PlotShiftedVariables1D` task. Instead of the ```--shift``` +parameter, this task implements the ```--shift-sources``` option and creates one plot per shift source +displaying the nominal distribution (black) compared to the shift source varied up (red) and down (blue). +The task can be called e.g. via +```shell +law run cf.PlotShiftedVariables1D --version v1 \ + --processes tt,st --variables jet1_pt,n_jet --shift-sources mu +``` +and produces the following plot: + +::::{grid} 1 1 2 2 +:::{figure} ../plots/cf.PlotShiftedVariables1D_tpl_config_analy__1__42b45aba89__plot__proc_2_a2211e799f__unc_mu__cat_incl__var_jet1_pt.pdf +:width: 100% +::: + +:::{figure} ../plots/cf.PlotShiftedVariables1D_tpl_config_analy__1__42b45aba89__plot__proc_2_a2211e799f__unc_mu__cat_incl__var_n_jet.pdf +:width: 100% +::: +:::: + +This produces per default only one plot containing the sum of all processes. To produce this plot +per process, you can use the {py:class}`~columnflow.tasks.plotting.PlotShiftedVariablesPerProcess1D` +task + + +## Directly displaying plots in the terminal + +All plotting tasks also include a ```--view-cmd``` parameter that allows directly printing the plot +during the runtime of the task: +```shell +law run cf.PlotVariables1D --version v1 \ + --processes tt,st --variables n_jet --view-cmd evince-previewer +``` + +(custom_plot_function)= +## Using your own plotting function + +While all plotting tasks provide default plotting functions which implement many parameters to +customize the plot, it might be necessary to write your own plotting functions if you want to create +a specific type of plot. In that case, you can simply write a function that follows the signature +of all other plotting functions and call a plotting task with this function using the +`--plot-function` parameter. + +An example on how to implement such a plotting function is shown in the following: + + +```{literalinclude} ../../analysis_templates/cms_minimal/__cf_module_name__/plotting/example.py +:language: python +:start-at: def my_plot1d_func( +:end-at: return fig, (ax,) +``` diff --git a/docs/user_guide/sandbox.md b/docs/user_guide/sandbox.md new file mode 100644 index 000000000..c36182d37 --- /dev/null +++ b/docs/user_guide/sandbox.md @@ -0,0 +1,59 @@ + +# Sandboxes and Their Use in Columnflow +Columnflow is a backend framework designed to orchestrate columnar analyses entirely with Python. +Various steps of an analysis necessitate distinct software environments. +To maintain flexibility and lightweight configurations [venv](https://docs.python.org/3/library/venv.html) (virtual environments) are employed. + +The following guide explains how to define, update, and utilize a sandbox within Columnflow. + +## Whats needs to be defined +A Columnflow environment consists of two files: +1. **setup.sh:**, sets up a virtual environment in `$CF_VENV_PATH`. +2. **requirement.txt:** , defines the required modules and their versions. + +## Where to Store the Corresponding Files + +For organizational purposes, it is recommended to store your setup and requirement files in `$ANALYSIS_BASE/sandboxes/`, where `$ANALYSIS_BASE` is the root directory for your analysis. +The latter is usually the abbreviated form you specified for your analysis in upper case, e.g. the root directory for analysis `hbt` is `$HBT_BASE`. +It is of good practice to follow the naming conventions used by Columnflow. +Begin your setup file with "venv," for example, `venv_YOUR_ENVIRONMENT.sh`, and use only the environment name (without "venv") for your requirement file, e.g., `YOUR_ENVIRONMENT.txt`. + +## The Setup File + +Begin your setup file by referencing an existing setup file within the `$ANALYSIS_BASE/sandboxes/` directory. +In this example, we start from a copy of `venv_columnar.sh`: +We start from `venv_columnar.sh` +```{literalinclude} ../../sandboxes/venv_columnar.sh +:language: bash +``` +You only need to change `CF_VENV_REQUIREMENTS` to point to your new requirement file. + +## The requirement file +The requirement.txt uses pip notation. +A quick overview is given of commonly used notation: +```bash +# version 1 +# Get Coffea from a GitHub repository with a specific commit/tag (@9ecdee5) as an egg file named coffea (#egg=coffea). +git+https://github.com/riga/coffea.git@9ecdee5#egg=coffea + +# Get the latest version of dask-awkward, but do not exceed version 2023.2. +dask-awkward~=2023.2 +``` +For more information, refer to the official [pip documentation](https://pip.pypa.io/en/stable/reference/requirements-file-format/). + +Columnflow manages sandboxes by using a version number at the very first line (`# version ANY_FLOAT`). +This version defines a software package, and it is good practice to change the version number whenever an environment is altered. + +## How to Use the Namespace +One may wonder how to work with namespaces of certain modules when it is not guaranteed that this module is available. +For this case, the {py:meth}`~columnflow.util.maybe_import` function is there. +This utility function handles the import for you and makes the namespace available. + +The input of {py:meth}`~columnflow.util.maybe_import` is the name of the module, provided as a string. +```python +# maybe_import version of: +# import tensorflow as tf +tf = maybe_import("tensorflow") +``` +It is good practice to use `maybe_import` within the local namespace if you use the module once, and at the global namespace level if you intend to use multiple times. + diff --git a/docs/user_guide/structure.md b/docs/user_guide/structure.md index e44167104..93c653edc 100644 --- a/docs/user_guide/structure.md +++ b/docs/user_guide/structure.md @@ -1,41 +1,25 @@ # Columnflow Structure -In this section, an overview to the structure of Columnflow is provided, starting with -a general introduction, followed by a description of the various tasks implemented in -columnflow and ending with an introduction on how to configure your analysis on top -of columnflow with the analysis and config object from the -[order](https://github.com/riga/order) package. +In this section, an overview to the structure of Columnflow is provided, starting with a general introduction, followed by a description of the various tasks implemented in columnflow and ending with an introduction on how to configure your analysis on top of columnflow with the analysis and config object from the [order](https://github.com/riga/order) package. ## General introduction -Columnflow is a fully orchestrated columnar analysis tool for HEP analyses with Python. The -workflow orchestration is managed by [law](https://github.com/riga/law) and the meta data and -configuration is managed by [order](https://github.com/riga/order). A short introduction to law is -given in the {doc}`law section `. If you have never used law before, this section is highly -recommended as a few very convenient commands are presented there. - - -The data processing in columnflow is based on columns in [awkward arrays](https://awkward-array.org/doc/main/) -with [coffea](https://coffeateam.github.io/coffea/)-generated behaviour. Fields like "Jet" exist -too, they contain columns with the same first dimension (the parameters of the field, e.g. Jet.pt). -A few additional functions for simplified handling of columns were defined in -{py:mod}`~columnflow.columnar_util`. - -As most of the information is conserved in the form of columns, it would be very inefficient -(and might not even fit in the memory) to use all columns and all events from a dataset at once for -each task. Therefore, in order to reduce the impact on the memory: -- a chunking of the datasets is implemented using [dask](https://www.dask.org/): not all events -from a dataset are inputed in a task at once, but only chunked in groups of events. -(100 000 events max per group is default as of 05.2023, default is set in the law.cfg file). -- the user needs to define for each {py:class}`~columnflow.production.Producer`, -{py:class}`~columnflow.calibration.Calibrator` and {py:class}`~columnflow.selection.Selector` which -columns are to be loaded (this happens by defining the ```uses``` set in the header of the -decorator of the class) and which new columns/fields are to be saved in parquet files after the -respective task (this happens by defining the ```produces``` set in the header of the decorator of -the class). The exact implementation for this feature is further detailed in -{doc}`building_blocks/selectors` and {doc}`building_blocks/producers` +Columnflow is a fully orchestrated columnar analysis tool for HEP analyses with Python. +The workflow orchestration is managed by [law](https://github.com/riga/law) and the meta data and configuration is managed by [order](https://github.com/riga/order). +A short introduction to law is given in the {doc}`law section `. +If you have never used law before, this section is highly recommended as a few very convenient commands are presented there. + + +The data processing in columnflow is based on columns in [awkward arrays](https://awkward-array.org/doc/main/) with [coffea](https://coffeateam.github.io/coffea/)-generated behaviour. +Fields like "Jet" exist too, they contain columns with the same first dimension (the parameters of the field, e.g. Jet.pt). A few additional functions for simplified handling of columns were defined in {py:mod}`~columnflow.columnar_util`. + +As most of the information is conserved in the form of columns, it would be very inefficient (and might not even fit in the memory) to use all columns and all events from a dataset at once for each task. +Therefore, in order to reduce the impact on the memory: +- a chunking of the datasets is implemented using [dask](https://www.dask.org/): not all events from a dataset are inputed in a task at once, but only chunked in groups of events. (100 000 events max per group is default as of 05.2023, default is set in the law.cfg file). +- the user needs to define for each {py:class}`~columnflow.production.Producer`, {py:class}`~columnflow.calibration.Calibrator` and {py:class}`~columnflow.selection.Selector` which columns are to be loaded (this happens by defining the ```uses``` set in the header of the decorator of the class) and which new columns/fields are to be saved in parquet files after the respective task (this happens by defining the ```produces``` set in the header of the decorator of the class). +The exact implementation for this feature is further detailed in {doc}`building_blocks/selectors` and {doc}`building_blocks/producers`. ## Tasks in columnflow @@ -53,172 +37,136 @@ experiment-specific tasks which are not present in this graph. However, these ar Further informations about tasks and law can be found in the {doc}`"Law Introduction" ` section of this documentation or in the [example section](https://github.com/riga/law#examples) of the law -Github repository. In general (at least for people working in the CMS experiment), -using columnflow requires a grid proxy, as the -dataset files are accessed through it. The grid proxy should be activated after the setup of the -default environment. It is however possible to create a custom law config file to be used by people -without a grid certificate, although someone with a grid proxy must run the tasks -{py:class}`~columnflow.tasks.external.GetDatasetLFNs` (and potentially the cms-specific -{py:class}`~columnflow.tasks.cms.external.CreatePileUpWeights`) for them. Once these -tasks are done, the local task outputs can be used without grid certificate by other users if -they are able to access them with the storage location declared in the custom law config file. -An example for such a custom config file can be found in the {doc}`examples` section of this -documentation. - - -While the name of each task is fairly descriptive of its purpose, a short introduction of the most -important facts and parameters about each task group is provided below. As some tasks require -others to run, the arguments for a task higher in the tree will also be required for tasks below -in the tree (sometimes in a slightly different version, e.g. with an "s" if the task allows several -instances of the parameter to be given at once (e.g. several dataset**s**)). - -It should be mentioned that in your analysis, the command line argument for running the columnflow -tasks described below will contain an additional "cf."-prefix -before the name of the task, as these are columnflow tasks and not new tasks created explicitely -for your analysis. For example, running the {py:class}`~columnflow.tasks.selection.SelectEvents` -task will require the following syntax: -```shell -law run cf.SelectEvents --your-parameters parameter_value -``` - -- {py:class}`~columnflow.tasks.external.GetDatasetLFNs`: This task looks for the logical file names -of the datasets to be used and saves them in a json file. The argument ```--dataset``` followed by -the name of the dataset to be searched for, as defined in the analysis config is needed for this -task to run. TODO: more infos? - -- {py:class}`~columnflow.tasks.calibration.CalibrateEvents`: Task to implement corrections to be -applied on the datasets, e.g. jet-energy corrections. This task uses objects of the -{py:class}`~columnflow.calibration.Calibrator` class to apply the calibration. The argument -```--calibrator``` followed by the name of the Calibrator -object to be run is needed for this task to run. A default value for this argument can be set in -the analysis config. Similarly, the ```--shift``` argument can be given, in order to choose which -corrections are to be used, e.g. which variation (up, down, nominal) of the jet-energy corrections -are to be used. TODO: more infos, e.g. output type of task? - -- {py:class}`~columnflow.tasks.selection.SelectEvents`: Task to implement selections to be applied -on the datssets. This task uses objects of the {py:class}`~columnflow.selection.Selector` class to -apply the selection. The output are masks for the events and objects to be selected, saved in a -parquet file, and some additional parameters stored in a dictionary format, like the statistics of -the selection (which are needed for the plotting tasks further down the task tree), saved in a json -file. The mask are not applied to the columns during this task. -The argument ```--selector``` followed by the name of the -Selector object to be run is needed for this task to run. -A default value for this argument can be set in the analysis config. From this task on, the -```--calibrator``` argument is replaced by ```--calibrators```. - -- {py:class}`~columnflow.tasks.reduction.ReduceEvents`: Task to apply the masks created in -{py:class}`~columnflow.tasks.selection.SelectEvents` on the datasets. All -tasks below ReduceEvents in the task graph use the parquet -file resulting from ReduceEvents to work on, not the -original dataset. The columns to be conserved after -ReduceEvents are to be given in the analysis config under -the ```config.x.keep_columns``` argument in a ```DotDict``` structure -(from {py:mod}`~columnflow.util`). - -- {py:class}`~columnflow.tasks.production.ProduceColumns`: Task to produce additional columns for -the reduced datasets, e.g. for new high level variables. This task uses objects of the -{py:class}`~columnflow.production.Producer` class to create the new columns. The new columns are -saved in a parquet file that can be used by the task below on the task graph. The argument -```--producer``` followed by the name of the Producer object -to be run is needed for this task to run. A default value for this argument can be set in the -analysis config. - -- {py:class}`~columnflow.tasks.ml.PrepareMLEvents`, {py:class}`~columnflow.tasks.ml.MLTraining`, -{py:class}`~columnflow.tasks.ml.MLEvaluation`: Tasks to -train, evaluate neural networks and plot (to be implemented) their results. -TODO: more informations? output type? -all tf based? or very general and almost everything must be set in the training scripts? -and if general, what is taken care of? - -- {py:class}`~columnflow.tasks.histograms.CreateHistograms`: Task to create histograms with the -python package [Hist](https://hist.readthedocs.io/en/latest/) which can be used by the tasks below -in the task graph. From this task on, the ```--producer``` argument is replaced by -```--producers```. The histograms are saved in a pickle file. -TODO: more informations? - -(PlotVariablesTasks)= -- ```PlotVariables*```, ```PlotShiftedVariables*``` (e.g. -{py:class}`~columnflow.tasks.plotting.PlotVariables1D`, -{py:class}`~columnflow.tasks.plotting.PlotVariables2D`, -{py:class}`~columnflow.tasks.plotting.PlotShiftedVariables1D`): Tasks to plot the histograms created by -{py:class}`~columnflow.tasks.histograms.CreateHistograms` using the python package -[matplotlib](https://matplotlib.org/) with [mplhep](https://mplhep.readthedocs.io/en/latest/) style. -Several plot types are possible, including -plots of variables for different physical processes or plots of variables for a single physical -process but different shifts (e.g. jet-energy correction variations). The argument ```--variables``` -followed by the name of the variables defined in the analysis config, separated by a comma, is -needed for these tasks to run. It is also possible to replace the ```--datasets``` argument -for these tasks by the ```--processes``` argument followed by the name of the physical processes to -be plotted, as defined in the analysis config. For the ```PlotShiftedVariables*``` plots, the -argument ```shift-sources``` is needed and replaces the argument ```shift```. The output format for -these plots can be given with the ```--file-types``` argument. It is possible to set a default for the -variables in the analysis config. - -- {py:class}`~columnflow.tasks.cms.inference.CreateDatacards`: TODO - -- ```PlotCutflow*``` (e.g. {py:class}`~columnflow.tasks.cutflow.PlotCutflow`, -{py:class}`~columnflow.tasks.cutflow.PlotCutflowVariables1D`): Tasks to plot the histograms created -by {py:class}`~columnflow.tasks.cutflow.CreateCutflowHistograms`. The -{py:class}`~columnflow.tasks.cutflow.PlotCutflowVariables1D` are plotted in a similar way to the -["PlotVariables*"](PlotVariablesTasks) tasks. The difference is that these plots show the selection -yields of the different selection steps defined in -{py:class}`~columnflow.tasks.selection.SelectEvents` instead of only after the -{py:class}`~columnflow.tasks.reduction.ReduceEvents` procedure. The selection steps to be shown -can be chosen with the ```--selector-steps``` argument. Without further argument, the outputs are -as much plots as the number of selector steps given. On the other hand, the -PlotCutflow task gives a single histograms containing only -the total event yields for each selection step given. - -- ```Merge``` tasks (e.g. {py:class}`~columnflow.tasks.reduction.MergeReducedEvents`, -{py:class}`~columnflow.tasks.histograms.MergeHistograms`): Tasks to merge the local outputs from -the various occurences of the corresponding tasks. TODO: details? why needed? Only convenience? -Or I/O? - - -There are also CMS-specialized tasks, like -{py:class}`~columnflow.tasks.cms.external.CreatePileUpWeights`, which are described in the -{ref}`CMS specializations section `. As a note, the CreatePileUpWeights task is -interesting from a workflow point of view as it is an example of a task required through an object -of the {py:class}`~columnflow.production.Producer` class. This behaviour can be observed in the -{py:meth}`~columnflow.production.cms.pileup.pu_weight_requires` method. - -TODO: maybe interesting to have examples e.g. for the usage of the -parameters for the 2d plots. Maybe in the example section, or after the next subsection, such that -all parameters are explained? If so, at least to be mentioned here. +Github repository. +For an overview of the tasks that are available with columnflow, please see the +[Task Overview](../task_overview/introduction.md). ## Important note on required parameters -It should also be added that there are additional parameters specific for the tasks in columnflow, -required by the fact that columnflow's purpose is for HEP analysis. These are the ```--analysis``` -and ```--config``` parameters, which defaults can be set in the law.cfg. These two parameters -respectively define the config file for the different analyses to be used (where the different -analyses and their parameters should be defined) and the name of the config file for the specific -analysis to be used. +It should also be added that there are additional parameters specific for the tasks in columnflow, required by the fact that columnflow's purpose is for HEP analysis. +These are the ```--analysis``` and ```-config``` parameters, which defaults can be set in the law.cfg. +These two parameters respectively define the config file for the different analyses to be used (where the different analyses and their parameters should be defined) and the name of the config file for the specific analysis to be used. -Similarly the ```--version``` parameter, which purpose is explained in the {doc}`law` section of -this documentation, is required to start a task. +Similarly the ```--version``` parameter, which purpose is explained in the {doc}`law` section of this documentation, is required to start a task. ## Important modules and configs -The standard syntax to access objects in columnflow is the dot syntax, usable for the -[order](https://github.com/riga/order) metavariables (e.g. campaign.x.year) as well as the -[awkward arrays](https://awkward-array.org/doc/main/) (e.g. events.Jet.pt). +The standard syntax to access objects in columnflow is the dot syntax, usable for the [order](https://github.com/riga/order) metavariables (e.g. campaign.x.year) as well as the [awkward arrays](https://awkward-array.org/doc/main/) (e.g. events.Jet.pt). TODO here mention the analysis template -### Campaigns - ### Law config +(analysis_campaign_config)= +### Analysis, Campaign and Config + +Columnflow uses the {external+order:py:class}`order.analysis.Analysis` class from the [order](https://github.com/riga/order) package to define a specific analysis. +This object does not contain most of the analysis information by itself. +It is to be linked to objects of the {external+order:py:class}`order.config.Campaign` and {external+order:py:class}`order.config.Config` classes, as described in [the Analysis, Campaign and Config section](https://python-order.readthedocs.io/en/latest/quickstart.html#analysis-campaign-and-config) of the Quickstart section of the order documentation. +An example of an analysis with its full Analysis, Campaign and Config definitions in the same directory is given in the [Analysis Grand Challenge Columnflow repository](https://github.com/columnflow/agc_cms_ttbar/) repository and [its config directory](https://github.com/columnflow/agc_cms_ttbar/tree/master/agc/config). + +A Campaign object contains the analysis-independent information related to a specific and well-defined experimental campaign. +We define an experimental campaign as a set of fixed specific conditions for the data-taking or simulations (for example, a period of data-taking for which no significant change to the detector setup or operation was made, like the data-taking period of the year 2017 for the CMS detector at CERN). +This means general information like the center of mass energy of the collisions (argument `ecm`) as well as the datasets created during/for the specific campaign. +An example of a Campaign declaration (from the AGC Columnflow repository linked above) might be: + +```python +from order import Campaign + +campaign_cms_opendata_2015_agc = cpn = Campaign( + name="cms_opendata_2015_agc", # the name of the campaign + id=1, # a unique id for this campaign + ecm=13, # center of mass energy + aux={ # additional, arbitrary information + "tier": "NanoAOD", # data format, e.g. NanoAOD + "year": 2015, # year of data-taking + "location": "https://xrootd-local.unl.edu:1094//store/user/AGC/nanoAOD", # url to base path of the nanoAODs + "wlcg_fs": "wlcg_fs_unl", # file system to use on the WLCG + }, +) +``` + +In order to define a {external+order:py:class}`order.dataset.Dataset`, which may be included in the Campaign object, an associated {external+order:py:class}`order.process.Process` object must be defined first. +An example of such a Process, describing the physical process of the top-antitop production associated with jets in a pp-collision is defined in `agc.config.processes` (in the AGC Columnflow repository linked above) as: + +```python +from scinum import Number +from order import Process + +tt = Process( + name="tt", # name of the process + id=1000, # unique id for this process + label=r"$t\bar{t}$ + Jets", # label to be used in e.g. plot legends + color=(128, 76, 153), # color to be used in plots for this process (e.g. in RGB format) + xsecs={ # cross sections for this process + # format of these numbers is + # { + # center of mass energy: Number object with nominal prediction + uncertainties + # } + 13: Number(831.76, { + "scale": (19.77, 29.20), + "pdf": 35.06, + "mtop": (23.18, 22.45), + }), + }, +) +``` -### Analysis config +Using the physical Process defined above, one may now create a dataset, which can be added to the Campaign object. +An example of dataset definition with scale variations for this campaign would then be (values taken from [the analysis-grand-challenge GitHub repository](https://github.com/iris-hep/analysis-grand-challenge/blob/be91d2c80225b7a91ce6b153591f8605167bf555/analyses/cms-open-data-ttbar/nanoaod_inputs.json)): +```python +import agc.config.processes as procs +from order import DatasetInfo + +# cpn is the Campaign object defined in the previous code block +# the following shows an example for a simulated tt dataset generated with the Powheg package +cpn.add_dataset( + name="tt_powheg", # name of the dataset + id=1, # unique id for this dataset + processes=[procs.tt], # link this dataset to physics processes (which are also order objects in this case) + info={ # add additional information to this dataset + # information regarding the 'nominal' generation of the dataset, + # i.e. with the recommended central values of the MC parameters + "nominal": DatasetInfo( + # identifier for this dataset in a meta information database (e.g. DAS at CMS) + keys=["TT_TuneCUETP8M1_13TeV-powheg-pythia8"], + n_files=242, # how many files to process, important for book keeping + n_events=276079127), # total number of events + # you can also add information about additional samples that belong to this dataset + # for example, add samples where a set of MC parameters (tune) is varied within uncertainties + "scale_down": DatasetInfo( + keys=["TT_TuneCUETP8M1_13TeV-powheg-scaledown-pythia8"], + n_files=32, + n_events=39329663), + "scale_up": DatasetInfo( + keys=["TT_TuneCUETP8M1_13TeV-powheg-scaleup-pythia8"], + n_files=33, + n_events=38424467), + }, + aux={ # additional, general information about this dataset + "agc_process": "ttbar", + "agc_shifts": { + "scale_down": "scaledown", + "scale_up": "scaleup", + }, + }, +) +``` -The analysis config defines all analysis specific variables and objects that need to be defined for -the analysis to run. Some of them are required for columnflow to run, some are additional and can -be useful, depending on the analysis. +A Config object is saving the variables specific to the analysis. +It is associated to an Analysis and a Campaign object. +It is to be created using the {external+order:py:meth}`order.analysis.Analysis.add_config` method on the analysis object, with the associated Campaign object as argument. +An example would be: +```python +cfg = analysis.add_config(campaign, name=config_name, id=config_id) +``` + +In this way, one Analysis can be tied to different Campaigns (e.g. corresponding to different years of data-taking) very naturally. + +Several classes of the order library are used for the organization and use of the metavariables. +A more detailed description of the most important objects to be defined in the Config object is presented in the {doc}`building_blocks/config_objects` section of this documentation. -TODO diff --git a/modules/order b/modules/order index 5179bf28e..12eedf2d5 160000 --- a/modules/order +++ b/modules/order @@ -1 +1 @@ -Subproject commit 5179bf28ec564b0b68a3e77cfe1c7dfeb30ca3f7 +Subproject commit 12eedf2d5d72afc85c780e638275e81036df4efe From 6b48871d25c9ab466ca9834ff3bec99e0acc2385 Mon Sep 17 00:00:00 2001 From: Mathis Frahm <49306645+mafrahm@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:04:50 +0100 Subject: [PATCH 07/43] allow masking events via preparation producer (#404) Co-authored-by: Marcel Rieger --- columnflow/tasks/ml.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/columnflow/tasks/ml.py b/columnflow/tasks/ml.py index 6c1652abb..34091609f 100644 --- a/columnflow/tasks/ml.py +++ b/columnflow/tasks/ml.py @@ -29,6 +29,7 @@ from columnflow.tasks.reduction import MergeReducedEventsUser, MergeReducedEvents from columnflow.tasks.production import ProduceColumns from columnflow.util import dev_sandbox, safe_div, DotDict, maybe_import +from columnflow.columnar_util import set_ak_column ak = maybe_import("awkward") @@ -226,16 +227,19 @@ def run(self): ) # generate fold indices - fold_indices = events.deterministic_seed % self.ml_model_inst.folds + events = set_ak_column(events, "fold_indices", events.deterministic_seed % self.ml_model_inst.folds) # invoke the optional producer if len(events) and self.preparation_producer_inst: events = self.preparation_producer_inst( events, stats=stats, - fold_indices=fold_indices, + fold_indices=events.fold_indices, ml_model_inst=self.ml_model_inst, ) + # read fold_indices from events array to allow masking training events + fold_indices = events.fold_indices + # remove columns events = route_filter(events) From b7c27f3c32c0602860993def917a73b9756551e3 Mon Sep 17 00:00:00 2001 From: Mathis Frahm <49306645+mafrahm@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:30:45 +0100 Subject: [PATCH 08/43] request only one workflow per variable in CreateDatacards (#405) * request only one workflow per dataset in CreateDatacards * add data back to workflow requirements * add docstrings and remove duplicated function * switch to req_different_branching --------- Co-authored-by: Marcel Rieger --- columnflow/tasks/cms/inference.py | 77 +++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/columnflow/tasks/cms/inference.py b/columnflow/tasks/cms/inference.py index 71e7e217f..e0929303d 100644 --- a/columnflow/tasks/cms/inference.py +++ b/columnflow/tasks/cms/inference.py @@ -4,7 +4,7 @@ Tasks related to the creation of datacards for inference purposes. """ -from collections import OrderedDict +from collections import OrderedDict, defaultdict import law @@ -39,30 +39,69 @@ class CreateDatacards( def create_branch_map(self): return list(self.inference_model_inst.categories) + def get_mc_datasets(self, proc_obj: dict) -> list[str]: + """ + Helper to find automatic datasets + + :param proc_obj: process object from an InferenceModel + :return: List of dataset names corresponding to the process *proc_obj* + """ + # when datasets are defined on the process object itself, return them + if proc_obj.config_mc_datasets: + return proc_obj.config_mc_datasets + + # if not, check the config + return [ + dataset_inst.name + for dataset_inst in get_datasets_from_process(self.config_inst, proc_obj.config_process) + ] + def workflow_requires(self): reqs = super().workflow_requires() - # simply require the requirements of all branch tasks right now - reqs["merged_hists"] = set(sum(( - law.util.flatten(t.requires()) - for t in self.get_branch_tasks().values() - ), [])) + # initialize defaultdict, mapping datasets to variables + shift_sources + mc_dataset_params = defaultdict(lambda: {"variables": set(), "shift_sources": set()}) + data_dataset_params = defaultdict(lambda: {"variables": set()}) + + for cat_obj in self.branch_map.values(): + for proc_obj in cat_obj.processes: + for dataset in self.get_mc_datasets(proc_obj): + # add all required variables and shifts per dataset + mc_dataset_params[dataset]["variables"].add(cat_obj.config_variable) + mc_dataset_params[dataset]["shift_sources"].update( + param_obj.config_shift_source + for param_obj in proc_obj.parameters + if self.inference_model_inst.require_shapes_for_parameter(param_obj) + ) + + if cat_obj.config_data_datasets: + for dataset in cat_obj.config_data_datasets: + data_dataset_params[dataset].add(cat_obj.config_variable) + + # set workflow requirements per mc dataset + reqs["merged_hists"] = set( + self.reqs.MergeShiftedHistograms.req_different_branching( + self, + dataset=dataset, + shift_sources=tuple(params["shift_sources"]), + variables=tuple(params["variables"]), + ) + for dataset, params in mc_dataset_params.items() + ) + + # add workflow requirements per data dataset + for dataset, params in data_dataset_params.items(): + reqs["merged_hists"].add( + self.reqs.MergeHistograms.req_different_branching( + self, + dataset=dataset, + variables=tuple(params["variables"]), + ), + ) return reqs def requires(self): - # helper to find automatic datasets - def get_mc_datasets(proc_obj: dict) -> list[str]: - # when datasets are defined on the process object itself, return them - if proc_obj.config_mc_datasets: - return proc_obj.config_mc_datasets - - # if not, check the config - return [ - dataset_inst.name - for dataset_inst in get_datasets_from_process(self.config_inst, proc_obj.config_process) - ] - cat_obj = self.branch_data reqs = { proc_obj.name: { @@ -78,7 +117,7 @@ def get_mc_datasets(proc_obj: dict) -> list[str]: branch=-1, _exclude={"branches"}, ) - for dataset in get_mc_datasets(proc_obj) + for dataset in self.get_mc_datasets(proc_obj) } for proc_obj in cat_obj.processes } From 8400f442cbc1cb18c63af6dd3f874a8c03285cd2 Mon Sep 17 00:00:00 2001 From: "Marcel R." Date: Thu, 14 Mar 2024 16:34:10 +0100 Subject: [PATCH 09/43] Update upstream modules. --- modules/law | 2 +- modules/order | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/law b/modules/law index 3f8b5c895..d585cf818 160000 --- a/modules/law +++ b/modules/law @@ -1 +1 @@ -Subproject commit 3f8b5c8954bfb1db7ba1c3bcf0265bc0f12eaa04 +Subproject commit d585cf8183c3e7b389ef86ed0dc4b2367d7eaae8 diff --git a/modules/order b/modules/order index 12eedf2d5..5179bf28e 160000 --- a/modules/order +++ b/modules/order @@ -1 +1 @@ -Subproject commit 12eedf2d5d72afc85c780e638275e81036df4efe +Subproject commit 5179bf28ec564b0b68a3e77cfe1c7dfeb30ca3f7 From 096af5692090300937f59a112779675da2696c87 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 14 Mar 2024 16:39:57 +0100 Subject: [PATCH 10/43] switch to req_different_branching --- columnflow/tasks/cms/inference.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/columnflow/tasks/cms/inference.py b/columnflow/tasks/cms/inference.py index e0929303d..248cfc6aa 100644 --- a/columnflow/tasks/cms/inference.py +++ b/columnflow/tasks/cms/inference.py @@ -105,7 +105,7 @@ def requires(self): cat_obj = self.branch_data reqs = { proc_obj.name: { - dataset: self.reqs.MergeShiftedHistograms.req( + dataset: self.reqs.MergeShiftedHistograms.req_different_branching( self, dataset=dataset, shift_sources=tuple( @@ -115,7 +115,7 @@ def requires(self): ), variables=(cat_obj.config_variable,), branch=-1, - _exclude={"branches"}, + workflow="local", ) for dataset in self.get_mc_datasets(proc_obj) } @@ -123,12 +123,12 @@ def requires(self): } if cat_obj.config_data_datasets: reqs["data"] = { - dataset: self.reqs.MergeHistograms.req( + dataset: self.reqs.MergeHistograms.req_different_branching( self, dataset=dataset, variables=(cat_obj.config_variable,), branch=-1, - _exclude={"branches"}, + workflow="local", ) for dataset in cat_obj.config_data_datasets } From 87e5c3ed0c094e7470bb2488f53727ddfdfbddb8 Mon Sep 17 00:00:00 2001 From: Mathis Frahm <49306645+mafrahm@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:51:42 +0100 Subject: [PATCH 11/43] update muon sf producer for run 3 (#399) Co-authored-by: Marcel Rieger --- columnflow/production/cms/muon.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/columnflow/production/cms/muon.py b/columnflow/production/cms/muon.py index 67a639d87..75809caf8 100644 --- a/columnflow/production/cms/muon.py +++ b/columnflow/production/cms/muon.py @@ -63,13 +63,27 @@ def muon_weights( abs_eta = flat_np_view(abs(events.Muon.eta[muon_mask]), axis=1) pt = flat_np_view(events.Muon.pt[muon_mask], axis=1) + variable_map = { + "year": self.year, + "abseta": abs_eta, + "pt": pt, + } + # loop over systematics for syst, postfix in [ ("sf", ""), ("systup", "_up"), ("systdown", "_down"), ]: - sf_flat = self.muon_sf_corrector(self.year, abs_eta, pt, syst) + # get the inputs for this type of variation + variable_map_syst = { + **variable_map, + "scale_factors": "nominal" if syst == "sf" else syst, # syst key in 2022 + "ValType": syst, # syst key in 2017 + } + inputs = [variable_map_syst[inp.name] for inp in self.muon_sf_corrector.inputs] + + sf_flat = self.muon_sf_corrector(*inputs) # add the correct layout to it sf = layout_ak_array(sf_flat, events.Muon.pt[muon_mask]) From 04db85a44f0451179d2801eeacb05efc536183b3 Mon Sep 17 00:00:00 2001 From: Mathis Frahm <49306645+mafrahm@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:30:32 +0100 Subject: [PATCH 12/43] add top_pt_weight producer (#400) * add top_pt_weight producer * rename top pt module and update docs --------- Co-authored-by: Marcel Rieger --- columnflow/production/cms/top_pt_weight.py | 140 +++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 columnflow/production/cms/top_pt_weight.py diff --git a/columnflow/production/cms/top_pt_weight.py b/columnflow/production/cms/top_pt_weight.py new file mode 100644 index 000000000..c8447f430 --- /dev/null +++ b/columnflow/production/cms/top_pt_weight.py @@ -0,0 +1,140 @@ +# coding: utf-8 + +""" +Column producers related to top quark pt reweighting. +""" + +import law + +from columnflow.production import Producer, producer +from columnflow.util import maybe_import +from columnflow.columnar_util import set_ak_column + +ak = maybe_import("awkward") +np = maybe_import("numpy") +coffea = maybe_import("coffea") +maybe_import("coffea.nanoevents.methods.nanoaod") + +logger = law.logger.get_logger(__name__) + + +@producer( + uses={"GenPart.pdgId", "GenPart.statusFlags"}, + # requested GenPartonTop columns, passed to the *uses* and *produces* + produced_top_columns={"pt"}, + mc_only=True, +) +def gen_parton_top(self: Producer, events: ak.Array, **kwargs) -> ak.Array: + """ + Produce parton-level top quarks (before showering and detector simulation). + Creates new collection named "GenPartonTop" + + *produced_top_columns* can be adapted to change the columns that will be produced + for the GenPartonTop collection. + + The function is skipped when the dataset is data or when it does not have the tag *has_top*. + + :param events: awkward array containing events to process + """ + # find parton-level top quarks + abs_id = abs(events.GenPart.pdgId) + t = events.GenPart[abs_id == 6] + t = t[t.hasFlags("isLastCopy")] + t = t[~ak.is_none(t, axis=1)] + + # save the column + events = set_ak_column(events, "GenPartonTop", t) + + return events + + +@gen_parton_top.init +def gen_parton_top_init(self: Producer) -> bool: + for col in self.produced_top_columns: + self.uses.add(f"GenPart.{col}") + self.produces.add(f"GenPartonTop.{col}") + + +@gen_parton_top.skip +def gen_parton_top_skip(self: Producer) -> bool: + """ + Custom skip function that checks whether the dataset is a MC simulation containing top + quarks in the first place. + """ + # never skip when there is not dataset + if not getattr(self, "dataset_inst", None): + return False + + return self.dataset_inst.is_data or not self.dataset_inst.has_tag("has_top") + + +@producer( + uses={ + "GenPartonTop.pt", + }, + produces={ + "top_pt_weight", "top_pt_weight_up", "top_pt_weight_down", + }, + get_top_pt_config=(lambda self: self.config_inst.x.top_pt_reweighting_params), +) +def top_pt_weight(self: Producer, events: ak.Array, **kwargs) -> ak.Array: + """ + Compute SF to be used for top pt reweighting. + + The *GenPartonTop.pt* column can be produced with the :py:class:`gen_parton_top` Producer. + + The SF should *only be applied in ttbar MC* as an event weight and is computed + based on the gen-level top quark transverse momenta. + + The function is skipped when the dataset is data or when it does not have the tag *is_ttbar*. + + The top pt reweighting parameters should be given as an auxiliary entry in the config: + + .. code-block:: python + + cfg.x.top_pt_reweighting_params = { + "a": 0.0615, + "a_up": 0.0615 * 1.5, + "a_down": 0.0615 * 0.5, + "b": -0.0005, + "b_up": -0.0005 * 1.5, + "b_down": -0.0005 * 0.5, + } + + *get_top_pt_config* can be adapted in a subclass in case it is stored differently in the config. + + :param events: awkward array containing events to process + """ + + # get SF function parameters from config + params = self.get_top_pt_config() + + # check the number of gen tops + if ak.any(ak.num(events.GenPartonTop, axis=1) != 2): + logger.warning("There are events with != 2 GenPartonTops. This producer should only run for ttbar") + + # clamp top pT < 500 GeV + pt_clamped = ak.where(events.GenPartonTop.pt > 500.0, 500.0, events.GenPartonTop.pt) + for variation in ("", "_up", "_down"): + # evaluate SF function + sf = np.exp(params[f"a{variation}"] + params[f"b{variation}"] * pt_clamped) + + # compute weight from SF product for top and anti-top + weight = np.sqrt(np.prod(sf, axis=1)) + + # write out weights + events = set_ak_column(events, f"top_pt_weight{variation}", ak.fill_none(weight, 1.0)) + + return events + + +@top_pt_weight.skip +def top_pt_weight_skip(self: Producer) -> bool: + """ + Skip if running on anything except ttbar MC simulation. + """ + # never skip when there is no dataset + if not getattr(self, "dataset_inst", None): + return False + + return self.dataset_inst.is_data or not self.dataset_inst.has_tag("is_ttbar") From c8516a5cccf23475830e998165ce7a478f35f2ef Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Tue, 16 Apr 2024 09:49:54 +0200 Subject: [PATCH 13/43] store fold_indices in events during cf.MLEvaluation --- columnflow/tasks/ml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/columnflow/tasks/ml.py b/columnflow/tasks/ml.py index 34091609f..59987bdd8 100644 --- a/columnflow/tasks/ml.py +++ b/columnflow/tasks/ml.py @@ -805,14 +805,14 @@ def run(self): ) # generate fold indices - fold_indices = events.deterministic_seed % self.ml_model_inst.folds + events = set_ak_column(events, "fold_indices", events.deterministic_seed % self.ml_model_inst.folds) # invoke the optional producer if len(events) and self.preparation_producer_inst: events = self.preparation_producer_inst( events, stats=stats, - fold_indices=fold_indices, + fold_indices=events.fold_indices, ml_model_inst=self.ml_model_inst, ) @@ -821,7 +821,7 @@ def run(self): self, events, models, - fold_indices, + events.fold_indices, events_used_in_training=events_used_in_training, ) From 66f009ad8db462895ae438e6f1931c7ae31c8413 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Fri, 19 Apr 2024 12:01:28 +0200 Subject: [PATCH 14/43] add parsing of tuples for Settings and MultiSettings --- columnflow/tasks/framework/parameters.py | 11 ++++++++++- tests/test_task_parameters.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/columnflow/tasks/framework/parameters.py b/columnflow/tasks/framework/parameters.py index 405b786a8..bb3d8833d 100644 --- a/columnflow/tasks/framework/parameters.py +++ b/columnflow/tasks/framework/parameters.py @@ -31,13 +31,22 @@ class SettingsParameter(law.CSVParameter): def parse_setting(cls, setting: str) -> tuple[str, float | bool | str]: pair = setting.split("=", 1) key, value = pair if len(pair) == 2 else (pair[0], "True") + if ";" in value: + # split by semicolon and parse each value + value = tuple(cls.parse_value(v) for v in value.split(";")) + else: + value = cls.parse_value(value) + return (key, value) + + @classmethod + def parse_value(cls, value): if try_float(value): value = float(value) elif value.lower() == "true": value = True elif value.lower() == "false": value = False - return (key, value) + return value @classmethod def serialize_setting(cls, name: str, value: str) -> str: diff --git a/tests/test_task_parameters.py b/tests/test_task_parameters.py index b4f444cbf..b5f17f05b 100644 --- a/tests/test_task_parameters.py +++ b/tests/test_task_parameters.py @@ -23,6 +23,11 @@ def test_settings_parameter(self): p.parse("param1=10,param2,param3=text,param4=false"), {"param1": 10.0, "param2": True, "param3": "text", "param4": False}, ) + self.assertEqual( + # parsing of semicolon separated values + p.parse("param1=1;2;3;4,param2=a;b;true;false"), + {"param1": (1, 2, 3, 4), "param2": ("a", "b", True, False)}, + ) self.assertEqual( # if a parameter is set multiple times, prioritize last one p.parse("A=1,B,A=2"), @@ -46,6 +51,14 @@ def test_multi_settings_parameter(self): p.parse("obj1,k1=10,k2,k3=text:obj2,k4=false"), {"obj1": {"k1": 10.0, "k2": True, "k3": "text"}, "obj2": {"k4": False}}, ) + self.assertEqual( + # parsing of semicolon separated values + p.parse("obj1,k1=1;2;3;4,k2=a;b;true;false:obj2,k3=5;6;x;y"), + { + "obj1": {"k1": (1, 2, 3, 4), "k2": ("a", "b", True, False)}, + "obj2": {"k3": (5, 6, "x", "y")}, + }, + ) self.assertEqual( # providing the same key twice results in once combined dict p.parse("tt,A=2:st,A=2:tt,B=True"), From afc47e0a0b68ce4f4fb9cc69b03b5cbbff5b95d3 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Fri, 19 Apr 2024 12:13:29 +0200 Subject: [PATCH 15/43] allow parsing complex numbers --- columnflow/tasks/framework/parameters.py | 4 +++- columnflow/util.py | 13 ++++++++++++- tests/test_task_parameters.py | 8 ++++---- tests/test_util.py | 9 ++++++++- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/columnflow/tasks/framework/parameters.py b/columnflow/tasks/framework/parameters.py index bb3d8833d..3c4501e4e 100644 --- a/columnflow/tasks/framework/parameters.py +++ b/columnflow/tasks/framework/parameters.py @@ -8,7 +8,7 @@ import law -from columnflow.util import try_float, DotDict +from columnflow.util import try_float, try_complex, DotDict class SettingsParameter(law.CSVParameter): @@ -42,6 +42,8 @@ def parse_setting(cls, setting: str) -> tuple[str, float | bool | str]: def parse_value(cls, value): if try_float(value): value = float(value) + elif try_complex(value): + value = complex(value) elif value.lower() == "true": value = True elif value.lower() == "false": diff --git a/columnflow/util.py b/columnflow/util.py index 0dec79fea..fc730ad58 100644 --- a/columnflow/util.py +++ b/columnflow/util.py @@ -10,7 +10,7 @@ "UNSET", "maybe_import", "import_plt", "import_ROOT", "import_file", "create_random_name", "expand_path", "real_path", "ensure_dir", "wget", "call_thread", "call_proc", "ensure_proxy", "dev_sandbox", - "safe_div", "try_float", "try_int", "is_pattern", "is_regex", "pattern_matcher", + "safe_div", "try_float", "try_complex", "try_int", "is_pattern", "is_regex", "pattern_matcher", "dict_add_strict", "get_source_code", "DotDict", "MockModule", "FunctionArgs", "ClassPropertyDescriptor", "classproperty", "DerivableMeta", "Derivable", @@ -412,6 +412,17 @@ def try_float(f: Any) -> bool: return False +def try_complex(f: Any) -> bool: + """ + Tests whether a value *f* can be converted to a complex number. + """ + try: + complex(f) + return True + except (ValueError, TypeError): + return False + + def try_int(i: Any) -> bool: """ Tests whether a value *i* can be converted to an integer. diff --git a/tests/test_task_parameters.py b/tests/test_task_parameters.py index b5f17f05b..6bca67460 100644 --- a/tests/test_task_parameters.py +++ b/tests/test_task_parameters.py @@ -25,8 +25,8 @@ def test_settings_parameter(self): ) self.assertEqual( # parsing of semicolon separated values - p.parse("param1=1;2;3;4,param2=a;b;true;false"), - {"param1": (1, 2, 3, 4), "param2": ("a", "b", True, False)}, + p.parse("param1=1;2;3j;4j,param2=a;b;true;false"), + {"param1": (1, 2, 3j, 4j), "param2": ("a", "b", True, False)}, ) self.assertEqual( # if a parameter is set multiple times, prioritize last one @@ -53,9 +53,9 @@ def test_multi_settings_parameter(self): ) self.assertEqual( # parsing of semicolon separated values - p.parse("obj1,k1=1;2;3;4,k2=a;b;true;false:obj2,k3=5;6;x;y"), + p.parse("obj1,k1=1;2;3j;4j,k2=a;b;true;false:obj2,k3=5;6;x;y"), { - "obj1": {"k1": (1, 2, 3, 4), "k2": ("a", "b", True, False)}, + "obj1": {"k1": (1, 2, 3j, 4j), "k2": ("a", "b", True, False)}, "obj2": {"k3": (5, 6, "x", "y")}, }, ) diff --git a/tests/test_util.py b/tests/test_util.py index 82373f9e1..30769c453 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -7,7 +7,7 @@ from columnflow.util import ( create_random_name, maybe_import, MockModule, DotDict, Derivable, - safe_div, try_float, try_int, is_regex, is_pattern, pattern_matcher, + safe_div, try_float, try_int, try_complex, is_regex, is_pattern, pattern_matcher, ) @@ -44,6 +44,13 @@ def test_try_int_try_float(self): self.assertFalse(try_number(1j)) self.assertFalse(try_number([1, 2])) + def test_try_complex(self): + self.assertTrue(try_complex("1.2+2.5j")) + self.assertFalse(try_complex("some_string")) + self.assertFalse(try_complex([1, 2])) + # real numbers are also complex number + self.assertTrue(try_complex("5.0")) + def test_is_regex(self): self.assertTrue(is_regex(r"^foo\d+.*$")) self.assertFalse(is_regex(r"^no$atEnd")) From d3cd106235a3dfb4522ea00902d3ad95c7b8acfe Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Fri, 19 Apr 2024 15:18:43 +0200 Subject: [PATCH 16/43] fix serializing of SettingsParameter --- columnflow/tasks/framework/parameters.py | 7 ++++--- tests/test_task_parameters.py | 9 +++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/columnflow/tasks/framework/parameters.py b/columnflow/tasks/framework/parameters.py index 3c4501e4e..6b9844c76 100644 --- a/columnflow/tasks/framework/parameters.py +++ b/columnflow/tasks/framework/parameters.py @@ -9,6 +9,7 @@ import law from columnflow.util import try_float, try_complex, DotDict +from columnflow.types import Iterable class SettingsParameter(law.CSVParameter): @@ -32,7 +33,7 @@ def parse_setting(cls, setting: str) -> tuple[str, float | bool | str]: pair = setting.split("=", 1) key, value = pair if len(pair) == 2 else (pair[0], "True") if ";" in value: - # split by semicolon and parse each value + # split by ";" and parse each value value = tuple(cls.parse_value(v) for v in value.split(";")) else: value = cls.parse_value(value) @@ -51,7 +52,8 @@ def parse_value(cls, value): return value @classmethod - def serialize_setting(cls, name: str, value: str) -> str: + def serialize_setting(cls, name: str, value: str | Iterable[str]) -> str: + value = ";".join(str(v) for v in law.util.make_tuple(value)) return f"{name}={value}" def __init__(self, **kwargs): @@ -114,7 +116,6 @@ def parse(self, inp): ) # next, merge dicts outputs = law.util.merge_dicts(*outputs, deep=True) - return outputs def serialize(self, value): diff --git a/tests/test_task_parameters.py b/tests/test_task_parameters.py index 6bca67460..51f0f5deb 100644 --- a/tests/test_task_parameters.py +++ b/tests/test_task_parameters.py @@ -24,7 +24,7 @@ def test_settings_parameter(self): {"param1": 10.0, "param2": True, "param3": "text", "param4": False}, ) self.assertEqual( - # parsing of semicolon separated values + # parsing of lists of values, separated via ";" p.parse("param1=1;2;3j;4j,param2=a;b;true;false"), {"param1": (1, 2, 3j, 4j), "param2": ("a", "b", True, False)}, ) @@ -39,6 +39,11 @@ def test_settings_parameter(self): p.serialize({"param1": 2, "param2": False}), "param1=2,param2=False", ) + print(p.serialize({"param1": [1, 2j, "A", True, False]})) + self.assertEqual( + p.serialize({"param1": [1, 2j, "A", True, False]}), + "param1=1;2j;A;True;False", + ) def test_multi_settings_parameter(self): p = MultiSettingsParameter() @@ -52,7 +57,7 @@ def test_multi_settings_parameter(self): {"obj1": {"k1": 10.0, "k2": True, "k3": "text"}, "obj2": {"k4": False}}, ) self.assertEqual( - # parsing of semicolon separated values + # parsing of lists of values, separated via ";" p.parse("obj1,k1=1;2;3j;4j,k2=a;b;true;false:obj2,k3=5;6;x;y"), { "obj1": {"k1": (1, 2, 3j, 4j), "k2": ("a", "b", True, False)}, From aee90fc757e6951b7f728485a7f81cb16521f465 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Fri, 19 Apr 2024 15:29:41 +0200 Subject: [PATCH 17/43] add slicing of histograms --- columnflow/plotting/plot_util.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/columnflow/plotting/plot_util.py b/columnflow/plotting/plot_util.py index afa238bea..647ca886d 100644 --- a/columnflow/plotting/plot_util.py +++ b/columnflow/plotting/plot_util.py @@ -14,7 +14,7 @@ import order as od -from columnflow.util import maybe_import, try_int +from columnflow.util import maybe_import, try_int, try_complex math = maybe_import("math") hist = maybe_import("hist") @@ -122,6 +122,17 @@ def apply_variable_settings( h = h[{var_inst.name: hist.rebin(rebin_factor)}] hists[proc_inst] = h + slices = getattr(var_inst, "slice", None) or var_inst.x("slice", None) + if ( + slices and isinstance(slices, Iterable) and len(slices) >= 2 and + try_complex(slices[0]) and try_complex(slices[1]) + ): + slice_0 = int(slices[0]) if try_int(slices[0]) else complex(slices[0]) + slice_1 = int(slices[1]) if try_int(slices[1]) else complex(slices[1]) + for proc_inst, h in list(hists.items()): + h = h[{var_inst.name: slice(slice_0, slice_1)}] + hists[proc_inst] = h + return hists From 1dcb7d4375b0f80fb291cf99ddbbd25f95f25b41 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 25 Apr 2024 16:58:14 +0200 Subject: [PATCH 18/43] allow passing different b-score columns to btag_weights Producer --- columnflow/production/cms/btag.py | 55 +++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/columnflow/production/cms/btag.py b/columnflow/production/cms/btag.py index 79131b743..b3a0d86fe 100644 --- a/columnflow/production/cms/btag.py +++ b/columnflow/production/cms/btag.py @@ -6,6 +6,8 @@ from __future__ import annotations +import law + from columnflow.production import Producer, producer from columnflow.util import maybe_import, InsertableDict from columnflow.columnar_util import set_ak_column, flat_np_view, layout_ak_array @@ -13,10 +15,12 @@ np = maybe_import("numpy") ak = maybe_import("awkward") +logger = law.logger.get_logger(__name__) + @producer( uses={ - "Jet.hadronFlavour", "Jet.eta", "Jet.pt", "Jet.btagDeepFlavB", + "Jet.hadronFlavour", "Jet.eta", "Jet.pt", }, # only run on mc mc_only=True, @@ -44,12 +48,13 @@ def btag_weights( *get_btag_file* can be adapted in a subclass in case it is stored differently in the external files. - The name of the correction set as well as a list of JEC uncertainty sources which should be - propagated through the weight calculation should be given as an auxiliary entry in the config: + The name of the correction set, a list of JEC uncertainty sources which should be + propagated through the weight calculation, and the column used for b-tagging should + be given as an auxiliary entry in the config: .. code-block:: python - cfg.x.btag_sf = ("deepJet_shape", ["Absolute", "FlavorQCD", ...]) + cfg.x.btag_sf = ("deepJet_shape", ["Absolute", "FlavorQCD", ...], "btagDeepFlavB") *get_btag_config* can be adapted in a subclass in case it is stored differently in the config. @@ -68,7 +73,7 @@ def btag_weights( flavor = flat_np_view(events.Jet.hadronFlavour[jet_mask], axis=1) abs_eta = flat_np_view(abs(events.Jet.eta[jet_mask]), axis=1) pt = flat_np_view(events.Jet.pt[jet_mask], axis=1) - b_discr = flat_np_view(events.Jet.btagDeepFlavB[jet_mask], axis=1) + b_discr = flat_np_view(events.Jet[self.b_score_column][jet_mask], axis=1) # helper to create and store the weight def add_weight(syst_name, syst_direction, column_name): @@ -135,6 +140,35 @@ def add_weight(syst_name, syst_direction, column_name): return events +def get_b_score_column(btag_config: tuple[str, list, str]) -> str: + """ + Helper function to resolve the btag score column from the btag configuration. + """ + corrector_name = btag_config[0] + if len(btag_config) >= 3: + b_score_column = btag_config[2] + else: + # resolve the column name from the corrector name + if "deepjet" in corrector_name.lower(): + b_score_column = "btagDeepFlavB" + elif "particlenet" in corrector_name.lower(): + b_score_column = "btagPNetB" + else: + raise NotImplementedError(f"Cannot automatically determine btag column for Corrector '{corrector_name}'") + logger.info(f"No btag column specified; defaulting to column '{b_score_column}'") + + # warn about potentially wrong column usage + if ( + "deepjet" in corrector_name.lower() and "pnet" in b_score_column or + "particlenet" in corrector_name.lower() and "deepflav" in b_score_column + ): + logger.warning( + f"Using btag column '{b_score_column}' for BTag Corrector '{corrector_name}' is highly discouraged", + ) + + return b_score_column + + @btag_weights.init def btag_weights_init(self: Producer) -> None: # depending on the requested shift_inst, there are three cases to handle: @@ -143,6 +177,11 @@ def btag_weights_init(self: Producer) -> None: # 2. when the nominal shift is requested, the central weight and all variations related to the # method-intrinsic shifts are produced # 3. when any other shift is requested, only create the central weight column + btag_config = self.get_btag_config() + self.b_score_column = get_b_score_column(btag_config) + + self.uses.add(f"Jet.{self.b_score_column}") + shift_inst = getattr(self, "local_shift_inst", None) if not shift_inst: return @@ -152,7 +191,7 @@ def btag_weights_init(self: Producer) -> None: btag_sf_jec_source = "" if self.jec_source == "Total" else self.jec_source self.shift_is_known_jec_source = ( self.jec_source and - btag_sf_jec_source in self.get_btag_config()[1] + btag_sf_jec_source in btag_config[1] ) # save names of method-intrinsic uncertainties @@ -210,7 +249,3 @@ def btag_weights_setup( ) corrector_name = self.get_btag_config()[0] self.btag_sf_corrector = correction_set[corrector_name] - - # check versions - if self.btag_sf_corrector.version not in (3,): - raise Exception(f"unsuppprted btag sf corrector version {self.btag_sf_corrector.version}") From 2dddd2b31d4df3bf746bba15713bf459b6ea9312 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Mon, 29 Apr 2024 12:51:10 +0200 Subject: [PATCH 19/43] review comments --- columnflow/production/cms/btag.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/columnflow/production/cms/btag.py b/columnflow/production/cms/btag.py index b3a0d86fe..04cf68183 100644 --- a/columnflow/production/cms/btag.py +++ b/columnflow/production/cms/btag.py @@ -140,9 +140,14 @@ def add_weight(syst_name, syst_direction, column_name): return events -def get_b_score_column(btag_config: tuple[str, list, str]) -> str: +def get_b_score_column(btag_config: tuple[str, list, str] | tuple[str, list]) -> str: """ Helper function to resolve the btag score column from the btag configuration. + + :param btag_config: Entry in auxiliary `config_inst.x.btag_sf`, see example + :py:meth:`~columflow.production.cms.btag.btag_weights`. If tuple has less + than 3 entries, the column name is derived from the name of the correction set. + :returns: Name of column that is required for the calculation of this set of corrections. """ corrector_name = btag_config[0] if len(btag_config) >= 3: From f679d897a1b8aa8a0261c2c0e0897192ac50484f Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Mon, 29 Apr 2024 13:46:27 +0200 Subject: [PATCH 20/43] add options and log modes for handling of negative b scores in btag weight producer --- columnflow/production/cms/btag.py | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/columnflow/production/cms/btag.py b/columnflow/production/cms/btag.py index 04cf68183..9ceaf1023 100644 --- a/columnflow/production/cms/btag.py +++ b/columnflow/production/cms/btag.py @@ -33,6 +33,8 @@ def btag_weights( self: Producer, events: ak.Array, jet_mask: ak.Array | type(Ellipsis) = Ellipsis, + negative_b_score_action: str = "ignore", + negative_b_score_log_mode: str = "warning", **kwargs, ) -> ak.Array: """ @@ -61,14 +63,68 @@ def btag_weights( Optionally, a *jet_mask* can be supplied to compute the scale factor weight based only on a subset of jets. + The *negative_b_score_action* defines the procedure of how to handle jets with a negative b-tag. + Supported modes are: + + - "ignore": the *jet_mask* is extended to exclude jets with b_score < 0 + - "remove": the btag_weight is set to 0 for jets with b_score < 0 + - "raise": an exception is raised + + The verbosity of the handling of jets with negative b-score can be + set via *negative_b_score_log_mode*, which offers the following options: + + - ``"none"``: no message is given + - ``"info"``: a `logger.info` message is given + - ``"debug"``: a `logger.debug` message is given + - ``"warning"``: a `logger.warning` message is given + Resources: - https://twiki.cern.ch/twiki/bin/view/CMS/BTagShapeCalibration?rev=26 - https://indico.cern.ch/event/1096988/contributions/4615134/attachments/2346047/4000529/Nov21_btaggingSFjsons.pdf """ + known_actions = ("ignore", "remove", "raise") + if negative_b_score_action not in known_actions: + raise ValueError( + f"unknown negative_b_score_action '{negative_b_score_action}', " + f"known values are {','.join(known_actions)}", + ) + + known_log_modes = ("none", "info", "debug", "warning") + if negative_b_score_log_mode not in known_log_modes: + raise ValueError( + f"unknown negative_b_score_log_mode '{negative_b_score_log_mode}', " + f"known values are {','.join(known_log_modes)}", + ) + # get the total number of jets in the chunk n_jets_all = ak.sum(ak.num(events.Jet, axis=1)) + # check that the b-tag score is not negative for all jets considered in the SF calculation + jets_negative_b_score = events.Jet[self.b_score_column][jet_mask] < 0 + if ak.any(jets_negative_b_score): + msg_func = { + "none": lambda msg: None, + "info": logger.info, + "warning": logger.warning, + "debug": logger.debug, + }[negative_b_score_log_mode] + msg = f"In dataset {self.dataset_inst.name}, {ak.sum(jets_negative_b_score)} jets have a negative b-tag score." + + if negative_b_score_action == "ignore": + msg_func( + f"{msg} The *jet_mask* will be adjusted to exclude these jets, resulting in a " + "*btag_weight* of 1 for these jets.", + ) + jet_mask = jet_mask & (events.Jet[self.b_score_column] >= 0) + elif negative_b_score_action == "remove": + msg_func( + f"{msg} The *btag_weight* will be set to 0 for these jets.", + ) + jet_mask = jet_mask & (events.Jet[self.b_score_column] >= 0) + elif negative_b_score_action == "raise": + raise Exception(msg) + # get flat inputs, evaluated at jet_mask flavor = flat_np_view(events.Jet.hadronFlavour[jet_mask], axis=1) abs_eta = flat_np_view(abs(events.Jet.eta[jet_mask]), axis=1) @@ -107,6 +163,11 @@ def add_weight(syst_name, syst_direction, column_name): # enforce the correct shape and create the product over all jets per event sf = layout_ak_array(sf_flat_all, events.Jet.pt) + + if negative_b_score_log_mode == "remove": + # set the weight to 0 for jets with negative btag score + sf = ak.where(jets_negative_b_score, 0, sf) + weight = ak.prod(sf, axis=1, mask_identity=False) # save the new column From 008606e27af48dec273ccfabb5035c49341fb551 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 2 May 2024 09:09:12 +0200 Subject: [PATCH 21/43] fix btag weight Producer when passing Ellipsis as jet_mask --- columnflow/production/cms/btag.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/columnflow/production/cms/btag.py b/columnflow/production/cms/btag.py index 9ceaf1023..e38a4831a 100644 --- a/columnflow/production/cms/btag.py +++ b/columnflow/production/cms/btag.py @@ -116,15 +116,19 @@ def btag_weights( f"{msg} The *jet_mask* will be adjusted to exclude these jets, resulting in a " "*btag_weight* of 1 for these jets.", ) - jet_mask = jet_mask & (events.Jet[self.b_score_column] >= 0) elif negative_b_score_action == "remove": msg_func( f"{msg} The *btag_weight* will be set to 0 for these jets.", ) - jet_mask = jet_mask & (events.Jet[self.b_score_column] >= 0) elif negative_b_score_action == "raise": raise Exception(msg) + # set jet mask to False when b_score is negative + if jet_mask is Ellipsis: + jet_mask = (events.Jet[self.b_score_column] >= 0) + else: + jet_mask = jet_mask & (events.Jet[self.b_score_column] >= 0) + # get flat inputs, evaluated at jet_mask flavor = flat_np_view(events.Jet.hadronFlavour[jet_mask], axis=1) abs_eta = flat_np_view(abs(events.Jet.eta[jet_mask]), axis=1) From e0ca98a2112c8ddd76fe9ac4f446a19ac1803c05 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 2 May 2024 10:56:21 +0200 Subject: [PATCH 22/43] add producers parameter to ProduceColumnsWrapper --- columnflow/tasks/production.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/columnflow/tasks/production.py b/columnflow/tasks/production.py index 1bc430bee..a53e055bb 100644 --- a/columnflow/tasks/production.py +++ b/columnflow/tasks/production.py @@ -3,12 +3,13 @@ """ Tasks related to producing new columns. """ +import itertools import law from columnflow.tasks.framework.base import Requirements, AnalysisTask, wrapper_factory from columnflow.tasks.framework.mixins import ( - CalibratorsMixin, SelectorStepsMixin, ProducerMixin, ChunkedIOMixin, + CalibratorsMixin, SelectorStepsMixin, ProducerMixin, ChunkedIOMixin, ProducersMixin, ) from columnflow.tasks.framework.remote import RemoteWorkflow from columnflow.tasks.reduction import MergeReducedEventsUser, MergeReducedEvents @@ -165,8 +166,25 @@ def run(self): ) -ProduceColumnsWrapper = wrapper_factory( +ProduceColumnsWrapperBase = wrapper_factory( base_cls=AnalysisTask, require_cls=ProduceColumns, enable=["configs", "skip_configs", "datasets", "skip_datasets", "shifts", "skip_shifts"], ) +ProduceColumnsWrapperBase.exclude_index = True + + +class ProduceColumnsWrapper( + ProduceColumnsWrapperBase, + ProducersMixin, + +): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # add the producers parameter + self.wrapper_fields.extend(["producer"]) + + combined_parameters = itertools.product(self.wrapper_parameters, self.producers) + combined_parameters = [params_tuple + (producer,) for params_tuple, producer in combined_parameters] + self.wrapper_parameters = combined_parameters From f4224b98511b4d1eef19a241d541fbf086271143 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 2 May 2024 14:44:47 +0200 Subject: [PATCH 23/43] add fallback weight producer --- .../__cf_module_name__/weight/example.py | 2 +- columnflow/weight/example.py | 54 +++++++++++++++++++ law.cfg | 2 +- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 columnflow/weight/example.py diff --git a/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py b/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py index 6c657a58b..6ede7c428 100644 --- a/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py +++ b/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py @@ -14,7 +14,7 @@ @weight_producer( - # both produced columns and dependent shifts are defined in init below + # both used columns and dependent shifts are defined in init below # only run on mc mc_only=True, ) diff --git a/columnflow/weight/example.py b/columnflow/weight/example.py new file mode 100644 index 000000000..d9fa8c1dc --- /dev/null +++ b/columnflow/weight/example.py @@ -0,0 +1,54 @@ +# coding: utf-8 + +""" +Exemplary event weight producer. +""" + +from columnflow.weight import WeightProducer, weight_producer +from columnflow.util import maybe_import +from columnflow.columnar_util import has_ak_column, Route + +np = maybe_import("numpy") +ak = maybe_import("awkward") + + +@weight_producer( + # only run on mc + mc_only=True, +) +def example(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: + # build the full event weight + weight = ak.Array(np.ones(len(events))) + if self.dataset_inst.is_mc and len(events): + for column in self.config_inst.x.event_weights: + weight = weight * Route(column).apply(events) + for column in self.dataset_inst.x("event_weights", []): + if has_ak_column(events, column): + weight = weight * Route(column).apply(events) + else: + self.logger.warning_once( + f"missing_dataset_weight_{column}", + f"weight '{column}' for dataset {self.dataset_inst.name} not found", + ) + + return weight + + +@example.init +def example_init(self: WeightProducer) -> None: + weight_columns = set() + + # add used weight columns and declare shifts that the produced event weight depends on + if self.config_inst.has_aux("event_weights"): + weight_columns |= {Route(column) for column in self.config_inst.x.event_weights} + for shift_insts in self.config_inst.x.event_weights.values(): + self.shifts |= {shift_inst.name for shift_inst in shift_insts} + + # optionally also for weights defined by a dataset + if self.dataset_inst.has_aux("event_weights"): + weight_columns |= {Route(column) for column in self.dataset_inst.x("event_weights", [])} + for shift_insts in self.dataset_inst.x.event_weights.values(): + self.shifts |= {shift_inst.name for shift_inst in shift_insts} + + # add weight columns to uses + self.uses |= weight_columns diff --git a/law.cfg b/law.cfg index c2cd89e15..af06fe557 100644 --- a/law.cfg +++ b/law.cfg @@ -24,7 +24,7 @@ production_modules: columnflow.production.{categories,processes,normalization} calibration_modules: columnflow.calibration selection_modules: columnflow.selection.{empty} categorization_modules: columnflow.categorization -weight_production_modules: columnflow.weight.{empty} +weight_production_modules: columnflow.weight.{empty,example} ml_modules: columnflow.ml inference_modules: columnflow.inference From 9031c5bd6d19648fe33ec8618700085786ba704b Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 2 May 2024 15:01:39 +0200 Subject: [PATCH 24/43] add message when not passing weight_producer --- columnflow/tasks/framework/mixins.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/columnflow/tasks/framework/mixins.py b/columnflow/tasks/framework/mixins.py index c3a80faab..b78d412b4 100644 --- a/columnflow/tasks/framework/mixins.py +++ b/columnflow/tasks/framework/mixins.py @@ -2070,6 +2070,17 @@ def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: default_str="default_weight_producer", multiple=False, ) + if params["weight_producer"] is None: + raise Exception( + f"no weight producer configured for task. {cls.task_family}. " + "As of 02.05.2024, it is required to pass a weight_producer for tasks creating " + "histograms. You can add a 'default_weight_producer' to your config or directly " + "add the weight_producer on command line via the '--weight_producer' parameter. " + "To reproduce results from before this date, you can use the " + "'example' weight_producer defined in columnflow.weight.example, e.g. by adding " + "The following line to your config: \n" + "config.x.default_weight_producer = \"example\"" + ) params["weight_producer_inst"] = cls.get_weight_producer_inst( params["weight_producer"], params, From 52c85d65bfb10ac7ca7072b3932bc9d08a6e3cc0 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 2 May 2024 15:26:37 +0200 Subject: [PATCH 25/43] use weight_producer requires in CreateHistograms task --- columnflow/tasks/histograms.py | 170 ++++++++++++++++++--------------- 1 file changed, 93 insertions(+), 77 deletions(-) diff --git a/columnflow/tasks/histograms.py b/columnflow/tasks/histograms.py index 954c0ed8c..0d22fcc24 100644 --- a/columnflow/tasks/histograms.py +++ b/columnflow/tasks/histograms.py @@ -77,6 +77,9 @@ def workflow_requires(self): for ml_model_inst in self.ml_model_insts ] + # add weight_producer dependent requirements + reqs["weight_producer"] = self.weight_producer_inst.run_requires() + return reqs def requires(self): @@ -96,6 +99,9 @@ def requires(self): for ml_model_inst in self.ml_model_insts ] + # add weight_producer dependent requirements + reqs["weight_producer"] = self.weight_producer_inst.run_requires() + return reqs @MergeReducedEventsUser.maybe_dummy @@ -114,11 +120,15 @@ def run(self): from columnflow.columnar_util import Route, update_ak_array, add_ak_aliases, has_ak_column # prepare inputs and outputs + reqs = self.requires() inputs = self.input() # declare output: dict of histograms histograms = {} + # run the weight_producer setup + reader_targets = self.weight_producer_inst.run_setup(reqs["weight_producer"], inputs["weight_producer"]) + # create a temp dir for saving intermediate files tmp_dir = law.LocalDirectoryTarget(is_tmp=True) tmp_dir.touch() @@ -149,89 +159,95 @@ def run(self): empty_f32 = ak.Array(np.array([], dtype=np.float32)) # iterate over chunks of events and diffs - files = [inputs["events"]["collection"][0]["events"].path] + files = [inputs["events"]["collection"][0]["events"]] if self.producer_insts: - files.extend([inp["columns"].path for inp in inputs["producers"]]) + files.extend([inp["columns"] for inp in inputs["producers"]]) if self.ml_model_insts: - files.extend([inp["mlcolumns"].path for inp in inputs["ml"]]) - for (events, *columns), pos in self.iter_chunked_io( - files, - source_type=len(files) * ["awkward_parquet"], - read_columns=len(files) * [read_columns], - ): - # optional check for overlapping inputs - if self.check_overlapping_inputs: - self.raise_if_overlapping([events] + list(columns)) - - # add additional columns - events = update_ak_array(events, *columns) - - # add aliases - events = add_ak_aliases( - events, - aliases, - remove_src=True, - missing_strategy=self.missing_column_alias_strategy, - ) + files.extend([inp["mlcolumns"] for inp in inputs["ml"]]) + + # prepare inputs for localization + with law.localize_file_targets( + [*files, *reader_targets.values()], + mode="r", + ) as inps: + for (events, *columns), pos in self.iter_chunked_io( + [inp.path for inp in inps], + source_type=len(files) * ["awkward_parquet"] + [None] * len(reader_targets), + read_columns=(len(files) + len(reader_targets)) * [read_columns], + ): + # optional check for overlapping inputs + if self.check_overlapping_inputs: + self.raise_if_overlapping([events] + list(columns)) + + # add additional columns + events = update_ak_array(events, *columns) + + # add aliases + events = add_ak_aliases( + events, + aliases, + remove_src=True, + missing_strategy=self.missing_column_alias_strategy, + ) - # build the full event weight - weight = ( - ak.Array(np.ones(len(events), dtype=np.float32)) - if self.weight_producer_inst.skip_func() - else self.weight_producer_inst(events) - ) + # build the full event weight + weight = ( + ak.Array(np.ones(len(events), dtype=np.float32)) + if self.weight_producer_inst.skip_func() + else self.weight_producer_inst(events) + ) - # define and fill histograms, taking into account multiple axes - for var_key, var_names in self.variable_tuples.items(): - # get variable instances - variable_insts = [self.config_inst.get_variable(var_name) for var_name in var_names] - - # create the histogram if not present yet - if var_key not in histograms: - h = ( - hist.Hist.new - .IntCat([], name="category", growth=True) - .IntCat([], name="process", growth=True) - .IntCat([], name="shift", growth=True) - ) - # add variable axes - for variable_inst in variable_insts: - h = h.Var( - variable_inst.bin_edges, - name=variable_inst.name, - label=variable_inst.get_full_x_title(), + # define and fill histograms, taking into account multiple axes + for var_key, var_names in self.variable_tuples.items(): + # get variable instances + variable_insts = [self.config_inst.get_variable(var_name) for var_name in var_names] + + # create the histogram if not present yet + if var_key not in histograms: + h = ( + hist.Hist.new + .IntCat([], name="category", growth=True) + .IntCat([], name="process", growth=True) + .IntCat([], name="shift", growth=True) ) - # enable weights and store it - histograms[var_key] = h.Weight() - - # merge category ids - category_ids = ak.concatenate( - [Route(c).apply(events) for c in self.category_id_columns], - axis=-1, - ) + # add variable axes + for variable_inst in variable_insts: + h = h.Var( + variable_inst.bin_edges, + name=variable_inst.name, + label=variable_inst.get_full_x_title(), + ) + # enable weights and store it + histograms[var_key] = h.Weight() + + # merge category ids + category_ids = ak.concatenate( + [Route(c).apply(events) for c in self.category_id_columns], + axis=-1, + ) - # broadcast arrays so that each event can be filled for all its categories - fill_kwargs = { - "category": category_ids, - "process": events.process_id, - "shift": np.ones(len(events), dtype=np.int32) * self.global_shift_inst.id, - "weight": weight, - } - for variable_inst in variable_insts: - # prepare the expression - expr = variable_inst.expression - if isinstance(expr, str): - route = Route(expr) - def expr(events, *args, **kwargs): - if len(events) == 0 and not has_ak_column(events, route): - return empty_f32 - return route.apply(events, null_value=variable_inst.null_value) - # apply it - fill_kwargs[variable_inst.name] = expr(events) - - # broadcast and fill - arrays = ak.flatten(ak.cartesian(fill_kwargs)) - histograms[var_key].fill(**{field: arrays[field] for field in arrays.fields}) + # broadcast arrays so that each event can be filled for all its categories + fill_kwargs = { + "category": category_ids, + "process": events.process_id, + "shift": np.ones(len(events), dtype=np.int32) * self.global_shift_inst.id, + "weight": weight, + } + for variable_inst in variable_insts: + # prepare the expression + expr = variable_inst.expression + if isinstance(expr, str): + route = Route(expr) + def expr(events, *args, **kwargs): + if len(events) == 0 and not has_ak_column(events, route): + return empty_f32 + return route.apply(events, null_value=variable_inst.null_value) + # apply it + fill_kwargs[variable_inst.name] = expr(events) + + # broadcast and fill + arrays = ak.flatten(ak.cartesian(fill_kwargs)) + histograms[var_key].fill(**{field: arrays[field] for field in arrays.fields}) # merge output files self.output()["hists"].dump(histograms, formatter="pickle") From 2b15f3e5825627e4abe70dc70ad1c9a3c1e52a83 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 2 May 2024 15:30:25 +0200 Subject: [PATCH 26/43] fix issue where ml_preparation reqs are loaded twice --- columnflow/tasks/ml.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/columnflow/tasks/ml.py b/columnflow/tasks/ml.py index 59987bdd8..e036e2f33 100644 --- a/columnflow/tasks/ml.py +++ b/columnflow/tasks/ml.py @@ -197,8 +197,6 @@ def run(self): files = [inputs["events"]["collection"][0]["events"]] if self.producer_insts: files.extend([inp["columns"] for inp in inputs["producers"]]) - if reader_targets: - files.extend(reader_targets.values()) # prepare inputs for localization with law.localize_file_targets( From e2559fd81be8fc9fb6f99900658ed74b5b43286a Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Thu, 2 May 2024 15:31:24 +0200 Subject: [PATCH 27/43] lint --- columnflow/tasks/framework/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/columnflow/tasks/framework/mixins.py b/columnflow/tasks/framework/mixins.py index b78d412b4..d5ebef07d 100644 --- a/columnflow/tasks/framework/mixins.py +++ b/columnflow/tasks/framework/mixins.py @@ -2079,7 +2079,7 @@ def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: "To reproduce results from before this date, you can use the " "'example' weight_producer defined in columnflow.weight.example, e.g. by adding " "The following line to your config: \n" - "config.x.default_weight_producer = \"example\"" + "config.x.default_weight_producer = \"example\"", ) params["weight_producer_inst"] = cls.get_weight_producer_inst( params["weight_producer"], From 0eaf1c0686da6f0d2f2a668d176240ca571d9a19 Mon Sep 17 00:00:00 2001 From: Daniel Savoiu Date: Thu, 2 May 2024 18:58:45 +0200 Subject: [PATCH 28/43] Make `categorizer_map` regular `dict` to prevent silent `KeyErrors`. --- columnflow/production/categories.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/columnflow/production/categories.py b/columnflow/production/categories.py index 5a730e770..415b5dbd7 100644 --- a/columnflow/production/categories.py +++ b/columnflow/production/categories.py @@ -89,3 +89,6 @@ def category_ids_init(self: Producer) -> None: self.produces.add(categorizer) self.categorizer_map[cat_inst].append(categorizer) + + # cast to normal dict to prevent silent failures on KeyError + self.categorizer_map = dict(self.categorizer_map) From fe81613dcebb3d0f2638f7ebb3caa30e18646af1 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Fri, 3 May 2024 09:26:42 +0200 Subject: [PATCH 29/43] review comments --- columnflow/tasks/histograms.py | 12 ++++++------ columnflow/tasks/ml.py | 12 ++++++------ columnflow/weight/example.py | 4 +++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/columnflow/tasks/histograms.py b/columnflow/tasks/histograms.py index 0d22fcc24..fbe6554b2 100644 --- a/columnflow/tasks/histograms.py +++ b/columnflow/tasks/histograms.py @@ -159,21 +159,21 @@ def run(self): empty_f32 = ak.Array(np.array([], dtype=np.float32)) # iterate over chunks of events and diffs - files = [inputs["events"]["collection"][0]["events"]] + file_targets = [inputs["events"]["collection"][0]["events"]] if self.producer_insts: - files.extend([inp["columns"] for inp in inputs["producers"]]) + file_targets.extend([inp["columns"] for inp in inputs["producers"]]) if self.ml_model_insts: - files.extend([inp["mlcolumns"] for inp in inputs["ml"]]) + file_targets.extend([inp["mlcolumns"] for inp in inputs["ml"]]) # prepare inputs for localization with law.localize_file_targets( - [*files, *reader_targets.values()], + [*file_targets, *reader_targets.values()], mode="r", ) as inps: for (events, *columns), pos in self.iter_chunked_io( [inp.path for inp in inps], - source_type=len(files) * ["awkward_parquet"] + [None] * len(reader_targets), - read_columns=(len(files) + len(reader_targets)) * [read_columns], + source_type=len(file_targets) * ["awkward_parquet"] + [None] * len(reader_targets), + read_columns=(len(file_targets) + len(reader_targets)) * [read_columns], ): # optional check for overlapping inputs if self.check_overlapping_inputs: diff --git a/columnflow/tasks/ml.py b/columnflow/tasks/ml.py index e036e2f33..f5815a899 100644 --- a/columnflow/tasks/ml.py +++ b/columnflow/tasks/ml.py @@ -771,21 +771,21 @@ def run(self): route_filter = RouteFilter(write_columns) # iterate over chunks of events and columns - files = [inputs["events"]["collection"][0]["events"]] + file_targets = [inputs["events"]["collection"][0]["events"]] if self.producer_insts: - files.extend([inp["columns"] for inp in inputs["producers"]]) + file_targets.extend([inp["columns"] for inp in inputs["producers"]]) if reader_targets: - files.extend(reader_targets.values()) + file_targets.extend(reader_targets.values()) # prepare inputs for localization with law.localize_file_targets( - [*files, *reader_targets.values()], + [*file_targets, *reader_targets.values()], mode="r", ) as inps: for (events, *columns), pos in self.iter_chunked_io( [inp.path for inp in inps], - source_type=len(files) * ["awkward_parquet"] + [None] * len(reader_targets), - read_columns=(len(files) + len(reader_targets)) * [read_columns], + source_type=len(file_targets) * ["awkward_parquet"] + [None] * len(reader_targets), + read_columns=(len(file_targets) + len(reader_targets)) * [read_columns], ): # optional check for overlapping inputs if self.check_overlapping_inputs: diff --git a/columnflow/weight/example.py b/columnflow/weight/example.py index d9fa8c1dc..431eebaa3 100644 --- a/columnflow/weight/example.py +++ b/columnflow/weight/example.py @@ -20,8 +20,11 @@ def example(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: # build the full event weight weight = ak.Array(np.ones(len(events))) if self.dataset_inst.is_mc and len(events): + # multiply weights from global config `event_weights` aux entry for column in self.config_inst.x.event_weights: weight = weight * Route(column).apply(events) + + # multiply weights from dataset-specific `event_weights` aux entry for column in self.dataset_inst.x("event_weights", []): if has_ak_column(events, column): weight = weight * Route(column).apply(events) @@ -30,7 +33,6 @@ def example(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: f"missing_dataset_weight_{column}", f"weight '{column}' for dataset {self.dataset_inst.name} not found", ) - return weight From 7a5ebf0c973f98baf4dc63b84714d06194a4348c Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Fri, 3 May 2024 09:35:33 +0200 Subject: [PATCH 30/43] use sets instead of lists in weight_producer example --- .../cms_minimal/__cf_module_name__/weight/example.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py b/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py index 6ede7c428..c58dc8538 100644 --- a/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py +++ b/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py @@ -30,14 +30,14 @@ def example(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: @example.init def example_init(self: WeightProducer) -> None: # store column names referring to weights to multiply - self.weight_columns = [ + self.weight_columns = { "normalization_weight", "muon_weight", - ] - self.uses |= set(self.weight_columns) + } + self.uses |= self.weight_columns # declare shifts that the produced event weight depends on - shift_sources = [ + shift_sources = { "mu", - ] + } self.shifts |= set(get_shifts_from_sources(self.config_inst, *shift_sources)) From ac21e512f7149ccdd5c9fcb09a16b549bf489fcd Mon Sep 17 00:00:00 2001 From: Ana A Date: Fri, 3 May 2024 15:05:12 +0200 Subject: [PATCH 31/43] updating datacard labels --- columnflow/tasks/cms/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/columnflow/tasks/cms/inference.py b/columnflow/tasks/cms/inference.py index 248cfc6aa..d5103246c 100644 --- a/columnflow/tasks/cms/inference.py +++ b/columnflow/tasks/cms/inference.py @@ -137,7 +137,7 @@ def requires(self): def output(self): cat_obj = self.branch_data - basename = lambda name, ext: f"{name}__cat_{cat_obj.config_category}__var_{cat_obj.config_variable}.{ext}" + basename = lambda name, ext: f"{name}__cat_{cat_obj.name}__var_{cat_obj.config_variable}.{ext}" return { "card": self.target(basename("datacard", "txt")), From e01e3576e01653d2fa68ccdcdc4477bf2217732e Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Mon, 6 May 2024 15:51:55 +0200 Subject: [PATCH 32/43] multiply cferr columns with nominal btag weight --- columnflow/production/cms/btag.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/columnflow/production/cms/btag.py b/columnflow/production/cms/btag.py index e38a4831a..436ecf3c9 100644 --- a/columnflow/production/cms/btag.py +++ b/columnflow/production/cms/btag.py @@ -191,6 +191,14 @@ def add_weight(syst_name, syst_direction, column_name): direction, f"btag_weight_{name}_{direction}", ) + if syst_name in ["cferr1", "cferr2"]: + # for c flavor uncertainties, multiply the uncertainty with the nominal btag weight + events = set_ak_column( + events, + f"btag_weight_{name}_{direction}", + events.btag_weight * events[f"btag_weight_{name}_{direction}"], + value_type=np.float32, + ) elif self.shift_is_known_jec_source: # TODO: year dependent jec variations fully covered? events = add_weight( From 72882f4f07e2988ffc85b11928582904f3a44035 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Mon, 6 May 2024 16:40:42 +0200 Subject: [PATCH 33/43] change expected signature of WeightProducer to (events, weight) --- .../cms_minimal/__cf_module_name__/weight/example.py | 2 +- columnflow/tasks/framework/mixins.py | 4 ++-- columnflow/tasks/histograms.py | 2 +- columnflow/weight/example.py | 8 ++++---- law.cfg | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py b/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py index c58dc8538..be7158119 100644 --- a/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py +++ b/analysis_templates/cms_minimal/__cf_module_name__/weight/example.py @@ -24,7 +24,7 @@ def example(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: for column in self.weight_columns: weight = weight * Route(column).apply(events) - return weight + return events, weight @example.init diff --git a/columnflow/tasks/framework/mixins.py b/columnflow/tasks/framework/mixins.py index d5ebef07d..0ffc8e288 100644 --- a/columnflow/tasks/framework/mixins.py +++ b/columnflow/tasks/framework/mixins.py @@ -2077,9 +2077,9 @@ def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: "histograms. You can add a 'default_weight_producer' to your config or directly " "add the weight_producer on command line via the '--weight_producer' parameter. " "To reproduce results from before this date, you can use the " - "'example' weight_producer defined in columnflow.weight.example, e.g. by adding " + "'all_weights' weight_producer defined in columnflow.weight.example, e.g. by adding " "The following line to your config: \n" - "config.x.default_weight_producer = \"example\"", + "config.x.default_weight_producer = \"all_weights\"", ) params["weight_producer_inst"] = cls.get_weight_producer_inst( params["weight_producer"], diff --git a/columnflow/tasks/histograms.py b/columnflow/tasks/histograms.py index fbe6554b2..9b1cbe365 100644 --- a/columnflow/tasks/histograms.py +++ b/columnflow/tasks/histograms.py @@ -191,7 +191,7 @@ def run(self): ) # build the full event weight - weight = ( + events, weight = ( ak.Array(np.ones(len(events), dtype=np.float32)) if self.weight_producer_inst.skip_func() else self.weight_producer_inst(events) diff --git a/columnflow/weight/example.py b/columnflow/weight/example.py index 431eebaa3..2ea09c2d9 100644 --- a/columnflow/weight/example.py +++ b/columnflow/weight/example.py @@ -16,7 +16,7 @@ # only run on mc mc_only=True, ) -def example(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: +def all_weights(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: # build the full event weight weight = ak.Array(np.ones(len(events))) if self.dataset_inst.is_mc and len(events): @@ -33,11 +33,11 @@ def example(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: f"missing_dataset_weight_{column}", f"weight '{column}' for dataset {self.dataset_inst.name} not found", ) - return weight + return events, weight -@example.init -def example_init(self: WeightProducer) -> None: +@all_weights.init +def all_weights_init(self: WeightProducer) -> None: weight_columns = set() # add used weight columns and declare shifts that the produced event weight depends on diff --git a/law.cfg b/law.cfg index af06fe557..617f33e99 100644 --- a/law.cfg +++ b/law.cfg @@ -24,7 +24,7 @@ production_modules: columnflow.production.{categories,processes,normalization} calibration_modules: columnflow.calibration selection_modules: columnflow.selection.{empty} categorization_modules: columnflow.categorization -weight_production_modules: columnflow.weight.{empty,example} +weight_production_modules: columnflow.weight.{empty,all_weights} ml_modules: columnflow.ml inference_modules: columnflow.inference From 7d91e8d282d68e29136ce67e23e19e486b617fb4 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Mon, 6 May 2024 16:42:52 +0200 Subject: [PATCH 34/43] pass correct weight module in law cfg --- law.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/law.cfg b/law.cfg index 617f33e99..af06fe557 100644 --- a/law.cfg +++ b/law.cfg @@ -24,7 +24,7 @@ production_modules: columnflow.production.{categories,processes,normalization} calibration_modules: columnflow.calibration selection_modules: columnflow.selection.{empty} categorization_modules: columnflow.categorization -weight_production_modules: columnflow.weight.{empty,all_weights} +weight_production_modules: columnflow.weight.{empty,example} ml_modules: columnflow.ml inference_modules: columnflow.inference From 06d5609eca96e78e7b832c2ee63c2719a3adf901 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Tue, 7 May 2024 08:52:37 +0200 Subject: [PATCH 35/43] rename file of all_weights WeightProducer --- columnflow/tasks/framework/mixins.py | 2 +- columnflow/weight/{example.py => all_weights.py} | 0 law.cfg | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename columnflow/weight/{example.py => all_weights.py} (100%) diff --git a/columnflow/tasks/framework/mixins.py b/columnflow/tasks/framework/mixins.py index 0ffc8e288..0861cc0d3 100644 --- a/columnflow/tasks/framework/mixins.py +++ b/columnflow/tasks/framework/mixins.py @@ -2077,7 +2077,7 @@ def resolve_param_values(cls, params: dict[str, Any]) -> dict[str, Any]: "histograms. You can add a 'default_weight_producer' to your config or directly " "add the weight_producer on command line via the '--weight_producer' parameter. " "To reproduce results from before this date, you can use the " - "'all_weights' weight_producer defined in columnflow.weight.example, e.g. by adding " + "'all_weights' weight_producer defined in columnflow.weight.all_weights, e.g. by adding " "The following line to your config: \n" "config.x.default_weight_producer = \"all_weights\"", ) diff --git a/columnflow/weight/example.py b/columnflow/weight/all_weights.py similarity index 100% rename from columnflow/weight/example.py rename to columnflow/weight/all_weights.py diff --git a/law.cfg b/law.cfg index af06fe557..617f33e99 100644 --- a/law.cfg +++ b/law.cfg @@ -24,7 +24,7 @@ production_modules: columnflow.production.{categories,processes,normalization} calibration_modules: columnflow.calibration selection_modules: columnflow.selection.{empty} categorization_modules: columnflow.categorization -weight_production_modules: columnflow.weight.{empty,example} +weight_production_modules: columnflow.weight.{empty,all_weights} ml_modules: columnflow.ml inference_modules: columnflow.inference From 61cb6c96f9bd624a833dc3ba7a7beeb1aba1db57 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Tue, 7 May 2024 08:53:13 +0200 Subject: [PATCH 36/43] add docstring to all_weights WeightProducer --- columnflow/weight/all_weights.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/columnflow/weight/all_weights.py b/columnflow/weight/all_weights.py index 2ea09c2d9..ef2c11abb 100644 --- a/columnflow/weight/all_weights.py +++ b/columnflow/weight/all_weights.py @@ -17,6 +17,29 @@ mc_only=True, ) def all_weights(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: + """ + WeightProducer that combines all event weights from the *event_weights* aux entry from either + the config or the dataset. The weights are multiplied together to form the full event weight. + + The expected structure of the *event_weights* aux entry is a dictionary with the weight column + name as key and a list of shift sources as value. The shift sources are used to declare the + shifts that the produced event weight depends on. Example: + + .. code-block:: python + + from columnflow.config_util import get_shifts_from_sources + # add weights and their corresponding shifts for all datasets + cfg.x.event_weights = { + "normalization_weight": [], + "muon_weight": get_shifts_from_sources(config, "mu_sf"), + "btag_weight": get_shifts_from_sources(config, f"btag_hf", "btag_lf"), + } + for dataset_inst in cfg.datasets: + # add dataset-specific weights and their corresponding shifts + dataset.x.event_weights = {} + if not dataset_inst.has_tag("skip_pdf"): + dataset_inst.event_weights["pdf_weight"] = get_shifts_from_sources(config, "pdf") + """ # build the full event weight weight = ak.Array(np.ones(len(events))) if self.dataset_inst.is_mc and len(events): From fd831704035e2d1f2b68d6be839f0401b059813c Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Tue, 7 May 2024 08:58:58 +0200 Subject: [PATCH 37/43] fix histogram task for when WeightProducer skip_func is true --- columnflow/tasks/histograms.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/columnflow/tasks/histograms.py b/columnflow/tasks/histograms.py index 9b1cbe365..322500249 100644 --- a/columnflow/tasks/histograms.py +++ b/columnflow/tasks/histograms.py @@ -191,11 +191,10 @@ def run(self): ) # build the full event weight - events, weight = ( - ak.Array(np.ones(len(events), dtype=np.float32)) - if self.weight_producer_inst.skip_func() - else self.weight_producer_inst(events) - ) + if not self.weight_producer_inst.skip_func(): + events, weight = self.weight_producer_inst(events) + else: + weight = ak.Array(np.ones(len(events), dtype=np.float32)) # define and fill histograms, taking into account multiple axes for var_key, var_names in self.variable_tuples.items(): From 084f5e4346b44898c6070d8938eed0ddd425f62c Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Tue, 7 May 2024 10:53:56 +0200 Subject: [PATCH 38/43] skip all_weights init function when no dataset_inst is present --- columnflow/weight/all_weights.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/columnflow/weight/all_weights.py b/columnflow/weight/all_weights.py index ef2c11abb..9cd25bb1e 100644 --- a/columnflow/weight/all_weights.py +++ b/columnflow/weight/all_weights.py @@ -61,6 +61,9 @@ def all_weights(self: WeightProducer, events: ak.Array, **kwargs) -> ak.Array: @all_weights.init def all_weights_init(self: WeightProducer) -> None: + if not getattr(self, "dataset_inst", None): + return + weight_columns = set() # add used weight columns and declare shifts that the produced event weight depends on From 39e031e75438a9f337fed8ce8c5f6c4eca983de4 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Tue, 7 May 2024 15:50:55 +0200 Subject: [PATCH 39/43] add eta key in muon_weights variable_map --- columnflow/production/cms/muon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/columnflow/production/cms/muon.py b/columnflow/production/cms/muon.py index 75809caf8..79436a5b6 100644 --- a/columnflow/production/cms/muon.py +++ b/columnflow/production/cms/muon.py @@ -66,6 +66,7 @@ def muon_weights( variable_map = { "year": self.year, "abseta": abs_eta, + "eta": abs_eta, "pt": pt, } From 4a27d54e88d6f8fafbf835498252805bdf72a928 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Mon, 13 May 2024 17:44:22 +0200 Subject: [PATCH 40/43] customize muon_weights output column names --- columnflow/production/cms/muon.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/columnflow/production/cms/muon.py b/columnflow/production/cms/muon.py index 79436a5b6..3b96dcf54 100644 --- a/columnflow/production/cms/muon.py +++ b/columnflow/production/cms/muon.py @@ -18,15 +18,14 @@ uses={ "Muon.pt", "Muon.eta", }, - produces={ - "muon_weight", "muon_weight_up", "muon_weight_down", - }, + # produces in the init # only run on mc mc_only=True, # function to determine the correction file get_muon_file=(lambda self, external_files: external_files.muon_sf), # function to determine the muon weight config get_muon_config=(lambda self: self.config_inst.x.muon_sf_names), + weight_name=(lambda self: "muon_weight"), ) def muon_weights( self: Producer, @@ -83,7 +82,6 @@ def muon_weights( "ValType": syst, # syst key in 2017 } inputs = [variable_map_syst[inp.name] for inp in self.muon_sf_corrector.inputs] - sf_flat = self.muon_sf_corrector(*inputs) # add the correct layout to it @@ -93,7 +91,7 @@ def muon_weights( weight = ak.prod(sf, axis=1, mask_identity=False) # store it - events = set_ak_column(events, f"muon_weight{postfix}", weight, value_type=np.float32) + events = set_ak_column(events, f"{self.weight_name()}{postfix}", weight, value_type=np.float32) return events @@ -128,3 +126,9 @@ def muon_weights_setup( # check versions if self.muon_sf_corrector.version not in (1,): raise Exception(f"unsuppprted muon sf corrector version {self.muon_sf_corrector.version}") + + +@muon_weights.init +def muon_weights_init(self: Producer, **kwargs) -> None: + weight_name = self.weight_name() + self.produces |= {weight_name, f"{weight_name}_up", f"{weight_name}_down"} From a22501afe9e338f4ddad1ce61def43b02b5d7125 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Tue, 14 May 2024 10:13:22 +0200 Subject: [PATCH 41/43] allow customizing supported versions --- columnflow/production/cms/muon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/columnflow/production/cms/muon.py b/columnflow/production/cms/muon.py index 3b96dcf54..69e3d85b1 100644 --- a/columnflow/production/cms/muon.py +++ b/columnflow/production/cms/muon.py @@ -26,6 +26,7 @@ # function to determine the muon weight config get_muon_config=(lambda self: self.config_inst.x.muon_sf_names), weight_name=(lambda self: "muon_weight"), + supported_versions=(1, 2), ) def muon_weights( self: Producer, @@ -124,7 +125,7 @@ def muon_weights_setup( self.muon_sf_corrector = correction_set[corrector_name] # check versions - if self.muon_sf_corrector.version not in (1,): + if self.supported_versions and self.muon_sf_corrector.version not in self.supported_versions: raise Exception(f"unsuppprted muon sf corrector version {self.muon_sf_corrector.version}") From 593a7dee9179e8787a7f9be51278fe302c2172b7 Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Fri, 17 May 2024 08:30:08 +0200 Subject: [PATCH 42/43] change muon weight_name attribute to simple string --- columnflow/production/cms/muon.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/columnflow/production/cms/muon.py b/columnflow/production/cms/muon.py index 69e3d85b1..c702b4339 100644 --- a/columnflow/production/cms/muon.py +++ b/columnflow/production/cms/muon.py @@ -25,7 +25,7 @@ get_muon_file=(lambda self, external_files: external_files.muon_sf), # function to determine the muon weight config get_muon_config=(lambda self: self.config_inst.x.muon_sf_names), - weight_name=(lambda self: "muon_weight"), + weight_name="muon_weight", supported_versions=(1, 2), ) def muon_weights( @@ -92,7 +92,7 @@ def muon_weights( weight = ak.prod(sf, axis=1, mask_identity=False) # store it - events = set_ak_column(events, f"{self.weight_name()}{postfix}", weight, value_type=np.float32) + events = set_ak_column(events, f"{self.weight_name}{postfix}", weight, value_type=np.float32) return events @@ -131,5 +131,5 @@ def muon_weights_setup( @muon_weights.init def muon_weights_init(self: Producer, **kwargs) -> None: - weight_name = self.weight_name() + weight_name = self.weight_name self.produces |= {weight_name, f"{weight_name}_up", f"{weight_name}_down"} From b484802067660b02c4ad09bebfdcbaa168d7833a Mon Sep 17 00:00:00 2001 From: Mathis Frahm Date: Fri, 17 May 2024 08:58:39 +0200 Subject: [PATCH 43/43] add class attribute for SettingsParameter delimiters --- columnflow/tasks/framework/parameters.py | 6 ++++-- tests/test_task_parameters.py | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/columnflow/tasks/framework/parameters.py b/columnflow/tasks/framework/parameters.py index 6b9844c76..2fedcd21e 100644 --- a/columnflow/tasks/framework/parameters.py +++ b/columnflow/tasks/framework/parameters.py @@ -27,14 +27,16 @@ class SettingsParameter(law.CSVParameter): p.serialize({"param1": 2, "param2": False}) => "param1=2,param2=False" """ + settings_delimiter = "=" + tuple_delimiter = ";" @classmethod def parse_setting(cls, setting: str) -> tuple[str, float | bool | str]: - pair = setting.split("=", 1) + pair = setting.split(cls.settings_delimiter, 1) key, value = pair if len(pair) == 2 else (pair[0], "True") if ";" in value: # split by ";" and parse each value - value = tuple(cls.parse_value(v) for v in value.split(";")) + value = tuple(cls.parse_value(v) for v in value.split(cls.tuple_delimiter)) else: value = cls.parse_value(value) return (key, value) diff --git a/tests/test_task_parameters.py b/tests/test_task_parameters.py index 51f0f5deb..e06040800 100644 --- a/tests/test_task_parameters.py +++ b/tests/test_task_parameters.py @@ -14,6 +14,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def test_settings_parameter(self): + # check that the default delimiters have not been changed + self.assertEqual(SettingsParameter.settings_delimiter, "=") + self.assertEqual(SettingsParameter.tuple_delimiter, ";") + + # initialize a SettingParameter p = SettingsParameter() # parsing @@ -46,6 +51,7 @@ def test_settings_parameter(self): ) def test_multi_settings_parameter(self): + # initialize a MultiSettingsParameter p = MultiSettingsParameter() # parsing