diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec83c94..07fc151 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,23 +23,6 @@ jobs: - name: Default test run: python3 af_benchmark/benchmark.py tests/test-default.yaml - nanoevents-processor: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - - name: Install dependencies - run: pip install -r requirements.txt - - - name: Test NanoEvents processor - run: python3 af_benchmark/benchmark.py tests/test-nanoevents.yaml - read-from-dir: runs-on: ubuntu-latest steps: diff --git a/README.rst b/README.rst index c2867ac..3265bdc 100644 --- a/README.rst +++ b/README.rst @@ -32,10 +32,7 @@ This benchmark is designed for generic but comprehensive performance tests of th * Parallelized via ``Dask`` using local cluster * Parallelized via ``Dask`` using Gateway cluster -* Multiple methods of loading and reading columns from NanoAOD ROOT files: - - * ``uproot`` - * ``coffea.nanoevents`` +* Loading and reading columns from NanoAOD ROOT files is done using ``uproot``. * Generic operations applied to data in columns: diff --git a/af_benchmark/benchmark.py b/af_benchmark/benchmark.py index 0932b6f..9f693bc 100644 --- a/af_benchmark/benchmark.py +++ b/af_benchmark/benchmark.py @@ -6,7 +6,7 @@ from profiling.timing import time_profiler as tp from data_access.loader import get_file_list -from processor.processor import processors +from processor.uproot_processor import UprootProcessor from executor.sequential import SequentialExecutor from executor.futures import FuturesExecutor @@ -77,14 +77,7 @@ def reset_executor(self, **kwargs): def reset_processor(self, **kwargs): - # Select processor method - self.method = self.config.get('processor.method') - if self.method in processors: - self.processor = processors[self.method](self.config) - else: - raise NotImplementedError( - f"Invalid method: {self.method}. Allowed values are: {processors.keys()}" - ) + self.processor = UprootProcessor(self.config) @tp.profile @tp.enable @@ -118,7 +111,6 @@ def update_report(self): report = { "n_files": self.n_files, "n_columns_read": n_cols_read, - "processor": self.method, "operation": self.config.get('processor.operation'), "executor": self.backend, "n_workers": self.executor.get_n_workers(), diff --git a/af_benchmark/dask-gateway-setup.ipynb b/af_benchmark/dask-gateway-setup.ipynb deleted file mode 100644 index b7e5de6..0000000 --- a/af_benchmark/dask-gateway-setup.ipynb +++ /dev/null @@ -1,85 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "966d766a-aed9-46ac-a61a-3e2a26fd04f1", - "metadata": {}, - "outputs": [], - "source": [ - "from dask_gateway import Gateway\n", - "gateway = Gateway()\n", - "\n", - "cluster = gateway.new_cluster(\n", - " conda_env = \"/depot/cms/kernels/python3\",\n", - " queue = \"cms\",\n", - " worker_cores = 1,\n", - " worker_memory = 4,\n", - " env = {\n", - " \"PYTHONPATH\": \"/depot/cms/users/dkondra/af-benchmark/af_benchmark/\",\n", - " \"X509_USER_PROXY\": \"/depot/cms/users/dkondra/x509up_u616617\",\n", - " },\n", - " cluster_start_timeout = 60,\n", - ")\n", - "\n", - "cluster" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "069b503a-68c9-4705-b1eb-d7793f2e701d", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "079f76e8-98d3-4fac-b2ae-f9b764cb4548", - "metadata": {}, - "outputs": [], - "source": [ - "# delete all clusters\n", - "for cluster_info in gateway.list_clusters():\n", - " gateway.connect(cluster_info.name).shutdown()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6100fd02-30bb-41be-a7e0-0f22d533b45b", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python3 kernel (default)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.10" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "state": {}, - "version_major": 2, - "version_minor": 0 - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/af_benchmark/example-config.yaml b/af_benchmark/example-config.yaml index d7fff9e..77ed588 100644 --- a/af_benchmark/example-config.yaml +++ b/af_benchmark/example-config.yaml @@ -13,8 +13,6 @@ data-access: processor: parallelize_over: files # parallelize_over: columns - method: uproot - # method: nanoevents columns: 5 # - event # - Muon_pt diff --git a/af_benchmark/example-configs/example-config-1.yaml b/af_benchmark/example-configs/example-config-1.yaml index cbcb42a..6e967d1 100644 --- a/af_benchmark/example-configs/example-config-1.yaml +++ b/af_benchmark/example-configs/example-config-1.yaml @@ -6,7 +6,6 @@ data-access: - /eos/purdue/store/data/Run2016B/SingleMuon/NANOAOD/02Apr2020_ver2-v1/20000/ processor: parallelize_over: files - method: uproot columns: - event operation: sum diff --git a/af_benchmark/example-configs/example-config-dbs-block.yaml b/af_benchmark/example-configs/example-config-dbs-block.yaml index d16e3aa..3c66c7c 100644 --- a/af_benchmark/example-configs/example-config-dbs-block.yaml +++ b/af_benchmark/example-configs/example-config-dbs-block.yaml @@ -8,6 +8,5 @@ data-access: block: /SingleMuon/Run2017C-02Apr2020-v1/NANOAOD#44236284-ff8d-4b95-b971-dcec15b5130f processor: parallelize_over: files - method: uproot columns: 5 operation: sum diff --git a/af_benchmark/example-configs/example-config-dbs-dataset.yaml b/af_benchmark/example-configs/example-config-dbs-dataset.yaml index 92e3d99..a5bc6fc 100644 --- a/af_benchmark/example-configs/example-config-dbs-dataset.yaml +++ b/af_benchmark/example-configs/example-config-dbs-dataset.yaml @@ -7,6 +7,5 @@ data-access: dataset: /SingleMuon/Run2016B-02Apr2020_ver2-v1/NANOAOD processor: parallelize_over: files - method: uproot columns: 5 operation: sum diff --git a/af_benchmark/example-configs/example-config-local-dirs.yaml b/af_benchmark/example-configs/example-config-local-dirs.yaml index 4f53b70..bf45c5a 100644 --- a/af_benchmark/example-configs/example-config-local-dirs.yaml +++ b/af_benchmark/example-configs/example-config-local-dirs.yaml @@ -6,6 +6,5 @@ data-access: - /eos/purdue/store/data/Run2016B/SingleMuon/NANOAOD/02Apr2020_ver2-v1/ processor: parallelize_over: files - method: uproot columns: 5 operation: sum diff --git a/af_benchmark/processor/processor.py b/af_benchmark/processor/uproot_processor.py similarity index 64% rename from af_benchmark/processor/processor.py rename to af_benchmark/processor/uproot_processor.py index e71db3e..ef544f3 100644 --- a/af_benchmark/processor/processor.py +++ b/af_benchmark/processor/uproot_processor.py @@ -1,28 +1,29 @@ -from abc import ABC, abstractmethod from profiling.timing import time_profiler as tp import pandas as pd import numpy as np import uproot -from coffea.nanoevents import NanoEventsFactory, NanoAODSchema - -class BaseProcessor(ABC): - """A base processor class - """ +class UprootProcessor: def __init__(self, config): - self.config=config - - @tp.enable - def open_nanoaod(self, files, executor, **kwargs): - return executor.execute(self.open_nanoaod_, files, **kwargs) + self.config = config - @abstractmethod - def open_nanoaod_(self, file_path, **kwargs): - return + def open_nanoaod(self, file_path, **kwargs): + tree = uproot.open(file_path)["Events"] + return tree - @abstractmethod def get_column_list(self, file): - return + columns_to_read = self.config.get('processor.columns') + tree = self.open_nanoaod(file) + if isinstance(columns_to_read, list): + if any(c not in tree.keys() for c in columns_to_read): + raise ValueError(f"Error reading column: {column}") + self.column = columns_to_read + elif isinstance(columns_to_read, int): + self.columns = list(tree.keys())[:columns_to_read] + if len(self.columns)GatewayCluster'), HBox(children=(HTML(value='\\n
\\n\n\n \n \n \n
Workers 0
Threads 0
Memory 0 B
\n
\n" + } + }, + "2d5c46dd7a5a4e48b6e64ff0ead3c262": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "IntTextModel", + "state": { + "description": "Minimum", + "layout": "IPY_MODEL_c38a8550718d4263a88dfec9a7a4a6de", + "step": 1, + "style": "IPY_MODEL_789c14110b9c4868b3649124669ed01b" + } + }, + "2ef4ac5f08784d29b199e03390cb2169": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "children": [ + "IPY_MODEL_f065129aa1d2442b9a1c37f381804af1", + "IPY_MODEL_346015e17c414276a1045cecff0e7468" + ], + "layout": "IPY_MODEL_b90da51f8e1b45f085ea107f24c72ae7" + } + }, + "2f639b5c15474f23a29e64ecd74bc150": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "31393012ab35475c8d8bd2d068cd4f07": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_f736caf312994effbac73a94f58ea027", + "style": "IPY_MODEL_6021ccdd6f45438092c687b5f5136f64", + "value": "

Dashboard: http://dask-gateway.geddes.rcac.purdue.edu/clusters/2e2db27ebe8f428a9f6a686a55f57d67/status

\n" + } + }, + "346015e17c414276a1045cecff0e7468": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonModel", + "state": { + "description": "Scale", + "layout": "IPY_MODEL_ca154fd1189d4f4aa72c6da96e9b2fc8", + "style": "IPY_MODEL_c9c01325ece240499b424cecafe98474", + "tooltip": null + } + }, + "34e31fe1c76e4a2fa3b98cd4eb277a28": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "min_width": "500px" + } + }, + "353690cf749c4a74bb067b8b6cc7291c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "35e9954d3af94d0383f040ae5b0faa2a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "3f30fbc21aa542c7a4bc04b54d20f617": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "AccordionModel", + "state": { + "children": [ + "IPY_MODEL_e37191caf3fd41e0ac3b497be5a8f7f2", + "IPY_MODEL_470f5475f3d843c0a7d50587e51b399d" + ], + "layout": "IPY_MODEL_34e31fe1c76e4a2fa3b98cd4eb277a28", + "titles": [ + "Manual Scaling", + "Adaptive Scaling" + ] + } + }, + "41d4260d73f8428193955f6a50b489f6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "VBoxModel", + "state": { + "children": [ + "IPY_MODEL_42417643576240228c0d168e20bf98ba", + "IPY_MODEL_1f6b63b051fe4650af57f6d8e3b58686", + "IPY_MODEL_ce425e2343c445e5ac15f820d8ee5156", + "IPY_MODEL_7de096a013094ab8a93d013645daf091" + ], + "layout": "IPY_MODEL_6c00cd536acb41bcb9cd50efb04a81d1" + } + }, + "42417643576240228c0d168e20bf98ba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_35e9954d3af94d0383f040ae5b0faa2a", + "style": "IPY_MODEL_b635c66aa68b49ea959755649d7177f2", + "value": "

GatewayCluster

" + } + }, + "4610eab51aad4674a2e7e1cedebcb6b1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "470f5475f3d843c0a7d50587e51b399d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "children": [ + "IPY_MODEL_80941d8a81b245ccb6a69274afd81396", + "IPY_MODEL_a5051f1f11f84c4cb0e3b8e8704ee9d6", + "IPY_MODEL_57bae7f460fa453fbd4daa15bda1d585" + ], + "layout": "IPY_MODEL_17733e0cc2894455a9b054c2bd42534b" + } + }, + "49901653748f432e84382b43efc1fb9b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonModel", + "state": { + "description": "Scale", + "layout": "IPY_MODEL_c38a8550718d4263a88dfec9a7a4a6de", + "style": "IPY_MODEL_de5f96363c9b4dab8e9d94572b5fbf24", + "tooltip": null + } + }, + "57bae7f460fa453fbd4daa15bda1d585": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonModel", + "state": { + "description": "Adapt", + "layout": "IPY_MODEL_998cba2663a1413cb6683a3260ce485b", + "style": "IPY_MODEL_00a1d649950344b693ec1947bc8867b7", + "tooltip": null + } + }, + "59af5cb508ae46c7ab59fd588d865c5e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "5a3263b964cb495db8818f70a45b11b5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "5c02753d660244f4b08bf58a7f144d42": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "min_width": "500px" + } + }, + "5f0d3c8866b646a19874b7f2a81af9e0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "min_width": "150px" + } + }, + "5ffb612a0ff342e5b90f4605e7d88975": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_f89136098dec469699e14b0291b80716", + "style": "IPY_MODEL_c13f017a047f4e98b544941caabcc88c", + "value": "

GatewayCluster

" + } + }, + "6021ccdd6f45438092c687b5f5136f64": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "61ee2ba90fb44ce5997fbd1fd559fab4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "65713ae394ed4f09aebcb6e5b7168fd1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonModel", + "state": { + "description": "Scale", + "layout": "IPY_MODEL_998cba2663a1413cb6683a3260ce485b", + "style": "IPY_MODEL_18e8086887404c5dabeffa04c2b7e389", + "tooltip": null + } + }, + "66ca263aa1fe434989feee67a07135d2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "VBoxModel", + "state": { + "children": [ + "IPY_MODEL_5ffb612a0ff342e5b90f4605e7d88975", + "IPY_MODEL_805eed48a7b84fb7be3fcfcf4d75f548", + "IPY_MODEL_de205e4359c84f6c82cc04a90b6570d9", + "IPY_MODEL_31393012ab35475c8d8bd2d068cd4f07" + ], + "layout": "IPY_MODEL_03c83e8b1f6744dfb0eb15090c7b3ac7" + } + }, + "68ab6e2b9182422b92d9f221c01c1fd7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonStyleModel", + "state": { + "font_family": null, + "font_size": null, + "font_style": null, + "font_variant": null, + "font_weight": null, + "text_color": null, + "text_decoration": null + } + }, + "6913e7704596494fb67dd87663893c1e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "IntTextModel", + "state": { + "description": "Workers", + "layout": "IPY_MODEL_c38a8550718d4263a88dfec9a7a4a6de", + "step": 1, + "style": "IPY_MODEL_af3325595f5841e2b3abc401c7465448" + } + }, + "6c00cd536acb41bcb9cd50efb04a81d1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "7513d6f9043940519ff16cf2de5ea3fa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_5f0d3c8866b646a19874b7f2a81af9e0", + "style": "IPY_MODEL_59af5cb508ae46c7ab59fd588d865c5e", + "value": "\n
\n\n\n \n \n \n
Workers 0
Threads 0
Memory 0 B
\n
\n" + } + }, + "789c14110b9c4868b3649124669ed01b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "DescriptionStyleModel", + "state": { + "description_width": "" + } + }, + "7ac87303e83848f19c4c6edc52262742": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "VBoxModel", + "state": { + "children": [ + "IPY_MODEL_17febe9b579e4ba4856d4ae2d076c792", + "IPY_MODEL_fec87d64d2414ff8b148799c539c1c23", + "IPY_MODEL_abd81ae5a910467bbd2e3c3126993f66", + "IPY_MODEL_9aacac07a19749e88c189e1c614fad0b" + ], + "layout": "IPY_MODEL_089c02d1161e4b62a99c7b4c9a353452" + } + }, + "7d6fa5bf25d84b27b2820d2490ffb842": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "7de096a013094ab8a93d013645daf091": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_ebd5562e9c46405e9e455e4ce00eb4c4", + "style": "IPY_MODEL_353690cf749c4a74bb067b8b6cc7291c", + "value": "

Dashboard: http://dask-gateway.geddes.rcac.purdue.edu/clusters/40550125cbdf4ddfb8a3c27fe4f2b84e/status

\n" + } + }, + "805eed48a7b84fb7be3fcfcf4d75f548": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "children": [ + "IPY_MODEL_ee452d91ac834a76a716cb22395c482b", + "IPY_MODEL_3f30fbc21aa542c7a4bc04b54d20f617" + ], + "layout": "IPY_MODEL_d102c34bda4040a79defb016e9d20241" + } + }, + "80941d8a81b245ccb6a69274afd81396": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "IntTextModel", + "state": { + "description": "Minimum", + "layout": "IPY_MODEL_998cba2663a1413cb6683a3260ce485b", + "step": 1, + "style": "IPY_MODEL_84c2dff8691944e6b70ab06a90d60bb5" + } + }, + "84c2dff8691944e6b70ab06a90d60bb5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "DescriptionStyleModel", + "state": { + "description_width": "" + } + }, + "888523d51fc04ec0aaba56a27195ef62": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonModel", + "state": { + "description": "Adapt", + "layout": "IPY_MODEL_c38a8550718d4263a88dfec9a7a4a6de", + "style": "IPY_MODEL_8d310fa77018464393b5fe1aec0f9ea5", + "tooltip": null + } + }, + "8aa9593642af45fa8a4c88b966fdcfe2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "DescriptionStyleModel", + "state": { + "description_width": "" + } + }, + "8d310fa77018464393b5fe1aec0f9ea5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonStyleModel", + "state": { + "font_family": null, + "font_size": null, + "font_style": null, + "font_variant": null, + "font_weight": null, + "text_color": null, + "text_decoration": null + } + }, + "910e905c62454c5f939e887e199f17c7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "9358a984277e4ad880c7a5b55d5531b4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "998cba2663a1413cb6683a3260ce485b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "width": "150px" + } + }, + "9aacac07a19749e88c189e1c614fad0b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_61ee2ba90fb44ce5997fbd1fd559fab4", + "style": "IPY_MODEL_7d6fa5bf25d84b27b2820d2490ffb842", + "value": "

Dashboard: http://dask-gateway.geddes.rcac.purdue.edu/clusters/1a19f83f1596412ea562324b719f7369/status

\n" + } + }, + "9b3c2aa151b347df92913cca9b1e0e14": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonModel", + "state": { + "description": "Adapt", + "layout": "IPY_MODEL_ca154fd1189d4f4aa72c6da96e9b2fc8", + "style": "IPY_MODEL_68ab6e2b9182422b92d9f221c01c1fd7", + "tooltip": null + } + }, + "a5051f1f11f84c4cb0e3b8e8704ee9d6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "IntTextModel", + "state": { + "description": "Maximum", + "layout": "IPY_MODEL_998cba2663a1413cb6683a3260ce485b", + "step": 1, + "style": "IPY_MODEL_005e8de381714168a77a2d849c1c640a" + } + }, + "a853d9b2073f46cb9f8edb664c42ef17": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "a913f9de833d4ad397d26fecaf9a89d2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "IntTextModel", + "state": { + "description": "Maximum", + "layout": "IPY_MODEL_ca154fd1189d4f4aa72c6da96e9b2fc8", + "step": 1, + "style": "IPY_MODEL_e769db00497e426fb469c16938e8f604" + } + }, + "aa5eb3d6dd7641be92b24c07d175a2d3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "aab3507777fd4b7cb69c6fad3cb66f5a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "IntTextModel", + "state": { + "description": "Minimum", + "layout": "IPY_MODEL_ca154fd1189d4f4aa72c6da96e9b2fc8", + "step": 1, + "style": "IPY_MODEL_ef0798fb94dc44a3b1d1643175754c8f" + } + }, + "abd81ae5a910467bbd2e3c3126993f66": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_b81fa76e2f36469ab9d91a156926a570", + "style": "IPY_MODEL_c0a8291ee47548f3bf0dc2525bb32262", + "value": "

Name: 1a19f83f1596412ea562324b719f7369

" + } + }, + "ae241b121c2c4efd897876c7928cb017": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "af3325595f5841e2b3abc401c7465448": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "DescriptionStyleModel", + "state": { + "description_width": "" + } + }, + "b1984d00e36a4279842bfa498b3b5ed7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "min_width": "150px" + } + }, + "b203ad170fea4c17ba4f324e3418719c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "min_width": "500px" + } + }, + "b6129e95763648f8bbbb05aebd2dc57b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "AccordionModel", + "state": { + "children": [ + "IPY_MODEL_ffad225b14864d1281e20451464b4703", + "IPY_MODEL_2be70b226bf94cb4a0c416a9160557e3" + ], + "layout": "IPY_MODEL_5c02753d660244f4b08bf58a7f144d42", + "titles": [ + "Manual Scaling", + "Adaptive Scaling" + ] + } + }, + "b635c66aa68b49ea959755649d7177f2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "b81fa76e2f36469ab9d91a156926a570": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "b90da51f8e1b45f085ea107f24c72ae7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "bdaa485b76fc45f3bfb370bd34d403f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "DescriptionStyleModel", + "state": { + "description_width": "" + } + }, + "c0a8291ee47548f3bf0dc2525bb32262": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "c13f017a047f4e98b544941caabcc88c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "c38a8550718d4263a88dfec9a7a4a6de": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "width": "150px" + } + }, + "c9c01325ece240499b424cecafe98474": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonStyleModel", + "state": { + "font_family": null, + "font_size": null, + "font_style": null, + "font_variant": null, + "font_weight": null, + "text_color": null, + "text_decoration": null + } + }, + "ca154fd1189d4f4aa72c6da96e9b2fc8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "width": "150px" + } + }, + "ce425e2343c445e5ac15f820d8ee5156": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_ae241b121c2c4efd897876c7928cb017", + "style": "IPY_MODEL_19f5bc00318848d4bb3d7601b6c4da77", + "value": "

Name: 40550125cbdf4ddfb8a3c27fe4f2b84e

" + } + }, + "d102c34bda4040a79defb016e9d20241": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "d6817188a5634e459732f05d8e660234": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "children": [ + "IPY_MODEL_aab3507777fd4b7cb69c6fad3cb66f5a", + "IPY_MODEL_a913f9de833d4ad397d26fecaf9a89d2", + "IPY_MODEL_9b3c2aa151b347df92913cca9b1e0e14" + ], + "layout": "IPY_MODEL_f3a290c4b4fa4cb299b83e9422bafbb2" + } + }, + "d874a3f8ef5f43968588f0f815fb6edc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "da9757cc604449bd8bc9eee334e79073": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "dd9f6246d6aa44e4b84868780b7e1eac": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "IntTextModel", + "state": { + "description": "Workers", + "layout": "IPY_MODEL_998cba2663a1413cb6683a3260ce485b", + "step": 1, + "style": "IPY_MODEL_faca509bad2c481fbc257db1aab0996e" + } + }, + "de205e4359c84f6c82cc04a90b6570d9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_eff46eef1a35420aad669b33d7a11943", + "style": "IPY_MODEL_da9757cc604449bd8bc9eee334e79073", + "value": "

Name: 2e2db27ebe8f428a9f6a686a55f57d67

" + } + }, + "de5f96363c9b4dab8e9d94572b5fbf24": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ButtonStyleModel", + "state": { + "font_family": null, + "font_size": null, + "font_style": null, + "font_variant": null, + "font_weight": null, + "text_color": null, + "text_decoration": null + } + }, + "e37191caf3fd41e0ac3b497be5a8f7f2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "children": [ + "IPY_MODEL_dd9f6246d6aa44e4b84868780b7e1eac", + "IPY_MODEL_65713ae394ed4f09aebcb6e5b7168fd1" + ], + "layout": "IPY_MODEL_aa5eb3d6dd7641be92b24c07d175a2d3" + } + }, + "e769db00497e426fb469c16938e8f604": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "DescriptionStyleModel", + "state": { + "description_width": "" + } + }, + "ebd5562e9c46405e9e455e4ce00eb4c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "ee151cb0447a4534bebe2ccddfed00c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "min_width": "150px" + } + }, + "ee452d91ac834a76a716cb22395c482b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "layout": "IPY_MODEL_b1984d00e36a4279842bfa498b3b5ed7", + "style": "IPY_MODEL_9358a984277e4ad880c7a5b55d5531b4", + "value": "\n
\n\n\n \n \n \n
Workers 0
Threads 0
Memory 0 B
\n
\n" + } + }, + "ef0798fb94dc44a3b1d1643175754c8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "DescriptionStyleModel", + "state": { + "description_width": "" + } + }, + "eff46eef1a35420aad669b33d7a11943": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "f065129aa1d2442b9a1c37f381804af1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "IntTextModel", + "state": { + "description": "Workers", + "layout": "IPY_MODEL_ca154fd1189d4f4aa72c6da96e9b2fc8", + "step": 1, + "style": "IPY_MODEL_bdaa485b76fc45f3bfb370bd34d403f7" + } + }, + "f3a290c4b4fa4cb299b83e9422bafbb2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "f736caf312994effbac73a94f58ea027": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "f89136098dec469699e14b0291b80716": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "faca509bad2c481fbc257db1aab0996e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "DescriptionStyleModel", + "state": { + "description_width": "" + } + }, + "fec87d64d2414ff8b148799c539c1c23": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "children": [ + "IPY_MODEL_2c869ba0171948a6b9b610287f7c4257", + "IPY_MODEL_b6129e95763648f8bbbb05aebd2dc57b" + ], + "layout": "IPY_MODEL_a853d9b2073f46cb9f8edb664c42ef17" + } + }, + "ffad225b14864d1281e20451464b4703": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "children": [ + "IPY_MODEL_6913e7704596494fb67dd87663893c1e", + "IPY_MODEL_49901653748f432e84382b43efc1fb9b" + ], + "layout": "IPY_MODEL_2f639b5c15474f23a29e64ecd74bc150" + } + }, + "fff8f741692f497188fe73c2beb49fbc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "AccordionModel", + "state": { + "children": [ + "IPY_MODEL_2ef4ac5f08784d29b199e03390cb2169", + "IPY_MODEL_d6817188a5634e459732f05d8e660234" + ], + "layout": "IPY_MODEL_b203ad170fea4c17ba4f324e3418719c", + "titles": [ + "Manual Scaling", + "Adaptive Scaling" + ] + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test-dask-local.yaml b/tests/test-dask-local.yaml index a53ee2b..483112c 100644 --- a/tests/test-dask-local.yaml +++ b/tests/test-dask-local.yaml @@ -5,7 +5,6 @@ data-access: files: - tests/data/nano_dimuon.root processor: - parallelize_over: columns - method: uproot + parallelize_over: files columns: 10 operation: sum diff --git a/tests/test-default.yaml b/tests/test-default.yaml index c999fb2..8cfd4a9 100644 --- a/tests/test-default.yaml +++ b/tests/test-default.yaml @@ -5,7 +5,6 @@ data-access: files: - tests/data/nano_dimuon.root processor: - parallelize_over: columns - method: uproot + parallelize_over: files columns: 10 operation: sum diff --git a/tests/test-explicit-columns.yaml b/tests/test-explicit-columns.yaml index 6049697..6a31c39 100644 --- a/tests/test-explicit-columns.yaml +++ b/tests/test-explicit-columns.yaml @@ -5,7 +5,7 @@ data-access: files: - tests/data/nano_dimuon.root processor: - method: uproot + parallelize_over: files columns: - event - Muon_pt diff --git a/tests/test-futures.yaml b/tests/test-futures.yaml index c999fb2..8cfd4a9 100644 --- a/tests/test-futures.yaml +++ b/tests/test-futures.yaml @@ -5,7 +5,6 @@ data-access: files: - tests/data/nano_dimuon.root processor: - parallelize_over: columns - method: uproot + parallelize_over: files columns: 10 operation: sum diff --git a/tests/test-nanoevents.yaml b/tests/test-nanoevents.yaml deleted file mode 100644 index fae3b78..0000000 --- a/tests/test-nanoevents.yaml +++ /dev/null @@ -1,12 +0,0 @@ -executor: - backend: sequential -data-access: - mode: local - files: - - tests/data/nano_dimuon.root -processor: - method: nanoevents - columns: - - event - - Muon_pt - operation: sum diff --git a/tests/test-read-by-column.yaml b/tests/test-read-by-column.yaml index e4d8703..8cfd4a9 100644 --- a/tests/test-read-by-column.yaml +++ b/tests/test-read-by-column.yaml @@ -5,6 +5,6 @@ data-access: files: - tests/data/nano_dimuon.root processor: - method: uproot + parallelize_over: files columns: 10 operation: sum diff --git a/tests/test-read-by-file.yaml b/tests/test-read-by-file.yaml index 982250a..8cfd4a9 100644 --- a/tests/test-read-by-file.yaml +++ b/tests/test-read-by-file.yaml @@ -6,6 +6,5 @@ data-access: - tests/data/nano_dimuon.root processor: parallelize_over: files - method: uproot columns: 10 operation: sum diff --git a/tests/test-read-from-dir.yaml b/tests/test-read-from-dir.yaml index dda3750..2f9ba6b 100644 --- a/tests/test-read-from-dir.yaml +++ b/tests/test-read-from-dir.yaml @@ -5,7 +5,6 @@ data-access: directories: - tests/data/ processor: - parallelize_over: columns - method: uproot + parallelize_over: files columns: 10 operation: sum