From 1974107bd3e44d9ab4b86c627858805b5c76588a Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 27 Mar 2024 17:31:43 +0100 Subject: [PATCH] cleanup & update docs --- README.rst | 11 ++--- .../modules/af_benchmark.data_access.rst | 21 --------- docs/source/modules/af_benchmark.executor.rst | 45 ------------------- .../source/modules/af_benchmark.executors.rst | 45 +++++++++++++++++++ .../modules/af_benchmark.processing.rst | 21 --------- .../source/modules/af_benchmark.profiling.rst | 21 --------- docs/source/modules/af_benchmark.rst | 29 ++++++++++-- tests/config-default.yaml | 1 - 8 files changed, 76 insertions(+), 118 deletions(-) delete mode 100644 docs/source/modules/af_benchmark.data_access.rst delete mode 100644 docs/source/modules/af_benchmark.executor.rst create mode 100644 docs/source/modules/af_benchmark.executors.rst delete mode 100644 docs/source/modules/af_benchmark.processing.rst delete mode 100644 docs/source/modules/af_benchmark.profiling.rst diff --git a/README.rst b/README.rst index 3265bdc..f1b4767 100644 --- a/README.rst +++ b/README.rst @@ -32,13 +32,14 @@ This benchmark is designed for generic but comprehensive performance tests of th * Parallelized via ``Dask`` using local cluster * Parallelized via ``Dask`` using Gateway cluster -* Loading and reading columns from NanoAOD ROOT files is done using ``uproot``. +* Multiple data access options: + + * Explicit list of files or directories in local or mounted filesystem + * List of files, blocks, or datasets at CMS DBS accessed via network, e.g. XRootD or XCache -* Generic operations applied to data in columns: +* Loading and reading columns from NanoAOD ROOT files is done using ``uproot``. - * Nothing (just open the ROOT file with a given method) - * Load column data into memory - * Perform a simple operation on a column, e.g. ``mean()`` +* Abstract "operation" with a given timeout executed by workers to emulate data processing during analysis. * Time profiling * Measuring size of columns in bytes diff --git a/docs/source/modules/af_benchmark.data_access.rst b/docs/source/modules/af_benchmark.data_access.rst deleted file mode 100644 index 595ef4b..0000000 --- a/docs/source/modules/af_benchmark.data_access.rst +++ /dev/null @@ -1,21 +0,0 @@ -af\_benchmark.data\_access package -================================== - -Submodules ----------- - -af\_benchmark.data\_access.loader module ----------------------------------------- - -.. automodule:: af_benchmark.data_access.loader - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: af_benchmark.data_access - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/modules/af_benchmark.executor.rst b/docs/source/modules/af_benchmark.executor.rst deleted file mode 100644 index 71f5851..0000000 --- a/docs/source/modules/af_benchmark.executor.rst +++ /dev/null @@ -1,45 +0,0 @@ -af\_benchmark.executor package -============================== - -Submodules ----------- - -af\_benchmark.executor.base module ----------------------------------- - -.. automodule:: af_benchmark.executor.base - :members: - :undoc-members: - :show-inheritance: - -af\_benchmark.executor.dask module ----------------------------------- - -.. automodule:: af_benchmark.executor.dask - :members: - :undoc-members: - :show-inheritance: - -af\_benchmark.executor.futures module -------------------------------------- - -.. automodule:: af_benchmark.executor.futures - :members: - :undoc-members: - :show-inheritance: - -af\_benchmark.executor.sequential module ----------------------------------------- - -.. automodule:: af_benchmark.executor.sequential - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: af_benchmark.executor - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/modules/af_benchmark.executors.rst b/docs/source/modules/af_benchmark.executors.rst new file mode 100644 index 0000000..be4571d --- /dev/null +++ b/docs/source/modules/af_benchmark.executors.rst @@ -0,0 +1,45 @@ +af\_benchmark.executors package +=============================== + +Submodules +---------- + +af\_benchmark.executors.base module +----------------------------------- + +.. automodule:: af_benchmark.executors.base + :members: + :undoc-members: + :show-inheritance: + +af\_benchmark.executors.dask module +----------------------------------- + +.. automodule:: af_benchmark.executors.dask + :members: + :undoc-members: + :show-inheritance: + +af\_benchmark.executors.futures module +-------------------------------------- + +.. automodule:: af_benchmark.executors.futures + :members: + :undoc-members: + :show-inheritance: + +af\_benchmark.executors.sequential module +----------------------------------------- + +.. automodule:: af_benchmark.executors.sequential + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: af_benchmark.executors + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/modules/af_benchmark.processing.rst b/docs/source/modules/af_benchmark.processing.rst deleted file mode 100644 index 215ee96..0000000 --- a/docs/source/modules/af_benchmark.processing.rst +++ /dev/null @@ -1,21 +0,0 @@ -af\_benchmark.processing package -================================ - -Submodules ----------- - -af\_benchmark.processing.handler module ---------------------------------------- - -.. automodule:: af_benchmark.processing.handler - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: af_benchmark.processing - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/modules/af_benchmark.profiling.rst b/docs/source/modules/af_benchmark.profiling.rst deleted file mode 100644 index 502478f..0000000 --- a/docs/source/modules/af_benchmark.profiling.rst +++ /dev/null @@ -1,21 +0,0 @@ -af\_benchmark.profiling package -=============================== - -Submodules ----------- - -af\_benchmark.profiling.timing module -------------------------------------- - -.. automodule:: af_benchmark.profiling.timing - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: af_benchmark.profiling - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/modules/af_benchmark.rst b/docs/source/modules/af_benchmark.rst index 0c2679a..ed5997e 100644 --- a/docs/source/modules/af_benchmark.rst +++ b/docs/source/modules/af_benchmark.rst @@ -7,10 +7,7 @@ Subpackages .. toctree:: :maxdepth: 4 - af_benchmark.data_access - af_benchmark.executor - af_benchmark.processing - af_benchmark.profiling + af_benchmark.executors Submodules ---------- @@ -23,6 +20,30 @@ af\_benchmark.benchmark module :undoc-members: :show-inheritance: +af\_benchmark.data\_loader module +--------------------------------- + +.. automodule:: af_benchmark.data_loader + :members: + :undoc-members: + :show-inheritance: + +af\_benchmark.time\_profiler module +----------------------------------- + +.. automodule:: af_benchmark.time_profiler + :members: + :undoc-members: + :show-inheritance: + +af\_benchmark.uproot\_processor module +-------------------------------------- + +.. automodule:: af_benchmark.uproot_processor + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/tests/config-default.yaml b/tests/config-default.yaml index 158a0df..625ab2c 100644 --- a/tests/config-default.yaml +++ b/tests/config-default.yaml @@ -7,7 +7,6 @@ executor: processor: parallelize_over: files columns: 5 - operation: sum load_columns_into_memory: True worker_operation_time: 0