diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c9dc4747..cf6763fe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,12 +16,12 @@ jobs: python-version: ["3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Need to clone everything for the git tags. fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} channels: conda-forge,defaults @@ -31,19 +31,19 @@ jobs: - name: Update pip/wheel infrastructure shell: bash -l {0} run: | - conda install -y -q pip wheel + conda install -y -q pip wheel uv - name: Install dependencies shell: bash -l {0} run: | - pip install -r requirements.txt + uv pip install -r requirements.txt # We have two cores so we can speed up the testing with xdist - name: Install pytest packages shell: bash -l {0} run: | conda install -y -q \ - pytest pytest-xdist pytest-openfiles pytest-cov + pytest pytest-xdist pytest-cov - name: List installed packages shell: bash -l {0} @@ -54,12 +54,12 @@ jobs: - name: Build and install shell: bash -l {0} run: | - pip install -v --no-deps -e . + uv pip install -v --no-deps -e . - name: Run tests shell: bash -l {0} run: | - pytest -r a -v -n 3 --open-files --cov=lsst.ctrl.mpexec --cov=tests --cov-report=xml --cov-report=term --cov-branch + pytest -r a -v -n 3 --cov=lsst.ctrl.mpexec --cov=tests --cov-report=xml --cov-report=term --cov-branch - name: Upload coverage to codecov uses: codecov/codecov-action@v4 with: @@ -72,13 +72,13 @@ jobs: if: startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Need to clone everything to embed the version. fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" diff --git a/.github/workflows/build_docs.yaml b/.github/workflows/build_docs.yaml index 29d5340e..8fb83212 100644 --- a/.github/workflows/build_docs.yaml +++ b/.github/workflows/build_docs.yaml @@ -10,13 +10,13 @@ jobs: build_sphinx_docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Need to clone everything for the git tags. fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.11" cache: "pip" diff --git a/.github/workflows/do_not_merge.yaml b/.github/workflows/do_not_merge.yaml index d602c38c..7bac8126 100644 --- a/.github/workflows/do_not_merge.yaml +++ b/.github/workflows/do_not_merge.yaml @@ -26,7 +26,7 @@ jobs: 'DO NOT MERGE'. Remove this commit from the branch before merging or change the commit summary." - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check requirements.txt for branches shell: bash diff --git a/.github/workflows/docstyle.yaml b/.github/workflows/docstyle.yaml index 98bed05c..4433bad8 100644 --- a/.github/workflows/docstyle.yaml +++ b/.github/workflows/docstyle.yaml @@ -14,10 +14,10 @@ jobs: numpydoc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 - name: Install numpydoc run: | diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 6c463eed..59bae3f1 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -12,5 +12,5 @@ jobs: ruff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: chartboost/ruff-action@v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1869627d..d557ca80 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - id: check-toml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.1 + rev: 24.4.2 hooks: - id: black # It is recommended to specify the latest version of Python @@ -22,10 +22,10 @@ repos: name: isort (python) - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.1.15 + rev: v0.4.6 hooks: - id: ruff - repo: https://github.com/numpy/numpydoc - rev: "v1.6.0" + rev: "v1.7.0" hooks: - id: numpydoc-validation diff --git a/doc/changes/DM-35599.bugfix.rst b/doc/changes/DM-35599.bugfix.rst deleted file mode 100644 index 1a727118..00000000 --- a/doc/changes/DM-35599.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Removed shadowing of ``pipetask build -t`` by ``pipetask qgraph -t``. -``-t`` now means ``--task`` rather than ``--transfer``. diff --git a/doc/changes/DM-37140.feature.md b/doc/changes/DM-37140.feature.md deleted file mode 100644 index 78b82b16..00000000 --- a/doc/changes/DM-37140.feature.md +++ /dev/null @@ -1,4 +0,0 @@ -Be more permissive about input/output collection consistency, and provide `--rebase` option to force consistency. - -An existing output collection is now considered consistent with a given sequence of input collections if the latter is a contiguous subsequence of the former. -When this is not the case, `--rebase` redefines the output collection such that it will be. diff --git a/doc/changes/DM-37231.feature.rst b/doc/changes/DM-37231.feature.rst deleted file mode 100644 index 16d9f522..00000000 --- a/doc/changes/DM-37231.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Updated the open-source license to allow for the code to be distributed with either GPLv3 or BSD 3-clause license. diff --git a/doc/changes/DM-39779.feature.md b/doc/changes/DM-39779.feature.md deleted file mode 100644 index 3eb205f4..00000000 --- a/doc/changes/DM-39779.feature.md +++ /dev/null @@ -1 +0,0 @@ -Add `pipeline-graph` and `task-graph` options for `pipetask build --show`, which provide text-art visualization of pipeline graphs. diff --git a/doc/changes/DM-41131.feature.md b/doc/changes/DM-41131.feature.md deleted file mode 100644 index ea2615ab..00000000 --- a/doc/changes/DM-41131.feature.md +++ /dev/null @@ -1,4 +0,0 @@ -Add `pipetask report` which reads a quantum graph and reports on the outputs -of failed, produced and missing quanta. This is a command-line incarnation of -`QuantumGraphExecutionReport.make_reports` in combination with -`QuantumGraphExecutionReport.write_summary_yaml`. diff --git a/doc/changes/DM-41542.feature.rst b/doc/changes/DM-41542.feature.rst deleted file mode 100644 index d16cfce5..00000000 --- a/doc/changes/DM-41542.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added --summary option to ``pipetask qgraph``. diff --git a/doc/changes/DM-41606.feature.md b/doc/changes/DM-41606.feature.md deleted file mode 100644 index d1b1b9f8..00000000 --- a/doc/changes/DM-41606.feature.md +++ /dev/null @@ -1,7 +0,0 @@ -Make option to output `pipetask report` information to the command-line using -astropy tables and set to default. -Unpack a more human-readable dictionary from -`lsst.pipe.base.QuantumGraphExecutionReports.to_summary_dict` and print summary -tables of quanta and datasets to the command-line. Save error messages and -associated data ids to a yaml file in the working directory, or optionally print -them to screen as well. diff --git a/doc/changes/DM-41832.removal.md b/doc/changes/DM-41832.removal.md deleted file mode 100644 index 6c45f834..00000000 --- a/doc/changes/DM-41832.removal.md +++ /dev/null @@ -1,3 +0,0 @@ -Support for fork option in `pipetask run` has been removed as unsafe. -Default start option now is `spawn`, `forkserver` is also available. -The `fork` option is still present in CLI for compatibility, but is deprecated and replaced by `spawn` if specified. diff --git a/doc/changes/DM-41962.bugfix.md b/doc/changes/DM-41962.bugfix.md deleted file mode 100644 index 5aaa7587..00000000 --- a/doc/changes/DM-41962.bugfix.md +++ /dev/null @@ -1,4 +0,0 @@ -Fix a storage class bug in registering dataset types in ``pipetask run``. - -Prior to this fix, the presence of multiple storage classes being associated with the same dataset type in a pipeline could cause the registered dataset type's storage class to be random and nondeterministic in regular `pipetask run` execution (but not quantum-backed butler execution). -It now follows the rules set by `PipelineGraph`, in which the definition in the task that produces the dataset wins. diff --git a/doc/changes/DM-42118.bugfix.rst b/doc/changes/DM-42118.bugfix.rst deleted file mode 100644 index 1bab217c..00000000 --- a/doc/changes/DM-42118.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure that the implicit threading options for ``run-qbb`` is used so that implicit threading can be disabled. diff --git a/doc/changes/DM-42302.misc.md b/doc/changes/DM-42302.misc.md deleted file mode 100644 index 35cc9621..00000000 --- a/doc/changes/DM-42302.misc.md +++ /dev/null @@ -1 +0,0 @@ -Drop support for Pydantic 1.x. diff --git a/doc/changes/DM-42376.bugfix.rst b/doc/changes/DM-42376.bugfix.rst deleted file mode 100644 index 955ca719..00000000 --- a/doc/changes/DM-42376.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``dump_kwargs`` `TypeError` caused by migration to Pydantic 2. diff --git a/doc/changes/DM-42751.api.rst b/doc/changes/DM-42751.api.rst deleted file mode 100644 index 1b93d1fb..00000000 --- a/doc/changes/DM-42751.api.rst +++ /dev/null @@ -1,2 +0,0 @@ -``SeparablePipelineExecutor.run_pipeline`` has been modified to take a ``num_proc`` parameter to specify how many subprocesses can be used to execute the pipeline. -The default is now ``1`` (no spawning), which is a change from the previous behavior of using 80% of the available cores. diff --git a/doc/changes/DM-43363.bugfix.md b/doc/changes/DM-43363.bugfix.md deleted file mode 100644 index cbbc0621..00000000 --- a/doc/changes/DM-43363.bugfix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix the `--show-errors` option in `pipetask report`. - -Correctly pass the option to the function as a flag. Then, in testing, use the `--show-errors` option to avoid saving yaml files to disk without adequate cleanup. diff --git a/doc/changes/DM-43484.bugfix.md b/doc/changes/DM-43484.bugfix.md deleted file mode 100644 index 00bc64ef..00000000 --- a/doc/changes/DM-43484.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fix BPS auto-retry functionality broken on DM-43060, by restoring support for repeated execution of already-successful quanta in `pipetask run-qbb`. diff --git a/doc/lsst.ctrl.mpexec/CHANGES.rst b/doc/lsst.ctrl.mpexec/CHANGES.rst index 7ceba9b2..71595310 100644 --- a/doc/lsst.ctrl.mpexec/CHANGES.rst +++ b/doc/lsst.ctrl.mpexec/CHANGES.rst @@ -1,3 +1,65 @@ +lsst-ctrl-mpexec 27.0.0 (2024-05-29) +==================================== + +New Features +------------ + +- Be more permissive about input/output collection consistency, and provided a ``--rebase`` option to ``pipetask run`` and ``pipetask qgraph`` to force consistency. + + An existing output collection is now considered consistent with a given sequence of input collections if the latter is a contiguous subsequence of the former. + When this is not the case, ``--rebase`` redefines the output collection such that it will be. (`DM-37140 `_) +- Updated the open-source license to allow for the code to be distributed with either GPLv3 or BSD 3-clause license. (`DM-37231 `_) +- Adde ``pipeline-graph`` and ``task-graph`` options for ``pipetask build --show``, which provide text-art visualization of pipeline graphs. (`DM-39779 `_) +- Added ``pipetask report`` which reads a quantum graph and reports on the outputs of failed, produced and missing quanta. + This is a command-line incarnation of + ``QuantumGraphExecutionReport.make_reports`` in combination with + ``QuantumGraphExecutionReport.write_summary_yaml``. (`DM-41131 `_) +- Added ``--summary`` option to ``pipetask qgraph``. (`DM-41542 `_) +- Made option to output ``pipetask report`` information to the command-line using Astropy tables and set to default. + Now unpack a more human-readable dictionary from + ``lsst.pipe.base.QuantumGraphExecutionReports.to_summary_dict`` and print summary tables of quanta and datasets to the command-line. + Can now save error messages and associated data ids to a YAML file in the working directory, or optionally print them to screen as well. (`DM-41606 `_) + + +API Changes +----------- + +- ``SeparablePipelineExecutor.run_pipeline`` has been modified to take a ``num_proc`` parameter to specify how many subprocesses can be used to execute the pipeline. + The default is now ``1`` (no spawning), which is a change from the previous behavior of using 80% of the available cores. (`DM-42751 `_) + + +Bug Fixes +--------- + +- Removed shadowing of ``pipetask build -t`` by ``pipetask qgraph -t``. + ``-t`` now means ``--task`` (the original meaning) rather than ``--transfer``. (`DM-35599 `_) +- Fixed a storage class bug in registering dataset types in ``pipetask run``. + + Prior to this fix, the presence of multiple storage classes being associated with the same dataset type in a pipeline could cause the registered dataset type's storage class to be random and nondeterministic in regular ``pipetask run`` execution (but not quantum-backed butler execution). + It now follows the rules set by ``PipelineGraph``, in which the definition in the task that produces the dataset wins. (`DM-41962 `_) +- Ensured that the implicit threading options for ``run-qbb`` is used so that implicit threading can be disabled. (`DM-42118 `_) +- Fixed ``dump_kwargs`` `TypeError` caused by migration to Pydantic 2. (`DM-42376 `_) +- Fixed the ``--show-errors`` option in ``pipetask report``. + + Correctly pass the option to the function as a flag. + Then, in testing, use the ``--show-errors`` option to avoid saving YAML files to disk without adequate cleanup. (`DM-43363 `_) +- Fixed BPS auto-retry functionality broken on `DM-43060 `_, by restoring support for repeated execution of already-successful quanta in ``pipetask run-qbb``. (`DM-43484 `_) + + +Other Changes and Additions +--------------------------- + +- Dropped support for Pydantic 1.x. (`DM-42302 `_) + + +An API Removal or Deprecation +----------------------------- + +- Support for fork option in ``pipetask run`` has been removed as unsafe. + Default start option now is ``spawn``, and ``forkserver`` is also available. + The ``fork`` option is still present in CLI for compatibility, but is deprecated and replaced by ``spawn`` if specified. (`DM-41832 `_) + + lsst-ctrl-mpexec v26.0.0 (2023-09-23) ===================================== @@ -8,18 +70,18 @@ New Features ``pipetask`` adds two new commands to support execution with Quantum-backed butler, mostly useful for BPS: * ``pre-exec-init-qbb`` which runs ``PreExecInit`` step of the execution to produce InitOutputs. - * ``run-qbb`` which executes ``QuantumGraph`` (or individual quanta) using Quantum-backed butler. (`DM-33497 `_) -- Added ``--coverage`` and ``--cov-packages`` to ``pipetask`` commands to allow for code coverage calculations when running (`DM-34420 `_) + * ``run-qbb`` which executes ``QuantumGraph`` (or individual quanta) using Quantum-backed butler. (`DM-33497 `_) +- Added ``--coverage`` and ``--cov-packages`` to ``pipetask`` commands to allow for code coverage calculations when running (`DM-34420 `_) - Added ``SeparablePipelineExecutor``, a pipeline executor midway in capability between ``SimplePipelineExecutor`` and ``CmdLineFwk``. ``SeparablePipelineExecutor`` is designed to be run from Python, and lets the caller decide when each pipeline processing step is carried out. - It also allows certain pipeline steps to be customized by passing alternate implementations of execution strategies (e.g., custom graph builder). (`DM-36162 `_) -- ``pipetask`` will now produce ``QuantumGraph`` with resolved output references, even with execution butler option. (`DM-37582 `_) + It also allows certain pipeline steps to be customized by passing alternate implementations of execution strategies (e.g., custom graph builder). (`DM-36162 `_) +- ``pipetask`` will now produce ``QuantumGraph`` with resolved output references, even with execution butler option. (`DM-37582 `_) - Added new command ``update-graph-run`` to ``pipetask``. - It updates existing quantum graph with new output run name and re-generates output dataset IDs. (`DM-38780 `_) + It updates existing quantum graph with new output run name and re-generates output dataset IDs. (`DM-38780 `_) - Added new command line options ``--cores-per-quantum`` and ``--memory-per-quantum``. - These can be used to pass some execution context into a quantum, allowing that quantum to change how it executes (maybe by using multiple threads). (`DM-39661 `_) -- Made it possible to force failures in mocked pipelines from the command-line. (`DM-39672 `_) -- The output of the ``pipetask ... --show=graph`` now includes extended information about dataset references and their related datastore records. (`DM-40254 `_) + These can be used to pass some execution context into a quantum, allowing that quantum to change how it executes (maybe by using multiple threads). (`DM-39661 `_) +- Made it possible to force failures in mocked pipelines from the command-line. (`DM-39672 `_) +- The output of the ``pipetask ... --show=graph`` now includes extended information about dataset references and their related datastore records. (`DM-40254 `_) API Changes @@ -31,16 +93,16 @@ API Changes * Abstract classes ``QuantumExecutor`` and ``QuantumGraphExecutor`` do not accept ``Butler`` instance in their ``execute()`` methods. * ``MPGraphExecutor`` and ``SingleQuantumExecutor`` methods updated to reflect above change and support execution with either full ``Butler`` or ``LimitedButler``. * New class ``PreExecInitLimited`` which performs pre-exec-init in case of Quantum-backed butler. - The code that it shares with a regular ``PreExecInit`` class is now in their common base class ``PreExecInitBase``. (`DM-33497 `_) + The code that it shares with a regular ``PreExecInit`` class is now in their common base class ``PreExecInitBase``. (`DM-33497 `_) - Added new ``resources`` parameter to ``SingleQuantumExecutor``, ``SimplePipelineExecutor``, and ``SeparablePipelineExecutor`` constructors. - This optional parameter is a `~lsst.pipe.base.ExecutionResources` object and allows the execution context to be passed into the `~lsst.pipe.base.PipelinesTask.runQuantum` method. (`DM-39661 `_) + This optional parameter is a `~lsst.pipe.base.ExecutionResources` object and allows the execution context to be passed into the `~lsst.pipe.base.PipelinesTask.runQuantum` method. (`DM-39661 `_) Bug Fixes --------- - Fixed ``SingleQuantumExecutor`` class to correctly handle the case with ``clobberOutputs=True`` and ``skipExistingIn=None``. - Documentation says that complete quantum outputs should be removed in this case, but they were not removed. (`DM-38601 `_) + Documentation says that complete quantum outputs should be removed in this case, but they were not removed. (`DM-38601 `_) Other Changes and Additions @@ -50,18 +112,18 @@ Other Changes and Additions This has been done so that the refs in the quantum graph are preserved when they are written to a normal Butler. * Fixed a race condition when ``pipetask run`` creates the graph with a timestamped output run and then executes it. Previously the graph creation and run execution phases calculated their own timestamped output run and it would be possible for the execution output run to be one second later than the graph run. - Previously this did not matter (the graph run was being ignored) but with the change to always use the ``DatasetRef`` from the graph it becomes critical that they match. (`DM-38779 `_) + Previously this did not matter (the graph run was being ignored) but with the change to always use the ``DatasetRef`` from the graph it becomes critical that they match. (`DM-38779 `_) - Revive the previously-bitrotted pipeline mocking system. - Most of the implementation has been moved to `pipe_base`, and the point at which mocking occurs has moved from execution to just before `QuantumGraph` generation, which changes which `pipetask` subcommands the `--mock` option is valid for. (`DM-38952 `_) + Most of the implementation has been moved to `pipe_base`, and the point at which mocking occurs has moved from execution to just before `QuantumGraph` generation, which changes which `pipetask` subcommands the `--mock` option is valid for. (`DM-38952 `_) - Updated the directed graph color scheme with an aim towards making node text easier to read. The previous pipeline directed graph nodes used dark gray as their background color. It had been reported that it is difficult to read the black text on the dark gray background. In the process of exploring what color schemes would be optimal to satisfy the aim of this ticket, it emerged that making use of the Rubin visual identity colors may be desirable. This will help to make LSST pipeline graphs more instantly recognizable as Rubin-associated products. - Colors: https://rubin.canto.com/g/RubinVisualIdentity (`DM-39294 `_) -- The ``saveMetadata`` configuration field is now ignored by executors in this package, metadata is assumed to be saved for each task. (`DM-39377 `_) -- Improved logging and removed some obsolete code paths in ``SingleQuantumExecutor``. (`DM-40332 `_) + Colors: https://rubin.canto.com/g/RubinVisualIdentity (`DM-39294 `_) +- The ``saveMetadata`` configuration field is now ignored by executors in this package, metadata is assumed to be saved for each task. (`DM-39377 `_) +- Improved logging and removed some obsolete code paths in ``SingleQuantumExecutor``. (`DM-40332 `_) - Command line help for ``pipetask run`` has been updated to reflect its correct clobbering behavior. @@ -71,26 +133,26 @@ lsst-ctrl-mpexec v25.0.0 (2023-02-28) New Features ------------ -- * Added support for transferring files into execution butler. (`DM-35494 `_) +- * Added support for transferring files into execution butler. (`DM-35494 `_) - * Added documentation on how to use ``--show`` and ``--config``. * A pipeline will now never execute if ``--show`` option is used with ``pipetask run``. * The ``--config`` option can now accept list configuration values (with or without square brackets), for example ``--config task:listItem=a,b`` or ``--config "task:listItem=[a,b]"``. - * The ``--config-file`` option can now take comma-separated file names for multiple config files. (`DM-35917 `_) -- * added additional quanta information to be displayed by the logger, showing number of quanta per task (`DM-36145 `_) + * The ``--config-file`` option can now take comma-separated file names for multiple config files. (`DM-35917 `_) +- * added additional quanta information to be displayed by the logger, showing number of quanta per task (`DM-36145 `_) - If ``pipetask`` is run with multiple processes and if a butler datastore cache is configured, all subprocesses will now share the same cache. For large numbers of simultaneous processes it may be necessary to significantly increase the number of datasets in the cache to make the cache usable. This can be done by using the ``$DAF_BUTLER_CACHE_EXPIRATION_MODE`` environment variable. - Previously each subprocess would get its own cache and if ``fork`` start method was used these cache directories would not be cleaned up. (`DM-36412 `_) + Previously each subprocess would get its own cache and if ``fork`` start method was used these cache directories would not be cleaned up. (`DM-36412 `_) - Always disable implicit threading (e.g. in OpenBLAS) by default in ``pipetask run``, even when not using ``-j``. - The new ``--enable-implicit-threading`` can be used to turn it back on. (`DM-36831 `_) + The new ``--enable-implicit-threading`` can be used to turn it back on. (`DM-36831 `_) API Changes ----------- -- ``SimplePipelineExecutor`` factory methods add ``bind`` parameter for bind values to use with the user expression. (`DM-36487 `_) +- ``SimplePipelineExecutor`` factory methods add ``bind`` parameter for bind values to use with the user expression. (`DM-36487 `_) lsst-ctrl-mpexec v24.0.0 (2022-08-26) @@ -100,28 +162,28 @@ New Features ------------ - Added ``--dataset-query-constraint`` option to ``pipetask qgraph`` command (and thus downstream commands) that allows a - user to control how `~lsst.pipe.base.QuantumGraph` creation is constrained by dataset existence. (`DM-31769 `_) -- Builds using ``setuptools`` now calculate versions from the Git repository, including the use of alpha releases for those associated with weekly tags. (`DM-32408 `_) -- Added ``--summary`` option to ``pipetask run`` command, it produces JSON report for execution status of the whole process and individual quanta. (`DM-33481 `_) -- Added ``pipetask`` CLI commands ``purge`` and ``cleanup``. (`DM-33634 `_) -- Removed dependency on the ``obs_base`` and ``afw`` packages. Now only depends on ``pipe_base`` and ``daf_butler`` (along with ``pex_config`` and ``utils``). (`DM-34105 `_) + user to control how `~lsst.pipe.base.QuantumGraph` creation is constrained by dataset existence. (`DM-31769 `_) +- Builds using ``setuptools`` now calculate versions from the Git repository, including the use of alpha releases for those associated with weekly tags. (`DM-32408 `_) +- Added ``--summary`` option to ``pipetask run`` command, it produces JSON report for execution status of the whole process and individual quanta. (`DM-33481 `_) +- Added ``pipetask`` CLI commands ``purge`` and ``cleanup``. (`DM-33634 `_) +- Removed dependency on the ``obs_base`` and ``afw`` packages. Now only depends on ``pipe_base`` and ``daf_butler`` (along with ``pex_config`` and ``utils``). (`DM-34105 `_) - Replaced the unused ``--do-raise`` option with ``--pdb``, which drops the user into the debugger (``pdb`` by default, but ``--pdb=ipdb`` also works if you have ``ipdb`` installed) - on an exception. (`DM-34215 `_) + on an exception. (`DM-34215 `_) Bug Fixes --------- -- The `click.Path` API should mostly be used with keyword arguments, changed from ordered arguments to keyword arguments when calling it. (`DM-34261 `_) -- Fixed a bug where dot graphs of pipelines did not correctly render edges between composite and component dataset types. (`DM-34811 `_) +- The `click.Path` API should mostly be used with keyword arguments, changed from ordered arguments to keyword arguments when calling it. (`DM-34261 `_) +- Fixed a bug where dot graphs of pipelines did not correctly render edges between composite and component dataset types. (`DM-34811 `_) Other Changes and Additions --------------------------- -- Added `lsst.ctrl.mpexec.SimplePipelineExecutor`, a minimal high-level Python interface for `~lsst.pipe.base.PipelineTask` execution intended primarily for unit testing. (`DM-31966 `_) +- Added `lsst.ctrl.mpexec.SimplePipelineExecutor`, a minimal high-level Python interface for `~lsst.pipe.base.PipelineTask` execution intended primarily for unit testing. (`DM-31966 `_) lsst-ctrl-mpexec v23.0.1 (2022-02-02) @@ -130,7 +192,7 @@ lsst-ctrl-mpexec v23.0.1 (2022-02-02) Miscellaneous Changes of Minor Interest --------------------------------------- -- Allow ``pipetask run`` execution to continue in single-process mode after failure of one or more tasks. Previously execution stopped on an exception from any task. (`DM-33339 `_) +- Allow ``pipetask run`` execution to continue in single-process mode after failure of one or more tasks. Previously execution stopped on an exception from any task. (`DM-33339 `_) lsst-ctrl-mpexec v23.0.0 (2021-12-10) @@ -144,28 +206,28 @@ New Features - New option ``--skip-existing-in`` which takes collection names(s), if output datasets already exist in those collections corresponding quanta is skipped. - A ``--skip-existing`` option is now equivalent to appending output run collection to the ``--skip-existing-in`` list. - An ``--extend-run`` option implicitly enables ``--skip-existing`` option. - - A ``--prune-replaced=unstore`` option only removes regular output datasets; InitOutputs, task configs, and package versions are not removed. (`DM-27492 `_) -- GraphViz dot files generated by pipetask now include more information (``RUN`` collection for datasets, dimensions for tasks, data IDs for quanta). (`DM-28111 `_) -- ``pipetask qgraph`` can now generate a standalone "execution butler" which is a SQLite registry with all the expected outputs pre-filled in registry. Using this registry allow ``pipetask run`` to execute without touching the main registry whilst still writing file artifacts to the standard location. It is not yet configured to allow a completely detached processing using a local datastore but this can be changed manually after creation to use a chained datastore. (`DM-28649 `_) + - A ``--prune-replaced=unstore`` option only removes regular output datasets; InitOutputs, task configs, and package versions are not removed. (`DM-27492 `_) +- GraphViz dot files generated by pipetask now include more information (``RUN`` collection for datasets, dimensions for tasks, data IDs for quanta). (`DM-28111 `_) +- ``pipetask qgraph`` can now generate a standalone "execution butler" which is a SQLite registry with all the expected outputs pre-filled in registry. Using this registry allow ``pipetask run`` to execute without touching the main registry whilst still writing file artifacts to the standard location. It is not yet configured to allow a completely detached processing using a local datastore but this can be changed manually after creation to use a chained datastore. (`DM-28649 `_) - * Log messages issued during quantum execution are now collected and stored in butler as ``tasklabel_log`` dataset types. - * New command line options for logging have been added to ``pipetask``. These include ``--log-file`` to write log messages to a file and ``--no-log-tty`` to disable log output to the terminal. (`DM-30977 `_) + * New command line options for logging have been added to ``pipetask``. These include ``--log-file`` to write log messages to a file and ``--no-log-tty`` to disable log output to the terminal. (`DM-30977 `_) - * Add the output run to the log record. - * Add ``--log-label`` option to ``pipetask`` command to allow extra information to be injected into the log record. (`DM-31884 `_) + * Add ``--log-label`` option to ``pipetask`` command to allow extra information to be injected into the log record. (`DM-31884 `_) Bug Fixes --------- -- Improve exception handling in ``ExecFixupDataId`` (`DM-29384 `_) -- Fix config comparison with ``--skip-existing``. (`DM-29580 `_) -- Include output collection in call to ``buildExecutionButler``. (`DM-31691 `_) -- Fix call to ``buildExecutionButler`` when chained input collection. (`DM-31711 `_) +- Improve exception handling in ``ExecFixupDataId`` (`DM-29384 `_) +- Fix config comparison with ``--skip-existing``. (`DM-29580 `_) +- Include output collection in call to ``buildExecutionButler``. (`DM-31691 `_) +- Fix call to ``buildExecutionButler`` when chained input collection. (`DM-31711 `_) Miscellaneous Changes of Minor Interest --------------------------------------- -- Add some of the pipetask command line options to QuantumGraph metadata (`DM-30702 `_) +- Add some of the pipetask command line options to QuantumGraph metadata (`DM-30702 `_) lsst-ctrl-mpexec v22.0 (2021-04-01) diff --git a/pyproject.toml b/pyproject.toml index df5d2440..c70793e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ pipetask = "lsst.ctrl.mpexec.cli.pipetask:main" filename = "doc/lsst.ctrl.mpexec/CHANGES.rst" directory = "doc/changes" title_format = "lsst-ctrl-mpexec {version} ({project_date})" - issue_format = "`{issue} `_" + issue_format = "`{issue} `_" [[tool.towncrier.type]] directory = "feature" @@ -133,9 +133,13 @@ exclude_lines = [ ] [tool.ruff] +line-length = 110 +target-version = "py311" exclude = [ "__init__.py", ] + +[tool.ruff.lint] ignore = [ "N802", "N803", @@ -153,7 +157,6 @@ ignore = [ "D205", "D400", ] -line-length = 110 select = [ "E", # pycodestyle "F", # pyflakes @@ -161,15 +164,14 @@ select = [ "W", # pycodestyle "D", # pydocstyle ] -target-version = "py311" extend-select = [ "RUF100", # Warn about unused noqa ] -[tool.ruff.pycodestyle] +[tool.ruff.lint.pycodestyle] max-doc-length = 79 -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "numpy" [tool.numpydoc_validation]