Merge branch 'main' into pending-status

Author: tobiasraabe

.github/workflows/main.yml

@@ -5,8 +5,6 @@ concurrency:
   group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
-env:
-  CONDA_EXE: mamba
 
 on:
   push:
@@ -25,13 +23,13 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v6
         with:
-          python-version-file: .python-version
-          allow-prereleases: true
-          cache: pip
-      - run: pip install tox-uv
-      - run: tox -e typing
+          enable-cache: true
+      - name: Install just
+        uses: extractions/setup-just@v2
+      - name: Run type checking
+        run: just typing
 
   run-tests:
 
@@ -42,33 +40,18 @@ jobs:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ matrix.python-version }}
-          cache: pip
-          allow-prereleases: true
-      - run: pip install tox
-
-      # Unit, integration, and end-to-end tests.
-
-      - name: Run unit tests and doctests.
-        shell: bash -l {0}
-        run: tox -e test -- tests -m "unit or (not integration and not end_to_end)" --cov=src --cov=tests --cov-report=xml
-
-      - name: Upload unit test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v4
-        with:
-          flags: unit
-
-      - name: Run end-to-end tests.
-        shell: bash -l {0}
-        run: tox -e test -- tests -m end_to_end --cov=src --cov=tests --cov-report=xml
-
-      - name: Upload end_to_end test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v4
-        with:
-          flags: end_to_end
+          enable-cache: true
+      - name: Install just
+        uses: extractions/setup-just@v2
+      - name: Run tests
+        run: just test
+
+      - name: Upload test coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v5

.github/workflows/publish-to-pypi.yml

@@ -67,7 +67,7 @@ jobs:
         name: python-package-distributions
         path: dist/
     - name: Sign the dists with Sigstore
-      uses: sigstore/gh-action-sigstore-python@v2.1.1
+      uses: sigstore/gh-action-sigstore-python@v3.0.1
       with:
         inputs: >-
           ./dist/*.tar.gz

.gitignore

@@ -21,3 +21,4 @@ tests/test_jupyter/*.txt
 .pytest_cache
 .ruff_cache
 .venv
+uv.lock

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
     -   id: check-added-large-files
         args: ['--maxkb=25']
@@ -23,20 +23,16 @@ repos:
     -   id: python-use-type-annotations
     -   id: text-unicode-replacement-char
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.4
+    rev: v0.12.4
     hooks:
       - id: ruff
       - id: ruff-format
--   repo: https://github.com/dosisod/refurb
-    rev: v2.0.0
-    hooks:
-    -   id: refurb
 -   repo: https://github.com/kynan/nbstripout
-    rev: 0.7.1
+    rev: 0.8.1
     hooks:
     -   id: nbstripout
 -   repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.22
     hooks:
     -   id: mdformat
         additional_dependencies: [
@@ -47,7 +43,7 @@ repos:
         args: [--wrap, "88"]
         files: (docs/.)
 -   repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.22
     hooks:
     -   id: mdformat
         additional_dependencies: [
@@ -57,7 +53,7 @@ repos:
         args: [--wrap, "88"]
         files: (README\.md)
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.2.6
+  rev: v2.4.1
   hooks:
     -   id: codespell
 -   repo: meta

.readthedocs.yaml

@@ -1,17 +1,18 @@
 version: 2
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.10"
+    python: "3.12"
+  jobs:
+    create_environment:
+      - asdf plugin add uv
+      - asdf install uv latest
+      - asdf global uv latest
+      - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --group docs
+    install:
+      - "true"
 
 sphinx:
   configuration: docs/source/conf.py
   fail_on_warning: true
-
-python:
-  install:
-  - method: pip
-    path: .
-    extra_requirements:
-     - docs

docs/source/api.md

@@ -0,0 +1,15 @@
+# API
+
+```{eval-rst}
+.. currentmodule:: pytask_parallel
+
+.. autoclass:: ParallelBackend
+.. autoclass:: ParallelBackendRegistry
+   :members:
+.. autoclass:: WorkerType
+.. autodata:: registry
+
+   An instantiated :class:`~pytask_parallel.ParallelBackendRegistry` to register or
+   overwrite parallel backends.
+
+```

docs/source/changes.md

@@ -5,7 +5,11 @@ chronological order. Releases follow [semantic versioning](https://semver.org/)
 releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and
 [Anaconda.org](https://anaconda.org/conda-forge/pytask-parallel).
 
-## 0.5.0 - 2024-xx-xx
+## 0.5.1 - 2025-03-09
+
+- {pull}`114` drops support for Python 3.8 and adds support for Python 3.13.
+
+## 0.5.0 - 2024-05-26
 
 - {pull}`85` simplifies code since loky is a dependency.
 - {pull}`86` adds support for dask.
@@ -29,6 +33,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and
 - {pull}`106` fixes {pull}`99` such that only when there are coiled functions, all ready
   tasks are submitted.
 - {pull}`107` removes status from `pytask_execute_task_log_start` hook call.
+- {pull}`109` improves the documentation.
+- {pull}`110` prepares the release of v0.5.
 
 ## 0.4.1 - 2024-01-12
 

docs/source/coiled.md

@@ -58,8 +58,13 @@ pytask -n 2
 pytask --parallel-backend loky
 ```
 
+```{note}
+When you build a project using coiled, you will see a message after pytask's startup
+that coiled is creating the remote software environment which takes 1-2m.
+```
+
 When you apply the {func}`@task <pytask.task>` decorator to the task, make sure the
-{func}`@coiled.function <coiled.function>` decorator is applied first, or is closer to
+{func}`@coiled.function <coiled.function>` decorator is applied first or is closer to
 the function. Otherwise, it will be ignored. Add more arguments to the decorator to
 configure the hardware and software environment.
 
@@ -71,7 +76,7 @@ By default, {func}`@coiled.function <coiled.function>`
 to the workload. It means that coiled infers from the number of submitted tasks and
 previous runtimes, how many additional remote workers it should deploy to handle the
 workload. It provides a convenient mechanism to scale without intervention. Also,
-workers launched by {func}`@coiled.function <coiled.function>` will shutdown quicker
+workers launched by {func}`@coiled.function <coiled.function>` will shut down quicker
 than a cluster.
 
 ```{seealso}
@@ -88,7 +93,8 @@ coiled. Usually, it is not necessary and you are better off using coiled's serve
 functions.
 
 If you want to launch a cluster managed by coiled, register a function that builds an
-executor using {class}`coiled.Cluster`.
+executor using {class}`coiled.Cluster`. Assign a name to the cluster to reuse it when
+you build your project again and the cluster has not been shut down.
 
 ```python
 import coiled
@@ -98,7 +104,11 @@ from concurrent.futures import Executor
 
 
 def _build_coiled_executor(n_workers: int) -> Executor:
-    return coiled.Cluster(n_workers=n_workers).get_client().get_executor()
+    return (
+        coiled.Cluster(n_workers=n_workers, name="coiled-project")
+        .get_client()
+        .get_executor()
+    )
 
 
 registry.register_parallel_backend(ParallelBackend.CUSTOM, _build_coiled_executor)
@@ -109,3 +119,12 @@ Then, execute your workflow with
 ```console
 pytask --parallel-backend custom
 ```
+
+## Tips
+
+When you are changing your project during executions and your cluster is still up and
+running, the local and the remote software environment can get out of sync. Then, you
+see errors in remote workers you have fixed locally.
+
+A quick solution is to stop the cluster in the coiled dashboard and create a new one
+with the next `pytask build`.

docs/source/conf.py

@@ -18,7 +18,7 @@
 import pytask_parallel
 
 if TYPE_CHECKING:
-    import sphinx
+    import sphinx  # ty: ignore[unresolved-import]
 
 
 # -- Project information ---------------------------------------------------------------
@@ -30,7 +30,7 @@
 # The version, including alpha/beta/rc tags, but not commit hash and datestamps
 release = version("pytask_parallel")
 # The short X.Y version.
-version = ".".join(release.split(".")[:2])
+version = ".".join(release.split(".")[:2])  # ty: ignore[invalid-assignment]
 
 # -- General configuration -------------------------------------------------------------
 
@@ -100,7 +100,7 @@
 
 
 # Linkcode, based on numpy doc/source/conf.py
-def linkcode_resolve(domain: str, info: dict[str, str]) -> str:  # noqa: C901, PLR0912
+def linkcode_resolve(domain: str, info: dict[str, str]) -> str | None:  # noqa: C901
     """Determine the URL corresponding to Python object."""
     if domain != "py":
         return None
@@ -123,10 +123,10 @@ def linkcode_resolve(domain: str, info: dict[str, str]) -> str:  # noqa: C901, P
             return None
 
     try:
-        fn = inspect.getsourcefile(inspect.unwrap(obj))
+        fn = inspect.getsourcefile(inspect.unwrap(obj))  # ty: ignore[invalid-argument-type]
     except TypeError:
         try:  # property
-            fn = inspect.getsourcefile(inspect.unwrap(obj.fget))
+            fn = inspect.getsourcefile(inspect.unwrap(obj.fget))  # ty: ignore[possibly-unbound-attribute,invalid-argument-type]
         except (AttributeError, TypeError):
             fn = None
     if not fn:
@@ -136,7 +136,7 @@ def linkcode_resolve(domain: str, info: dict[str, str]) -> str:  # noqa: C901, P
         source, lineno = inspect.getsourcelines(obj)
     except TypeError:
         try:  # property
-            source, lineno = inspect.getsourcelines(obj.fget)
+            source, lineno = inspect.getsourcelines(obj.fget)  # ty: ignore[possibly-unbound-attribute]
         except (AttributeError, TypeError):
             lineno = None
     except OSError:
@@ -202,7 +202,7 @@ def linkcode_resolve(domain: str, info: dict[str, str]) -> str:  # noqa: C901, P
 }
 
 
-def setup(app: sphinx.application.Sphinx) -> None:
+def setup(app: sphinx.application.Sphinx) -> None:  # ty: ignore[unresolved-attribute]
     """Configure sphinx."""
     app.add_object_type(
         "confval",

docs/source/custom_executors.md

@@ -1,33 +1,35 @@
 # Custom Executors
 
-```{caution}
-The interface for custom executors is rudimentary right now. Please, give some feedback
-if you managed to implement a custom executor or have suggestions for improvement.
-
-Please, also consider contributing your executor to pytask-parallel if you believe it
+```{note}
+Please, consider contributing your executor to pytask-parallel if you believe it
 could be helpful to other people. Start by creating an issue or a draft PR.
 ```
 
-pytask-parallel allows you to use your parallel backend as long as it follows the
-interface defined by {class}`~concurrent.futures.Executor`.
+pytask-parallel allows you to use any parallel backend as long as it follows the
+interface defined by {class}`concurrent.futures.Executor`.
 
 In some cases, adding a new backend can be as easy as registering a builder function
-that receives some arguments (currently only `n_workers`) and returns the instantiated
-executor.
+that receives `n_workers` and returns the instantiated executor.
+
+```{important}
+Place the following code in any module that will be imported when you are executing
+pytask. For example, the `src/project/config.py` in your project, the
+`src/project/__init__.py` or the task module directly.
+```
 
 ```{literalinclude} ../../docs_src/custom_executors.py
 ```
 
-Given {class}`pytask_parallel.WorkerType` pytask applies automatic wrappers around the
-task function to collect tracebacks, capture stdout/stderr and their like. The `remote`
-keyword allows pytask to handle local paths automatically for remote clusters.
+Given the optional {class}`~pytask_parallel.WorkerType` pytask applies automatic
+wrappers around the task function to collect tracebacks, capture stdout/stderr and their
+like. Possible values are `WorkerType.PROCESSES` (default) or `WorkerType.THREADS`.
 
-Now, build the project requesting your custom backend.
+The `remote` keyword signals pytask that tasks are executed in remote workers without
+access to the local filesystem. pytask will then automatically sync local files to the
+workers. By default, pytask assumes workers have access to the local filesystem.
+
+Now, build the project with your custom backend.
 
 ```console
 pytask --parallel-backend custom
 ```
-
-```{important}
-pytask applies automatic wrappers
-```