Modernize the package. (#119)

tobiasraabe · web-flow · commit 225cb1982e52 · 2025-07-27T10:58:50.000+02:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -5,8 +5,6 @@ concurrency:
   group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
-env:
-  CONDA_EXE: mamba
 
 on:
   push:
@@ -25,13 +23,13 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v6
         with:
-          python-version-file: .python-version
-          allow-prereleases: true
-          cache: pip
-      - run: pip install tox-uv
-      - run: tox -e typing
+          enable-cache: true
+      - name: Install just
+        uses: extractions/setup-just@v2
+      - name: Run type checking
+        run: just typing
 
   run-tests:
 
@@ -46,29 +44,14 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ matrix.python-version }}
-          cache: pip
-          allow-prereleases: true
-      - run: pip install tox
+          enable-cache: true
+      - name: Install just
+        uses: extractions/setup-just@v2
+      - name: Run tests
+        run: just test
 
-      # Unit, integration, and end-to-end tests.
-
-      - name: Run unit tests and doctests.
-        shell: bash -l {0}
-        run: tox -e test -- tests -m "unit or (not integration and not end_to_end)" --cov=src --cov=tests --cov-report=xml
-
-      - name: Upload unit test coverage reports to Codecov with GitHub Action
+      - name: Upload test coverage reports to Codecov with GitHub Action
         uses: codecov/codecov-action@v5
-        with:
-          flags: unit
-
-      - name: Run end-to-end tests.
-        shell: bash -l {0}
-        run: tox -e test -- tests -m end_to_end --cov=src --cov=tests --cov-report=xml
-
-      - name: Upload end_to_end test coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v5
-        with:
-          flags: end_to_end
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -27,10 +27,6 @@ repos:
     hooks:
       - id: ruff
       - id: ruff-format
--   repo: https://github.com/dosisod/refurb
-    rev: v2.1.0
-    hooks:
-    -   id: refurb
 -   repo: https://github.com/kynan/nbstripout
     rev: 0.8.1
     hooks:
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -1,17 +1,18 @@
 version: 2
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.10"
+    python: "3.12"
+  jobs:
+    create_environment:
+      - asdf plugin add uv
+      - asdf install uv latest
+      - asdf global uv latest
+      - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --group docs
+    install:
+      - "true"
 
 sphinx:
   configuration: docs/source/conf.py
   fail_on_warning: true
-
-python:
-  install:
-  - method: pip
-    path: .
-    extra_requirements:
-     - docs
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -18,7 +18,7 @@
 import pytask_parallel
 
 if TYPE_CHECKING:
-    import sphinx
+    import sphinx  # ty: ignore[unresolved-import]
 
 
 # -- Project information ---------------------------------------------------------------
@@ -30,7 +30,7 @@
 # The version, including alpha/beta/rc tags, but not commit hash and datestamps
 release = version("pytask_parallel")
 # The short X.Y version.
-version = ".".join(release.split(".")[:2])
+version = ".".join(release.split(".")[:2])  # ty: ignore[invalid-assignment]
 
 # -- General configuration -------------------------------------------------------------
 
@@ -100,7 +100,7 @@
 
 
 # Linkcode, based on numpy doc/source/conf.py
-def linkcode_resolve(domain: str, info: dict[str, str]) -> str:  # noqa: C901
+def linkcode_resolve(domain: str, info: dict[str, str]) -> str | None:  # noqa: C901
     """Determine the URL corresponding to Python object."""
     if domain != "py":
         return None
@@ -123,10 +123,10 @@ def linkcode_resolve(domain: str, info: dict[str, str]) -> str:  # noqa: C901
             return None
 
     try:
-        fn = inspect.getsourcefile(inspect.unwrap(obj))
+        fn = inspect.getsourcefile(inspect.unwrap(obj))  # ty: ignore[invalid-argument-type]
     except TypeError:
         try:  # property
-            fn = inspect.getsourcefile(inspect.unwrap(obj.fget))
+            fn = inspect.getsourcefile(inspect.unwrap(obj.fget))  # ty: ignore[possibly-unbound-attribute,invalid-argument-type]
         except (AttributeError, TypeError):
             fn = None
     if not fn:
@@ -136,7 +136,7 @@ def linkcode_resolve(domain: str, info: dict[str, str]) -> str:  # noqa: C901
         source, lineno = inspect.getsourcelines(obj)
     except TypeError:
         try:  # property
-            source, lineno = inspect.getsourcelines(obj.fget)
+            source, lineno = inspect.getsourcelines(obj.fget)  # ty: ignore[possibly-unbound-attribute]
         except (AttributeError, TypeError):
             lineno = None
     except OSError:
@@ -202,7 +202,7 @@ def linkcode_resolve(domain: str, info: dict[str, str]) -> str:  # noqa: C901
 }
 
 
-def setup(app: sphinx.application.Sphinx) -> None:
+def setup(app: sphinx.application.Sphinx) -> None:  # ty: ignore[unresolved-attribute]
     """Configure sphinx."""
     app.add_object_type(
         "confval",
diff --git a/docs_src/custom_executors.py b/docs_src/custom_executors.py
@@ -1,6 +1,6 @@
 from concurrent.futures import Executor
 
-from my_project.executor import CustomExecutor
+from my_project.executor import CustomExecutor  # ty: ignore[unresolved-import]
 
 from pytask_parallel import ParallelBackend
 from pytask_parallel import WorkerType
diff --git a/justfile b/justfile
@@ -0,0 +1,34 @@
+# Install all dependencies
+install:
+    uv sync --all-groups
+
+# Run tests
+test *args="":
+    uv run --group test pytest --cov=src --cov=tests --cov-report=xml {{args}}
+
+# Run tests with lowest dependency resolution
+test-lowest *args="":
+    uv run --group test --resolution lowest-direct pytest {{args}}
+
+# Run tests with highest dependency resolution
+test-highest *args="":
+    uv run --group test --resolution highest pytest {{args}}
+
+# Run type checking
+typing:
+    uv run --group typing --group test --isolated ty check
+
+# Run linting and formatting
+lint:
+    uvx --with pre-commit-uv pre-commit run -a
+
+# Build documentation
+docs:
+    uv run --group docs sphinx-build docs/source docs/build
+
+# Serve documentation with auto-reload
+docs-serve:
+    uv run --group docs sphinx-autobuild docs/source docs/build
+
+# Run all checks (format, lint, typing, test)
+check: lint typing test
diff --git a/pyproject.toml b/pyproject.toml
@@ -11,7 +11,7 @@ classifiers = [
 requires-python = ">=3.9"
 dependencies = [
     "attrs>=21.3.0",
-    "click",
+    "click>=8.1.8,!=8.2.0",
     "cloudpickle",
     "loky",
     "pluggy>=1.0.0",
@@ -24,7 +24,7 @@ dynamic = ["version"]
 name = "Tobias Raabe"
 email = "raabe@posteo.de"
 
-[project.optional-dependencies]
+[dependency-groups]
 coiled = ["coiled>=0.9.4"]
 dask = ["dask[complete]", "distributed"]
 docs = [
@@ -34,13 +34,26 @@ docs = [
     "myst-parser",
     "nbsphinx",
     "sphinx",
+    "sphinx-autobuild",
     "sphinx-click",
     "sphinx-copybutton",
     "sphinx-design>=0.3",
     "sphinx-toolbox",
     "sphinxext-opengraph",
 ]
-test = ["pytask-parallel[coiled,dask]", "nbmake", "pytest", "pytest-cov"]
+test = [
+    "nbmake",
+    "pytest>=8.4.0",
+    "pytest-cov>=5.0.0",
+    {include-group = "coiled"},
+    {include-group = "dask"},
+]
+typing = [
+    "pytask-parallel",
+    "ty",
+    {include-group = "coiled"},
+    {include-group = "dask"},
+]
 
 [project.readme]
 file = "README.md"
@@ -63,14 +76,6 @@ pytask_parallel = "pytask_parallel.plugin"
 requires = ["hatchling", "hatch_vcs"]
 build-backend = "hatchling.build"
 
-[tool.rye]
-managed = true
-dev-dependencies = ["s3fs>=2024.3.1", "tox-uv>=1.7.0"]
-
-[tool.rye.scripts]
-clean-docs = { cmd = "rm -rf docs/build" }
-build-docs = { cmd = "sphinx-build -b html docs/source docs/build" }
-
 [tool.hatch.build.hooks.vcs]
 version-file = "src/pytask_parallel/_version.py"
 
@@ -88,24 +93,7 @@ source = "vcs"
 [tool.hatch.metadata]
 allow-direct-references = true
 
-[tool.mypy]
-files = ["src", "tests"]
-check_untyped_defs = true
-disallow_any_generics = true
-disallow_incomplete_defs = true
-disallow_untyped_defs = true
-no_implicit_optional = true
-warn_redundant_casts = true
-warn_unused_ignores = true
-ignore_missing_imports = true
-
-[[tool.mypy.overrides]]
-module = "tests.*"
-disallow_untyped_defs = false
-ignore_errors = true
-
 [tool.ruff]
-target-version = "py39"
 fix = true
 unsafe-fixes = true
 
@@ -132,13 +120,6 @@ convention = "numpy"
 addopts = ["--nbmake"]
 # Do not add src since it messes with the loading of pytask-parallel as a plugin.
 testpaths = ["tests"]
-markers = [
-    "wip: Tests that are work-in-progress.",
-    "unit: Flag for unit tests which target mainly a single function.",
-    "integration: Flag for integration tests which may comprise of multiple unit tests.",
-    "end_to_end: Flag for tests that cover the whole program.",
-]
-norecursedirs = [".idea", ".tox"]
 
 [tool.coverage.report]
 exclude_also = [
diff --git a/src/pytask_parallel/backends.py b/src/pytask_parallel/backends.py
@@ -47,10 +47,13 @@ def submit(
 def _get_dask_executor(n_workers: int) -> Executor:
     """Get an executor from a dask client."""
     _rich_traceback_guard = True
-    from pytask import import_optional_dependency  # noqa: PLC0415
 
-    distributed = import_optional_dependency("distributed")
-    assert distributed  # noqa: S101
+    try:
+        import distributed  # noqa: PLC0415
+    except ImportError:
+        msg = "The distributed package is not installed. Please install it."
+        raise ImportError(msg) from None
+
     try:
         client = distributed.Client.current()
     except ValueError:
diff --git a/src/pytask_parallel/execute.py b/src/pytask_parallel/execute.py
@@ -209,7 +209,7 @@ def pytask_execute_task(session: Session, task: PTask) -> Future[WrapperResult]:
         task_module = get_module(task.function, getattr(task, "path", None))
         cloudpickle.register_pickle_by_value(task_module)
 
-        return wrapper_func.submit(
+        return wrapper_func.submit(  # ty: ignore[possibly-unbound-attribute,invalid-return-type]
             task=task,
             console_options=console.options,
             kwargs=kwargs,
diff --git a/src/pytask_parallel/wrappers.py b/src/pytask_parallel/wrappers.py
@@ -192,8 +192,8 @@ def _patch_set_trace_and_breakpoint() -> None:
     import pdb  # noqa: PLC0415, T100
     import sys  # noqa: PLC0415
 
-    pdb.set_trace = _raise_exception_on_breakpoint
-    sys.breakpointhook = _raise_exception_on_breakpoint
+    pdb.set_trace = _raise_exception_on_breakpoint  # ty: ignore[invalid-assignment]
+    sys.breakpointhook = _raise_exception_on_breakpoint  # ty: ignore[invalid-assignment]
 
 
 def _render_traceback_to_string(
@@ -205,7 +205,7 @@ def _render_traceback_to_string(
     traceback = Traceback(exc_info, show_locals=show_locals)
     segments = console.render(traceback, options=console_options)
     text = "".join(segment.text for segment in segments)
-    return (*exc_info[:2], text)
+    return (*exc_info[:2], text)  # ty: ignore[invalid-return-type]
 
 
 def _handle_function_products(
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -79,3 +79,10 @@ def pytest_collection_modifyitems(session, config, items) -> None:  # noqa: ARG0
     for item in items:
         if isinstance(item, NotebookItem):
             item.add_marker(pytest.mark.xfail(reason="The tests are flaky."))
+
+
+skip_if_deadlock = pytest.mark.skipif(
+    (sys.version_info[:2] in [(3, 12), (3, 13)] and sys.platform == "win32")
+    or (sys.version_info[:2] == (3, 13) and sys.platform == "linux"),
+    reason="Deadlock in loky/backend/resource_tracker.py, line 181, maybe related to https://github.com/joblib/loky/pull/450",
+)
diff --git a/tests/test_backends.py b/tests/test_backends.py
@@ -1,22 +1,19 @@
 import textwrap
 
-import pytest
 from pytask import ExitCode
 from pytask import cli
 
 from pytask_parallel import ParallelBackend
 from pytask_parallel import registry
 
 
-@pytest.mark.end_to_end
 def test_error_requesting_custom_backend_without_registration(runner, tmp_path):
     tmp_path.joinpath("task_example.py").write_text("def task_example(): pass")
     result = runner.invoke(cli, [tmp_path.as_posix(), "--parallel-backend", "custom"])
     assert result.exit_code == ExitCode.FAILED
     assert "No registered parallel backend found" in result.output
 
 
-@pytest.mark.end_to_end
 def test_error_while_instantiating_custom_backend(runner, tmp_path):
     hook_source = """
     from pytask_parallel import ParallelBackend, registry
@@ -35,7 +32,6 @@ def task_example(): pass
     assert "Could not instantiate parallel backend 'custom'." in result.output
 
 
-@pytest.mark.end_to_end
 def test_register_custom_backend(runner, tmp_path):
     source = """
     from loky import get_reusable_executor
diff --git a/tests/test_capture.py b/tests/test_capture.py
diff --git a/tests/test_config.py b/tests/test_config.py
diff --git a/tests/test_execute.py b/tests/test_execute.py
diff --git a/tests/test_jupyter/test_functional_interface.ipynb b/tests/test_jupyter/test_functional_interface.ipynb
diff --git a/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb
diff --git a/tests/test_remote.py b/tests/test_remote.py
