diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5bfb4cebf..ebc658db8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,15 @@ repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.287 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black rev: 22.3.0 hooks: - id: black language_version: python3 exclude: versioneer.py - - repo: https://github.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - language_version: python3 - repo: https://github.com/rapidsai/frigate/ rev: v0.4.0 # pre-commit autoupdate - to keep the version up to date hooks: diff --git a/ci/pre-commit-crd.py b/ci/pre-commit-crd.py index c16212ba1..b1fadd862 100755 --- a/ci/pre-commit-crd.py +++ b/ci/pre-commit-crd.py @@ -1,10 +1,10 @@ #!/usr/bin/env python import os import pathlib -import tempfile +import shutil import subprocess import sys -import shutil +import tempfile ROOT_DIR = pathlib.Path(__file__).parent.parent.absolute() diff --git a/dask_kubernetes/__init__.py b/dask_kubernetes/__init__.py index f277bd661..941ff6629 100644 --- a/dask_kubernetes/__init__.py +++ b/dask_kubernetes/__init__.py @@ -3,15 +3,15 @@ from . import config from .common.auth import ( + AutoRefreshConfiguration, + AutoRefreshKubeConfigLoader, ClusterAuth, + InCluster, KubeAuth, KubeConfig, - InCluster, - AutoRefreshKubeConfigLoader, - AutoRefreshConfiguration, ) +from .common.objects import clean_pod_template, make_pod_from_dict, make_pod_spec from .helm import HelmCluster -from .common.objects import make_pod_spec, make_pod_from_dict, clean_pod_template __all__ = ["HelmCluster", "KubeCluster"] diff --git a/dask_kubernetes/classic/__init__.py b/dask_kubernetes/classic/__init__.py index bb63f5686..d1aaad827 100644 --- a/dask_kubernetes/classic/__init__.py +++ b/dask_kubernetes/classic/__init__.py @@ -1,2 +1,2 @@ -from .kubecluster import KubeCluster from ..common.objects import make_pod_spec +from .kubecluster import KubeCluster diff --git a/dask_kubernetes/classic/kubecluster.py b/dask_kubernetes/classic/kubecluster.py index efd9d2ddf..6f5b8a299 100644 --- a/dask_kubernetes/classic/kubecluster.py +++ b/dask_kubernetes/classic/kubecluster.py @@ -8,31 +8,31 @@ import warnings import aiohttp -import yaml import dask import dask.distributed import distributed.security -from distributed.deploy import SpecCluster, ProcessInterface -from distributed.utils import format_dashboard_link, Log, Logs import kubernetes_asyncio as kubernetes +import yaml +from distributed.deploy import ProcessInterface, SpecCluster +from distributed.utils import Log, Logs, format_dashboard_link from kubernetes_asyncio.client.rest import ApiException +from ..common.auth import ClusterAuth +from ..common.networking import ( + get_external_address_for_scheduler_service, + get_scheduler_address, +) from ..common.objects import ( - make_pod_from_dict, - make_service_from_dict, - make_pdb_from_dict, + clean_pdb_template, clean_pod_template, clean_service_template, - clean_pdb_template, + make_pdb_from_dict, + make_pod_from_dict, + make_service_from_dict, ) -from ..common.auth import ClusterAuth from ..common.utils import ( - get_current_namespace, escape, -) -from ..common.networking import ( - get_external_address_for_scheduler_service, - get_scheduler_address, + get_current_namespace, ) logger = logging.getLogger(__name__) diff --git a/dask_kubernetes/classic/tests/test_async.py b/dask_kubernetes/classic/tests/test_async.py index 5ea317d20..b6c2a6201 100644 --- a/dask_kubernetes/classic/tests/test_async.py +++ b/dask_kubernetes/classic/tests/test_async.py @@ -3,29 +3,28 @@ import getpass import os import random -from time import time -import yaml import sys +from time import time +import dask import kubernetes_asyncio as kubernetes import pytest - -import dask +import yaml from dask.distributed import Client, wait +from dask.utils import tmpfile +from distributed.utils_test import captured_logger + import dask_kubernetes from dask_kubernetes import ( - KubeCluster, - make_pod_spec, - clean_pod_template, ClusterAuth, - KubeConfig, KubeAuth, + KubeCluster, + KubeConfig, + clean_pod_template, + make_pod_spec, ) -from dask.utils import tmpfile -from distributed.utils_test import captured_logger - -from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME from dask_kubernetes.common.utils import get_current_namespace +from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME TEST_DIR = os.path.abspath(os.path.join(__file__, "..")) CONFIG_DEMO = os.path.join(TEST_DIR, "config-demo.yaml") diff --git a/dask_kubernetes/classic/tests/test_sync.py b/dask_kubernetes/classic/tests/test_sync.py index 7bd2ea6b5..ce7fed9d1 100644 --- a/dask_kubernetes/classic/tests/test_sync.py +++ b/dask_kubernetes/classic/tests/test_sync.py @@ -1,12 +1,12 @@ import os from time import sleep, time -import yaml import dask import pytest +import yaml from dask.distributed import Client, wait -from distributed.utils_test import captured_logger from dask.utils import tmpfile +from distributed.utils_test import captured_logger from dask_kubernetes.classic import KubeCluster, make_pod_spec from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME @@ -275,10 +275,10 @@ def test_pod_template_with_custom_container_name(docker_image): def test_bad_args(): - with pytest.raises(FileNotFoundError) as info: + with pytest.raises(FileNotFoundError): KubeCluster("myfile.yaml") - with pytest.raises((ValueError, TypeError, AttributeError)) as info: + with pytest.raises((ValueError, TypeError, AttributeError)): KubeCluster({"kind": "Pod"}) diff --git a/dask_kubernetes/cli/cli.py b/dask_kubernetes/cli/cli.py index 28c2c107a..0237db3e2 100644 --- a/dask_kubernetes/cli/cli.py +++ b/dask_kubernetes/cli/cli.py @@ -1,10 +1,11 @@ -import click -import yaml import json import time + +import click +import yaml from rich.console import Console -from dask_kubernetes.operator import make_cluster_spec, KubeCluster +from dask_kubernetes.operator import KubeCluster, make_cluster_spec console = Console() @@ -86,7 +87,7 @@ def cluster(**kwargs): @main.command(help="Port-forward the scheduler of a DaskCluster resource") @click.argument("cluster") def port_forward(cluster): - with console.status(f"Connecting to cluster {cluster}") as status: + with console.status(f"Connecting to cluster {cluster}"): try: kcluster = KubeCluster.from_name( cluster, shutdown_on_close=False, quiet=True diff --git a/dask_kubernetes/common/auth.py b/dask_kubernetes/common/auth.py index ee80ed113..1ebdff89f 100644 --- a/dask_kubernetes/common/auth.py +++ b/dask_kubernetes/common/auth.py @@ -12,11 +12,10 @@ import kubernetes import kubernetes_asyncio - from kubernetes_asyncio.client import Configuration -from kubernetes_asyncio.config.kube_config import KubeConfigLoader, KubeConfigMerger -from kubernetes_asyncio.config.google_auth import google_auth_credentials from kubernetes_asyncio.config.dateutil import parse_rfc3339 +from kubernetes_asyncio.config.google_auth import google_auth_credentials +from kubernetes_asyncio.config.kube_config import KubeConfigLoader, KubeConfigMerger logger = logging.getLogger(__name__) diff --git a/dask_kubernetes/common/networking.py b/dask_kubernetes/common/networking.py index 04886a8d3..30dc3ff9f 100644 --- a/dask_kubernetes/common/networking.py +++ b/dask_kubernetes/common/networking.py @@ -1,16 +1,16 @@ import asyncio -from contextlib import suppress import random import socket import subprocess import time +from contextlib import suppress from weakref import finalize -import kubernetes_asyncio as kubernetes -from tornado.iostream import StreamClosedError import kr8s -from kr8s.asyncio.objects import Pod +import kubernetes_asyncio as kubernetes from distributed.core import rpc +from kr8s.asyncio.objects import Pod +from tornado.iostream import StreamClosedError from dask_kubernetes.common.utils import check_dependency from dask_kubernetes.exceptions import CrashLoopBackOffError diff --git a/dask_kubernetes/common/objects.py b/dask_kubernetes/common/objects.py index 99cf1ae4a..44c90fd56 100644 --- a/dask_kubernetes/common/objects.py +++ b/dask_kubernetes/common/objects.py @@ -1,11 +1,11 @@ """ Convenience functions for creating pod templates. """ -from collections import namedtuple import copy -from kubernetes import client import json +from collections import namedtuple +from kubernetes import client from kubernetes.client.configuration import Configuration from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME diff --git a/dask_kubernetes/common/tests/test_kind.py b/dask_kubernetes/common/tests/test_kind.py index d7241593b..2af77bfe6 100644 --- a/dask_kubernetes/common/tests/test_kind.py +++ b/dask_kubernetes/common/tests/test_kind.py @@ -1,7 +1,7 @@ -import pytest from subprocess import check_output import kubernetes_asyncio as kubernetes +import pytest from dask_kubernetes.common.auth import ClusterAuth from dask_kubernetes.common.utils import get_current_namespace diff --git a/dask_kubernetes/common/tests/test_objects.py b/dask_kubernetes/common/tests/test_objects.py index 72fac2da7..61ef99b3f 100644 --- a/dask_kubernetes/common/tests/test_objects.py +++ b/dask_kubernetes/common/tests/test_objects.py @@ -1,5 +1,5 @@ -from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME from dask_kubernetes.common.objects import make_pod_from_dict +from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME def test_make_pod_from_dict(): diff --git a/dask_kubernetes/config.py b/dask_kubernetes/config.py index 8e8136a85..7e53a40b3 100644 --- a/dask_kubernetes/config.py +++ b/dask_kubernetes/config.py @@ -1,11 +1,10 @@ -from __future__ import print_function, division, absolute_import +from __future__ import absolute_import, division, print_function import os import dask import yaml - fn = os.path.join(os.path.dirname(__file__), "kubernetes.yaml") dask.config.ensure_file(source=fn) diff --git a/dask_kubernetes/conftest.py b/dask_kubernetes/conftest.py index 087b9a321..454802849 100644 --- a/dask_kubernetes/conftest.py +++ b/dask_kubernetes/conftest.py @@ -1,12 +1,11 @@ -import pytest - import logging -import pathlib import os +import pathlib import subprocess import tempfile import uuid +import pytest from kopf.testing import KopfRunner from pytest_kind.cluster import KindCluster diff --git a/dask_kubernetes/helm/helmcluster.py b/dask_kubernetes/helm/helmcluster.py index e4fa1318a..b3390b664 100644 --- a/dask_kubernetes/helm/helmcluster.py +++ b/dask_kubernetes/helm/helmcluster.py @@ -1,21 +1,21 @@ import asyncio -import aiohttp +import json import subprocess import warnings from contextlib import suppress -import json +import aiohttp +import kubernetes_asyncio as kubernetes +from distributed.core import Status, rpc from distributed.deploy import Cluster -from distributed.core import rpc, Status from distributed.utils import Log, Logs -import kubernetes_asyncio as kubernetes from ..common.auth import ClusterAuth +from ..common.networking import get_external_address_for_scheduler_service from ..common.utils import ( - get_current_namespace, check_dependency, + get_current_namespace, ) -from ..common.networking import get_external_address_for_scheduler_service class HelmCluster(Cluster): diff --git a/dask_kubernetes/helm/tests/test_helm.py b/dask_kubernetes/helm/tests/test_helm.py index abbd1a350..0ac9e824a 100644 --- a/dask_kubernetes/helm/tests/test_helm.py +++ b/dask_kubernetes/helm/tests/test_helm.py @@ -1,16 +1,15 @@ -import pytest - -import subprocess import os.path +import subprocess import dask.config -from distributed import Client -from distributed.core import Status +import pytest from dask_ctl.discovery import ( - list_discovery_methods, discover_cluster_names, discover_clusters, + list_discovery_methods, ) +from distributed import Client +from distributed.core import Status ############### # Fixtures @@ -131,9 +130,10 @@ def sync_cluster(k8s_cluster, release, test_namespace): def test_import(): - from dask_kubernetes import HelmCluster from distributed.deploy import Cluster + from dask_kubernetes import HelmCluster + assert issubclass(HelmCluster, Cluster) diff --git a/dask_kubernetes/operator/_objects.py b/dask_kubernetes/operator/_objects.py index 3f7ce6f00..79b96f333 100644 --- a/dask_kubernetes/operator/_objects.py +++ b/dask_kubernetes/operator/_objects.py @@ -1,7 +1,8 @@ from __future__ import annotations + from typing import List -from kr8s.asyncio.objects import APIObject, Pod, Deployment, Service +from kr8s.asyncio.objects import APIObject, Deployment, Pod, Service class DaskCluster(APIObject): diff --git a/dask_kubernetes/operator/controller/controller.py b/dask_kubernetes/operator/controller/controller.py index 5cc38c484..00c47aeed 100644 --- a/dask_kubernetes/operator/controller/controller.py +++ b/dask_kubernetes/operator/controller/controller.py @@ -1,26 +1,26 @@ import asyncio -from collections import defaultdict import time +from collections import defaultdict from contextlib import suppress from datetime import datetime from uuid import uuid4 import aiohttp +import dask.config import kopf import kr8s -from kr8s.asyncio.objects import Pod, Deployment, Service +from distributed.core import clean_exception, rpc +from distributed.protocol.pickle import dumps from importlib_metadata import entry_points +from kr8s.asyncio.objects import Deployment, Pod, Service from dask_kubernetes.operator._objects import ( - DaskCluster, DaskAutoscaler, - DaskWorkerGroup, + DaskCluster, DaskJob, + DaskWorkerGroup, ) from dask_kubernetes.operator.networking import get_scheduler_address -from distributed.core import rpc, clean_exception -from distributed.protocol.pickle import dumps -import dask.config _ANNOTATION_NAMESPACES_TO_IGNORE = ( "kopf.zalando.org", @@ -840,7 +840,7 @@ async def daskcluster_autoshutdown(spec, name, namespace, logger, **kwargs): namespace=namespace, logger=logger, ) - except Exception as e: + except Exception: logger.warn("Unable to connect to scheduler, skipping autoshutdown check.") return if idle_since and time.time() > idle_since + spec["idleTimeout"]: diff --git a/dask_kubernetes/operator/controller/tests/test_controller.py b/dask_kubernetes/operator/controller/tests/test_controller.py index 7e0b772ac..0bc3b0997 100644 --- a/dask_kubernetes/operator/controller/tests/test_controller.py +++ b/dask_kubernetes/operator/controller/tests/test_controller.py @@ -5,17 +5,17 @@ from contextlib import asynccontextmanager from datetime import datetime, timedelta +import dask.config import pytest import yaml from dask.distributed import Client -import dask.config +from kr8s.asyncio.objects import Deployment, Pod, Service -from kr8s.asyncio.objects import Pod, Deployment, Service +from dask_kubernetes.operator._objects import DaskCluster, DaskJob, DaskWorkerGroup from dask_kubernetes.operator.controller import ( KUBERNETES_DATETIME_FORMAT, get_job_runner_pod_name, ) -from dask_kubernetes.operator._objects import DaskCluster, DaskWorkerGroup, DaskJob DIR = pathlib.Path(__file__).parent.absolute() @@ -105,7 +105,7 @@ def test_operator_plugins(kopf_runner): @pytest.mark.timeout(180) @pytest.mark.anyio async def test_simplecluster(k8s_cluster, kopf_runner, gen_cluster): - with kopf_runner as runner: + with kopf_runner: async with gen_cluster() as (cluster_name, ns): scheduler_deployment_name = "simple-scheduler" worker_pod_name = "simple-default-worker" @@ -249,7 +249,7 @@ async def test_simplecluster(k8s_cluster, kopf_runner, gen_cluster): @pytest.mark.anyio async def test_scalesimplecluster(k8s_cluster, kopf_runner, gen_cluster): - with kopf_runner as runner: + with kopf_runner: async with gen_cluster() as (cluster_name, ns): scheduler_deployment_name = "simple-scheduler" worker_pod_name = "simple-default-worker" @@ -296,7 +296,7 @@ async def test_scalesimplecluster(k8s_cluster, kopf_runner, gen_cluster): async def test_scalesimplecluster_from_cluster_spec( k8s_cluster, kopf_runner, gen_cluster ): - with kopf_runner as runner: + with kopf_runner: async with gen_cluster() as (cluster_name, ns): scheduler_deployment_name = "simple-scheduler" worker_pod_name = "simple-default-worker" @@ -341,7 +341,7 @@ async def test_scalesimplecluster_from_cluster_spec( @pytest.mark.anyio async def test_recreate_scheduler_pod(k8s_cluster, kopf_runner, gen_cluster): - with kopf_runner as runner: + with kopf_runner: async with gen_cluster() as (cluster_name, ns): scheduler_deployment_name = "simple-scheduler" worker_pod_name = "simple-default-worker" @@ -380,7 +380,7 @@ async def test_recreate_scheduler_pod(k8s_cluster, kopf_runner, gen_cluster): @pytest.mark.anyio @pytest.mark.skip(reason="Flaky in CI") async def test_recreate_worker_pods(k8s_cluster, kopf_runner, gen_cluster): - with kopf_runner as runner: + with kopf_runner: async with gen_cluster() as (cluster_name, ns): cluster = await DaskCluster.get(cluster_name, namespace=ns) # Get the default worker group @@ -411,7 +411,7 @@ async def test_recreate_worker_pods(k8s_cluster, kopf_runner, gen_cluster): async def test_simplecluster_batched_worker_deployments( k8s_cluster, kopf_runner, gen_cluster ): - with kopf_runner as runner: + with kopf_runner: with dask.config.set( { "kubernetes.controller.worker-allocation.batch-size": 1, @@ -624,7 +624,7 @@ async def test_failed_job(k8s_cluster, kopf_runner, gen_job): @pytest.mark.anyio async def test_object_dask_cluster(k8s_cluster, kopf_runner, gen_cluster): - with kopf_runner as runner: + with kopf_runner: async with gen_cluster() as (cluster_name, ns): cluster = await DaskCluster.get(cluster_name, namespace=ns) @@ -648,7 +648,7 @@ async def test_object_dask_cluster(k8s_cluster, kopf_runner, gen_cluster): @pytest.mark.anyio async def test_object_dask_worker_group(k8s_cluster, kopf_runner, gen_cluster): - with kopf_runner as runner: + with kopf_runner: async with gen_cluster() as (cluster_name, ns): cluster = await DaskCluster.get(cluster_name, namespace=ns) @@ -678,7 +678,7 @@ async def test_object_dask_worker_group(k8s_cluster, kopf_runner, gen_cluster): @pytest.mark.anyio @pytest.mark.skip(reason="Flaky in CI") async def test_object_dask_job(k8s_cluster, kopf_runner, gen_job): - with kopf_runner as runner: + with kopf_runner: async with gen_job("simplejob.yaml") as (job_name, ns): job = await DaskJob.get(job_name, namespace=ns) diff --git a/dask_kubernetes/operator/kubecluster/discovery.py b/dask_kubernetes/operator/kubecluster/discovery.py index e4e17b6fc..da31b911e 100644 --- a/dask_kubernetes/operator/kubecluster/discovery.py +++ b/dask_kubernetes/operator/kubecluster/discovery.py @@ -1,9 +1,9 @@ -from typing import Callable, AsyncIterator, Tuple +from typing import AsyncIterator, Callable, Tuple import kr8s -from dask_kubernetes.operator.kubecluster import KubeCluster from dask_kubernetes.operator._objects import DaskCluster # noqa +from dask_kubernetes.operator.kubecluster import KubeCluster async def discover() -> AsyncIterator[Tuple[str, Callable]]: diff --git a/dask_kubernetes/operator/kubecluster/tests/test_discovery.py b/dask_kubernetes/operator/kubecluster/tests/test_discovery.py index a94db1027..4ace81ba0 100644 --- a/dask_kubernetes/operator/kubecluster/tests/test_discovery.py +++ b/dask_kubernetes/operator/kubecluster/tests/test_discovery.py @@ -1,8 +1,7 @@ import pytest - from dask.distributed import Client -from dask_kubernetes.operator import KubeCluster -from dask_kubernetes.operator import discover + +from dask_kubernetes.operator import KubeCluster, discover @pytest.mark.anyio diff --git a/dask_kubernetes/operator/kubecluster/tests/test_kubecluster.py b/dask_kubernetes/operator/kubecluster/tests/test_kubecluster.py index 105231dd9..931f24b90 100644 --- a/dask_kubernetes/operator/kubecluster/tests/test_kubecluster.py +++ b/dask_kubernetes/operator/kubecluster/tests/test_kubecluster.py @@ -1,10 +1,9 @@ import pytest - from dask.distributed import Client from distributed.utils import TimeoutError -from dask_kubernetes.operator import KubeCluster, make_cluster_spec from dask_kubernetes.exceptions import SchedulerStartupError +from dask_kubernetes.operator import KubeCluster, make_cluster_spec def test_experimental_shim(): diff --git a/dask_kubernetes/operator/networking.py b/dask_kubernetes/operator/networking.py index 417e3647d..640646991 100644 --- a/dask_kubernetes/operator/networking.py +++ b/dask_kubernetes/operator/networking.py @@ -1,14 +1,14 @@ import asyncio -from contextlib import suppress import random import socket -import time import threading -from tornado.iostream import StreamClosedError +import time +from contextlib import suppress import kr8s -from kr8s.asyncio.objects import Pod, Service from distributed.core import rpc +from kr8s.asyncio.objects import Pod, Service +from tornado.iostream import StreamClosedError from dask_kubernetes.exceptions import CrashLoopBackOffError diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..50583738a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[tool.ruff] +# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E", "F", "W", "I"] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "__init__.py", + "versioneer.py", + "distributed/_concurrent_futures_thread.py", +] + +per-file-ignores = {} + +ignore = [ + "E4", # Import formatting + "E721", # Comparing types instead of isinstance + "E731", # Assigning lambda expression + "E741", # Ambiguous variable names + "F811", # redefinition of unused 'loop' from line 10 + "F841", # local variable is assigned to but never used +] + +line-length = 120 + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +# Assume Python 3.8 +target-version = "py38" diff --git a/setup.py b/setup.py index cc7c87d83..00496efa4 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ #!/usr/bin/env python from os.path import exists -from setuptools import setup, find_packages + +from setuptools import find_packages, setup import versioneer