Add ruff to pre-commit and run against all files (#815)

.pre-commit-config.yaml

@@ -1,15 +1,15 @@
 repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.287
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
   - repo: https://github.com/psf/black
     rev: 22.3.0
     hooks:
       - id: black
         language_version: python3
         exclude: versioneer.py
-  - repo: https://github.com/pycqa/flake8
-    rev: 3.9.2
-    hooks:
-      - id: flake8
-        language_version: python3
   - repo: https://github.com/rapidsai/frigate/
     rev: v0.4.0 #  pre-commit autoupdate  - to keep the version up to date
     hooks:

ci/pre-commit-crd.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 import os
 import pathlib
-import tempfile
+import shutil
 import subprocess
 import sys
-import shutil
+import tempfile
 
 ROOT_DIR = pathlib.Path(__file__).parent.parent.absolute()
 

dask_kubernetes/__init__.py

@@ -3,15 +3,15 @@
 
 from . import config
 from .common.auth import (
+    AutoRefreshConfiguration,
+    AutoRefreshKubeConfigLoader,
     ClusterAuth,
+    InCluster,
     KubeAuth,
     KubeConfig,
-    InCluster,
-    AutoRefreshKubeConfigLoader,
-    AutoRefreshConfiguration,
 )
+from .common.objects import clean_pod_template, make_pod_from_dict, make_pod_spec
 from .helm import HelmCluster
-from .common.objects import make_pod_spec, make_pod_from_dict, clean_pod_template
 
 __all__ = ["HelmCluster", "KubeCluster"]
 

dask_kubernetes/classic/__init__.py

@@ -1,2 +1,2 @@
-from .kubecluster import KubeCluster
 from ..common.objects import make_pod_spec
+from .kubecluster import KubeCluster

dask_kubernetes/classic/kubecluster.py

@@ -8,31 +8,31 @@
 import warnings
 
 import aiohttp
-import yaml
 import dask
 import dask.distributed
 import distributed.security
-from distributed.deploy import SpecCluster, ProcessInterface
-from distributed.utils import format_dashboard_link, Log, Logs
 import kubernetes_asyncio as kubernetes
+import yaml
+from distributed.deploy import ProcessInterface, SpecCluster
+from distributed.utils import Log, Logs, format_dashboard_link
 from kubernetes_asyncio.client.rest import ApiException
 
+from ..common.auth import ClusterAuth
+from ..common.networking import (
+    get_external_address_for_scheduler_service,
+    get_scheduler_address,
+)
 from ..common.objects import (
-    make_pod_from_dict,
-    make_service_from_dict,
-    make_pdb_from_dict,
+    clean_pdb_template,
     clean_pod_template,
     clean_service_template,
-    clean_pdb_template,
+    make_pdb_from_dict,
+    make_pod_from_dict,
+    make_service_from_dict,
 )
-from ..common.auth import ClusterAuth
 from ..common.utils import (
-    get_current_namespace,
     escape,
-)
-from ..common.networking import (
-    get_external_address_for_scheduler_service,
-    get_scheduler_address,
+    get_current_namespace,
 )
 
 logger = logging.getLogger(__name__)

dask_kubernetes/classic/tests/test_async.py

@@ -3,29 +3,28 @@
 import getpass
 import os
 import random
-from time import time
-import yaml
 import sys
+from time import time
 
+import dask
 import kubernetes_asyncio as kubernetes
 import pytest
-
-import dask
+import yaml
 from dask.distributed import Client, wait
+from dask.utils import tmpfile
+from distributed.utils_test import captured_logger
+
 import dask_kubernetes
 from dask_kubernetes import (
-    KubeCluster,
-    make_pod_spec,
-    clean_pod_template,
     ClusterAuth,
-    KubeConfig,
     KubeAuth,
+    KubeCluster,
+    KubeConfig,
+    clean_pod_template,
+    make_pod_spec,
 )
-from dask.utils import tmpfile
-from distributed.utils_test import captured_logger
-
-from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME
 from dask_kubernetes.common.utils import get_current_namespace
+from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME
 
 TEST_DIR = os.path.abspath(os.path.join(__file__, ".."))
 CONFIG_DEMO = os.path.join(TEST_DIR, "config-demo.yaml")

dask_kubernetes/classic/tests/test_sync.py

@@ -1,12 +1,12 @@
 import os
 from time import sleep, time
-import yaml
 
 import dask
 import pytest
+import yaml
 from dask.distributed import Client, wait
-from distributed.utils_test import captured_logger
 from dask.utils import tmpfile
+from distributed.utils_test import captured_logger
 
 from dask_kubernetes.classic import KubeCluster, make_pod_spec
 from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME
@@ -275,10 +275,10 @@ def test_pod_template_with_custom_container_name(docker_image):
 
 
 def test_bad_args():
-    with pytest.raises(FileNotFoundError) as info:
+    with pytest.raises(FileNotFoundError):
         KubeCluster("myfile.yaml")
 
-    with pytest.raises((ValueError, TypeError, AttributeError)) as info:
+    with pytest.raises((ValueError, TypeError, AttributeError)):
         KubeCluster({"kind": "Pod"})
 
 

dask_kubernetes/cli/cli.py

@@ -1,10 +1,11 @@
-import click
-import yaml
 import json
 import time
+
+import click
+import yaml
 from rich.console import Console
 
-from dask_kubernetes.operator import make_cluster_spec, KubeCluster
+from dask_kubernetes.operator import KubeCluster, make_cluster_spec
 
 console = Console()
 
@@ -86,7 +87,7 @@ def cluster(**kwargs):
 @main.command(help="Port-forward the scheduler of a DaskCluster resource")
 @click.argument("cluster")
 def port_forward(cluster):
-    with console.status(f"Connecting to cluster {cluster}") as status:
+    with console.status(f"Connecting to cluster {cluster}"):
         try:
             kcluster = KubeCluster.from_name(
                 cluster, shutdown_on_close=False, quiet=True

dask_kubernetes/common/auth.py

@@ -12,11 +12,10 @@
 
 import kubernetes
 import kubernetes_asyncio
-
 from kubernetes_asyncio.client import Configuration
-from kubernetes_asyncio.config.kube_config import KubeConfigLoader, KubeConfigMerger
-from kubernetes_asyncio.config.google_auth import google_auth_credentials
 from kubernetes_asyncio.config.dateutil import parse_rfc3339
+from kubernetes_asyncio.config.google_auth import google_auth_credentials
+from kubernetes_asyncio.config.kube_config import KubeConfigLoader, KubeConfigMerger
 
 logger = logging.getLogger(__name__)
 

dask_kubernetes/common/networking.py

@@ -1,16 +1,16 @@
 import asyncio
-from contextlib import suppress
 import random
 import socket
 import subprocess
 import time
+from contextlib import suppress
 from weakref import finalize
-import kubernetes_asyncio as kubernetes
-from tornado.iostream import StreamClosedError
 
 import kr8s
-from kr8s.asyncio.objects import Pod
+import kubernetes_asyncio as kubernetes
 from distributed.core import rpc
+from kr8s.asyncio.objects import Pod
+from tornado.iostream import StreamClosedError
 
 from dask_kubernetes.common.utils import check_dependency
 from dask_kubernetes.exceptions import CrashLoopBackOffError

dask_kubernetes/common/objects.py

@@ -1,11 +1,11 @@
 """
 Convenience functions for creating pod templates.
 """
-from collections import namedtuple
 import copy
-from kubernetes import client
 import json
+from collections import namedtuple
 
+from kubernetes import client
 from kubernetes.client.configuration import Configuration
 
 from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME

dask_kubernetes/common/tests/test_kind.py

@@ -1,7 +1,7 @@
-import pytest
 from subprocess import check_output
 
 import kubernetes_asyncio as kubernetes
+import pytest
 
 from dask_kubernetes.common.auth import ClusterAuth
 from dask_kubernetes.common.utils import get_current_namespace

dask_kubernetes/common/tests/test_objects.py

@@ -1,5 +1,5 @@
-from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME
 from dask_kubernetes.common.objects import make_pod_from_dict
+from dask_kubernetes.constants import KUBECLUSTER_CONTAINER_NAME
 
 
 def test_make_pod_from_dict():

dask_kubernetes/config.py

@@ -1,11 +1,10 @@
-from __future__ import print_function, division, absolute_import
+from __future__ import absolute_import, division, print_function
 
 import os
 
 import dask
 import yaml
 
-
 fn = os.path.join(os.path.dirname(__file__), "kubernetes.yaml")
 dask.config.ensure_file(source=fn)
 

dask_kubernetes/conftest.py

@@ -1,12 +1,11 @@
-import pytest
-
 import logging
-import pathlib
 import os
+import pathlib
 import subprocess
 import tempfile
 import uuid
 
+import pytest
 from kopf.testing import KopfRunner
 from pytest_kind.cluster import KindCluster
 

dask_kubernetes/helm/helmcluster.py

@@ -1,21 +1,21 @@
 import asyncio
-import aiohttp
+import json
 import subprocess
 import warnings
 from contextlib import suppress
-import json
 
+import aiohttp
+import kubernetes_asyncio as kubernetes
+from distributed.core import Status, rpc
 from distributed.deploy import Cluster
-from distributed.core import rpc, Status
 from distributed.utils import Log, Logs
-import kubernetes_asyncio as kubernetes
 
 from ..common.auth import ClusterAuth
+from ..common.networking import get_external_address_for_scheduler_service
 from ..common.utils import (
-    get_current_namespace,
     check_dependency,
+    get_current_namespace,
 )
-from ..common.networking import get_external_address_for_scheduler_service
 
 
 class HelmCluster(Cluster):

dask_kubernetes/helm/tests/test_helm.py

@@ -1,16 +1,15 @@
-import pytest
-
-import subprocess
 import os.path
+import subprocess
 
 import dask.config
-from distributed import Client
-from distributed.core import Status
+import pytest
 from dask_ctl.discovery import (
-    list_discovery_methods,
     discover_cluster_names,
     discover_clusters,
+    list_discovery_methods,
 )
+from distributed import Client
+from distributed.core import Status
 
 ###############
 # Fixtures
@@ -131,9 +130,10 @@ def sync_cluster(k8s_cluster, release, test_namespace):
 
 
 def test_import():
-    from dask_kubernetes import HelmCluster
     from distributed.deploy import Cluster
 
+    from dask_kubernetes import HelmCluster
+
     assert issubclass(HelmCluster, Cluster)
 
 

dask_kubernetes/operator/_objects.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
+
 from typing import List
 
-from kr8s.asyncio.objects import APIObject, Pod, Deployment, Service
+from kr8s.asyncio.objects import APIObject, Deployment, Pod, Service
 
 
 class DaskCluster(APIObject):