Added ruff (#850)

Replace black and isort with ruff. Update Github Actions

.github/workflows/build.yml

@@ -14,49 +14,38 @@ on:
 
 env:
   BUILD_INCREMENT: 150
+  PIP_DISABLE_PIP_VERSION_CHECK: on
+  PIP_DEFAULT_TIMEOUT: 10
+  PIP_PROGRESS_BAR: off
 
 jobs:
   python-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.11
-      - name: Run isort
-        uses: isort/isort-action@master
-        with:
-          isort-version: 5.12.0
-          configuration: "--settings-file pyconfig.toml --check-only --diff"
-          sort-paths: "src"
-      - name: Run black
-        uses: psf/black@stable
-        with:
-          options: "--config pyconfig.toml --check --verbose"
-          src: "./src"
-          version: "~= 22.0"
+      - run: python -m pip install pre-commit
+      - run: pre-commit run -a --show-diff-on-failure
 
   python-test:
     needs: [ python-lint ]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ macos-latest, ubuntu-latest]
-        python-version: [ 3.8, 3.9, "3.10", 3.11 ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install '.[all]'
-          pip install -r requirements_dev.txt
+        run: pip install -U '.[all]' -r requirements_dev.txt
       - name: Run pytest
-        run: |
-          pytest src/tests
+        run: pytest src/tests
 
   update-get-dstack:
     needs: [ python-test ]

.github/workflows/release.yml

@@ -5,48 +5,40 @@ on:
     tags:
       - '*'
 
+env:
+  BUILD_INCREMENT: 150
+  PIP_DISABLE_PIP_VERSION_CHECK: on
+  PIP_DEFAULT_TIMEOUT: 10
+  PIP_PROGRESS_BAR: off
+
 jobs:
   python-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.11
-      - name: Run isort
-        uses: isort/isort-action@master
-        with:
-          isort-version: 5.12.0
-          configuration: "--settings-file pyconfig.toml --check-only --diff"
-          sort-paths: "src"
-      - name: Run black
-        uses: psf/black@stable
-        with:
-          options: "--config pyconfig.toml --check --verbose"
-          src: "./src"
-          version: "~= 22.0"
+      - run: python -m pip install pre-commit
+      - run: pre-commit run -a --show-diff-on-failure
 
   python-test:
     needs: [ python-lint ]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ macos-latest, ubuntu-latest]
-        python-version: [ 3.8, 3.9, "3.10", 3.11 ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install '.[all]'
-          pip install -r requirements_dev.txt
+        run: pip install -U '.[all]' -r requirements_dev.txt
       - name: Run pytest
-        run: |
-          pytest src/tests
+        run: pytest src/tests
 
   runner-test:
     defaults:

.pre-commit-config.yaml

@@ -1,13 +1,7 @@
 repos:
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.2.0
     hooks:
-      - id: black
-        language_version: python3.11
-        args: ['--config', 'pyconfig.toml']
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        name: isort (python)
-        args: ['--settings-file', 'pyconfig.toml']
+      - id: ruff
+        args: ['--fix']
+      - id: ruff-format

CONTRIBUTING.md

@@ -35,13 +35,13 @@
 
 ## Making changes
 
-We use [`black`](https://github.com/psf/black) to format Python code and [`isort`](https://pycqa.github.io/isort/index.html) to sort Python imports. Before committing your changes, run:
+We use [`ruff`](https://docs.astral.sh/ruff/) to format Python code and to sort Python imports. Before committing your changes, run:
 
-1. `isort --settings-file pyconfig.toml cli `
-2. `black --config pyconfig.toml cli`
+1. `ruff --fix src`
+2. `ruff format src`
 
 
-There are also helper pre-commits installed for [`black`](https://black.readthedocs.io/en/stable/integrations/source_version_control.html) and [`isort`](https://pycqa.github.io/isort/docs/configuration/pre-commit.html) that make commits fail if the code is not formatted or the imports are not sorted. They also change the code as required so that you can review the changes and commit again.
+There are also helper pre-commits installed for [`ruff`](https://docs.astral.sh/ruff/integrations/#pre-commit) that make commits fail if the code is not formatted or the imports are not sorted. They also change the code as required so that you can review the changes and commit again.
 
 ## Adding a new backend
 

gateway/src/dstack/gateway/registry/schemas.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import BaseModel
 
 import dstack.gateway.schemas

gateway/src/dstack/gateway/services/tunnel.py

@@ -78,7 +78,7 @@ def start(self):
 
     def stop(self):
         logger.info("Stopping SSH tunnel for %s", self.sock_path)
-        r = subprocess.run(self.exit_cmd, timeout=5, capture_output=True)
+        subprocess.run(self.exit_cmd, timeout=5, capture_output=True)
         shutil.rmtree(self.temp_dir, ignore_errors=True)
 
     # TODO(egor-s): retry ssh tunnel

requirements_dev.txt

@@ -1,7 +1,6 @@
-black~=22.0
 pre-commit
-isort~=5.0
 httpx>=0.23
 pytest~=7.2
 pytest-asyncio>=0.21
 freezegun>=1.2.0
+ruff

ruff.toml

@@ -0,0 +1,13 @@
+target-version = "py38"
+line-length = 99
+
+[lint]
+select = ['E', 'F', 'I' ,'Q', 'W', 'PGH', 'FLY']
+ignore =[
+  'E501',
+  'E712',
+]
+
+[lint.isort]
+known-first-party = ["dstack"]
+known-third-party = ["mkdocs_gen_files"]

src/dstack/_internal/cli/commands/__init__.py

@@ -1,15 +1,12 @@
 import argparse
 import os
 from abc import ABC, abstractmethod
-from pathlib import Path
 
 from rich_argparse import RichHelpFormatter
 
-import dstack._internal.core.services.api_client as api_client_service
 from dstack._internal.cli.utils.common import configure_logging
 from dstack._internal.core.errors import CLIError, ConfigurationError
 from dstack.api import Client
-from dstack.api.server import APIClient
 
 
 class BaseCommand(ABC):

src/dstack/_internal/cli/commands/config.py

@@ -40,7 +40,7 @@ def _command(self, args: argparse.Namespace):
         if args.remove:
             config_manager.delete_project(args.project)
             config_manager.save()
-            console.print(f"[grey58]OK[/]")
+            console.print("[grey58]OK[/]")
             return
 
         if not args.url:

src/dstack/_internal/cli/commands/logs.py

@@ -4,7 +4,6 @@
 
 from dstack._internal.cli.commands import APIBaseCommand
 from dstack._internal.core.errors import CLIError
-from dstack._internal.core.services.ssh.ports import PortUsedError
 
 
 class LogsCommand(APIBaseCommand):

src/dstack/_internal/cli/utils/run.py

@@ -3,7 +3,7 @@
 from rich.table import Table
 
 from dstack._internal.cli.utils.common import console
-from dstack._internal.core.models.instances import InstanceAvailability, InstanceType, Resources
+from dstack._internal.core.models.instances import InstanceAvailability, InstanceType
 from dstack._internal.core.models.runs import RunPlan
 from dstack._internal.utils.common import pretty_date
 from dstack.api import Run

src/dstack/_internal/core/backends/aws/compute.py

@@ -6,7 +6,6 @@
 import botocore.exceptions
 
 import dstack._internal.core.backends.aws.resources as aws_resources
-import dstack.version as version
 from dstack._internal import settings
 from dstack._internal.core.backends.aws.config import AWSConfig
 from dstack._internal.core.backends.base.compute import (

src/dstack/_internal/core/backends/base/__init__.py

@@ -1,5 +1,4 @@
 from abc import abstractmethod
-from typing import List
 
 from dstack._internal.core.backends.base.compute import Compute
 from dstack._internal.core.models.backends.base import BackendType

src/dstack/_internal/core/backends/datacrunch/compute.py

@@ -35,9 +35,7 @@ def get_offers(
     def _get_offers_with_availability(
         self, offers: List[InstanceOffer]
     ) -> List[InstanceOfferWithAvailability]:
-        raw_availabilities: List[
-            Dict
-        ] = self.api_client.client.instances.get_availabilities()  # type: ignore
+        raw_availabilities: List[Dict] = self.api_client.client.instances.get_availabilities()
 
         region_availabilities = {}
         for location in raw_availabilities:

src/dstack/_internal/core/backends/gcp/compute.py

@@ -1,5 +1,5 @@
 from collections import defaultdict
-from typing import Callable, Dict, List, Optional, Tuple
+from typing import Callable, Dict, List, Optional
 
 import google.api_core.exceptions
 import google.cloud.compute_v1 as compute_v1

src/dstack/_internal/core/backends/gcp/resources.py

@@ -112,7 +112,7 @@ def create_runner_firewall_rules(
     network: str = "global/networks/default",
 ):
     firewall_rule = compute_v1.Firewall()
-    firewall_rule.name = f"dstack-ssh-in-" + network.replace("/", "-")
+    firewall_rule.name = "dstack-ssh-in-" + network.replace("/", "-")
     firewall_rule.direction = "INGRESS"
 
     allowed_ssh_port = compute_v1.Allowed()

src/dstack/_internal/core/backends/kubernetes/client.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional
+from typing import Dict, List
 
 import kubernetes
 import yaml

src/dstack/_internal/core/backends/kubernetes/compute.py

@@ -109,7 +109,7 @@ def run_job(
                 "jump_pod_port": self.config.networking.ssh_port,
             },
         ).start()
-        pod_response = self.api.create_namespaced_pod(
+        self.api.create_namespaced_pod(
             namespace=DEFAULT_NAMESPACE,
             body=client.V1Pod(
                 metadata=client.V1ObjectMeta(
@@ -201,7 +201,7 @@ def create_gateway(
         # Consider deploying an NLB. It seems it requires some extra configuration on the cluster:
         # https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html
         commands = _get_gateway_commands(authorized_keys=[ssh_key_pub])
-        pod_response = self.api.create_namespaced_pod(
+        self.api.create_namespaced_pod(
             namespace=DEFAULT_NAMESPACE,
             body=client.V1Pod(
                 metadata=client.V1ObjectMeta(
@@ -231,7 +231,7 @@ def create_gateway(
                 ),
             ),
         )
-        service_response = self.api.create_namespaced_service(
+        self.api.create_namespaced_service(
             namespace=DEFAULT_NAMESPACE,
             body=client.V1Service(
                 metadata=client.V1ObjectMeta(
@@ -360,7 +360,7 @@ def _create_jump_pod_service(
     # TODO use restricted ssh-forwarding-only user for jump pod instead of root.
     commands = _get_jump_pod_commands(authorized_keys=[project_ssh_public_key])
     pod_name = _get_jump_pod_name(project_name)
-    pod_response = api.create_namespaced_pod(
+    api.create_namespaced_pod(
         namespace=DEFAULT_NAMESPACE,
         body=client.V1Pod(
             metadata=client.V1ObjectMeta(
@@ -385,7 +385,7 @@ def _create_jump_pod_service(
             ),
         ),
     )
-    service_response = api.create_namespaced_service(
+    api.create_namespaced_service(
         namespace=DEFAULT_NAMESPACE,
         body=client.V1Service(
             metadata=client.V1ObjectMeta(name=_get_jump_pod_service_name(project_name)),

src/dstack/_internal/core/backends/lambdalabs/config.py

@@ -1,7 +1,6 @@
 from dstack._internal.core.backends.base.config import BackendConfig
 from dstack._internal.core.models.backends.lambdalabs import (
     AnyLambdaCreds,
-    LambdaConfigInfoWithCreds,
     LambdaStoredConfig,
 )
 

src/dstack/_internal/core/backends/local/compute.py

@@ -1,4 +1,3 @@
-from abc import ABC, abstractmethod
 from typing import List, Optional
 
 from dstack._internal.core.backends.base.compute import Compute, get_dstack_runner_version
@@ -46,12 +45,7 @@ def run_job(
         project_ssh_public_key: str,
         project_ssh_private_key: str,
     ) -> LaunchedInstanceInfo:
-        authorized_keys = "\\n".join(
-            [
-                run.run_spec.ssh_key_pub.strip(),
-                project_ssh_public_key.strip(),
-            ]
-        )
+        authorized_keys = f"{run.run_spec.ssh_key_pub.strip()}\\n{project_ssh_public_key.strip()}"
         logger.info(
             "Running job in LocalBackend. To start processing, run: `"
             f"DSTACK_BACKEND=local "

src/dstack/_internal/core/backends/nebius/types.py

@@ -1,4 +1,4 @@
-from typing import List, Literal, Optional, TypedDict
+from typing import TypedDict
 
 
 class ServiceAccount(TypedDict):

src/dstack/_internal/core/backends/vastai/api_client.py

@@ -1,4 +1,3 @@
-import shlex
 import threading
 import time
 from typing import List, Optional, Union
@@ -29,7 +28,7 @@ def __init__(self, api_key: str):
         self.instances_cache: List[dict] = []
 
     def get_bundle(self, bundle_id: Union[str, int]) -> Optional[dict]:
-        resp = self.s.post(self._url(f"/bundles/"), json={"id": {"eq": bundle_id}})
+        resp = self.s.post(self._url("/bundles/"), json={"id": {"eq": bundle_id}})
         resp.raise_for_status()
         data = resp.json()
         offers = data["offers"]
@@ -103,7 +102,7 @@ def destroy_instance(self, instance_id: Union[str, int]) -> bool:
     def get_instances(self, cache_ttl: float = 3.0) -> List[dict]:
         with self.lock:
             if time.time() - self.instances_cache_ts > cache_ttl:
-                resp = self.s.get(self._url(f"/instances/"))
+                resp = self.s.get(self._url("/instances/"))
                 resp.raise_for_status()
                 data = resp.json()
                 self.instances_cache_ts = time.time()