From 731f524bb85349bea53b898c20af52ae86c340e2 Mon Sep 17 00:00:00 2001 From: Sergey Mezentsev Date: Mon, 5 Feb 2024 18:25:46 +0300 Subject: [PATCH] Added ruff (#850) Replace black and isort with ruff. Update Github Actions --- .github/workflows/build.yml | 35 ++++++----------- .github/workflows/release.yml | 38 ++++++++----------- .pre-commit-config.yaml | 16 +++----- CONTRIBUTING.md | 8 ++-- .../src/dstack/gateway/registry/schemas.py | 2 - gateway/src/dstack/gateway/services/tunnel.py | 2 +- pyconfig.toml | 7 ---- requirements_dev.txt | 3 +- ruff.toml | 13 +++++++ src/dstack/_internal/cli/commands/__init__.py | 3 -- src/dstack/_internal/cli/commands/config.py | 2 +- src/dstack/_internal/cli/commands/logs.py | 1 - src/dstack/_internal/cli/utils/run.py | 2 +- .../_internal/core/backends/aws/compute.py | 1 - .../_internal/core/backends/base/__init__.py | 1 - .../core/backends/datacrunch/compute.py | 4 +- .../_internal/core/backends/gcp/compute.py | 2 +- .../_internal/core/backends/gcp/resources.py | 2 +- .../core/backends/kubernetes/client.py | 2 +- .../core/backends/kubernetes/compute.py | 10 ++--- .../core/backends/lambdalabs/config.py | 1 - .../_internal/core/backends/local/compute.py | 8 +--- .../_internal/core/backends/nebius/types.py | 2 +- .../core/backends/vastai/api_client.py | 5 +-- .../_internal/core/models/backends/dstack.py | 2 +- .../core/models/backends/lambdalabs.py | 2 +- src/dstack/_internal/core/models/profiles.py | 2 +- .../_internal/core/models/repos/__init__.py | 8 +++- .../_internal/core/models/repos/local.py | 1 - src/dstack/_internal/core/models/runs.py | 4 +- .../_internal/core/services/api_client.py | 4 +- .../core/services/configs/__init__.py | 1 - src/dstack/_internal/core/services/repos.py | 2 +- .../_internal/core/services/ssh/__init__.py | 2 - src/dstack/_internal/server/app.py | 3 +- .../112753bc17dd_remove_nullable_fields.py | 1 - .../14f2cb002fc2_add_jobmodel_removed_flag.py | 1 - .../23e01c56279a_make_blob_nullable.py | 1 - .../3dbdce90d0e0_fix_code_uq_constraint.py | 2 - ...ecbaea2_do_not_delete_projects_and_runs.py | 1 - .../versions/686fb8341ea5_add_user_emails.py | 1 - .../migrations/versions/a060e2440936_.py | 2 +- .../migrations/versions/bfba43f6def2_.py | 1 - ...e8af4786fa_gateway_compute_flag_deleted.py | 1 - src/dstack/_internal/server/routers/runs.py | 2 +- .../_internal/server/routers/secrets.py | 1 - .../_internal/server/security/permissions.py | 2 +- .../server/services/backends/__init__.py | 1 - .../services/backends/configurators/aws.py | 4 +- .../services/backends/configurators/azure.py | 6 +-- .../services/backends/configurators/gcp.py | 2 +- .../backends/configurators/kubernetes.py | 3 +- .../backends/configurators/lambdalabs.py | 4 -- .../server/services/gateways/pool.py | 1 - .../server/services/jobs/__init__.py | 1 - .../services/jobs/configurators/base.py | 2 +- .../server/services/jobs/configurators/dev.py | 4 +- .../jobs/configurators/extensions/vscode.py | 10 ++--- src/dstack/_internal/server/services/logs.py | 4 +- src/dstack/_internal/server/services/repos.py | 2 +- .../_internal/server/services/runner/ssh.py | 4 +- .../_internal/server/services/storage.py | 5 +-- src/dstack/_internal/server/settings.py | 2 - src/dstack/_internal/server/utils/common.py | 2 +- src/dstack/_internal/server/utils/logging.py | 1 - src/dstack/_internal/server/utils/routers.py | 3 +- src/dstack/_internal/utils/common.py | 2 +- src/dstack/_internal/utils/ssh.py | 2 +- src/dstack/api/_public/__init__.py | 2 +- .../huggingface/completions/__init__.py | 2 +- src/dstack/api/_public/repos.py | 4 +- src/dstack/api/_public/runs.py | 3 +- src/dstack/api/server/_group.py | 2 +- src/dstack/api/server/_runs.py | 2 +- src/dstack/api/server/_secrets.py | 2 +- src/dstack/api/utils.py | 6 +-- .../tasks/test_process_running_jobs.py | 4 +- .../tasks/test_process_submitted_jobs.py | 2 - .../tasks/test_process_terminating_jobs.py | 3 +- .../_internal/server/routers/test_backends.py | 6 +-- .../_internal/server/routers/test_gateways.py | 4 +- .../_internal/server/routers/test_projects.py | 2 +- .../_internal/server/routers/test_runs.py | 8 ++-- .../_internal/server/services/test_config.py | 4 -- 84 files changed, 134 insertions(+), 204 deletions(-) delete mode 100644 pyconfig.toml create mode 100644 ruff.toml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4ad7a594..5cc4837a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,27 +14,20 @@ on: env: BUILD_INCREMENT: 150 + PIP_DISABLE_PIP_VERSION_CHECK: on + PIP_DEFAULT_TIMEOUT: 10 + PIP_PROGRESS_BAR: off jobs: python-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.11 - - name: Run isort - uses: isort/isort-action@master - with: - isort-version: 5.12.0 - configuration: "--settings-file pyconfig.toml --check-only --diff" - sort-paths: "src" - - name: Run black - uses: psf/black@stable - with: - options: "--config pyconfig.toml --check --verbose" - src: "./src" - version: "~= 22.0" + - run: python -m pip install pre-commit + - run: pre-commit run -a --show-diff-on-failure python-test: needs: [ python-lint ] @@ -42,21 +35,17 @@ jobs: strategy: matrix: os: [ macos-latest, ubuntu-latest] - python-version: [ 3.8, 3.9, "3.10", 3.11 ] + python-version: [ "3.8", "3.9", "3.10", "3.11" ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install '.[all]' - pip install -r requirements_dev.txt + run: pip install -U '.[all]' -r requirements_dev.txt - name: Run pytest - run: | - pytest src/tests + run: pytest src/tests update-get-dstack: needs: [ python-test ] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 087824784..d6ecbb36c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,26 +5,22 @@ on: tags: - '*' +env: + BUILD_INCREMENT: 150 + PIP_DISABLE_PIP_VERSION_CHECK: on + PIP_DEFAULT_TIMEOUT: 10 + PIP_PROGRESS_BAR: off + jobs: python-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.11 - - name: Run isort - uses: isort/isort-action@master - with: - isort-version: 5.12.0 - configuration: "--settings-file pyconfig.toml --check-only --diff" - sort-paths: "src" - - name: Run black - uses: psf/black@stable - with: - options: "--config pyconfig.toml --check --verbose" - src: "./src" - version: "~= 22.0" + - run: python -m pip install pre-commit + - run: pre-commit run -a --show-diff-on-failure python-test: needs: [ python-lint ] @@ -32,21 +28,17 @@ jobs: strategy: matrix: os: [ macos-latest, ubuntu-latest] - python-version: [ 3.8, 3.9, "3.10", 3.11 ] + python-version: [ "3.8", "3.9", "3.10", "3.11" ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install '.[all]' - pip install -r requirements_dev.txt + run: pip install -U '.[all]' -r requirements_dev.txt - name: Run pytest - run: | - pytest src/tests + run: pytest src/tests runner-test: defaults: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0593575f2..80ecadba6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,7 @@ repos: - - repo: https://github.com/psf/black - rev: 22.12.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.2.0 hooks: - - id: black - language_version: python3.11 - args: ['--config', 'pyconfig.toml'] - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - name: isort (python) - args: ['--settings-file', 'pyconfig.toml'] + - id: ruff + args: ['--fix'] + - id: ruff-format diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82a7aff87..da6c605e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,13 +35,13 @@ ## Making changes -We use [`black`](https://github.com/psf/black) to format Python code and [`isort`](https://pycqa.github.io/isort/index.html) to sort Python imports. Before committing your changes, run: +We use [`ruff`](https://docs.astral.sh/ruff/) to format Python code and to sort Python imports. Before committing your changes, run: -1. `isort --settings-file pyconfig.toml cli ` -2. `black --config pyconfig.toml cli` +1. `ruff --fix src` +2. `ruff format src` -There are also helper pre-commits installed for [`black`](https://black.readthedocs.io/en/stable/integrations/source_version_control.html) and [`isort`](https://pycqa.github.io/isort/docs/configuration/pre-commit.html) that make commits fail if the code is not formatted or the imports are not sorted. They also change the code as required so that you can review the changes and commit again. +There are also helper pre-commits installed for [`ruff`](https://docs.astral.sh/ruff/integrations/#pre-commit) that make commits fail if the code is not formatted or the imports are not sorted. They also change the code as required so that you can review the changes and commit again. ## Adding a new backend diff --git a/gateway/src/dstack/gateway/registry/schemas.py b/gateway/src/dstack/gateway/registry/schemas.py index 337356243..9ff3a11aa 100644 --- a/gateway/src/dstack/gateway/registry/schemas.py +++ b/gateway/src/dstack/gateway/registry/schemas.py @@ -1,5 +1,3 @@ -from typing import Optional - from pydantic import BaseModel import dstack.gateway.schemas diff --git a/gateway/src/dstack/gateway/services/tunnel.py b/gateway/src/dstack/gateway/services/tunnel.py index dabca762b..7574081c5 100644 --- a/gateway/src/dstack/gateway/services/tunnel.py +++ b/gateway/src/dstack/gateway/services/tunnel.py @@ -78,7 +78,7 @@ def start(self): def stop(self): logger.info("Stopping SSH tunnel for %s", self.sock_path) - r = subprocess.run(self.exit_cmd, timeout=5, capture_output=True) + subprocess.run(self.exit_cmd, timeout=5, capture_output=True) shutil.rmtree(self.temp_dir, ignore_errors=True) # TODO(egor-s): retry ssh tunnel diff --git a/pyconfig.toml b/pyconfig.toml deleted file mode 100644 index 5e454b969..000000000 --- a/pyconfig.toml +++ /dev/null @@ -1,7 +0,0 @@ -[tool.black] -line-length = 99 - -[isort] -profile = "black" -known_local_folder = ["dstack", "tests"] -line_length = 99 diff --git a/requirements_dev.txt b/requirements_dev.txt index 53a602c5d..d54fdfa1c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,7 +1,6 @@ -black~=22.0 pre-commit -isort~=5.0 httpx>=0.23 pytest~=7.2 pytest-asyncio>=0.21 freezegun>=1.2.0 +ruff diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..32e7b000d --- /dev/null +++ b/ruff.toml @@ -0,0 +1,13 @@ +target-version = "py38" +line-length = 99 + +[lint] +select = ['E', 'F', 'I' ,'Q', 'W', 'PGH', 'FLY'] +ignore =[ + 'E501', + 'E712', +] + +[lint.isort] +known-first-party = ["dstack"] +known-third-party = ["mkdocs_gen_files"] diff --git a/src/dstack/_internal/cli/commands/__init__.py b/src/dstack/_internal/cli/commands/__init__.py index 0c048513c..6a6167cdf 100644 --- a/src/dstack/_internal/cli/commands/__init__.py +++ b/src/dstack/_internal/cli/commands/__init__.py @@ -1,15 +1,12 @@ import argparse import os from abc import ABC, abstractmethod -from pathlib import Path from rich_argparse import RichHelpFormatter -import dstack._internal.core.services.api_client as api_client_service from dstack._internal.cli.utils.common import configure_logging from dstack._internal.core.errors import CLIError, ConfigurationError from dstack.api import Client -from dstack.api.server import APIClient class BaseCommand(ABC): diff --git a/src/dstack/_internal/cli/commands/config.py b/src/dstack/_internal/cli/commands/config.py index 6a3392737..c251baa54 100644 --- a/src/dstack/_internal/cli/commands/config.py +++ b/src/dstack/_internal/cli/commands/config.py @@ -40,7 +40,7 @@ def _command(self, args: argparse.Namespace): if args.remove: config_manager.delete_project(args.project) config_manager.save() - console.print(f"[grey58]OK[/]") + console.print("[grey58]OK[/]") return if not args.url: diff --git a/src/dstack/_internal/cli/commands/logs.py b/src/dstack/_internal/cli/commands/logs.py index bf6c7ace2..7e6aa2493 100644 --- a/src/dstack/_internal/cli/commands/logs.py +++ b/src/dstack/_internal/cli/commands/logs.py @@ -4,7 +4,6 @@ from dstack._internal.cli.commands import APIBaseCommand from dstack._internal.core.errors import CLIError -from dstack._internal.core.services.ssh.ports import PortUsedError class LogsCommand(APIBaseCommand): diff --git a/src/dstack/_internal/cli/utils/run.py b/src/dstack/_internal/cli/utils/run.py index 06bc08242..c2509be70 100644 --- a/src/dstack/_internal/cli/utils/run.py +++ b/src/dstack/_internal/cli/utils/run.py @@ -3,7 +3,7 @@ from rich.table import Table from dstack._internal.cli.utils.common import console -from dstack._internal.core.models.instances import InstanceAvailability, InstanceType, Resources +from dstack._internal.core.models.instances import InstanceAvailability, InstanceType from dstack._internal.core.models.runs import RunPlan from dstack._internal.utils.common import pretty_date from dstack.api import Run diff --git a/src/dstack/_internal/core/backends/aws/compute.py b/src/dstack/_internal/core/backends/aws/compute.py index 4e2839710..bfee1ba89 100644 --- a/src/dstack/_internal/core/backends/aws/compute.py +++ b/src/dstack/_internal/core/backends/aws/compute.py @@ -6,7 +6,6 @@ import botocore.exceptions import dstack._internal.core.backends.aws.resources as aws_resources -import dstack.version as version from dstack._internal import settings from dstack._internal.core.backends.aws.config import AWSConfig from dstack._internal.core.backends.base.compute import ( diff --git a/src/dstack/_internal/core/backends/base/__init__.py b/src/dstack/_internal/core/backends/base/__init__.py index bf6783d29..42b779169 100644 --- a/src/dstack/_internal/core/backends/base/__init__.py +++ b/src/dstack/_internal/core/backends/base/__init__.py @@ -1,5 +1,4 @@ from abc import abstractmethod -from typing import List from dstack._internal.core.backends.base.compute import Compute from dstack._internal.core.models.backends.base import BackendType diff --git a/src/dstack/_internal/core/backends/datacrunch/compute.py b/src/dstack/_internal/core/backends/datacrunch/compute.py index 19b60fb96..aac58296c 100644 --- a/src/dstack/_internal/core/backends/datacrunch/compute.py +++ b/src/dstack/_internal/core/backends/datacrunch/compute.py @@ -35,9 +35,7 @@ def get_offers( def _get_offers_with_availability( self, offers: List[InstanceOffer] ) -> List[InstanceOfferWithAvailability]: - raw_availabilities: List[ - Dict - ] = self.api_client.client.instances.get_availabilities() # type: ignore + raw_availabilities: List[Dict] = self.api_client.client.instances.get_availabilities() region_availabilities = {} for location in raw_availabilities: diff --git a/src/dstack/_internal/core/backends/gcp/compute.py b/src/dstack/_internal/core/backends/gcp/compute.py index 8942f44e1..46f989543 100644 --- a/src/dstack/_internal/core/backends/gcp/compute.py +++ b/src/dstack/_internal/core/backends/gcp/compute.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Callable, Dict, List, Optional, Tuple +from typing import Callable, Dict, List, Optional import google.api_core.exceptions import google.cloud.compute_v1 as compute_v1 diff --git a/src/dstack/_internal/core/backends/gcp/resources.py b/src/dstack/_internal/core/backends/gcp/resources.py index bea0370f0..b6d0659ef 100644 --- a/src/dstack/_internal/core/backends/gcp/resources.py +++ b/src/dstack/_internal/core/backends/gcp/resources.py @@ -112,7 +112,7 @@ def create_runner_firewall_rules( network: str = "global/networks/default", ): firewall_rule = compute_v1.Firewall() - firewall_rule.name = f"dstack-ssh-in-" + network.replace("/", "-") + firewall_rule.name = "dstack-ssh-in-" + network.replace("/", "-") firewall_rule.direction = "INGRESS" allowed_ssh_port = compute_v1.Allowed() diff --git a/src/dstack/_internal/core/backends/kubernetes/client.py b/src/dstack/_internal/core/backends/kubernetes/client.py index 4aa43a2eb..4d5d9e2a5 100644 --- a/src/dstack/_internal/core/backends/kubernetes/client.py +++ b/src/dstack/_internal/core/backends/kubernetes/client.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import Dict, List import kubernetes import yaml diff --git a/src/dstack/_internal/core/backends/kubernetes/compute.py b/src/dstack/_internal/core/backends/kubernetes/compute.py index d3457d13a..b21d9b4b7 100644 --- a/src/dstack/_internal/core/backends/kubernetes/compute.py +++ b/src/dstack/_internal/core/backends/kubernetes/compute.py @@ -109,7 +109,7 @@ def run_job( "jump_pod_port": self.config.networking.ssh_port, }, ).start() - pod_response = self.api.create_namespaced_pod( + self.api.create_namespaced_pod( namespace=DEFAULT_NAMESPACE, body=client.V1Pod( metadata=client.V1ObjectMeta( @@ -201,7 +201,7 @@ def create_gateway( # Consider deploying an NLB. It seems it requires some extra configuration on the cluster: # https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html commands = _get_gateway_commands(authorized_keys=[ssh_key_pub]) - pod_response = self.api.create_namespaced_pod( + self.api.create_namespaced_pod( namespace=DEFAULT_NAMESPACE, body=client.V1Pod( metadata=client.V1ObjectMeta( @@ -231,7 +231,7 @@ def create_gateway( ), ), ) - service_response = self.api.create_namespaced_service( + self.api.create_namespaced_service( namespace=DEFAULT_NAMESPACE, body=client.V1Service( metadata=client.V1ObjectMeta( @@ -360,7 +360,7 @@ def _create_jump_pod_service( # TODO use restricted ssh-forwarding-only user for jump pod instead of root. commands = _get_jump_pod_commands(authorized_keys=[project_ssh_public_key]) pod_name = _get_jump_pod_name(project_name) - pod_response = api.create_namespaced_pod( + api.create_namespaced_pod( namespace=DEFAULT_NAMESPACE, body=client.V1Pod( metadata=client.V1ObjectMeta( @@ -385,7 +385,7 @@ def _create_jump_pod_service( ), ), ) - service_response = api.create_namespaced_service( + api.create_namespaced_service( namespace=DEFAULT_NAMESPACE, body=client.V1Service( metadata=client.V1ObjectMeta(name=_get_jump_pod_service_name(project_name)), diff --git a/src/dstack/_internal/core/backends/lambdalabs/config.py b/src/dstack/_internal/core/backends/lambdalabs/config.py index 08e8a40c1..4fbea18e1 100644 --- a/src/dstack/_internal/core/backends/lambdalabs/config.py +++ b/src/dstack/_internal/core/backends/lambdalabs/config.py @@ -1,7 +1,6 @@ from dstack._internal.core.backends.base.config import BackendConfig from dstack._internal.core.models.backends.lambdalabs import ( AnyLambdaCreds, - LambdaConfigInfoWithCreds, LambdaStoredConfig, ) diff --git a/src/dstack/_internal/core/backends/local/compute.py b/src/dstack/_internal/core/backends/local/compute.py index 8bfa3077d..31643cbaa 100644 --- a/src/dstack/_internal/core/backends/local/compute.py +++ b/src/dstack/_internal/core/backends/local/compute.py @@ -1,4 +1,3 @@ -from abc import ABC, abstractmethod from typing import List, Optional from dstack._internal.core.backends.base.compute import Compute, get_dstack_runner_version @@ -46,12 +45,7 @@ def run_job( project_ssh_public_key: str, project_ssh_private_key: str, ) -> LaunchedInstanceInfo: - authorized_keys = "\\n".join( - [ - run.run_spec.ssh_key_pub.strip(), - project_ssh_public_key.strip(), - ] - ) + authorized_keys = f"{run.run_spec.ssh_key_pub.strip()}\\n{project_ssh_public_key.strip()}" logger.info( "Running job in LocalBackend. To start processing, run: `" f"DSTACK_BACKEND=local " diff --git a/src/dstack/_internal/core/backends/nebius/types.py b/src/dstack/_internal/core/backends/nebius/types.py index b59aff736..177a98389 100644 --- a/src/dstack/_internal/core/backends/nebius/types.py +++ b/src/dstack/_internal/core/backends/nebius/types.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional, TypedDict +from typing import TypedDict class ServiceAccount(TypedDict): diff --git a/src/dstack/_internal/core/backends/vastai/api_client.py b/src/dstack/_internal/core/backends/vastai/api_client.py index ab3e7613e..d4988072a 100644 --- a/src/dstack/_internal/core/backends/vastai/api_client.py +++ b/src/dstack/_internal/core/backends/vastai/api_client.py @@ -1,4 +1,3 @@ -import shlex import threading import time from typing import List, Optional, Union @@ -29,7 +28,7 @@ def __init__(self, api_key: str): self.instances_cache: List[dict] = [] def get_bundle(self, bundle_id: Union[str, int]) -> Optional[dict]: - resp = self.s.post(self._url(f"/bundles/"), json={"id": {"eq": bundle_id}}) + resp = self.s.post(self._url("/bundles/"), json={"id": {"eq": bundle_id}}) resp.raise_for_status() data = resp.json() offers = data["offers"] @@ -103,7 +102,7 @@ def destroy_instance(self, instance_id: Union[str, int]) -> bool: def get_instances(self, cache_ttl: float = 3.0) -> List[dict]: with self.lock: if time.time() - self.instances_cache_ts > cache_ttl: - resp = self.s.get(self._url(f"/instances/")) + resp = self.s.get(self._url("/instances/")) resp.raise_for_status() data = resp.json() self.instances_cache_ts = time.time() diff --git a/src/dstack/_internal/core/models/backends/dstack.py b/src/dstack/_internal/core/models/backends/dstack.py index f3ac97212..3760c6e30 100644 --- a/src/dstack/_internal/core/models/backends/dstack.py +++ b/src/dstack/_internal/core/models/backends/dstack.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel from typing_extensions import Literal diff --git a/src/dstack/_internal/core/models/backends/lambdalabs.py b/src/dstack/_internal/core/models/backends/lambdalabs.py index 344f0797e..7f1674768 100644 --- a/src/dstack/_internal/core/models/backends/lambdalabs.py +++ b/src/dstack/_internal/core/models/backends/lambdalabs.py @@ -1,6 +1,6 @@ from typing import List, Optional, Union -from pydantic import BaseModel, Field +from pydantic import BaseModel from typing_extensions import Literal from dstack._internal.core.models.backends.base import ConfigMultiElement diff --git a/src/dstack/_internal/core/models/profiles.py b/src/dstack/_internal/core/models/profiles.py index 35a9717ff..9677c734f 100644 --- a/src/dstack/_internal/core/models/profiles.py +++ b/src/dstack/_internal/core/models/profiles.py @@ -1,6 +1,6 @@ import re from enum import Enum -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Union from pydantic import Field, confloat, root_validator, validator from typing_extensions import Annotated, Literal diff --git a/src/dstack/_internal/core/models/repos/__init__.py b/src/dstack/_internal/core/models/repos/__init__.py index adf1966e4..371563ffb 100644 --- a/src/dstack/_internal/core/models/repos/__init__.py +++ b/src/dstack/_internal/core/models/repos/__init__.py @@ -2,8 +2,12 @@ from pydantic import BaseModel, Field -from dstack._internal.core.models.repos.local import LocalRepo, LocalRepoInfo, LocalRunRepoData -from dstack._internal.core.models.repos.remote import ( +from dstack._internal.core.models.repos.local import ( # noqa: F401 + LocalRepo, + LocalRepoInfo, + LocalRunRepoData, +) +from dstack._internal.core.models.repos.remote import ( # noqa: F401 RemoteRepo, RemoteRepoCreds, RemoteRepoInfo, diff --git a/src/dstack/_internal/core/models/repos/local.py b/src/dstack/_internal/core/models/repos/local.py index 9598282d4..5b03d9490 100644 --- a/src/dstack/_internal/core/models/repos/local.py +++ b/src/dstack/_internal/core/models/repos/local.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import BinaryIO, Optional -from pydantic import BaseModel from typing_extensions import Literal from dstack._internal.core.models.repos.base import BaseRepoInfo, Repo diff --git a/src/dstack/_internal/core/models/runs.py b/src/dstack/_internal/core/models/runs.py index 6d22471d7..7eb8638a3 100644 --- a/src/dstack/_internal/core/models/runs.py +++ b/src/dstack/_internal/core/models/runs.py @@ -1,9 +1,9 @@ from datetime import datetime, timedelta from enum import Enum -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional from pydantic import UUID4, BaseModel, Field -from typing_extensions import Annotated, Literal +from typing_extensions import Annotated from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import AnyRunConfiguration, RegistryAuth diff --git a/src/dstack/_internal/core/services/api_client.py b/src/dstack/_internal/core/services/api_client.py index acf59f18a..a3a2c097b 100644 --- a/src/dstack/_internal/core/services/api_client.py +++ b/src/dstack/_internal/core/services/api_client.py @@ -1,7 +1,7 @@ from typing import Optional, Tuple import dstack._internal.core.services.configs as configs -from dstack._internal.core.errors import ConfigurationError, DstackError +from dstack._internal.core.errors import ConfigurationError from dstack.api.server import APIClient @@ -11,5 +11,5 @@ def get_api_client(project_name: Optional[str] = None) -> Tuple[APIClient, str]: if project is None: if project_name is not None: raise ConfigurationError(f"Project {project_name} is not configured") - raise ConfigurationError(f"No default project, specify project name") + raise ConfigurationError("No default project, specify project name") return APIClient(project.url, project.token), project.name diff --git a/src/dstack/_internal/core/services/configs/__init__.py b/src/dstack/_internal/core/services/configs/__init__.py index 1c90f4496..f4cd82164 100644 --- a/src/dstack/_internal/core/services/configs/__init__.py +++ b/src/dstack/_internal/core/services/configs/__init__.py @@ -7,7 +7,6 @@ import yaml from pydantic import ValidationError from rich import print -from rich.prompt import Confirm from dstack._internal.cli.utils.common import colors, confirm_ask from dstack._internal.core.models.config import GlobalConfig, ProjectConfig, RepoConfig diff --git a/src/dstack/_internal/core/services/repos.py b/src/dstack/_internal/core/services/repos.py index d1edfa0f7..e28736c0c 100644 --- a/src/dstack/_internal/core/services/repos.py +++ b/src/dstack/_internal/core/services/repos.py @@ -15,7 +15,7 @@ RemoteRepoCreds, RemoteRunRepoData, ) -from dstack._internal.core.models.repos.base import Repo, RepoProtocol +from dstack._internal.core.models.repos.base import RepoProtocol from dstack._internal.utils.path import PathLike from dstack._internal.utils.ssh import ( get_host_config, diff --git a/src/dstack/_internal/core/services/ssh/__init__.py b/src/dstack/_internal/core/services/ssh/__init__.py index 7a9bf535d..d10dc862e 100644 --- a/src/dstack/_internal/core/services/ssh/__init__.py +++ b/src/dstack/_internal/core/services/ssh/__init__.py @@ -1,5 +1,3 @@ -from typing import re - from dstack._internal.core.errors import ( SSHConnectionRefusedError, SSHError, diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index e1d6e0152..8ad67c0e9 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -49,7 +49,6 @@ def create_app() -> FastAPI: - if settings.SENTRY_DSN is not None: sentry_sdk.init( dsn=settings.SENTRY_DSN, @@ -78,7 +77,7 @@ async def lifespan(app: FastAPI): os.path.expanduser("~"), "~", 1 ) if not config_loaded: - with console.status(f"Initializing the default configuration..."): + with console.status("Initializing the default configuration..."): await server_config_manager.init_config(session=session) console.print( f"[code]✓[/] Initialized the default configuration at [code]{server_config_dir}[/]" diff --git a/src/dstack/_internal/server/migrations/versions/112753bc17dd_remove_nullable_fields.py b/src/dstack/_internal/server/migrations/versions/112753bc17dd_remove_nullable_fields.py index 6f528b47e..4faf9c6b6 100644 --- a/src/dstack/_internal/server/migrations/versions/112753bc17dd_remove_nullable_fields.py +++ b/src/dstack/_internal/server/migrations/versions/112753bc17dd_remove_nullable_fields.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa -import sqlalchemy_utils from alembic import op # revision identifiers, used by Alembic. diff --git a/src/dstack/_internal/server/migrations/versions/14f2cb002fc2_add_jobmodel_removed_flag.py b/src/dstack/_internal/server/migrations/versions/14f2cb002fc2_add_jobmodel_removed_flag.py index 995173ca2..b7e3fed71 100644 --- a/src/dstack/_internal/server/migrations/versions/14f2cb002fc2_add_jobmodel_removed_flag.py +++ b/src/dstack/_internal/server/migrations/versions/14f2cb002fc2_add_jobmodel_removed_flag.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa -import sqlalchemy_utils from alembic import op from dstack._internal.core.models.runs import JobStatus diff --git a/src/dstack/_internal/server/migrations/versions/23e01c56279a_make_blob_nullable.py b/src/dstack/_internal/server/migrations/versions/23e01c56279a_make_blob_nullable.py index 9086a8eaf..1b6209c93 100644 --- a/src/dstack/_internal/server/migrations/versions/23e01c56279a_make_blob_nullable.py +++ b/src/dstack/_internal/server/migrations/versions/23e01c56279a_make_blob_nullable.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa -import sqlalchemy_utils from alembic import op # revision identifiers, used by Alembic. diff --git a/src/dstack/_internal/server/migrations/versions/3dbdce90d0e0_fix_code_uq_constraint.py b/src/dstack/_internal/server/migrations/versions/3dbdce90d0e0_fix_code_uq_constraint.py index e0114d59f..6cbe4a51b 100644 --- a/src/dstack/_internal/server/migrations/versions/3dbdce90d0e0_fix_code_uq_constraint.py +++ b/src/dstack/_internal/server/migrations/versions/3dbdce90d0e0_fix_code_uq_constraint.py @@ -5,8 +5,6 @@ Create Date: 2023-11-14 10:41:52.084668 """ -import sqlalchemy as sa -import sqlalchemy_utils from alembic import op # revision identifiers, used by Alembic. diff --git a/src/dstack/_internal/server/migrations/versions/48ad3ecbaea2_do_not_delete_projects_and_runs.py b/src/dstack/_internal/server/migrations/versions/48ad3ecbaea2_do_not_delete_projects_and_runs.py index 1455b93f3..edeeef2ea 100644 --- a/src/dstack/_internal/server/migrations/versions/48ad3ecbaea2_do_not_delete_projects_and_runs.py +++ b/src/dstack/_internal/server/migrations/versions/48ad3ecbaea2_do_not_delete_projects_and_runs.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa -import sqlalchemy_utils from alembic import op # revision identifiers, used by Alembic. diff --git a/src/dstack/_internal/server/migrations/versions/686fb8341ea5_add_user_emails.py b/src/dstack/_internal/server/migrations/versions/686fb8341ea5_add_user_emails.py index 5493a3f29..f07a0060c 100644 --- a/src/dstack/_internal/server/migrations/versions/686fb8341ea5_add_user_emails.py +++ b/src/dstack/_internal/server/migrations/versions/686fb8341ea5_add_user_emails.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa -import sqlalchemy_utils from alembic import op # revision identifiers, used by Alembic. diff --git a/src/dstack/_internal/server/migrations/versions/a060e2440936_.py b/src/dstack/_internal/server/migrations/versions/a060e2440936_.py index 81f00e638..520ef316a 100644 --- a/src/dstack/_internal/server/migrations/versions/a060e2440936_.py +++ b/src/dstack/_internal/server/migrations/versions/a060e2440936_.py @@ -1,7 +1,7 @@ """empty message Revision ID: a060e2440936 -Revises: +Revises: Create Date: 2023-09-20 16:34:34.649303 """ diff --git a/src/dstack/_internal/server/migrations/versions/bfba43f6def2_.py b/src/dstack/_internal/server/migrations/versions/bfba43f6def2_.py index 6066752d7..efd39c7f9 100644 --- a/src/dstack/_internal/server/migrations/versions/bfba43f6def2_.py +++ b/src/dstack/_internal/server/migrations/versions/bfba43f6def2_.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa -import sqlalchemy_utils from alembic import op # revision identifiers, used by Alembic. diff --git a/src/dstack/_internal/server/migrations/versions/d3e8af4786fa_gateway_compute_flag_deleted.py b/src/dstack/_internal/server/migrations/versions/d3e8af4786fa_gateway_compute_flag_deleted.py index 691935ae4..b8fc30d0f 100644 --- a/src/dstack/_internal/server/migrations/versions/d3e8af4786fa_gateway_compute_flag_deleted.py +++ b/src/dstack/_internal/server/migrations/versions/d3e8af4786fa_gateway_compute_flag_deleted.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa -import sqlalchemy_utils from alembic import op # revision identifiers, used by Alembic. diff --git a/src/dstack/_internal/server/routers/runs.py b/src/dstack/_internal/server/routers/runs.py index 4c7a0df1a..318ad4490 100644 --- a/src/dstack/_internal/server/routers/runs.py +++ b/src/dstack/_internal/server/routers/runs.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple +from typing import List, Tuple from fastapi import APIRouter, Depends from sqlalchemy.ext.asyncio import AsyncSession diff --git a/src/dstack/_internal/server/routers/secrets.py b/src/dstack/_internal/server/routers/secrets.py index 54d9e4f20..3732a0b1c 100644 --- a/src/dstack/_internal/server/routers/secrets.py +++ b/src/dstack/_internal/server/routers/secrets.py @@ -4,7 +4,6 @@ from dstack._internal.core.models.runs import Run from dstack._internal.core.models.secrets import Secret -from dstack._internal.server.schemas.runs import SubmitRunRequest from dstack._internal.server.schemas.secrets import ( AddSecretRequest, DeleteSecretsRequest, diff --git a/src/dstack/_internal/server/security/permissions.py b/src/dstack/_internal/server/security/permissions.py index 6722efa30..b5555bc10 100644 --- a/src/dstack/_internal/server/security/permissions.py +++ b/src/dstack/_internal/server/security/permissions.py @@ -1,6 +1,6 @@ from typing import Tuple -from fastapi import Depends, Security, status +from fastapi import Depends, Security from fastapi.security import HTTPBearer from fastapi.security.http import HTTPAuthorizationCredentials from sqlalchemy.ext.asyncio import AsyncSession diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py index 0d1fd0e26..09a54cc0a 100644 --- a/src/dstack/_internal/server/services/backends/__init__.py +++ b/src/dstack/_internal/server/services/backends/__init__.py @@ -1,6 +1,5 @@ import asyncio import heapq -import time from typing import List, Optional, Tuple, Type, Union from sqlalchemy import delete, update diff --git a/src/dstack/_internal/server/services/backends/configurators/aws.py b/src/dstack/_internal/server/services/backends/configurators/aws.py index 3b2555e6a..f9d489021 100644 --- a/src/dstack/_internal/server/services/backends/configurators/aws.py +++ b/src/dstack/_internal/server/services/backends/configurators/aws.py @@ -1,5 +1,5 @@ import json -from typing import List, Optional +from typing import List from dstack._internal.core.backends.aws import AWSBackend, auth from dstack._internal.core.backends.aws.config import AWSConfig @@ -73,7 +73,7 @@ def get_config_values(self, config: AWSConfigInfoWithCredsPartial) -> AWSConfigV raise_invalid_credentials_error(fields=[["creds"]]) try: auth.authenticate(creds=config.creds, region=MAIN_REGION) - except: + except Exception: if isinstance(config.creds, AWSAccessKeyCreds): raise_invalid_credentials_error( fields=[ diff --git a/src/dstack/_internal/server/services/backends/configurators/azure.py b/src/dstack/_internal/server/services/backends/configurators/azure.py index c6e4fba83..cdc2da1f7 100644 --- a/src/dstack/_internal/server/services/backends/configurators/azure.py +++ b/src/dstack/_internal/server/services/backends/configurators/azure.py @@ -69,7 +69,7 @@ ("(Asia Pacific) East Asia", "eastasia"), ("(South America) Brazil South", "brazilsouth"), ] -LOCATION_VALUES = [l[1] for l in LOCATIONS] +LOCATION_VALUES = [loc[1] for loc in LOCATIONS] DEFAULT_LOCATIONS = LOCATION_VALUES MAIN_LOCATION = "eastus" @@ -269,8 +269,8 @@ def _get_subscription_id_element( def _get_locations_element(self, selected: List[str]) -> ConfigMultiElement: element = ConfigMultiElement() - for l in LOCATION_VALUES: - element.values.append(ConfigElementValue(value=l, label=l)) + for loc in LOCATION_VALUES: + element.values.append(ConfigElementValue(value=loc, label=loc)) element.selected = selected return element diff --git a/src/dstack/_internal/server/services/backends/configurators/gcp.py b/src/dstack/_internal/server/services/backends/configurators/gcp.py index 22fc912c7..3b6fdc9a9 100644 --- a/src/dstack/_internal/server/services/backends/configurators/gcp.py +++ b/src/dstack/_internal/server/services/backends/configurators/gcp.py @@ -112,7 +112,7 @@ "default_zone": "australia-southeast1-c", }, ] -REGIONS = [r for l in LOCATIONS for r in l["regions"]] +REGIONS = [r for loc in LOCATIONS for r in loc["regions"]] DEFAULT_REGIONS = REGIONS MAIN_REGION = "us-east1" diff --git a/src/dstack/_internal/server/services/backends/configurators/kubernetes.py b/src/dstack/_internal/server/services/backends/configurators/kubernetes.py index bc639ee61..8585fad6e 100644 --- a/src/dstack/_internal/server/services/backends/configurators/kubernetes.py +++ b/src/dstack/_internal/server/services/backends/configurators/kubernetes.py @@ -1,9 +1,8 @@ -from typing import List, Optional +from typing import List from dstack._internal.core.backends.kubernetes import KubernetesBackend from dstack._internal.core.backends.kubernetes.client import get_api_from_config_data from dstack._internal.core.backends.kubernetes.config import KubernetesConfig -from dstack._internal.core.errors import BackendInvalidCredentialsError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.backends.kubernetes import ( AnyKubernetesConfigInfo, diff --git a/src/dstack/_internal/server/services/backends/configurators/lambdalabs.py b/src/dstack/_internal/server/services/backends/configurators/lambdalabs.py index 6ea12ebe8..0d0c094e6 100644 --- a/src/dstack/_internal/server/services/backends/configurators/lambdalabs.py +++ b/src/dstack/_internal/server/services/backends/configurators/lambdalabs.py @@ -1,12 +1,8 @@ import json from typing import List -from requests import HTTPError - -from dstack._internal.core.backends.base import Backend from dstack._internal.core.backends.lambdalabs import LambdaBackend, api_client from dstack._internal.core.backends.lambdalabs.config import LambdaConfig -from dstack._internal.core.errors import BackendInvalidCredentialsError from dstack._internal.core.models.backends.base import ( BackendType, ConfigElementValue, diff --git a/src/dstack/_internal/server/services/gateways/pool.py b/src/dstack/_internal/server/services/gateways/pool.py index 86213d210..11215c2a6 100644 --- a/src/dstack/_internal/server/services/gateways/pool.py +++ b/src/dstack/_internal/server/services/gateways/pool.py @@ -1,7 +1,6 @@ import asyncio from typing import Dict, List, Optional -from dstack._internal.core.errors import SSHError from dstack._internal.server.services.gateways.connection import GatewayConnection from dstack._internal.utils.logging import get_logger diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index b92a75fd2..7f8f5148d 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -4,7 +4,6 @@ from datetime import timezone from typing import Dict, List, Optional -from sqlalchemy import update from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.errors import SSHError diff --git a/src/dstack/_internal/server/services/jobs/configurators/base.py b/src/dstack/_internal/server/services/jobs/configurators/base.py index c62d55474..d2a6b25fc 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/base.py +++ b/src/dstack/_internal/server/services/jobs/configurators/base.py @@ -77,7 +77,7 @@ def _commands(self) -> List[str]: entrypoint = ["/bin/sh", "-i", "-c"] commands = [_join_shell_commands(self._shell_commands())] else: # custom docker image without commands - raise NotImplemented() # TODO read docker image manifest + raise NotImplementedError() # TODO read docker image manifest return entrypoint + commands def _app_specs(self) -> List[AppSpec]: diff --git a/src/dstack/_internal/server/services/jobs/configurators/dev.py b/src/dstack/_internal/server/services/jobs/configurators/dev.py index 0163b2f5e..35676ab92 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/dev.py +++ b/src/dstack/_internal/server/services/jobs/configurators/dev.py @@ -9,8 +9,8 @@ DEFAULT_MAX_DURATION_SECONDS = 6 * 3600 INSTALL_IPYKERNEL = ( - f"(echo pip install ipykernel... && pip install -q --no-cache-dir ipykernel 2> /dev/null) || " - f'echo "no pip, ipykernel was not installed"' + "(echo pip install ipykernel... && pip install -q --no-cache-dir ipykernel 2> /dev/null) || " + 'echo "no pip, ipykernel was not installed"' ) diff --git a/src/dstack/_internal/server/services/jobs/configurators/extensions/vscode.py b/src/dstack/_internal/server/services/jobs/configurators/extensions/vscode.py index f1050436b..f38c4fcec 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/extensions/vscode.py +++ b/src/dstack/_internal/server/services/jobs/configurators/extensions/vscode.py @@ -20,8 +20,8 @@ def get_install_commands(self) -> List[str]: target = f'~/.vscode-server/bin/"{self.version}"' commands.extend( [ - f'if [ $(uname -m) = "aarch64" ]; then arch="arm64"; else arch="x64"; fi', - f"mkdir -p /tmp", + 'if [ $(uname -m) = "aarch64" ]; then arch="arm64"; else arch="x64"; fi', + "mkdir -p /tmp", f'wget -q --show-progress "{url}" -O "/tmp/{archive}"', f"mkdir -vp {target}", f'tar --no-same-owner -xz --strip-components=1 -C {target} -f "/tmp/{archive}"', @@ -35,8 +35,8 @@ def get_install_commands(self) -> List[str]: def get_print_readme_commands(self) -> List[str]: return [ - f"echo To open in VS Code Desktop, use link below:", - f"echo ''", + "echo To open in VS Code Desktop, use link below:", + "echo ''", f"echo ' vscode://vscode-remote/ssh-remote+{self.run_name}/workflow'", # TODO use $REPO_DIR - f"echo ''", + "echo ''", ] diff --git a/src/dstack/_internal/server/services/logs.py b/src/dstack/_internal/server/services/logs.py index bc191b89b..364d434f3 100644 --- a/src/dstack/_internal/server/services/logs.py +++ b/src/dstack/_internal/server/services/logs.py @@ -44,10 +44,10 @@ def _write_logs( log_file_path: Path, log_events: List[RunnerLogEvent], ): - log_events_parsed = [_runner_log_event_to_log_event(l) for l in log_events] + log_events_parsed = [_runner_log_event_to_log_event(log) for log in log_events] log_file_path.parent.mkdir(exist_ok=True, parents=True) with open(log_file_path, "a") as f: - f.writelines(l.json() + "\n" for l in log_events_parsed) + f.writelines(log.json() + "\n" for log in log_events_parsed) def poll_logs( diff --git a/src/dstack/_internal/server/services/repos.py b/src/dstack/_internal/server/services/repos.py index cc9163d6a..c741c4d40 100644 --- a/src/dstack/_internal/server/services/repos.py +++ b/src/dstack/_internal/server/services/repos.py @@ -3,7 +3,7 @@ import sqlalchemy.exc from fastapi import UploadFile -from sqlalchemy import delete, insert, select, update +from sqlalchemy import delete, select, update from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.errors import RepoDoesNotExistError, ServerClientError diff --git a/src/dstack/_internal/server/services/runner/ssh.py b/src/dstack/_internal/server/services/runner/ssh.py index ba7a486ce..959bff6f1 100644 --- a/src/dstack/_internal/server/services/runner/ssh.py +++ b/src/dstack/_internal/server/services/runner/ssh.py @@ -20,14 +20,14 @@ def runner_ssh_tunnel( ports: List[int], retries: int = 3, retry_interval: float = 1 ) -> Callable[[Callable[P, bool]], Callable[Concatenate[str, JobProvisioningData, P], bool]]: def decorator( - func: Callable[P, bool] + func: Callable[P, bool], ) -> Callable[Concatenate[str, JobProvisioningData, P], bool]: @functools.wraps(func) def wrapper( ssh_private_key: str, job_provisioning_data: JobProvisioningData, *args: P.args, - **kwargs: P.kwargs + **kwargs: P.kwargs, ) -> bool: """ Returns: diff --git a/src/dstack/_internal/server/services/storage.py b/src/dstack/_internal/server/services/storage.py index c66671f1d..0745ab36e 100644 --- a/src/dstack/_internal/server/services/storage.py +++ b/src/dstack/_internal/server/services/storage.py @@ -1,15 +1,14 @@ from typing import Optional -BOTO_AVAILABLE = True +from dstack._internal.server import settings +BOTO_AVAILABLE = True try: import botocore.exceptions from boto3 import Session except ImportError: BOTO_AVAILABLE = False -from dstack._internal.server import settings - class S3Storage: def __init__( diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index 01bd9f537..a305d8fa8 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -1,8 +1,6 @@ import os from pathlib import Path -from dstack import version - DSTACK_DIR_PATH = Path("~/.dstack/").expanduser() SERVER_DIR_PATH = Path(os.getenv("DSTACK_SERVER_DIR", DSTACK_DIR_PATH / "server")) diff --git a/src/dstack/_internal/server/utils/common.py b/src/dstack/_internal/server/utils/common.py index a4a80c0c2..b35adc02d 100644 --- a/src/dstack/_internal/server/utils/common.py +++ b/src/dstack/_internal/server/utils/common.py @@ -21,7 +21,7 @@ async def gather_map_async( items: Sequence[ItemT], func: Callable[[ItemT], Awaitable[ResultT]], *, - return_exceptions: bool = False + return_exceptions: bool = False, ) -> List[Tuple[ItemT, Union[ResultT, Exception]]]: """ A parallel wrapper around asyncio.gather that returns a list of tuples (item, result). diff --git a/src/dstack/_internal/server/utils/logging.py b/src/dstack/_internal/server/utils/logging.py index 3c4a45f14..9a171c7c6 100644 --- a/src/dstack/_internal/server/utils/logging.py +++ b/src/dstack/_internal/server/utils/logging.py @@ -1,6 +1,5 @@ import asyncio import logging -import os import sys from dstack._internal.server import settings diff --git a/src/dstack/_internal/server/utils/routers.py b/src/dstack/_internal/server/utils/routers.py index 66022d958..9e286a4c0 100644 --- a/src/dstack/_internal/server/utils/routers.py +++ b/src/dstack/_internal/server/utils/routers.py @@ -1,11 +1,10 @@ from typing import Dict, List, Optional -from fastapi import HTTPException, Request, Response, status +from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse from packaging import version from dstack._internal.core.errors import ServerClientError -from dstack._internal.server import settings def error_detail(msg: str, code: Optional[str] = None, **kwargs) -> Dict: diff --git a/src/dstack/_internal/utils/common.py b/src/dstack/_internal/utils/common.py index eb9de218e..a3e724b92 100644 --- a/src/dstack/_internal/utils/common.py +++ b/src/dstack/_internal/utils/common.py @@ -2,7 +2,7 @@ import time from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Optional, Tuple, Union +from typing import Any, Optional, Union def get_dstack_dir() -> Path: diff --git a/src/dstack/_internal/utils/ssh.py b/src/dstack/_internal/utils/ssh.py index 6142d590f..88bde95a7 100644 --- a/src/dstack/_internal/utils/ssh.py +++ b/src/dstack/_internal/utils/ssh.py @@ -91,7 +91,7 @@ def update_ssh_config(path: PathLike, host: str, options: Dict[str, str]): if os.path.exists(path): with open(path, "r") as f: for line in f: - m = re.match(rf"^Host\s+(\S+)$", line.strip()) + m = re.match(r"^Host\s+(\S+)$", line.strip()) if m is not None: copy_mode = m.group(1) != host if copy_mode: diff --git a/src/dstack/api/_public/__init__.py b/src/dstack/api/_public/__init__.py index ffcc6da95..1c1490619 100644 --- a/src/dstack/api/_public/__init__.py +++ b/src/dstack/api/_public/__init__.py @@ -66,7 +66,7 @@ def from_config( """ if server_url is not None and user_token is not None: if project_name is None: - raise ConfigurationError(f"The project name is not specified") + raise ConfigurationError("The project name is not specified") api_client = APIClient(server_url, user_token) else: api_client, project_name = api_client_service.get_api_client(project_name=project_name) diff --git a/src/dstack/api/_public/huggingface/completions/__init__.py b/src/dstack/api/_public/huggingface/completions/__init__.py index 7ae27c92c..f4902a840 100644 --- a/src/dstack/api/_public/huggingface/completions/__init__.py +++ b/src/dstack/api/_public/huggingface/completions/__init__.py @@ -21,7 +21,7 @@ def get_configuration( "The quantize argument can be one of the following: 'awq', 'eetq', 'gptq', or 'bitsandbytes'." ) env["MODEL_ID"] = model_name - launcher_command = f"text-generation-launcher --hostname 0.0.0.0 --port 80 --trust-remote-code" + launcher_command = "text-generation-launcher --hostname 0.0.0.0 --port 80 --trust-remote-code" if quantize: launcher_command += f" --quantize {quantize}" if dtype: diff --git a/src/dstack/api/_public/repos.py b/src/dstack/api/_public/repos.py index bec23a06a..bb0ac8f12 100644 --- a/src/dstack/api/_public/repos.py +++ b/src/dstack/api/_public/repos.py @@ -132,7 +132,7 @@ def load( repo_config = config.get_repo_config(repo_dir) if repo_config is None: raise ConfigurationError( - f"The repo is not initialized. Run `dstack init` to initialize the repo." + "The repo is not initialized. Run `dstack init` to initialize the repo." ) repo = load_repo(repo_config) try: @@ -140,7 +140,7 @@ def load( except requests.HTTPError as e: if "404" in e.args[0]: raise ConfigurationError( - f"The repo is not initialized. Run `dstack init` to initialize the repo." + "The repo is not initialized. Run `dstack init` to initialize the repo." ) raise else: diff --git a/src/dstack/api/_public/runs.py b/src/dstack/api/_public/runs.py index 89558743a..bd3faab9e 100644 --- a/src/dstack/api/_public/runs.py +++ b/src/dstack/api/_public/runs.py @@ -19,10 +19,9 @@ from dstack._internal.core.models.profiles import Profile, ProfileRetryPolicy, SpotPolicy from dstack._internal.core.models.repos.base import Repo from dstack._internal.core.models.resources import ResourcesSpec -from dstack._internal.core.models.runs import JobSpec +from dstack._internal.core.models.runs import JobSpec, RunPlan, RunSpec from dstack._internal.core.models.runs import JobStatus as RunStatus from dstack._internal.core.models.runs import Run as RunModel -from dstack._internal.core.models.runs import RunPlan, RunSpec from dstack._internal.core.services.logs import URLReplacer from dstack._internal.core.services.ssh.attach import SSHAttach from dstack._internal.core.services.ssh.ports import PortsLock diff --git a/src/dstack/api/server/_group.py b/src/dstack/api/server/_group.py index 19a391edf..5f4052fe7 100644 --- a/src/dstack/api/server/_group.py +++ b/src/dstack/api/server/_group.py @@ -1,4 +1,4 @@ -from typing import Optional, Union +from typing import Optional import requests from typing_extensions import Protocol diff --git a/src/dstack/api/server/_runs.py b/src/dstack/api/server/_runs.py index 0f1bbe1ca..63bc79367 100644 --- a/src/dstack/api/server/_runs.py +++ b/src/dstack/api/server/_runs.py @@ -16,7 +16,7 @@ class RunsAPIClient(APIClientGroup): def list(self, project_name: Optional[str], repo_id: Optional[str]) -> List[Run]: body = ListRunsRequest(project_name=project_name, repo_id=repo_id) - resp = self._request(f"/api/runs/list", body=body.json()) + resp = self._request("/api/runs/list", body=body.json()) return parse_obj_as(List[Run], resp.json()) def get(self, project_name: str, run_name: str) -> Run: diff --git a/src/dstack/api/server/_secrets.py b/src/dstack/api/server/_secrets.py index fbbcb0580..2b16010ec 100644 --- a/src/dstack/api/server/_secrets.py +++ b/src/dstack/api/server/_secrets.py @@ -19,7 +19,7 @@ def list(self, project_name: str, repo_id: str) -> List[Secret]: return parse_obj_as(List[Secret], resp.json()) def get(self, project_name: str, repo_id: str, secret_name: str) -> Secret: - raise NotImplemented() + raise NotImplementedError() body = GetSecretsRequest(repo_id=repo_id) resp = self._request(f"/api/project/{project_name}/secrets/get", body=body.json()) return parse_obj_as(Secret, resp.json()) diff --git a/src/dstack/api/utils.py b/src/dstack/api/utils.py index ad5c976d1..d8a64e28c 100644 --- a/src/dstack/api/utils.py +++ b/src/dstack/api/utils.py @@ -4,7 +4,7 @@ import yaml from dstack._internal.core.errors import ConfigurationError -from dstack._internal.core.models.configurations import AnyRunConfiguration, BaseConfiguration +from dstack._internal.core.models.configurations import AnyRunConfiguration from dstack._internal.core.models.configurations import parse as parse_configuration from dstack._internal.core.models.profiles import Profile, ProfilesConfig from dstack._internal.utils.path import PathLike, path_in_dir @@ -52,7 +52,7 @@ def load_configuration( repo_dir = Path(repo_dir) work_dir = repo_dir / (work_dir or ".") if not path_in_dir(work_dir, repo_dir): - raise ConfigurationError(f"Working directory is outside of the repo") + raise ConfigurationError("Working directory is outside of the repo") if configuration_file is None: configuration_path = work_dir / ".dstack.yml" @@ -61,7 +61,7 @@ def load_configuration( else: configuration_path = repo_dir / configuration_file if not path_in_dir(configuration_path, repo_dir): - raise ConfigurationError(f"Configuration file is outside of the repo") + raise ConfigurationError("Configuration file is outside of the repo") try: with open(configuration_path, "r") as f: diff --git a/src/tests/_internal/server/background/tasks/test_process_running_jobs.py b/src/tests/_internal/server/background/tasks/test_process_running_jobs.py index 29bd6a272..c539d3c6a 100644 --- a/src/tests/_internal/server/background/tasks/test_process_running_jobs.py +++ b/src/tests/_internal/server/background/tasks/test_process_running_jobs.py @@ -147,9 +147,7 @@ async def test_updates_running_job(self, test_db, session: AsyncSession, tmp_pat "dstack._internal.server.services.runner.ssh.RunnerTunnel" ) as RunnerTunnelMock, patch( "dstack._internal.server.services.runner.client.RunnerClient" - ) as RunnerClientMock, patch.object( - settings, "SERVER_DIR_PATH", tmp_path - ): + ) as RunnerClientMock, patch.object(settings, "SERVER_DIR_PATH", tmp_path): runner_client_mock = RunnerClientMock.return_value runner_client_mock.pull.return_value = PullResponse( job_states=[JobStateEvent(timestamp=1, state=JobStatus.RUNNING)], diff --git a/src/tests/_internal/server/background/tasks/test_process_submitted_jobs.py b/src/tests/_internal/server/background/tasks/test_process_submitted_jobs.py index 2bb8a6b2f..bb0e175ee 100644 --- a/src/tests/_internal/server/background/tasks/test_process_submitted_jobs.py +++ b/src/tests/_internal/server/background/tasks/test_process_submitted_jobs.py @@ -1,7 +1,5 @@ -import json from datetime import datetime, timezone from unittest.mock import Mock, patch -from uuid import UUID import pytest from sqlalchemy.ext.asyncio import AsyncSession diff --git a/src/tests/_internal/server/background/tasks/test_process_terminating_jobs.py b/src/tests/_internal/server/background/tasks/test_process_terminating_jobs.py index 08688261e..bb4173eef 100644 --- a/src/tests/_internal/server/background/tasks/test_process_terminating_jobs.py +++ b/src/tests/_internal/server/background/tasks/test_process_terminating_jobs.py @@ -1,5 +1,4 @@ -from datetime import datetime, timezone -from unittest.mock import Mock, patch +from unittest.mock import patch import pytest from sqlalchemy.ext.asyncio import AsyncSession diff --git a/src/tests/_internal/server/routers/test_backends.py b/src/tests/_internal/server/routers/test_backends.py index ddfa6d4a4..c6c259199 100644 --- a/src/tests/_internal/server/routers/test_backends.py +++ b/src/tests/_internal/server/routers/test_backends.py @@ -643,7 +643,7 @@ async def test_creates_aws_backend(self, test_db, session: AsyncSession): "dstack._internal.core.backends.aws.auth.default_creds_available" ) as default_creds_available_mock, patch( "dstack._internal.core.backends.aws.auth.authenticate" - ) as authenticate_mock: + ) as authenticate_mock: # noqa: F841 default_creds_available_mock.return_value = False response = client.post( f"/api/project/{project.name}/backends/create", @@ -816,7 +816,7 @@ async def test_returns_400_if_backend_exists(self, test_db, session: AsyncSessio "dstack._internal.core.backends.aws.auth.default_creds_available" ) as default_creds_available_mock, patch( "dstack._internal.core.backends.aws.auth.authenticate" - ) as authenticate_mock: + ) as authenticate_mock: # noqa: F841 default_creds_available_mock.return_value = False response = client.post( f"/api/project/{project.name}/backends/create", @@ -866,7 +866,7 @@ async def test_updates_backend(self, test_db, session: AsyncSession): "dstack._internal.core.backends.aws.auth.default_creds_available" ) as default_creds_available_mock, patch( "dstack._internal.core.backends.aws.auth.authenticate" - ) as authenticate_mock: + ) as authenticate_mock: # noqa: F841 default_creds_available_mock.return_value = False response = client.post( f"/api/project/{project.name}/backends/update", diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py index 2aa8e5377..a1c715138 100644 --- a/src/tests/_internal/server/routers/test_gateways.py +++ b/src/tests/_internal/server/routers/test_gateways.py @@ -29,7 +29,7 @@ class TestListAndGetGateways: @pytest.mark.asyncio async def test_returns_40x_if_not_authenticated(self, test_db, session: AsyncSession): - response = client.post(f"/api/project/main/gateways/list") + response = client.post("/api/project/main/gateways/list") assert response.status_code == 403 @pytest.mark.asyncio @@ -281,7 +281,7 @@ async def test_get_default_gateway(self, test_db, session: AsyncSession): async def test_default_gateway_is_missing(self, test_db, session: AsyncSession): project = await create_project(session) backend = await create_backend(session, project.id) - gateway = await create_gateway(session, project.id, backend.id) + await create_gateway(session, project.id, backend.id) res = await get_project_default_gateway(session, project) assert res is None diff --git a/src/tests/_internal/server/routers/test_projects.py b/src/tests/_internal/server/routers/test_projects.py index 7d1c4a526..5ca1e7e8d 100644 --- a/src/tests/_internal/server/routers/test_projects.py +++ b/src/tests/_internal/server/routers/test_projects.py @@ -137,7 +137,7 @@ async def test_returns_400_if_user_project_quota_exceeded( response = client.post( "/api/projects/create", headers=get_auth_headers(user.token), - json={"project_name": f"project4"}, + json={"project_name": "project4"}, ) assert response.status_code == 400 assert response.json() == { diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 814319013..62122417f 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -263,7 +263,7 @@ def get_dev_env_run_dict( class TestListRuns: @pytest.mark.asyncio async def test_returns_40x_if_not_authenticated(self, test_db, session: AsyncSession): - response = client.post(f"/api/runs/list") + response = client.post("/api/runs/list") assert response.status_code in [401, 403] @pytest.mark.asyncio @@ -294,7 +294,7 @@ async def test_lists_runs(self, test_db, session: AsyncSession): ) job_spec = JobSpec.parse_raw(job.job_spec_data) response = client.post( - f"/api/runs/list", + "/api/runs/list", headers=get_auth_headers(user.token), json={}, ) @@ -651,6 +651,8 @@ async def test_deletes_runs(self, test_db, session: AsyncSession): assert response.status_code == 200 await session.refresh(run) assert run.deleted + await session.refresh(job) + assert job.status == JobStatus.FAILED @pytest.mark.asyncio async def test_returns_400_if_runs_active(self, test_db, session: AsyncSession): @@ -669,7 +671,7 @@ async def test_returns_400_if_runs_active(self, test_db, session: AsyncSession): repo=repo, user=user, ) - job = await create_job( + await create_job( session=session, run=run, ) diff --git a/src/tests/_internal/server/services/test_config.py b/src/tests/_internal/server/services/test_config.py index 4f337b420..27e55243a 100644 --- a/src/tests/_internal/server/services/test_config.py +++ b/src/tests/_internal/server/services/test_config.py @@ -1,4 +1,3 @@ -import json from pathlib import Path from unittest.mock import patch @@ -13,10 +12,7 @@ from dstack._internal.server.models import BackendModel from dstack._internal.server.services.config import AzureConfig, ServerConfigManager from dstack._internal.server.testing.common import ( - create_backend, create_project, - create_user, - get_auth_headers, )