Merge branch 'master' into droplet

.github/workflows/format.yml

@@ -21,29 +21,35 @@ jobs:
         python-version: ["3.8"]
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+    - name: Install the latest version of uv
+      uses: astral-sh/setup-uv@v4
       with:
+        version: "latest"
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install yapf==0.32.0
-        pip install toml==0.10.2
-        pip install black==22.10.0
-        pip install isort==5.12.0
+        uv venv --seed ~/test-env
+        source ~/test-env/bin/activate
+        uv pip install yapf==0.32.0
+        uv pip install toml==0.10.2
+        uv pip install black==22.10.0
+        uv pip install isort==5.12.0
     - name: Running yapf
       run: |
+        source ~/test-env/bin/activate
         yapf --diff --recursive ./ --exclude 'sky/skylet/ray_patches/**' \
             --exclude 'sky/skylet/providers/ibm/**'
     - name: Running black
       run: |
+        source ~/test-env/bin/activate
         black --diff --check sky/skylet/providers/ibm/
     - name: Running isort for black formatted files
       run: |
+        source ~/test-env/bin/activate
         isort --diff --check --profile black -l 88 -m 3 \
             sky/skylet/providers/ibm/
     - name: Running isort for yapf formatted files
       run: |
+        source ~/test-env/bin/activate
         isort --diff --check ./ --sg 'sky/skylet/ray_patches/**' \
             --sg 'sky/skylet/providers/ibm/**'

.github/workflows/mypy.yml

@@ -11,6 +11,8 @@ on:
     branches:
       - master
       - 'releases/**'
+  merge_group:
+
 jobs:
   mypy:
     runs-on: ubuntu-latest
@@ -19,15 +21,18 @@ jobs:
         python-version: ["3.8"]
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+    - name: Install the latest version of uv
+      uses: astral-sh/setup-uv@v4
       with:
+        version: "latest"
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install mypy==$(grep mypy requirements-dev.txt | cut -d'=' -f3)
-        pip install $(grep types- requirements-dev.txt | tr '\n' ' ')
+        uv venv --seed ~/test-env
+        source ~/test-env/bin/activate
+        uv pip install mypy==$(grep mypy requirements-dev.txt | cut -d'=' -f3)
+        uv pip install $(grep types- requirements-dev.txt | tr '\n' ' ')
     - name: Running mypy
       run: |
+        source ~/test-env/bin/activate
         mypy $(cat tests/mypy_files.txt)

.github/workflows/pylint.yml

@@ -21,16 +21,20 @@ jobs:
         python-version: ["3.8"]
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+    - name: Install the latest version of uv
+      uses: astral-sh/setup-uv@v4
       with:
+        version: "latest"
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install ".[all]"
-        pip install pylint==2.14.5
-        pip install pylint-quotes==0.2.3
+        uv venv --seed ~/test-env
+        source ~/test-env/bin/activate
+        uv pip install --prerelease=allow "azure-cli>=2.65.0"
+        uv pip install ".[all]"
+        uv pip install pylint==2.14.5
+        uv pip install pylint-quotes==0.2.3
     - name: Analysing the code with pylint
       run: |
+        source ~/test-env/bin/activate
         pylint --load-plugins pylint_quotes sky

.github/workflows/pytest.yml

@@ -35,26 +35,21 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-
-      - name: Install Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v4
         with:
+          version: "latest"
           python-version: ${{ matrix.python-version }}
-
-      - name: Cache dependencies
-        uses: actions/cache@v3
-        if: startsWith(runner.os, 'Linux')
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-pytest-${{ matrix.python-version }}
-          restore-keys: |
-            ${{ runner.os }}-pip-pytest-${{ matrix.python-version }}
-
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[all]"
-          pip install pytest pytest-xdist pytest-env>=0.6 memory-profiler==0.61.0
-
+          uv venv --seed ~/test-env
+          source ~/test-env/bin/activate
+          uv pip install --prerelease=allow "azure-cli>=2.65.0"
+          # Use -e to include examples and tests folder in the path for unit
+          # tests to access them.
+          uv pip install -e ".[all]"
+          uv pip install pytest pytest-xdist pytest-env>=0.6 memory-profiler==0.61.0
       - name: Run tests with pytest
-        run: SKYPILOT_DISABLE_USAGE_COLLECTION=1 SKYPILOT_SKIP_CLOUD_IDENTITY_CHECK=1 pytest -n 0 --dist no ${{ matrix.test-path }}
+        run: |
+          source ~/test-env/bin/activate
+          SKYPILOT_DISABLE_USAGE_COLLECTION=1 SKYPILOT_SKIP_CLOUD_IDENTITY_CHECK=1 pytest -n 0 --dist no ${{ matrix.test-path }}

.github/workflows/test-doc-build.yml

@@ -14,24 +14,28 @@ on:
   merge_group:
 
 jobs:
-  format:
+  doc-build:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.10"]
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+    - name: Install the latest version of uv
+      uses: astral-sh/setup-uv@v4
       with:
+        version: "latest"
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install .
+        uv venv --seed ~/test-env
+        source ~/test-env/bin/activate
+        uv pip install --prerelease=allow "azure-cli>=2.65.0"
+        uv pip install ".[all]"
         cd docs
-        pip install -r ./requirements-docs.txt
+        uv pip install -r ./requirements-docs.txt
     - name: Build documentation
       run: |
+        source ~/test-env/bin/activate
         cd ./docs
         ./build.sh

.pre-commit-config.yaml

@@ -0,0 +1,74 @@
+# Ensure this configuration aligns with format.sh and requirements.txt
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+
+-   repo: https://github.com/psf/black
+    rev: 22.10.0  # Match the version from requirements
+    hooks:
+    -   id: black
+        name: black (IBM specific)
+        files: "^sky/skylet/providers/ibm/.*"  # Match only files in the IBM directory
+
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0  # Match the version from requirements
+    hooks:
+    # First isort command
+    -   id: isort
+        name: isort (general)
+        args:
+          - "--sg=build/**"  # Matches "${ISORT_YAPF_EXCLUDES[@]}"
+          - "--sg=sky/skylet/providers/ibm/**"
+        files: "^(sky|tests|examples|llm|docs)/.*"  # Only match these directories
+    # Second isort command
+    -   id: isort
+        name: isort (IBM specific)
+        args:
+          - "--profile=black"
+          - "-l=88"
+          - "-m=3"
+        files: "^sky/skylet/providers/ibm/.*"  # Only match IBM-specific directory
+
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991  # Match the version from requirements
+    hooks:
+    -   id: mypy
+        args:
+            # From tests/mypy_files.txt
+            - "sky"
+            - "--exclude"
+            - "sky/benchmark|sky/callbacks|sky/skylet/providers/azure|sky/resources.py|sky/backends/monkey_patches"
+        pass_filenames: false
+        additional_dependencies:
+            - types-PyYAML
+            - types-requests<2.31  # Match the condition in requirements.txt
+            - types-setuptools
+            - types-cachetools
+            - types-pyvmomi
+
+-   repo: https://github.com/google/yapf
+    rev: v0.32.0  # Match the version from requirements
+    hooks:
+    -   id: yapf
+        name: yapf
+        exclude: (build/.*|sky/skylet/providers/ibm/.*)  # Matches exclusions from the script
+        args: ['--recursive', '--parallel']  # Only necessary flags
+        additional_dependencies: [toml==0.10.2]
+
+-   repo: https://github.com/pylint-dev/pylint
+    rev: v2.14.5  # Match the version from requirements
+    hooks:
+    -   id: pylint
+        additional_dependencies:
+            - pylint-quotes==0.2.3  # Match the version from requirements
+        name: pylint
+        args:
+            - --rcfile=.pylintrc  # Use your custom pylint configuration
+            - --load-plugins=pylint_quotes  # Load the pylint-quotes plugin
+        files: ^sky/  # Only include files from the 'sky/' directory
+        exclude: ^sky/skylet/providers/ibm/

CONTRIBUTING.md

@@ -78,6 +78,7 @@ It has some convenience features which you might find helpful (see [Dockerfile](
 - If relevant, add tests for your changes. For changes that touch the core system, run the [smoke tests](#testing) and ensure they pass.
 - Follow the [Google style guide](https://google.github.io/styleguide/pyguide.html).
 - Ensure code is properly formatted by running [`format.sh`](https://github.com/skypilot-org/skypilot/blob/master/format.sh).
+  - [Optional] You can also install pre-commit hooks by running `pre-commit install` to automatically format your code on commit.
 - Push your changes to your fork and open a pull request in the SkyPilot repository.
 - In the PR description, write a `Tested:` section to describe relevant tests performed.
 

README.md

@@ -155,10 +155,11 @@ SkyPilot then performs the heavy-lifting for you, including:
 Refer to [Quickstart](https://skypilot.readthedocs.io/en/latest/getting-started/quickstart.html) to get started with SkyPilot.
 
 ## More Information
-To learn more, see our [documentation](https://skypilot.readthedocs.io/en/latest/), [blog](https://blog.skypilot.co/), and [community integrations](https://blog.skypilot.co/community/).
+To learn more, see [Concept: Sky Computing](https://docs.skypilot.co/en/latest/sky-computing.html), [SkyPilot docs](https://skypilot.readthedocs.io/en/latest/), and [SkyPilot blog](https://blog.skypilot.co/).
 
 <!-- Keep this section in sync with index.rst in SkyPilot Docs -->
 Runnable examples:
+- [**AI Gallery**](https://docs.skypilot.co/en/latest/gallery/index.html)
 - LLMs on SkyPilot
   - [Llama 3.2: lightweight and vision models](./llm/llama-3_2/)
   - [Pixtral](./llm/pixtral/)

docs/source/docs/index.rst

@@ -74,10 +74,14 @@ You can chat with the SkyPilot team and community on the `SkyPilot Slack <http:/
 Learn more
 --------------------------
 
+To learn more, see :ref:`Concept: Sky Computing <sky-computing>` and `SkyPilot blog <https://blog.skypilot.co/>`_.
+
 Runnable examples:
 
 .. Keep this section in sync with README.md in SkyPilot repo
 
+* :ref:`AI Gallery <ai-gallery>`
+
 * **LLMs on SkyPilot**
 
   * `Llama 3.2: lightweight and vision models <https://github.com/skypilot-org/skypilot/tree/master/llm/llama-3_2>`_

sky/__init__.py

@@ -105,6 +105,7 @@ def set_proxy_env_var(proxy_var: str, urllib_var: Optional[str]):
 from sky.data import StoreType
 from sky.execution import exec  # pylint: disable=redefined-builtin
 from sky.execution import launch
+from sky.jobs import ManagedJobStatus
 # TODO (zhwu): These imports are for backward compatibility, and spot APIs
 # should be called with `sky.spot.xxx` instead. Remove in release 0.8.0
 from sky.jobs.core import spot_cancel
@@ -163,6 +164,7 @@ def set_proxy_env_var(proxy_var: str, urllib_var: Optional[str]):
     'StoreType',
     'ClusterStatus',
     'JobStatus',
+    'ManagedJobStatus',
     # APIs
     'Dag',
     'Task',