Merge branch 'master' into continuous-sync-alpha-1

merge master
skypilot-org · Sep 17, 2023 · 7cbf729 · 7cbf729
2 parents 6e1e9f2 + dd9c5d2
commit 7cbf729
Show file tree

Hide file tree

Showing 13 changed files with 87 additions and 42 deletions.
diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -18,9 +18,9 @@ jobs:
       matrix:
         python-version: ["3.8"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -18,9 +18,9 @@ jobs:
       matrix:
         python-version: ["3.8"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -19,9 +19,9 @@ jobs:
       matrix:
         python-version: ["3.8"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

diff --git a/.github/workflows/pypi-nightly-build.yml b/.github/workflows/pypi-nightly-build.yml
@@ -11,7 +11,7 @@ jobs:
     outputs:
       should_run: ${{ steps.should_run.outputs.should_run }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: print latest_commit
         run: echo ${{ github.sha }}
       - id: should_run

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -24,17 +24,16 @@ jobs:
           - tests/test_list_accelerators.py
           - tests/test_optimizer_dryruns.py
           - tests/test_optimizer_random_dag.py
-          - tests/test_pycryptodome_version.py
           - tests/test_storage.py
           - tests/test_wheels.py
           - tests/test_spot.py
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Install Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-poetry-build.yml b/.github/workflows/test-poetry-build.yml
@@ -0,0 +1,61 @@
+name: Poetry Test
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - master
+      - 'releases/**'
+  pull_request:
+    branches:
+      - master
+      - 'releases/**'
+jobs:
+  poetry-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+          echo "$HOME/.poetry/bin" >> $GITHUB_PATH
+      - name: Create foo package
+        run: |
+          mkdir foo
+          MASTER_REPO_URL=${{ github.server_url }}/${{ github.repository }}
+          REPO_URL=${{ github.event.pull_request.head.repo.html_url }}
+          if [ -z "$REPO_URL" ]; then
+            # This is a push, not a PR, so use the repo URL
+            REPO_URL=$MASTER_REPO_URL
+          fi
+          echo Master repo URL: $MASTER_REPO_URL
+          echo Using repo URL: $REPO_URL
+          cat <<EOF > foo/pyproject.toml
+          [tool.poetry]
+          name = "foo"
+          version = "1.0.0"
+          authors = ["skypilot-bot"]
+          description = ""
+
+          [tool.poetry.dependencies]
+          python = "3.10.x"
+
+          [tool.poetry.group.dev.dependencies]
+          skypilot = {git = "${REPO_URL}.git", branch = "${{ github.head_ref }}"}
+
+          [build-system]
+          requires = ["poetry-core"]
+          build-backend = "poetry.core.masonry.api"
+          
+          EOF
+
+      - name: Check poetry lock time
+        run: |
+          cd foo
+          poetry lock --no-update
+        timeout-minutes: 2
+
+
diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt
@@ -1,5 +1,5 @@
 sphinx==4.3.2
-sphinx-click==3.0.2
+sphinx-click==5.0.1
 sphinx-copybutton==0.5.0
 pydata-sphinx-theme==0.7.2
 sphinx-autodoc-typehints==1.17.0

diff --git a/docs/source/reference/storage.rst b/docs/source/reference/storage.rst
@@ -173,7 +173,7 @@ and storage mounting:
       # s3://sky-multisource-storage/datasets/. ~/mydir/myfile.txt will appear
       # at s3://sky-multisource-storage/myfile.txt.
       /datasets-multisource-storage:
-        name: sky-multisource-storage2 # Make sure this name is unique or you own this bucket
+        name: sky-multisource-storage # Make sure this name is unique or you own this bucket
         source: [~/mydir/myfile.txt, ~/datasets]
 
 

diff --git a/docs/source/reference/yaml-spec.rst b/docs/source/reference/yaml-spec.rst
@@ -140,7 +140,7 @@ Available fields:
       image_id: ami-0868a20f5a3bf9702
       # GCP
       # To find GCP images: https://cloud.google.com/compute/docs/images
-      # image_id: projects/deeplearning-platform-release/global/images/family/tf2-ent-2-1-cpu-ubuntu-2004
+      # image_id: projects/deeplearning-platform-release/global/images/common-cpu-v20230615-debian-11-py310
       # Or machine image: https://cloud.google.com/compute/docs/machine-images
       # image_id: projects/my-project/global/machineImages/my-machine-image
       #

diff --git a/llm/localgpt/localgpt.yaml b/llm/localgpt/localgpt.yaml
@@ -7,7 +7,7 @@
 #   * Open http://localhost:5111 in your browser
 
 resources:
-  accelerators: T4:1
+  accelerators: A100:1
 
 setup: |
   conda activate py39
@@ -16,17 +16,13 @@ setup: |
   else
     conda create -y -n py39 python=3.9
     conda activate py39
-    git clone https://github.com/romilbhardwaj/localGPT.git
+    git clone https://github.com/PromtEngineer/localGPT.git
     cd localGPT/
-    git checkout localui_update
+    git checkout 5ab5e1921adb45a2df8d61f189a3fb4e9b401e58
     pip install -r requirements.txt
-    cd ..
-
-    # Install AutoGPTQ
-    git clone https://github.com/PanQiWei/AutoGPTQ.git
-    cd AutoGPTQ
-    git checkout v0.2.2
-    pip install .
+    # bitsandbytes>0.39.1 fails on GCP, see issue:
+    # https://github.com/TimDettmers/bitsandbytes/issues/620
+    pip install -U bitsandbytes==0.39.1
   fi
 
 run: |

diff --git a/sky/data/data_transfer.py b/sky/data/data_transfer.py
@@ -47,12 +47,12 @@ def s3_to_gcs(s3_bucket_name: str, gs_bucket_name: str) -> None:
       gs_bucket_name: str; Name of the Google Cloud Storage Bucket
     """
     # pylint: disable=import-outside-toplevel
-    from oauth2client.client import GoogleCredentials
+    import google.auth
 
-    oauth_credentials = GoogleCredentials.get_application_default()
+    credentials, _ = google.auth.default()
     storagetransfer = gcp.build('storagetransfer',
                                 'v1',
-                                credentials=oauth_credentials)
+                                credentials=credentials)
 
     session = aws.session()
     aws_credentials = session.get_credentials().get_frozen_credentials()

diff --git a/sky/setup_files/setup.py b/sky/setup_files/setup.py
@@ -56,11 +56,8 @@ def parse_readme(readme: str) -> str:
 
 install_requires = [
     'wheel',
-    # NOTE: ray requires click>=7.0. Also, click 8.1.x makes our rendered CLI
-    # docs display weird blockquotes.
-    # TODO(zongheng): investigate how to make click 8.1.x display nicely and
-    # remove the upper bound.
-    'click <= 8.0.4, >= 7.0',
+    # NOTE: ray requires click>=7.0.
+    'click >= 7.0',
     # NOTE: required by awscli. To avoid ray automatically installing
     # the latest version.
     'colorama < 0.4.5',
@@ -71,7 +68,6 @@ def parse_readme(readme: str) -> str:
     'jinja2 >= 3.0',
     'jsonschema',
     'networkx',
-    'oauth2client',
     'pandas',
     'pendulum',
     # PrettyTable with version >=2.0.0 is required for the support of
@@ -109,17 +105,15 @@ def parse_readme(readme: str) -> str:
     # >=1.10.8 is needed for ray>=2.6. See
     # https://github.com/ray-project/ray/issues/35661
     'pydantic <2.0, >=1.10.8',
-    # Cython 3.0 release breaks PyYAML installed by aws-cli.
-    # https://github.com/yaml/pyyaml/issues/601
-    # https://github.com/aws/aws-cli/issues/8036
+    # Cython 3.0 release breaks PyYAML 5.4.* (https://github.com/yaml/pyyaml/issues/601)
     # <= 3.13 may encounter https://github.com/ultralytics/yolov5/issues/414
-    'pyyaml > 3.13, <= 5.3.1'
+    'pyyaml > 3.13, != 5.4.*'
 ]
 
 # NOTE: Change the templates/spot-controller.yaml.j2 file if any of the
 # following packages dependencies are changed.
 aws_dependencies = [
-    # botocore does not work with urllib3>=2.0.0, accuroding to https://github.com/boto/botocore/issues/2926
+    # botocore does not work with urllib3>=2.0.0, according to https://github.com/boto/botocore/issues/2926
     # We have to explicitly pin the version to optimize the time for
     # poetry install. See https://github.com/orgs/python-poetry/discussions/7937
     'urllib3<2',
@@ -129,8 +123,6 @@ def parse_readme(readme: str) -> str:
     'awscli>=1.27.10',
     'botocore>=1.29.10',
     'boto3>=1.26.1',
-    # 'Crypto' module used in authentication.py for AWS.
-    'pycryptodome==3.12.0',
 ]
 extras_require: Dict[str, List[str]] = {
     'aws': aws_dependencies,

diff --git a/tests/test_pycryptodome_version.py b/tests/test_pycryptodome_version.py