From a8100f448feb5d3d24aab530f8d568cdf2df3f8c Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Mon, 4 Sep 2023 23:49:17 -0700 Subject: [PATCH 1/7] [Doc] Fix the image_id example in yaml spec (#2512) Fix the image_id example in yaml spec --- docs/source/reference/yaml-spec.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference/yaml-spec.rst b/docs/source/reference/yaml-spec.rst index 4a567160c78..94e6a9e8f97 100644 --- a/docs/source/reference/yaml-spec.rst +++ b/docs/source/reference/yaml-spec.rst @@ -140,7 +140,7 @@ Available fields: image_id: ami-0868a20f5a3bf9702 # GCP # To find GCP images: https://cloud.google.com/compute/docs/images - # image_id: projects/deeplearning-platform-release/global/images/family/tf2-ent-2-1-cpu-ubuntu-2004 + # image_id: projects/deeplearning-platform-release/global/images/common-cpu-v20230615-debian-11-py310 # Or machine image: https://cloud.google.com/compute/docs/machine-images # image_id: projects/my-project/global/machineImages/my-machine-image # From 7d59ff9b4c2306c85ef61aa8026a0f2a24b8608c Mon Sep 17 00:00:00 2001 From: Steve McClain <85196623+steve-marmalade@users.noreply.github.com> Date: Wed, 6 Sep 2023 00:53:49 -0400 Subject: [PATCH 2/7] Remove version upper bound on `pyyaml` (#2514) * Remove version upper bound on `pyyaml` * Disallow `5.4.*` due to incompatibility with cython3 Co-authored-by: Romil Bhardwaj * Update comment for PyYAML pinned versions --------- Co-authored-by: Romil Bhardwaj --- sky/setup_files/setup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sky/setup_files/setup.py b/sky/setup_files/setup.py index ae314a8ce90..5bc9abc57a6 100644 --- a/sky/setup_files/setup.py +++ b/sky/setup_files/setup.py @@ -109,11 +109,9 @@ def parse_readme(readme: str) -> str: # >=1.10.8 is needed for ray>=2.6. See # https://github.com/ray-project/ray/issues/35661 'pydantic <2.0, >=1.10.8', - # Cython 3.0 release breaks PyYAML installed by aws-cli. - # https://github.com/yaml/pyyaml/issues/601 - # https://github.com/aws/aws-cli/issues/8036 + # Cython 3.0 release breaks PyYAML 5.4.* (https://github.com/yaml/pyyaml/issues/601) # <= 3.13 may encounter https://github.com/ultralytics/yolov5/issues/414 - 'pyyaml > 3.13, <= 5.3.1' + 'pyyaml > 3.13, != 5.4.*' ] # NOTE: Change the templates/spot-controller.yaml.j2 file if any of the From eb8599dcadd64b928c9ccfaf0f1bc99763bf9c73 Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Thu, 7 Sep 2023 15:46:32 -0700 Subject: [PATCH 3/7] [Docs] Typo fix storage.rst (#2511) --- docs/source/reference/storage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference/storage.rst b/docs/source/reference/storage.rst index dcf9958b5bf..cda5f7b0a47 100644 --- a/docs/source/reference/storage.rst +++ b/docs/source/reference/storage.rst @@ -173,7 +173,7 @@ and storage mounting: # s3://sky-multisource-storage/datasets/. ~/mydir/myfile.txt will appear # at s3://sky-multisource-storage/myfile.txt. /datasets-multisource-storage: - name: sky-multisource-storage2 # Make sure this name is unique or you own this bucket + name: sky-multisource-storage # Make sure this name is unique or you own this bucket source: [~/mydir/myfile.txt, ~/datasets] From 6f1d4e8118f04b010000bd294bf69d6b396b1d6b Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Thu, 7 Sep 2023 21:39:42 -0700 Subject: [PATCH 4/7] Cleanup dependencies (#2515) * cleaup some dependencies * Remove upper bound for click * upgrade actions * upgrade requirements docs * Add test for poetry --lock * fix poetry test * rename poetry test * fix * fix path * fix github env var * fix * fix poetry * Adopt changes from #2514 * increase poetry build time --- .github/workflows/format.yml | 4 +- .github/workflows/mypy.yml | 4 +- .github/workflows/pylint.yml | 4 +- .github/workflows/pypi-nightly-build.yml | 2 +- .github/workflows/pytest.yml | 5 +-- .github/workflows/test-poetry-build.yml | 53 ++++++++++++++++++++++++ docs/requirements-docs.txt | 2 +- sky/data/data_transfer.py | 6 +-- sky/setup_files/setup.py | 12 ++---- tests/test_pycryptodome_version.py | 3 -- 10 files changed, 69 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/test-poetry-build.yml delete mode 100644 tests/test_pycryptodome_version.py diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 776df2014d0..11ee63f71d3 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -18,9 +18,9 @@ jobs: matrix: python-version: ["3.8"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 15d7a5d4c60..d59e90a9e99 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -18,9 +18,9 @@ jobs: matrix: python-version: ["3.8"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index e5c2c98f64f..e1dd092c82d 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -19,9 +19,9 @@ jobs: matrix: python-version: ["3.8"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/pypi-nightly-build.yml b/.github/workflows/pypi-nightly-build.yml index 55ad4978ebb..d83fd5e6069 100644 --- a/.github/workflows/pypi-nightly-build.yml +++ b/.github/workflows/pypi-nightly-build.yml @@ -11,7 +11,7 @@ jobs: outputs: should_run: ${{ steps.should_run.outputs.should_run }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: print latest_commit run: echo ${{ github.sha }} - id: should_run diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 77020384f12..02375cc8093 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,17 +24,16 @@ jobs: - tests/test_list_accelerators.py - tests/test_optimizer_dryruns.py - tests/test_optimizer_random_dag.py - - tests/test_pycryptodome_version.py - tests/test_storage.py - tests/test_wheels.py - tests/test_spot.py runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test-poetry-build.yml b/.github/workflows/test-poetry-build.yml new file mode 100644 index 00000000000..4fea60ccf02 --- /dev/null +++ b/.github/workflows/test-poetry-build.yml @@ -0,0 +1,53 @@ +name: Poetry Test +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - master + - 'releases/**' + pull_request: + branches: + - master + - 'releases/**' +jobs: + poetry-build-test: + runs-on: ubuntu-latest + steps: + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python - + echo "$HOME/.poetry/bin" >> $GITHUB_PATH + - name: Create foo package + run: | + mkdir foo + cat < foo/pyproject.toml + [tool.poetry] + name = "foo" + version = "1.0.0" + authors = ["skypilot-bot"] + description = "" + + [tool.poetry.dependencies] + python = "3.10.x" + + [tool.poetry.group.dev.dependencies] + skypilot = {git = "https://github.com/skypilot-org/skypilot.git", branch = "${{ github.head_ref }}"} + + [build-system] + requires = ["poetry-core"] + build-backend = "poetry.core.masonry.api" + + EOF + + - name: Check poetry lock time + run: | + cd foo + poetry lock --no-update + timeout-minutes: 2 + + diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 675e4477d93..5e2522b47ca 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ sphinx==4.3.2 -sphinx-click==3.0.2 +sphinx-click==5.0.1 sphinx-copybutton==0.5.0 pydata-sphinx-theme==0.7.2 sphinx-autodoc-typehints==1.17.0 diff --git a/sky/data/data_transfer.py b/sky/data/data_transfer.py index 80c24d8c6da..374871031cb 100644 --- a/sky/data/data_transfer.py +++ b/sky/data/data_transfer.py @@ -47,12 +47,12 @@ def s3_to_gcs(s3_bucket_name: str, gs_bucket_name: str) -> None: gs_bucket_name: str; Name of the Google Cloud Storage Bucket """ # pylint: disable=import-outside-toplevel - from oauth2client.client import GoogleCredentials + import google.auth - oauth_credentials = GoogleCredentials.get_application_default() + credentials, _ = google.auth.default() storagetransfer = gcp.build('storagetransfer', 'v1', - credentials=oauth_credentials) + credentials=credentials) session = aws.session() aws_credentials = session.get_credentials().get_frozen_credentials() diff --git a/sky/setup_files/setup.py b/sky/setup_files/setup.py index 5bc9abc57a6..a8e1770d88f 100644 --- a/sky/setup_files/setup.py +++ b/sky/setup_files/setup.py @@ -56,11 +56,8 @@ def parse_readme(readme: str) -> str: install_requires = [ 'wheel', - # NOTE: ray requires click>=7.0. Also, click 8.1.x makes our rendered CLI - # docs display weird blockquotes. - # TODO(zongheng): investigate how to make click 8.1.x display nicely and - # remove the upper bound. - 'click <= 8.0.4, >= 7.0', + # NOTE: ray requires click>=7.0. + 'click >= 7.0', # NOTE: required by awscli. To avoid ray automatically installing # the latest version. 'colorama < 0.4.5', @@ -71,7 +68,6 @@ def parse_readme(readme: str) -> str: 'jinja2 >= 3.0', 'jsonschema', 'networkx', - 'oauth2client', 'pandas', 'pendulum', # PrettyTable with version >=2.0.0 is required for the support of @@ -117,7 +113,7 @@ def parse_readme(readme: str) -> str: # NOTE: Change the templates/spot-controller.yaml.j2 file if any of the # following packages dependencies are changed. aws_dependencies = [ - # botocore does not work with urllib3>=2.0.0, accuroding to https://github.com/boto/botocore/issues/2926 + # botocore does not work with urllib3>=2.0.0, according to https://github.com/boto/botocore/issues/2926 # We have to explicitly pin the version to optimize the time for # poetry install. See https://github.com/orgs/python-poetry/discussions/7937 'urllib3<2', @@ -127,8 +123,6 @@ def parse_readme(readme: str) -> str: 'awscli>=1.27.10', 'botocore>=1.29.10', 'boto3>=1.26.1', - # 'Crypto' module used in authentication.py for AWS. - 'pycryptodome==3.12.0', ] extras_require: Dict[str, List[str]] = { 'aws': aws_dependencies, diff --git a/tests/test_pycryptodome_version.py b/tests/test_pycryptodome_version.py deleted file mode 100644 index 30086bf1f28..00000000000 --- a/tests/test_pycryptodome_version.py +++ /dev/null @@ -1,3 +0,0 @@ -def test_pycryptodome_version(): - from Crypto.PublicKey import RSA - print(RSA.__file__) From e164dc29fc9aee2b14c4abe634af9b3b84e9a806 Mon Sep 17 00:00:00 2001 From: Romil Bhardwaj Date: Fri, 8 Sep 2023 14:27:14 -0700 Subject: [PATCH 5/7] [Examples] Update localgpt example (#2530) * wip * fix * A100 --- llm/localgpt/localgpt.yaml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/llm/localgpt/localgpt.yaml b/llm/localgpt/localgpt.yaml index 121e2ad9b29..7bb4e2a0d5d 100644 --- a/llm/localgpt/localgpt.yaml +++ b/llm/localgpt/localgpt.yaml @@ -7,7 +7,7 @@ # * Open http://localhost:5111 in your browser resources: - accelerators: T4:1 + accelerators: A100:1 setup: | conda activate py39 @@ -16,17 +16,13 @@ setup: | else conda create -y -n py39 python=3.9 conda activate py39 - git clone https://github.com/romilbhardwaj/localGPT.git + git clone https://github.com/PromtEngineer/localGPT.git cd localGPT/ - git checkout localui_update + git checkout 5ab5e1921adb45a2df8d61f189a3fb4e9b401e58 pip install -r requirements.txt - cd .. - - # Install AutoGPTQ - git clone https://github.com/PanQiWei/AutoGPTQ.git - cd AutoGPTQ - git checkout v0.2.2 - pip install . + # bitsandbytes>0.39.1 fails on GCP, see issue: + # https://github.com/TimDettmers/bitsandbytes/issues/620 + pip install -U bitsandbytes==0.39.1 fi run: | From a609a19464b79661bba36d3633d6df8a65d54467 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Sat, 9 Sep 2023 14:37:30 -0700 Subject: [PATCH 6/7] [CI] Fix poetry CI url for forked repository (#2537) * Fix poetry test url * Add .git --- .github/workflows/test-poetry-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-poetry-build.yml b/.github/workflows/test-poetry-build.yml index 4fea60ccf02..b43f7b57811 100644 --- a/.github/workflows/test-poetry-build.yml +++ b/.github/workflows/test-poetry-build.yml @@ -36,7 +36,7 @@ jobs: python = "3.10.x" [tool.poetry.group.dev.dependencies] - skypilot = {git = "https://github.com/skypilot-org/skypilot.git", branch = "${{ github.head_ref }}"} + skypilot = {git = "${{ github.event.pull_request.head.repo.html_url }}.git", branch = "${{ github.head_ref }}"} [build-system] requires = ["poetry-core"] From dd9c5d28437393b02f1efbd34ba7696e99578762 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Sat, 9 Sep 2023 19:06:25 -0700 Subject: [PATCH 7/7] [CI] Fix poetry test for master branch (#2538) * Fix poetry test for master branch * fix * fix * syntax --- .github/workflows/test-poetry-build.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-poetry-build.yml b/.github/workflows/test-poetry-build.yml index b43f7b57811..d72b9754e10 100644 --- a/.github/workflows/test-poetry-build.yml +++ b/.github/workflows/test-poetry-build.yml @@ -25,6 +25,14 @@ jobs: - name: Create foo package run: | mkdir foo + MASTER_REPO_URL=${{ github.server_url }}/${{ github.repository }} + REPO_URL=${{ github.event.pull_request.head.repo.html_url }} + if [ -z "$REPO_URL" ]; then + # This is a push, not a PR, so use the repo URL + REPO_URL=$MASTER_REPO_URL + fi + echo Master repo URL: $MASTER_REPO_URL + echo Using repo URL: $REPO_URL cat < foo/pyproject.toml [tool.poetry] name = "foo" @@ -36,7 +44,7 @@ jobs: python = "3.10.x" [tool.poetry.group.dev.dependencies] - skypilot = {git = "${{ github.event.pull_request.head.repo.html_url }}.git", branch = "${{ github.head_ref }}"} + skypilot = {git = "${REPO_URL}.git", branch = "${{ github.head_ref }}"} [build-system] requires = ["poetry-core"]