From 9f3ec4b552ed9acb44facc684a2872b284b549da Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 13:45:04 -0500 Subject: [PATCH 01/11] add workflow to upload conda packages to legate channel --- .github/workflows/build.yaml | 8 +- .github/workflows/conda-upload-packages.yaml | 39 ++++ .github/workflows/github-actions.yml | 179 ++++++++++--------- ci/upload-to-anaconda.sh | 36 ++++ 4 files changed, 173 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/conda-upload-packages.yaml create mode 100755 ci/upload-to-anaconda.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7b593402..756ef8f2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -10,9 +10,8 @@ on: branches: - "main" - "release/v[0-9][0-9].[0-9][0-9].[0-9][0-9]" - # run on pushes of new release tags + # run on pushes of any tags tags: - - v[0-9][0-9].[0-9][0-9].[0-9][0-9] # run by clicking buttons in the GitHub Actions UI workflow_dispatch: @@ -22,6 +21,11 @@ jobs: with: script: "ci/build_python.sh" secrets: inherit + upload-conda: + needs: + - conda-python-build + uses: ./.github/workflows/conda-upload-packages.yaml + secrets: inherit docs-build: needs: - conda-python-build diff --git a/.github/workflows/conda-upload-packages.yaml b/.github/workflows/conda-upload-packages.yaml new file mode 100644 index 00000000..a19a030f --- /dev/null +++ b/.github/workflows/conda-upload-packages.yaml @@ -0,0 +1,39 @@ +# adopted from https://github.com/rapidsai/shared-workflows/blob/branch-24.12/.github/workflows/conda-upload-packages.yaml +# with notable exceptions: +# +# * assumes packages were uploaded to GitHub artifact store, not Amazon S3 +# * always publishes to the same channel, but uses different label for non-release packages +# + +on: + # run only when called by other workflows + workflow_call: + +env: + # where jobs that download conda packages store the local channel + RAPIDS_LOCAL_CONDA_CHANNEL: /tmp/local-conda-packages + +jobs: + upload: + runs-on: linux-amd64-cpu4 + container: + image: rapidsai/ci-conda:latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # the notebooks and PNG files stored in git LFS aren't necessary for package uploads + lfs: false + - name: download conda packages + uses: actions/download-artifact@v4 + with: + # omitting 'name' here means "download all artifacts from this run"... useful to + # avoid having to list the matrix of CUDA / Python versions here + path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + - name: Upload packages + run: "ci/upload-to-anaconda.sh" + env: + CONDA_LEGATE_TOKEN: ${{ secrets.CONDA_LEGATE_TOKEN }} diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index b4bfc70a..7cd4e87e 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -23,8 +23,6 @@ jobs: # group together all jobs that must pass for a PR to be merged # (for use by branch protections) pr-builder: - # skip on builds from merges to main - if: github.ref != 'refs/heads/main' needs: - pre-commit - conda-python-build @@ -47,92 +45,99 @@ jobs: script: "ci/build_python.sh" secrets: inherit - conda-python-tests-cpu: + # TODO: remove this from the PR workflow before merging + upload-conda: needs: - - pre-commit - conda-python-build - strategy: - fail-fast: false - matrix: - include: - - ARCH: "amd64" - CUDA_VER: "12.5.1" - PY_VER: "3.11" - runs-on: linux-${{ matrix.ARCH }}-cpu16 - container: - image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}" - steps: - - name: install git-lfs - run: | - conda install --yes \ - -c conda-forge \ - git-lfs - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - lfs: true - - name: download conda packages - uses: actions/download-artifact@v4 - with: - name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }} - path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} - github-token: ${{ secrets.GITHUB_TOKEN }} - repository: ${{ github.repository }} - run-id: ${{ github.run_id }} - - name: test - run: | - ci/test_python_cpu.sh + uses: ./.github/workflows/conda-upload-packages.yaml + secrets: inherit - conda-python-tests-gpu: - needs: - - pre-commit - - conda-python-build - strategy: - fail-fast: false - # As of the last time this was updated, legate / cunumeric packages were published for only: - # - # * architectures: amd64 only - # * CUDA: >=12.2 - # * Python: 3.10, 3.11, 3.12 - # - # Valid set of RAPIDS ci-conda image tags: https://hub.docker.com/r/rapidsai/ci-conda/tags - matrix: - include: - - ARCH: "amd64" - CUDA_VER: "12.5.1" - PY_VER: "3.10" - runs-on: linux-${{ matrix.ARCH }}-gpu-v100-latest-1 - container: - image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}" - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - steps: - - name: install git-lfs - run: | - conda install --yes \ - -c conda-forge \ - git-lfs - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - lfs: true - - name: download conda packages - uses: actions/download-artifact@v4 - with: - name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }} - path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} - github-token: ${{ secrets.GITHUB_TOKEN }} - repository: ${{ github.repository }} - run-id: ${{ github.run_id }} - - name: test - run: | - ci/test_python_gpu.sh + # conda-python-tests-cpu: + # needs: + # - pre-commit + # - conda-python-build + # strategy: + # fail-fast: false + # matrix: + # include: + # - ARCH: "amd64" + # CUDA_VER: "12.5.1" + # PY_VER: "3.11" + # runs-on: linux-${{ matrix.ARCH }}-cpu16 + # container: + # image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}" + # steps: + # - name: install git-lfs + # run: | + # conda install --yes \ + # -c conda-forge \ + # git-lfs + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # lfs: true + # - name: download conda packages + # uses: actions/download-artifact@v4 + # with: + # name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }} + # path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} + # github-token: ${{ secrets.GITHUB_TOKEN }} + # repository: ${{ github.repository }} + # run-id: ${{ github.run_id }} + # - name: test + # run: | + # ci/test_python_cpu.sh - docs-build: - needs: - - conda-python-build - uses: ./.github/workflows/docs-build.yaml - with: - script: "ci/build_docs.sh" - deploy: false - secrets: inherit + # conda-python-tests-gpu: + # needs: + # - pre-commit + # - conda-python-build + # strategy: + # fail-fast: false + # # As of the last time this was updated, legate / cunumeric packages were published for only: + # # + # # * architectures: amd64 only + # # * CUDA: >=12.2 + # # * Python: 3.10, 3.11, 3.12 + # # + # # Valid set of RAPIDS ci-conda image tags: https://hub.docker.com/r/rapidsai/ci-conda/tags + # matrix: + # include: + # - ARCH: "amd64" + # CUDA_VER: "12.5.1" + # PY_VER: "3.10" + # runs-on: linux-${{ matrix.ARCH }}-gpu-v100-latest-1 + # container: + # image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}" + # env: + # NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + # steps: + # - name: install git-lfs + # run: | + # conda install --yes \ + # -c conda-forge \ + # git-lfs + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # lfs: true + # - name: download conda packages + # uses: actions/download-artifact@v4 + # with: + # name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }} + # path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} + # github-token: ${{ secrets.GITHUB_TOKEN }} + # repository: ${{ github.repository }} + # run-id: ${{ github.run_id }} + # - name: test + # run: | + # ci/test_python_gpu.sh + + # docs-build: + # needs: + # - conda-python-build + # uses: ./.github/workflows/docs-build.yaml + # with: + # script: "ci/build_docs.sh" + # deploy: false + # secrets: inherit diff --git a/ci/upload-to-anaconda.sh b/ci/upload-to-anaconda.sh new file mode 100755 index 00000000..a81f18d9 --- /dev/null +++ b/ci/upload-to-anaconda.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# adopted from https://github.com/rapidsai/gha-tools/blob/main/tools/rapids-upload-to-anaconda, with some exceptions: +# +# * assumes artifacts are on GitHub Actions artifact store, not Amazon S3 +# * assumes packages have been unpacked to env variable RAPIDS_LOCAL_CONDA_CHANNEL +# * does not differentiate between pull request and nightly branches +# (relies on workflow triggers to just not run this script when it isn't needed) + +# publish to the 'experimental' label on the 'legate' channel, for all cases except +# releases (builds triggered by pushing a tag matching this regex exactly, e.g. 'v24.09.00') +if [[ "${GITHUB_REF}" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; + declare -r conda_label="main" +else + declare -r conda_label="experimental" +fi + +PKGS_TO_UPLOAD=$(rapids-find-anaconda-uploads.py "${RAPIDS_LOCAL_CONDA_CHANNEL}") + +if [ -z "${PKGS_TO_UPLOAD}" ]; then + rapids-echo-stderr "Couldn't find any packages to upload in: ${RAPIDS_LOCAL_CONDA_CHANNEL}" + ls -l "${RAPIDS_LOCAL_CONDA_CHANNEL}/"* + continue +fi + +rapids-echo-stderr "Uploading packages to Anaconda.org (channel='legate', label='${conda_label}'): ${PKGS_TO_UPLOAD}" + +# export RAPIDS_RETRY_SLEEP=180 +# # shellcheck disable=SC2086 +# rapids-retry anaconda \ +# -t "${CONDA_LEGATE_TOKEN}" \ +# upload \ +# --label "${RAPIDS_CONDA_UPLOAD_LABEL:-main}" \ +# --skip-existing \ +# --no-progress \ +# ${PKGS_TO_UPLOAD} From ea1567593373d2b95a2f538076ef131bb49630e2 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 14:00:02 -0500 Subject: [PATCH 02/11] fix pr-builder --- .github/workflows/github-actions.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 7cd4e87e..7cf46db4 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -26,9 +26,9 @@ jobs: needs: - pre-commit - conda-python-build - - conda-python-tests-cpu - - conda-python-tests-gpu - - docs-build + # - conda-python-tests-cpu + # - conda-python-tests-gpu + # - docs-build uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 pre-commit: From 2392376d5f0d03948ec57fb0f897350b84030056 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 14:03:52 -0500 Subject: [PATCH 03/11] fix upload-to-anaconda script --- ci/upload-to-anaconda.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/upload-to-anaconda.sh b/ci/upload-to-anaconda.sh index a81f18d9..109dfbf0 100755 --- a/ci/upload-to-anaconda.sh +++ b/ci/upload-to-anaconda.sh @@ -9,7 +9,7 @@ # publish to the 'experimental' label on the 'legate' channel, for all cases except # releases (builds triggered by pushing a tag matching this regex exactly, e.g. 'v24.09.00') -if [[ "${GITHUB_REF}" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; +if [[ "${GITHUB_REF}" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then declare -r conda_label="main" else declare -r conda_label="experimental" @@ -20,7 +20,6 @@ PKGS_TO_UPLOAD=$(rapids-find-anaconda-uploads.py "${RAPIDS_LOCAL_CONDA_CHANNEL}" if [ -z "${PKGS_TO_UPLOAD}" ]; then rapids-echo-stderr "Couldn't find any packages to upload in: ${RAPIDS_LOCAL_CONDA_CHANNEL}" ls -l "${RAPIDS_LOCAL_CONDA_CHANNEL}/"* - continue fi rapids-echo-stderr "Uploading packages to Anaconda.org (channel='legate', label='${conda_label}'): ${PKGS_TO_UPLOAD}" From 0b4aaf8d0c87dda5588d07104a63826a27b3e098 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 14:48:01 -0500 Subject: [PATCH 04/11] temporarily skip import tests --- conda/recipes/legate-boost/meta.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/conda/recipes/legate-boost/meta.yaml b/conda/recipes/legate-boost/meta.yaml index 10156ca7..f83708e7 100644 --- a/conda/recipes/legate-boost/meta.yaml +++ b/conda/recipes/legate-boost/meta.yaml @@ -118,13 +118,7 @@ requirements: test: # import tests at build time do not work for the CUDA packages, # because builds happen on machines without a GPU - {% if not gpu_enabled_bool %} - imports: - - legateboost - - legateboost.metrics - - legateboost.models - - legateboost.objectives - {% endif %} + # TODO: put thee import tests back before merging commands: - pip show legate-boost From d2fafbc2da89ea1a8f3fbb8dab78895e4e922d8e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 15:31:41 -0500 Subject: [PATCH 05/11] use .dev, pin back to older legate and cunumeric nightlies --- conda/recipes/legate-boost/meta.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/conda/recipes/legate-boost/meta.yaml b/conda/recipes/legate-boost/meta.yaml index f83708e7..331a1c32 100644 --- a/conda/recipes/legate-boost/meta.yaml +++ b/conda/recipes/legate-boost/meta.yaml @@ -79,12 +79,13 @@ requirements: {% if gpu_enabled_bool %} - cuda-version ={{ cuda_version }} - cuda-cudart-dev - - cunumeric {{ legate_version }} =*_gpu* - - legate {{ legate_version }} =*_gpu* + # TODO: reset these + - cunumeric=24.09.00.dev109=*_gpu* + - legate=24.09.00.dev296=*_gpu* - libcublas-dev {% else %} - - cunumeric {{ legate_version }} =*_cpu* - - legate {{ legate_version }} =*_cpu* + - cunumeric=24.09.00.dev109=*_cpu* + - legate=24.09.00.dev296=*_cpu* {% endif %} - openblas - python @@ -99,9 +100,9 @@ requirements: # cuda-version is used to constrain __cuda - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - __cuda - - cunumeric {{ legate_version }} =*_gpu* + - cunumeric=24.09.00.dev109=*_gpu* {% else %} - - cunumeric {{ legate_version }} =*_cpu* + - cunumeric=24.09.00.dev109=*_cpu* {% endif %} # Relying on run_exports from legate to pin an appropriate range of versions and # GPU vs. CPU selector. From b63549b350aaab4b2995932245268114e1bd96e7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 15:54:18 -0500 Subject: [PATCH 06/11] try actually uploading packages --- ci/build_python.sh | 2 +- ci/upload-to-anaconda.sh | 21 +++++++++++---------- contributing.md | 10 +++++----- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 16dac9cf..0e057cd7 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -8,7 +8,7 @@ rapids-generate-version > ./VERSION CMAKE_GENERATOR=Ninja \ CONDA_OVERRIDE_CUDA="${RAPIDS_CUDA_VERSION}" \ -LEGATEBOOST_PACKAGE_VERSION=$(head -1 ./VERSION) \ +LEGATEBOOST_PACKAGE_VERSION="24.09.00dev0" \ rapids-conda-retry mambabuild \ --channel legate \ --channel legate/label/experimental \ diff --git a/ci/upload-to-anaconda.sh b/ci/upload-to-anaconda.sh index 109dfbf0..0b4643d7 100755 --- a/ci/upload-to-anaconda.sh +++ b/ci/upload-to-anaconda.sh @@ -22,14 +22,15 @@ if [ -z "${PKGS_TO_UPLOAD}" ]; then ls -l "${RAPIDS_LOCAL_CONDA_CHANNEL}/"* fi -rapids-echo-stderr "Uploading packages to Anaconda.org (channel='legate', label='${conda_label}'): ${PKGS_TO_UPLOAD}" +rapids-echo-stderr "Uploading packages to Anaconda.org (channel='legate', label='${conda_label}'):" +rapids-echo-stderr "${PKGS_TO_UPLOAD}" -# export RAPIDS_RETRY_SLEEP=180 -# # shellcheck disable=SC2086 -# rapids-retry anaconda \ -# -t "${CONDA_LEGATE_TOKEN}" \ -# upload \ -# --label "${RAPIDS_CONDA_UPLOAD_LABEL:-main}" \ -# --skip-existing \ -# --no-progress \ -# ${PKGS_TO_UPLOAD} +# shellcheck disable=SC2086 +RAPIDS_RETRY_SLEEP=180 \ + rapids-retry anaconda \ + -t "${CONDA_LEGATE_TOKEN}" \ + upload \ + --label "${conda_label}" \ + --skip-existing \ + --no-progress \ + ${PKGS_TO_UPLOAD} diff --git a/contributing.md b/contributing.md index b33e10a2..1244ba28 100644 --- a/contributing.md +++ b/contributing.md @@ -223,8 +223,8 @@ git push origin update-version ```shell git checkout main git pull upstream main -git tag -a v24.12.00dev -m "v24.12.00dev" -git push upstream v24.12.00dev +git tag -a v24.12.00.dev -m "v24.12.00.dev" +git push upstream v24.12.00.dev ``` From that point forward, all packages produced by CI from the `main` branch will have versions like `v24.12.00.dev{n}`, @@ -251,8 +251,8 @@ git commit -m 'start v24.09.01 [skip ci]' git push upstream release/24.09 # tag the first commit on the new branch as the beginning of the 24.09.01 series -git tag -a v24.09.01dev -m 'v24.09.01dev' -git push upstream v24.09.01dev +git tag -a v24.09.01.dev -m 'v24.09.01.dev' +git push upstream v24.09.01.dev ``` 2. Open pull requests targeting that branch and merge them into that branch. @@ -283,7 +283,7 @@ git cherry-pick release/v24.09 NOTE: The use of `cherry-pick` here is important because it re-writes the commit IDs. That avoids the situation where e.g. the `v24.09.01` hotfix tag points to commits on the `main` branch during `v24.12` development (which could lead to those packages -incorrectly getting `v24.09.01dev{n}` versions). +incorrectly getting `v24.09.01.dev{n}` versions). 3. Open a pull request to merge that branch into `main`. 4. Perform a non-squash merge of that pull request. From f394171a7b6d8d9c74776ba15d90b647d71a03bc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 16:43:35 -0500 Subject: [PATCH 07/11] do not publish on every PR --- .github/workflows/github-actions.yml | 183 +++++++++++++-------------- ci/build_python.sh | 2 +- conda/recipes/legate-boost/meta.yaml | 8 +- 3 files changed, 96 insertions(+), 97 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 7cf46db4..880fd090 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -26,9 +26,9 @@ jobs: needs: - pre-commit - conda-python-build - # - conda-python-tests-cpu - # - conda-python-tests-gpu - # - docs-build + - conda-python-tests-cpu + - conda-python-tests-gpu + - docs-build uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 pre-commit: @@ -45,99 +45,92 @@ jobs: script: "ci/build_python.sh" secrets: inherit - # TODO: remove this from the PR workflow before merging - upload-conda: + conda-python-tests-cpu: needs: + - pre-commit - conda-python-build - uses: ./.github/workflows/conda-upload-packages.yaml - secrets: inherit - - # conda-python-tests-cpu: - # needs: - # - pre-commit - # - conda-python-build - # strategy: - # fail-fast: false - # matrix: - # include: - # - ARCH: "amd64" - # CUDA_VER: "12.5.1" - # PY_VER: "3.11" - # runs-on: linux-${{ matrix.ARCH }}-cpu16 - # container: - # image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}" - # steps: - # - name: install git-lfs - # run: | - # conda install --yes \ - # -c conda-forge \ - # git-lfs - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - # lfs: true - # - name: download conda packages - # uses: actions/download-artifact@v4 - # with: - # name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }} - # path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} - # github-token: ${{ secrets.GITHUB_TOKEN }} - # repository: ${{ github.repository }} - # run-id: ${{ github.run_id }} - # - name: test - # run: | - # ci/test_python_cpu.sh + strategy: + fail-fast: false + matrix: + include: + - ARCH: "amd64" + CUDA_VER: "12.5.1" + PY_VER: "3.11" + runs-on: linux-${{ matrix.ARCH }}-cpu16 + container: + image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}" + steps: + - name: install git-lfs + run: | + conda install --yes \ + -c conda-forge \ + git-lfs + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + - name: download conda packages + uses: actions/download-artifact@v4 + with: + name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }} + path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + - name: test + run: | + ci/test_python_cpu.sh - # conda-python-tests-gpu: - # needs: - # - pre-commit - # - conda-python-build - # strategy: - # fail-fast: false - # # As of the last time this was updated, legate / cunumeric packages were published for only: - # # - # # * architectures: amd64 only - # # * CUDA: >=12.2 - # # * Python: 3.10, 3.11, 3.12 - # # - # # Valid set of RAPIDS ci-conda image tags: https://hub.docker.com/r/rapidsai/ci-conda/tags - # matrix: - # include: - # - ARCH: "amd64" - # CUDA_VER: "12.5.1" - # PY_VER: "3.10" - # runs-on: linux-${{ matrix.ARCH }}-gpu-v100-latest-1 - # container: - # image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}" - # env: - # NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - # steps: - # - name: install git-lfs - # run: | - # conda install --yes \ - # -c conda-forge \ - # git-lfs - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - # lfs: true - # - name: download conda packages - # uses: actions/download-artifact@v4 - # with: - # name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }} - # path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} - # github-token: ${{ secrets.GITHUB_TOKEN }} - # repository: ${{ github.repository }} - # run-id: ${{ github.run_id }} - # - name: test - # run: | - # ci/test_python_gpu.sh + conda-python-tests-gpu: + needs: + - pre-commit + - conda-python-build + strategy: + fail-fast: false + # As of the last time this was updated, legate / cunumeric packages were published for only: + # + # * architectures: amd64 only + # * CUDA: >=12.2 + # * Python: 3.10, 3.11, 3.12 + # + # Valid set of RAPIDS ci-conda image tags: https://hub.docker.com/r/rapidsai/ci-conda/tags + matrix: + include: + - ARCH: "amd64" + CUDA_VER: "12.5.1" + PY_VER: "3.10" + runs-on: linux-${{ matrix.ARCH }}-gpu-v100-latest-1 + container: + image: "rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-ubuntu22.04-py${{ matrix.PY_VER }}" + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + steps: + - name: install git-lfs + run: | + conda install --yes \ + -c conda-forge \ + git-lfs + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + - name: download conda packages + uses: actions/download-artifact@v4 + with: + name: legate-boost-conda-cuda${{ matrix.CUDA_VER }}-${{ matrix.ARCH }}-py${{ matrix.PY_VER }} + path: ${{ env.RAPIDS_LOCAL_CONDA_CHANNEL }} + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + - name: test + run: | + ci/test_python_gpu.sh - # docs-build: - # needs: - # - conda-python-build - # uses: ./.github/workflows/docs-build.yaml - # with: - # script: "ci/build_docs.sh" - # deploy: false - # secrets: inherit + docs-build: + needs: + - conda-python-build + uses: ./.github/workflows/docs-build.yaml + with: + script: "ci/build_docs.sh" + deploy: false + secrets: inherit diff --git a/ci/build_python.sh b/ci/build_python.sh index 0e057cd7..16dac9cf 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -8,7 +8,7 @@ rapids-generate-version > ./VERSION CMAKE_GENERATOR=Ninja \ CONDA_OVERRIDE_CUDA="${RAPIDS_CUDA_VERSION}" \ -LEGATEBOOST_PACKAGE_VERSION="24.09.00dev0" \ +LEGATEBOOST_PACKAGE_VERSION=$(head -1 ./VERSION) \ rapids-conda-retry mambabuild \ --channel legate \ --channel legate/label/experimental \ diff --git a/conda/recipes/legate-boost/meta.yaml b/conda/recipes/legate-boost/meta.yaml index 331a1c32..71245129 100644 --- a/conda/recipes/legate-boost/meta.yaml +++ b/conda/recipes/legate-boost/meta.yaml @@ -119,7 +119,13 @@ requirements: test: # import tests at build time do not work for the CUDA packages, # because builds happen on machines without a GPU - # TODO: put thee import tests back before merging + {% if not gpu_enabled_bool %} + imports: + - legateboost + - legateboost.metrics + - legateboost.models + - legateboost.objectives + {% endif %} commands: - pip show legate-boost From 1cce078c03ed9fb4e85d69b59a4c31b2d61072f9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 16:53:32 -0500 Subject: [PATCH 08/11] more cleanup --- .github/workflows/conda-upload-packages.yaml | 2 +- ci/upload-to-anaconda.sh | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda-upload-packages.yaml b/.github/workflows/conda-upload-packages.yaml index a19a030f..48fabde7 100644 --- a/.github/workflows/conda-upload-packages.yaml +++ b/.github/workflows/conda-upload-packages.yaml @@ -1,5 +1,5 @@ # adopted from https://github.com/rapidsai/shared-workflows/blob/branch-24.12/.github/workflows/conda-upload-packages.yaml -# with notable exceptions: +# with some notable differences: # # * assumes packages were uploaded to GitHub artifact store, not Amazon S3 # * always publishes to the same channel, but uses different label for non-release packages diff --git a/ci/upload-to-anaconda.sh b/ci/upload-to-anaconda.sh index 0b4643d7..35a57e0a 100755 --- a/ci/upload-to-anaconda.sh +++ b/ci/upload-to-anaconda.sh @@ -1,12 +1,15 @@ #!/bin/bash -# adopted from https://github.com/rapidsai/gha-tools/blob/main/tools/rapids-upload-to-anaconda, with some exceptions: +# adopted from https://github.com/rapidsai/gha-tools/blob/main/tools/rapids-upload-to-anaconda, +# with some notable differences: # # * assumes artifacts are on GitHub Actions artifact store, not Amazon S3 # * assumes packages have been unpacked to env variable RAPIDS_LOCAL_CONDA_CHANNEL # * does not differentiate between pull request and nightly branches # (relies on workflow triggers to just not run this script when it isn't needed) +set -e -E -u -o pipefail + # publish to the 'experimental' label on the 'legate' channel, for all cases except # releases (builds triggered by pushing a tag matching this regex exactly, e.g. 'v24.09.00') if [[ "${GITHUB_REF}" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then From 046e95a18ece76043b50cdae2e1cf7f0d38abdef Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 28 Oct 2024 17:03:32 -0500 Subject: [PATCH 09/11] update install instructions --- README.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cf3e4f50..fb038698 100644 --- a/README.md +++ b/README.md @@ -75,13 +75,25 @@ model = lb.LBRegressor(base_models=(lb.models.KRR(sigma=0.5), lb.models.Tree(max ## Installation -If you already have `cunumeric` and `legate` installed, run the following: +Install using `conda`. ```shell -pip install \ - --no-build-isolation \ - --no-deps \ - . +# stable release +conda install -c legate -c conda-forge -c nvidia legate-boost + +# nightly release +conda install -c legate/label/experimental -c legate -c conda-forge -c nvidia legate-boost +``` + +On systems without a GPU, the CPU-only package should automatically be installed. +On systems with a GPU and compatible CUDA version, the GPU package should automatically be installed. + +To force `conda` to prefer one, pass the build strings `*_cpu*` or `*_gpu*`, for example: + +```shell +# nightly release (CPU-only) +conda install --dry-run -c legate/label/experimental -c legate -c conda-forge -c nvidia \ + 'legate-boost=*=*_cpu*' ``` -For more details on customizing the build and setting up a development environment, see [`contributing.md`](./contributing.md). +For more details on building from source and setting up a development environment, see [`contributing.md`](./contributing.md). From 010bf51859c5f753f361cee58651a6850346f4f1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 30 Oct 2024 08:05:08 -0700 Subject: [PATCH 10/11] skip latest nightlies --- conda/recipes/legate-boost/conda_build_config.yaml | 5 ++++- conda/recipes/legate-boost/meta.yaml | 13 ++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/conda/recipes/legate-boost/conda_build_config.yaml b/conda/recipes/legate-boost/conda_build_config.yaml index 64607b10..38e34ef9 100644 --- a/conda/recipes/legate-boost/conda_build_config.yaml +++ b/conda/recipes/legate-boost/conda_build_config.yaml @@ -25,5 +25,8 @@ gpu_enabled: - true - false +cunumeric_version: + - "=24.09.*,>=0.0.0.dev0,!=24.09.00.dev116" + legate_version: - - "=24.09.*,>=0.0.0.dev0" + - "=24.09.*,>=0.0.0.dev0,!=24.09.00.dev329" diff --git a/conda/recipes/legate-boost/meta.yaml b/conda/recipes/legate-boost/meta.yaml index 71245129..1db98b4c 100644 --- a/conda/recipes/legate-boost/meta.yaml +++ b/conda/recipes/legate-boost/meta.yaml @@ -79,13 +79,12 @@ requirements: {% if gpu_enabled_bool %} - cuda-version ={{ cuda_version }} - cuda-cudart-dev - # TODO: reset these - - cunumeric=24.09.00.dev109=*_gpu* - - legate=24.09.00.dev296=*_gpu* + - cunumeric {{ cunumeric_version }} =*_gpu* + - legate {{ legate_version }} =*_gpu* - libcublas-dev {% else %} - - cunumeric=24.09.00.dev109=*_cpu* - - legate=24.09.00.dev296=*_cpu* + - cunumeric {{ cunumeric_version }} =*_cpu* + - legate {{ legate_version }} =*_cpu* {% endif %} - openblas - python @@ -100,9 +99,9 @@ requirements: # cuda-version is used to constrain __cuda - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - __cuda - - cunumeric=24.09.00.dev109=*_gpu* + - cunumeric {{ cunumeric_version }} =*_gpu* {% else %} - - cunumeric=24.09.00.dev109=*_cpu* + - cunumeric {{ cunumeric_version }} =*_cpu* {% endif %} # Relying on run_exports from legate to pin an appropriate range of versions and # GPU vs. CPU selector. From e593ae69fbbd9e6a49a0719d0bb7f47f07da0848 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 30 Oct 2024 08:22:23 -0700 Subject: [PATCH 11/11] skip more nightlies --- conda/recipes/legate-boost/conda_build_config.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda/recipes/legate-boost/conda_build_config.yaml b/conda/recipes/legate-boost/conda_build_config.yaml index 38e34ef9..c612cd10 100644 --- a/conda/recipes/legate-boost/conda_build_config.yaml +++ b/conda/recipes/legate-boost/conda_build_config.yaml @@ -26,7 +26,10 @@ gpu_enabled: - false cunumeric_version: + # .dev116: https://github.com/nv-legate/legate.core.internal/issues/1409 - "=24.09.*,>=0.0.0.dev0,!=24.09.00.dev116" legate_version: - - "=24.09.*,>=0.0.0.dev0,!=24.09.00.dev329" + # .dev319: https://github.com/nv-legate/legate.core.internal/pull/1401 + # .dev329: https://github.com/nv-legate/legate.core.internal/issues/1409 + - "=24.09.*,>=0.0.0.dev0,!=24.09.00.dev319,!=24.09.00.dev329"