From 27c529f699961a4497e4bc4319019ac64c4ea025 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 Sep 2024 08:25:34 -0700 Subject: [PATCH 01/27] First pass at build/publishes through GHA --- .github/workflows/build.yml | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..92b8645 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,91 @@ +name: Docker build + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + +jobs: + docker: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + cuda: ["11.8.0"] + python: ["3.10", "3.11", "3.12"] + linux: ["ubuntu20.04"] + rapids: ["24.10", "24.12"] + image: + - tag: "gpuci/dask" + context: "./dask" + - tag: "gpuci/distributed" + context: "./distributed" + - tag: "gpuci/dask_image" + context: "./dask_image" + # dask-image gpuCI isn't dependent on RAPIDS + exclude: + - image: "gpuci/dask_image" + rapids: "24.10" + + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + if: github.repository == 'rapidsai/dask-build-environment' && github.event_name == 'push' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Compute ucx-py version + id: ucx_py + env: + rapids: ${{ matrix.rapids }} + run: | + echo "version=$(curl -sL https://version.gpuci.io/rapids/${rapids})" >> $GITHUB_ENV + + - name: Generate tag + id: tag + env: + cuda: ${{ matrix.cuda }} + python: ${{ matrix.python }} + linux: ${{ matrix.linux }} + rapids: ${{ matrix.rapids }} + image: ${{ matrix.image.tag }} + run: | + case ${image} in + "gpuci/dask_image") # doesn't depend on RAPIDS / ucx-py for gpuCI + tag="${image}:cuda${cuda}-devel-${linux}-py${python}" + ;; + *) + tag="${image}:${rapids}-cuda${cuda}-devel-${linux}-py${python}" + ;; + esac + + echo "tag=${tag}" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.image.context }} + push: ${{ github.repository == 'rapidsai/dask-build-environment' && github.event_name == 'push' }} + platforms: linux/amd64 + tags: ${{ steps.tag.outputs.tag }} + build-args: | + RAPIDS_VER=${{ matrix.rapids }} + UCX_PY_VER=${{ steps.ucx_py.outputs.version }} + CUDA_VER=${{ matrix.cuda }} + LINUX_VER=${{ matrix.linux }} + PYTHON_VER=${{ matrix.python }} + + - name: Report + run: echo Built ${{ steps.tag.outputs.tag }} From f7c702a681e7e4b5fab2faaf5d7b83f6eb94f0b1 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 Sep 2024 08:54:14 -0700 Subject: [PATCH 02/27] Merge RUN commands to reduce storage footprint of build --- dask/Dockerfile | 49 ++++++++++++++++-------------------------- dask_image/Dockerfile | 33 +++++++++++----------------- distributed/Dockerfile | 45 +++++++++++++++----------------------- 3 files changed, 48 insertions(+), 79 deletions(-) diff --git a/dask/Dockerfile b/dask/Dockerfile index 23443ec..749ecf5 100644 --- a/dask/Dockerfile +++ b/dask/Dockerfile @@ -11,38 +11,27 @@ ARG UCX_PY_VER=unset COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/dask/main/continuous_integration/environment-$PYTHON_VER.yaml /dask.yml - ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz -RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ - && rm /tools.tar.gz - -RUN conda config --set ssl_verify false - -RUN rapids-mamba-retry install conda-merge git - -RUN cat /rapids.yml \ - | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ - | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \ - | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \ - > /rapids_pinned.yml -# unpin problematic CI dependencies -RUN cat /dask.yml \ - | sed -r "s/pyarrow=/pyarrow>=/g" \ - | sed -r "s/pandas=/pandas>=/g" \ - | sed -r "s/numpy=/numpy>=/g" \ - > /dask_unpinned.yml - -RUN conda-merge /rapids_pinned.yml /dask_unpinned.yml > /dask.yml - -RUN rapids-mamba-retry env create -n dask --file /dask.yml - -# Clean up pkgs to reduce image size and chmod for all users -RUN chmod -R ugo+rw /opt/conda \ +RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ + && rm /tools.tar.gz \ + && conda config --set ssl_verify false \ + && rapids-mamba-retry install conda-merge git \ + && cat /rapids.yml \ + | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ + | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \ + | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \ + > /rapids_pinned.yml \ + && cat /dask.yml \ + | sed -r "s/pyarrow=/pyarrow>=/g" \ + | sed -r "s/pandas=/pandas>=/g" \ + | sed -r "s/numpy=/numpy>=/g" \ + > /dask_unpinned.yml \ + && conda-merge /rapids_pinned.yml /dask_unpinned.yml > /dask.yml \ + && rapids-mamba-retry env create -n dask --file /dask.yml \ + && chmod -R ugo+rw /opt/conda \ && conda clean -tipy \ - && chmod -R ugo+rw /opt/conda - -# need a user with access to conda -RUN useradd -r -g conda -u 10000 dask + && chmod -R ugo+rw /opt/conda \ + && useradd -r -g conda -u 10000 dask CMD [ "/bin/bash" ] diff --git a/dask_image/Dockerfile b/dask_image/Dockerfile index 87e52ed..da1cf70 100644 --- a/dask_image/Dockerfile +++ b/dask_image/Dockerfile @@ -9,29 +9,20 @@ ARG PYTHON_VER=unset COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/dask-image/main/continuous_integration/environment-$PYTHON_VER.yml /dask.yml - ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz -RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ - && rm /tools.tar.gz - -RUN conda config --set ssl_verify false - -RUN rapids-mamba-retry install conda-merge git - -RUN cat /rapids.yml \ - | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ - > /rapids_pinned.yml -RUN conda-merge /rapids_pinned.yml /dask.yml > /dask_image.yml - -RUN rapids-mamba-retry env create -n dask_image --file /dask_image.yml - -# Clean up pkgs to reduce image size and chmod for all users -RUN chmod -R ugo+rw /opt/conda \ +RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ + && rm /tools.tar.gz \ + && conda config --set ssl_verify false \ + && rapids-mamba-retry install conda-merge git \ + && cat /rapids.yml \ + | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ + > /rapids_pinned.yml \ + && conda-merge /rapids_pinned.yml /dask.yml > /dask_image.yml \ + && rapids-mamba-retry env create -n dask_image --file /dask_image.yml \ + && chmod -R ugo+rw /opt/conda \ && conda clean -tipy \ - && chmod -R ugo+rw /opt/conda - -# need a user with access to conda -RUN useradd -r -g conda -u 10000 dask + && chmod -R ugo+rw /opt/conda \ + && useradd -r -g conda -u 10000 dask CMD [ "/bin/bash" ] diff --git a/distributed/Dockerfile b/distributed/Dockerfile index 6a63f4e..1e1d886 100644 --- a/distributed/Dockerfile +++ b/distributed/Dockerfile @@ -11,36 +11,25 @@ ARG UCX_PY_VER=unset COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/distributed/main/continuous_integration/environment-$PYTHON_VER.yaml /dask.yml - ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz -RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ - && rm /tools.tar.gz - -RUN conda config --set ssl_verify false - -RUN rapids-mamba-retry install conda-merge git - -RUN cat /rapids.yml \ - | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ - | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \ - | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \ - > /rapids_pinned.yml -# unpin problematic CI dependencies -RUN cat /dask.yml \ - | sed -r "s/pyarrow=/pyarrow>=/g" \ - > /dask_unpinned.yml - -RUN conda-merge /rapids_pinned.yml /dask_unpinned.yml > /distributed.yml - -RUN rapids-mamba-retry env create -n dask --file /distributed.yml - -# Clean up pkgs to reduce image size and chmod for all users -RUN chmod -R ugo+rw /opt/conda \ +RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ + && rm /tools.tar.gz \ + && conda config --set ssl_verify false \ + && rapids-mamba-retry install conda-merge git \ + && cat /rapids.yml \ + | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ + | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \ + | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \ + > /rapids_pinned.yml \ + && cat /dask.yml \ + | sed -r "s/pyarrow=/pyarrow>=/g" \ + > /dask_unpinned.yml \ + && conda-merge /rapids_pinned.yml /dask_unpinned.yml > /distributed.yml \ + && rapids-mamba-retry env create -n dask --file /distributed.yml \ + && chmod -R ugo+rw /opt/conda \ && conda clean -tipy \ - && chmod -R ugo+rw /opt/conda - -# need a user with access to conda -RUN useradd -r -g conda -u 10000 dask + && chmod -R ugo+rw /opt/conda \ + && useradd -r -g conda -u 10000 dask CMD [ "/bin/bash" ] From 6469620f89e346991ddfc9dc2e8f17c50e7c7571 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:54:57 -0700 Subject: [PATCH 03/27] Modify trigger conditions --- .github/workflows/build.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92b8645..6454efa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,9 +4,8 @@ on: push: branches: - main - tags: - - "*" pull_request: + workflow_dispatch: jobs: docker: @@ -31,7 +30,6 @@ jobs: - image: "gpuci/dask_image" rapids: "24.10" - steps: - name: Checkout uses: actions/checkout@v3 @@ -41,7 +39,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v3 - if: github.repository == 'rapidsai/dask-build-environment' && github.event_name == 'push' + if: github.repository == 'rapidsai/dask-build-environment' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -77,7 +75,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ${{ matrix.image.context }} - push: ${{ github.repository == 'rapidsai/dask-build-environment' && github.event_name == 'push' }} + push: ${{ github.repository == 'rapidsai/dask-build-environment' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} platforms: linux/amd64 tags: ${{ steps.tag.outputs.tag }} build-args: | From 23e9fd90e0666dcf04e3294044ee73e30a2e7c48 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:11:27 -0700 Subject: [PATCH 04/27] Run builds on a schedule --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6454efa..8d9f13b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,6 +5,8 @@ on: branches: - main pull_request: + schedule: + - cron: "0 1 * * *" # Daily “At 01:00” UTC workflow_dispatch: jobs: From 200bfc1bd39a0d0cc8f6c8a47b3f935aaacd9827 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:13:37 -0700 Subject: [PATCH 05/27] Update GPU CI updating workflow to parse information from/update GHA workflow --- .github/workflows/update-gpuci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml index 7fa9c66..b92926f 100644 --- a/.github/workflows/update-gpuci.yml +++ b/.github/workflows/update-gpuci.yml @@ -13,19 +13,23 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Parse current axis YAML + - name: Fetch current RAPIDS versions from build matrix + id: parse_yaml uses: the-coding-turtle/ga-yaml-parser@v0.1.2 with: - file: ci/axis/dask.yaml - export_to_envs: true - return_to_outputs: false + file: .github/workflows/build.yml + export_to_envs: false + return_to_outputs: true - name: Get old RAPIDS / UCX-Py versions + env: + OLD_RAPIDS_VER: ${{ steps.parse_yaml.outputs.jobs_docker_strategy_matrix_rapids_0 }} + RAPIDS_VER: ${{ steps.parse_yaml.outputs.jobs_docker_strategy_matrix_rapids_1 }} run: | - echo OLD_RAPIDS_VER=$RAPIDS_VER_0 >> $GITHUB_ENV - echo OLD_UCX_PY_VER=$(curl -sL https://version.gpuci.io/rapids/$RAPIDS_VER_0) >> $GITHUB_ENV - echo RAPIDS_VER=$RAPIDS_VER_1 >> $GITHUB_ENV - echo UCX_PY_VER=$(curl -sL https://version.gpuci.io/rapids/$RAPIDS_VER_1) >> $GITHUB_ENV + echo OLD_RAPIDS_VER=$OLD_RAPIDS_VER >> $GITHUB_ENV + echo OLD_UCX_PY_VER=$(curl -sL https://version.gpuci.io/rapids/$OLD_RAPIDS_VER) >> $GITHUB_ENV + echo RAPIDS_VER=$RAPIDS_VER >> $GITHUB_ENV + echo UCX_PY_VER=$(curl -sL https://version.gpuci.io/rapids/$RAPIDS_VER) >> $GITHUB_ENV - name: Get latest cuDF nightly version id: cudf_latest @@ -54,7 +58,6 @@ jobs: - name: Update new RAPIDS versions uses: jacobtomlinson/gha-find-replace@v2 with: - include: 'ci/**' find: "${{ env.RAPIDS_VER }}" replace: "${{ env.NEW_RAPIDS_VER }}" regex: false @@ -62,7 +65,6 @@ jobs: - name: Update old RAPIDS versions uses: jacobtomlinson/gha-find-replace@v3 with: - include: 'ci/**' find: "${{ env.OLD_RAPIDS_VER }}" replace: "${{ env.RAPIDS_VER }}" regex: false @@ -70,7 +72,6 @@ jobs: - name: Update new UCX-Py versions uses: jacobtomlinson/gha-find-replace@v3 with: - include: 'ci/**' find: "${{ env.UCX_PY_VER }}" replace: "${{ env.NEW_UCX_PY_VER }}" regex: false @@ -78,7 +79,6 @@ jobs: - name: Update old UCX-Py versions uses: jacobtomlinson/gha-find-replace@v3 with: - include: 'ci/**' find: "${{ env.OLD_UCX_PY_VER }}" replace: "${{ env.UCX_PY_VER }}" regex: false From d1141c1d6ab4e4d1cd35a898ca1ba969ec47abc1 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:03:22 -0700 Subject: [PATCH 06/27] Use heredocs to clean up multi-line RUN --- dask/Dockerfile | 42 ++++++++++++++++++++++-------------------- dask_image/Dockerfile | 28 +++++++++++++++------------- distributed/Dockerfile | 38 ++++++++++++++++++++------------------ 3 files changed, 57 insertions(+), 51 deletions(-) diff --git a/dask/Dockerfile b/dask/Dockerfile index 749ecf5..7c53c90 100644 --- a/dask/Dockerfile +++ b/dask/Dockerfile @@ -13,25 +13,27 @@ COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/dask/main/continuous_integration/environment-$PYTHON_VER.yaml /dask.yml ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz -RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ - && rm /tools.tar.gz \ - && conda config --set ssl_verify false \ - && rapids-mamba-retry install conda-merge git \ - && cat /rapids.yml \ - | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ - | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \ - | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \ - > /rapids_pinned.yml \ - && cat /dask.yml \ - | sed -r "s/pyarrow=/pyarrow>=/g" \ - | sed -r "s/pandas=/pandas>=/g" \ - | sed -r "s/numpy=/numpy>=/g" \ - > /dask_unpinned.yml \ - && conda-merge /rapids_pinned.yml /dask_unpinned.yml > /dask.yml \ - && rapids-mamba-retry env create -n dask --file /dask.yml \ - && chmod -R ugo+rw /opt/conda \ - && conda clean -tipy \ - && chmod -R ugo+rw /opt/conda \ - && useradd -r -g conda -u 10000 dask +RUN < /rapids_pinned.yml + cat /dask.yml + | sed -r "s/pyarrow=/pyarrow>=/g" + | sed -r "s/pandas=/pandas>=/g" + | sed -r "s/numpy=/numpy>=/g" + > /dask_unpinned.yml + conda-merge /rapids_pinned.yml /dask_unpinned.yml > /dask.yml + rapids-mamba-retry env create -n dask --file /dask.yml + chmod -R ugo+rw /opt/conda + conda clean -tipy + chmod -R ugo+rw /opt/conda + useradd -r -g conda -u 10000 dask +EOT CMD [ "/bin/bash" ] diff --git a/dask_image/Dockerfile b/dask_image/Dockerfile index da1cf70..30d94bb 100644 --- a/dask_image/Dockerfile +++ b/dask_image/Dockerfile @@ -11,18 +11,20 @@ COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/dask-image/main/continuous_integration/environment-$PYTHON_VER.yml /dask.yml ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz -RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ - && rm /tools.tar.gz \ - && conda config --set ssl_verify false \ - && rapids-mamba-retry install conda-merge git \ - && cat /rapids.yml \ - | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ - > /rapids_pinned.yml \ - && conda-merge /rapids_pinned.yml /dask.yml > /dask_image.yml \ - && rapids-mamba-retry env create -n dask_image --file /dask_image.yml \ - && chmod -R ugo+rw /opt/conda \ - && conda clean -tipy \ - && chmod -R ugo+rw /opt/conda \ - && useradd -r -g conda -u 10000 dask +RUN < /rapids_pinned.yml + conda-merge /rapids_pinned.yml /dask.yml > /dask_image.yml + rapids-mamba-retry env create -n dask_image --file /dask_image.yml + chmod -R ugo+rw /opt/conda + conda clean -tipy + chmod -R ugo+rw /opt/conda + useradd -r -g conda -u 10000 dask +EOT CMD [ "/bin/bash" ] diff --git a/distributed/Dockerfile b/distributed/Dockerfile index 1e1d886..c760762 100644 --- a/distributed/Dockerfile +++ b/distributed/Dockerfile @@ -13,23 +13,25 @@ COPY environment.yml /rapids.yml ADD https://raw.githubusercontent.com/dask/distributed/main/continuous_integration/environment-$PYTHON_VER.yaml /dask.yml ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz -RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \ - && rm /tools.tar.gz \ - && conda config --set ssl_verify false \ - && rapids-mamba-retry install conda-merge git \ - && cat /rapids.yml \ - | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \ - | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \ - | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \ - > /rapids_pinned.yml \ - && cat /dask.yml \ - | sed -r "s/pyarrow=/pyarrow>=/g" \ - > /dask_unpinned.yml \ - && conda-merge /rapids_pinned.yml /dask_unpinned.yml > /distributed.yml \ - && rapids-mamba-retry env create -n dask --file /distributed.yml \ - && chmod -R ugo+rw /opt/conda \ - && conda clean -tipy \ - && chmod -R ugo+rw /opt/conda \ - && useradd -r -g conda -u 10000 dask +RUN < /rapids_pinned.yml + cat /dask.yml + | sed -r "s/pyarrow=/pyarrow>=/g" + > /dask_unpinned.yml + conda-merge /rapids_pinned.yml /dask_unpinned.yml > /distributed.yml + rapids-mamba-retry env create -n dask --file /distributed.yml + chmod -R ugo+rw /opt/conda + conda clean -tipy + chmod -R ugo+rw /opt/conda + useradd -r -g conda -u 10000 dask +EOT CMD [ "/bin/bash" ] From 2534e614a5bbf36fe388bcdff1f62a73a49e28dd Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:05:34 -0700 Subject: [PATCH 07/27] Fix typo in heredocs --- dask/Dockerfile | 8 ++++---- dask_image/Dockerfile | 4 ++-- distributed/Dockerfile | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dask/Dockerfile b/dask/Dockerfile index 7c53c90..b711ebe 100644 --- a/dask/Dockerfile +++ b/dask/Dockerfile @@ -18,10 +18,10 @@ RUN < /rapids_pinned.yml cat /dask.yml | sed -r "s/pyarrow=/pyarrow>=/g" diff --git a/dask_image/Dockerfile b/dask_image/Dockerfile index 30d94bb..fc33945 100644 --- a/dask_image/Dockerfile +++ b/dask_image/Dockerfile @@ -16,8 +16,8 @@ RUN < /rapids_pinned.yml conda-merge /rapids_pinned.yml /dask.yml > /dask_image.yml rapids-mamba-retry env create -n dask_image --file /dask_image.yml diff --git a/distributed/Dockerfile b/distributed/Dockerfile index c760762..83cb3b5 100644 --- a/distributed/Dockerfile +++ b/distributed/Dockerfile @@ -18,10 +18,10 @@ RUN < /rapids_pinned.yml cat /dask.yml | sed -r "s/pyarrow=/pyarrow>=/g" From 4a0f0f37d504897cea119338f985ab2f4f3618e2 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:07:48 -0700 Subject: [PATCH 08/27] Fix remaining typo in dask/distributed heredocs --- dask/Dockerfile | 8 ++++---- distributed/Dockerfile | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dask/Dockerfile b/dask/Dockerfile index b711ebe..795c885 100644 --- a/dask/Dockerfile +++ b/dask/Dockerfile @@ -23,10 +23,10 @@ RUN < /rapids_pinned.yml - cat /dask.yml - | sed -r "s/pyarrow=/pyarrow>=/g" - | sed -r "s/pandas=/pandas>=/g" - | sed -r "s/numpy=/numpy>=/g" + cat /dask.yml \ + | sed -r "s/pyarrow=/pyarrow>=/g" \ + | sed -r "s/pandas=/pandas>=/g" \ + | sed -r "s/numpy=/numpy>=/g" \ > /dask_unpinned.yml conda-merge /rapids_pinned.yml /dask_unpinned.yml > /dask.yml rapids-mamba-retry env create -n dask --file /dask.yml diff --git a/distributed/Dockerfile b/distributed/Dockerfile index 83cb3b5..68ba84b 100644 --- a/distributed/Dockerfile +++ b/distributed/Dockerfile @@ -23,8 +23,8 @@ RUN < /rapids_pinned.yml - cat /dask.yml - | sed -r "s/pyarrow=/pyarrow>=/g" + cat /dask.yml \ + | sed -r "s/pyarrow=/pyarrow>=/g" \ > /dask_unpinned.yml conda-merge /rapids_pinned.yml /dask_unpinned.yml > /distributed.yml rapids-mamba-retry env create -n dask --file /distributed.yml From 1e7f5525495a9ff5730480bcc8a44ccf1f2296f2 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:46:02 -0700 Subject: [PATCH 09/27] Update dockerhub credentials --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d9f13b..95fabca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,8 +43,8 @@ jobs: uses: docker/login-action@v3 if: github.repository == 'rapidsai/dask-build-environment' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} + password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} - name: Compute ucx-py version id: ucx_py From 6376f9178654392e4ab9132fb2419ca99c77fa68 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:53:55 -0700 Subject: [PATCH 10/27] Fix ucx-py step to write to GITHUB_OUTPUT --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95fabca..f8c2db0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: env: rapids: ${{ matrix.rapids }} run: | - echo "version=$(curl -sL https://version.gpuci.io/rapids/${rapids})" >> $GITHUB_ENV + echo "version=$(curl -sL https://version.gpuci.io/rapids/${rapids})" >> $GITHUB_OUTPUT - name: Generate tag id: tag From 37a765f6bf5094b519616d5d963e447c1bb04b80 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:05:01 -0700 Subject: [PATCH 11/27] Update image tags --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8c2db0..b990784 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,15 +21,15 @@ jobs: linux: ["ubuntu20.04"] rapids: ["24.10", "24.12"] image: - - tag: "gpuci/dask" + - tag: "rapidsai/dask" context: "./dask" - - tag: "gpuci/distributed" + - tag: "rapidsai/distributed" context: "./distributed" - - tag: "gpuci/dask_image" + - tag: "rapidsai/dask_image" context: "./dask_image" # dask-image gpuCI isn't dependent on RAPIDS exclude: - - image: "gpuci/dask_image" + - image: "rapidsai/dask_image" rapids: "24.10" steps: @@ -63,7 +63,7 @@ jobs: image: ${{ matrix.image.tag }} run: | case ${image} in - "gpuci/dask_image") # doesn't depend on RAPIDS / ucx-py for gpuCI + "rapidsai/dask_image") # doesn't depend on RAPIDS / ucx-py for gpuCI tag="${image}:cuda${cuda}-devel-${linux}-py${python}" ;; *) From 909c17ace3766c0142aeffb61a4682066352aa4f Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:11:19 -0700 Subject: [PATCH 12/27] Update image publishing conditions --- .github/workflows/build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b990784..9aeb996 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,9 @@ on: - cron: "0 1 * * *" # Daily “At 01:00” UTC workflow_dispatch: +env: + push: ${{ github.repository == 'rapidsai/dask-build-environment' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} + jobs: docker: runs-on: ubuntu-latest @@ -41,7 +44,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v3 - if: github.repository == 'rapidsai/dask-build-environment' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + if: env.push with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} @@ -77,7 +80,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ${{ matrix.image.context }} - push: ${{ github.repository == 'rapidsai/dask-build-environment' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} + push: ${{ env.push }} platforms: linux/amd64 tags: ${{ steps.tag.outputs.tag }} build-args: | From 1c7f52873fb1196dd28ef4ffd48288b52562b60c Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:13:24 -0700 Subject: [PATCH 13/27] Explicit bool comparison instead of truthy value --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9aeb996..d4eb824 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v3 - if: env.push + if: ${{ env.push == true }} with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} @@ -80,7 +80,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ${{ matrix.image.context }} - push: ${{ env.push }} + push: ${{ env.push == true }} platforms: linux/amd64 tags: ${{ steps.tag.outputs.tag }} build-args: | From fe60490f63ad346677697e3fdf67803e46c4deca Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:34:36 -0700 Subject: [PATCH 14/27] Test condition parsing --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4eb824..2241e80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: env: - push: ${{ github.repository == 'rapidsai/dask-build-environment' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} + push: ${{ github.repository == 'rapidsai/dask-build-environment' && contains(fromJSON('["pull_request"]'), github.event_name) }} jobs: docker: From b9493d51ec6ab45571b7f2d9a975ca90af97def0 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:35:52 -0700 Subject: [PATCH 15/27] Use string 'true' instead of boolean --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2241e80..b96284d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v3 - if: ${{ env.push == true }} + if: ${{ env.push == 'true' }} with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} @@ -80,7 +80,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ${{ matrix.image.context }} - push: ${{ env.push == true }} + push: ${{ env.push == 'true' }} platforms: linux/amd64 tags: ${{ steps.tag.outputs.tag }} build-args: | From 1b43c7fd88d7476c6af9625149d1054689fd4507 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:38:12 -0700 Subject: [PATCH 16/27] Revert publishing condition --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b96284d..23a2c8d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: env: - push: ${{ github.repository == 'rapidsai/dask-build-environment' && contains(fromJSON('["pull_request"]'), github.event_name) }} + push: ${{ github.repository == 'rapidsai/dask-build-environment' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} jobs: docker: From 127da2a749cc3cb189740381b81062ed2bd2b049 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:50:38 -0700 Subject: [PATCH 17/27] Fix matrix exclude rule --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23a2c8d..70fe3d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,8 @@ jobs: context: "./dask_image" # dask-image gpuCI isn't dependent on RAPIDS exclude: - - image: "rapidsai/dask_image" + - image: + tag: "rapidsai/dask_image" rapids: "24.10" steps: From 5a31da0ff23e848a1bff83be6769fe2225ed8775 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:32:41 -0700 Subject: [PATCH 18/27] Hardcode old and current RAPIDS/UCX-Py versions --- .github/workflows/build.yml | 17 +++++++---------- .github/workflows/update-gpuci.yml | 10 ++++++---- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70fe3d3..12049ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,10 @@ on: env: push: ${{ github.repository == 'rapidsai/dask-build-environment' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} + OLD_RAPIDS_VER: "24.10" + OLD_UCX_PY_VER: "0.40" + RAPIDS_VER: "24.12" + UCX_PY_VER: "0.41" jobs: docker: @@ -22,7 +26,7 @@ jobs: cuda: ["11.8.0"] python: ["3.10", "3.11", "3.12"] linux: ["ubuntu20.04"] - rapids: ["24.10", "24.12"] + rapids: [${{ env.OLD_RAPIDS_VER }}, ${{ env.RAPIDS_VER }}] image: - tag: "rapidsai/dask" context: "./dask" @@ -34,7 +38,7 @@ jobs: exclude: - image: tag: "rapidsai/dask_image" - rapids: "24.10" + rapids: ${{ env.OLD_RAPIDS_VER }} steps: - name: Checkout @@ -50,13 +54,6 @@ jobs: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} - - name: Compute ucx-py version - id: ucx_py - env: - rapids: ${{ matrix.rapids }} - run: | - echo "version=$(curl -sL https://version.gpuci.io/rapids/${rapids})" >> $GITHUB_OUTPUT - - name: Generate tag id: tag env: @@ -86,7 +83,7 @@ jobs: tags: ${{ steps.tag.outputs.tag }} build-args: | RAPIDS_VER=${{ matrix.rapids }} - UCX_PY_VER=${{ steps.ucx_py.outputs.version }} + UCX_PY_VER=${{ matrix.rapids == env.OLD_RAPIDS_VER && env.OLD_UCX_PY_VER || env.UCX_PY_VER }} CUDA_VER=${{ matrix.cuda }} LINUX_VER=${{ matrix.linux }} PYTHON_VER=${{ matrix.python }} diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml index b92926f..bcb5ef7 100644 --- a/.github/workflows/update-gpuci.yml +++ b/.github/workflows/update-gpuci.yml @@ -23,13 +23,15 @@ jobs: - name: Get old RAPIDS / UCX-Py versions env: - OLD_RAPIDS_VER: ${{ steps.parse_yaml.outputs.jobs_docker_strategy_matrix_rapids_0 }} - RAPIDS_VER: ${{ steps.parse_yaml.outputs.jobs_docker_strategy_matrix_rapids_1 }} + OLD_RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_OLD_RAPIDS_VER }} + OLD_UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_OLD_UCX_PY_VER }} + RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_RAPIDS_VER }} + UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_UCX_PY_VER }} run: | echo OLD_RAPIDS_VER=$OLD_RAPIDS_VER >> $GITHUB_ENV - echo OLD_UCX_PY_VER=$(curl -sL https://version.gpuci.io/rapids/$OLD_RAPIDS_VER) >> $GITHUB_ENV + echo OLD_UCX_PY_VER=$OLD_UCX_PY_VER >> $GITHUB_ENV echo RAPIDS_VER=$RAPIDS_VER >> $GITHUB_ENV - echo UCX_PY_VER=$(curl -sL https://version.gpuci.io/rapids/$RAPIDS_VER) >> $GITHUB_ENV + echo UCX_PY_VER=$UCX_PY_VER >> $GITHUB_ENV - name: Get latest cuDF nightly version id: cudf_latest From bbf65f4b462d78aa736f2699fe09e8a8b1b8456e Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:33:12 -0700 Subject: [PATCH 19/27] Update checkout step version --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 12049ad..a039ebc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 8cbe7da97634cf1bb3c1c1422c849495c1abd050 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:34:14 -0700 Subject: [PATCH 20/27] Move build platform to GHA matrix --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a039ebc..ca4e0d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,7 @@ jobs: strategy: fail-fast: false matrix: + platform: ["linux/amd64"] cuda: ["11.8.0"] python: ["3.10", "3.11", "3.12"] linux: ["ubuntu20.04"] @@ -79,7 +80,7 @@ jobs: with: context: ${{ matrix.image.context }} push: ${{ env.push == 'true' }} - platforms: linux/amd64 + platforms: ${{ matrix.platform }} tags: ${{ steps.tag.outputs.tag }} build-args: | RAPIDS_VER=${{ matrix.rapids }} From 3a044dd8999de2229d8fbcff9f8e8a713b39aa46 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:35:10 -0700 Subject: [PATCH 21/27] Clarify image tag in final report step --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca4e0d3..1c056f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,4 +90,4 @@ jobs: PYTHON_VER=${{ matrix.python }} - name: Report - run: echo Built ${{ steps.tag.outputs.tag }} + run: echo "Built image '${{ steps.tag.outputs.tag }}'" From fec0a76360a86e3e61d53cca0a0d1da80741eb94 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:37:40 -0700 Subject: [PATCH 22/27] Ensure GPU CI updating workflow only touches .github/workflows/build.yml --- .github/workflows/update-gpuci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml index bcb5ef7..bbb47fe 100644 --- a/.github/workflows/update-gpuci.yml +++ b/.github/workflows/update-gpuci.yml @@ -58,10 +58,11 @@ jobs: echo "NEW_UCX_PY_VER=$(echo $FULL_UCX_PY_VER | cut -d'.' -f1,2)" >> $GITHUB_ENV - name: Update new RAPIDS versions - uses: jacobtomlinson/gha-find-replace@v2 + uses: jacobtomlinson/gha-find-replace@v3 with: find: "${{ env.RAPIDS_VER }}" replace: "${{ env.NEW_RAPIDS_VER }}" + include: ".github/workflows/build.yml" regex: false - name: Update old RAPIDS versions @@ -69,6 +70,7 @@ jobs: with: find: "${{ env.OLD_RAPIDS_VER }}" replace: "${{ env.RAPIDS_VER }}" + include: ".github/workflows/build.yml" regex: false - name: Update new UCX-Py versions @@ -76,6 +78,7 @@ jobs: with: find: "${{ env.UCX_PY_VER }}" replace: "${{ env.NEW_UCX_PY_VER }}" + include: ".github/workflows/build.yml" regex: false - name: Update old UCX-Py versions @@ -83,6 +86,7 @@ jobs: with: find: "${{ env.OLD_UCX_PY_VER }}" replace: "${{ env.UCX_PY_VER }}" + include: ".github/workflows/build.yml" regex: false - name: Create pull request From f9dc7b474c07b50aba7889140b50ffd8553b8cb7 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:43:03 -0700 Subject: [PATCH 23/27] Use fromJSON to construct RAPIDS matrix array --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c056f7..6792a5b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: cuda: ["11.8.0"] python: ["3.10", "3.11", "3.12"] linux: ["ubuntu20.04"] - rapids: [${{ env.OLD_RAPIDS_VER }}, ${{ env.RAPIDS_VER }}] + rapids: ${{ fromJSON('["$OLD_RAPIDS_VER", "$RAPIDS_VER"]') }} image: - tag: "rapidsai/dask" context: "./dask" From 0af7f24e726b2170de32201e1518fc3c5d26449a Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:10:29 -0700 Subject: [PATCH 24/27] Use hard-coded RAPIDS versions in build matrix --- .github/workflows/build.yml | 30 +++++++++++++++++++++++------- .github/workflows/update-gpuci.yml | 20 ++++++++++---------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6792a5b..1e70215 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,8 +13,8 @@ env: push: ${{ github.repository == 'rapidsai/dask-build-environment' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} OLD_RAPIDS_VER: "24.10" OLD_UCX_PY_VER: "0.40" - RAPIDS_VER: "24.12" - UCX_PY_VER: "0.41" + CURRENT_RAPIDS_VER: "24.12" + CURRENT_UCX_PY_VER: "0.41" jobs: docker: @@ -27,7 +27,7 @@ jobs: cuda: ["11.8.0"] python: ["3.10", "3.11", "3.12"] linux: ["ubuntu20.04"] - rapids: ${{ fromJSON('["$OLD_RAPIDS_VER", "$RAPIDS_VER"]') }} + rapids: ["old", "current"] image: - tag: "rapidsai/dask" context: "./dask" @@ -39,12 +39,28 @@ jobs: exclude: - image: tag: "rapidsai/dask_image" - rapids: ${{ env.OLD_RAPIDS_VER }} + rapids: "old" steps: - name: Checkout uses: actions/checkout@v4 + - name: Compute RAPIDS / UCX-Py versions + run: | + case ${{ matrix.rapids }} in + "old") + rapids_ver=${{ env.OLD_RAPIDS_VER }} + ucx_py_ver=${{ env.OLD_UCX_PY_VER }} + ;; + *) + rapids_ver=${{ env.CURRENT_RAPIDS_VER }} + ucx_py_ver=${{ env.CURRENT_UCX_PY_VER }} + ;; + esac + + echo "RAPIDS_VER=${rapids_ver}" >> $GITHUB_ENV + echo "UCX_PY_VER=${ucx_py_ver}" >> $GITHUB_ENV + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -61,7 +77,7 @@ jobs: cuda: ${{ matrix.cuda }} python: ${{ matrix.python }} linux: ${{ matrix.linux }} - rapids: ${{ matrix.rapids }} + rapids: ${{ env.RAPIDS_VER }} image: ${{ matrix.image.tag }} run: | case ${image} in @@ -83,8 +99,8 @@ jobs: platforms: ${{ matrix.platform }} tags: ${{ steps.tag.outputs.tag }} build-args: | - RAPIDS_VER=${{ matrix.rapids }} - UCX_PY_VER=${{ matrix.rapids == env.OLD_RAPIDS_VER && env.OLD_UCX_PY_VER || env.UCX_PY_VER }} + RAPIDS_VER=${{ env.RAPIDS_VER }} + UCX_PY_VER=${{ env.UCX_PY_VER }} CUDA_VER=${{ matrix.cuda }} LINUX_VER=${{ matrix.linux }} PYTHON_VER=${{ matrix.python }} diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml index bbb47fe..b1bb443 100644 --- a/.github/workflows/update-gpuci.yml +++ b/.github/workflows/update-gpuci.yml @@ -25,12 +25,12 @@ jobs: env: OLD_RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_OLD_RAPIDS_VER }} OLD_UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_OLD_UCX_PY_VER }} - RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_RAPIDS_VER }} - UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_UCX_PY_VER }} + CURRENT_RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_CURRENT_RAPIDS_VER }} + CURRENT_UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_CURRENT_UCX_PY_VER }} run: | echo OLD_RAPIDS_VER=$OLD_RAPIDS_VER >> $GITHUB_ENV - echo OLD_UCX_PY_VER=$OLD_UCX_PY_VER >> $GITHUB_ENV - echo RAPIDS_VER=$RAPIDS_VER >> $GITHUB_ENV + echo OLD_UCX_PY_VER=$OLD_UCX_PY_VER >> $GITHUB_E + echo CURRENT_RAPIDS_VER=$CURRENT_RAPIDS_VER >> $GITHUB_ENV echo UCX_PY_VER=$UCX_PY_VER >> $GITHUB_ENV - name: Get latest cuDF nightly version @@ -60,7 +60,7 @@ jobs: - name: Update new RAPIDS versions uses: jacobtomlinson/gha-find-replace@v3 with: - find: "${{ env.RAPIDS_VER }}" + find: "${{ env.CURRENT_RAPIDS_VER }}" replace: "${{ env.NEW_RAPIDS_VER }}" include: ".github/workflows/build.yml" regex: false @@ -69,7 +69,7 @@ jobs: uses: jacobtomlinson/gha-find-replace@v3 with: find: "${{ env.OLD_RAPIDS_VER }}" - replace: "${{ env.RAPIDS_VER }}" + replace: "${{ env.CURRENT_RAPIDS_VER }}" include: ".github/workflows/build.yml" regex: false @@ -90,15 +90,15 @@ jobs: regex: false - name: Create pull request - if: ${{ env.RAPIDS_VER != env.NEW_RAPIDS_VER && env.UCX_PY_VER != env.NEW_UCX_PY_VER }} + if: ${{ env.CURRENT_RAPIDS_VER != env.NEW_RAPIDS_VER && env.UCX_PY_VER != env.NEW_UCX_PY_VER }} uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "Update gpuCI `RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`, `UCX_PY_VER` to `${{ env.NEW_UCX_PY_VER }}`" - title: "Update gpuCI `RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`" + commit-message: "Update gpuCI `CURRENT_RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`, `UCX_PY_VER` to `${{ env.NEW_UCX_PY_VER }}`" + title: "Update gpuCI `CURRENT_RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`" author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> branch: "upgrade-gpuci-rapids" body: | New cuDF and ucx-py nightly versions have been detected. - Updated `dask.yaml` and `run.sh` to use `RAPIDS_VER=${{ env.NEW_RAPIDS_VER }}` and `UCX_PY_VER=${{ env.NEW_UCX_PY_VER }}`. + Updated `dask.yaml` and `run.sh` to use `CURRENT_RAPIDS_VER=${{ env.NEW_RAPIDS_VER }}` and `UCX_PY_VER=${{ env.NEW_UCX_PY_VER }}`. From ec564f39979add3582c53bc4088c4e10144627e3 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:13:53 -0700 Subject: [PATCH 25/27] Update PR message --- .github/workflows/update-gpuci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml index b1bb443..d138e05 100644 --- a/.github/workflows/update-gpuci.yml +++ b/.github/workflows/update-gpuci.yml @@ -31,7 +31,7 @@ jobs: echo OLD_RAPIDS_VER=$OLD_RAPIDS_VER >> $GITHUB_ENV echo OLD_UCX_PY_VER=$OLD_UCX_PY_VER >> $GITHUB_E echo CURRENT_RAPIDS_VER=$CURRENT_RAPIDS_VER >> $GITHUB_ENV - echo UCX_PY_VER=$UCX_PY_VER >> $GITHUB_ENV + echo CURRENT_UCX_PY_VER=$CURRENT_UCX_PY_VER >> $GITHUB_ENV - name: Get latest cuDF nightly version id: cudf_latest @@ -76,7 +76,7 @@ jobs: - name: Update new UCX-Py versions uses: jacobtomlinson/gha-find-replace@v3 with: - find: "${{ env.UCX_PY_VER }}" + find: "${{ env.CURRENT_UCX_PY_VER }}" replace: "${{ env.NEW_UCX_PY_VER }}" include: ".github/workflows/build.yml" regex: false @@ -85,20 +85,20 @@ jobs: uses: jacobtomlinson/gha-find-replace@v3 with: find: "${{ env.OLD_UCX_PY_VER }}" - replace: "${{ env.UCX_PY_VER }}" + replace: "${{ env.CURRENT_UCX_PY_VER }}" include: ".github/workflows/build.yml" regex: false - name: Create pull request - if: ${{ env.CURRENT_RAPIDS_VER != env.NEW_RAPIDS_VER && env.UCX_PY_VER != env.NEW_UCX_PY_VER }} + if: ${{ env.CURRENT_RAPIDS_VER != env.NEW_RAPIDS_VER && env.CURRENT_UCX_PY_VER != env.NEW_UCX_PY_VER }} uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "Update gpuCI `CURRENT_RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`, `UCX_PY_VER` to `${{ env.NEW_UCX_PY_VER }}`" - title: "Update gpuCI `CURRENT_RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`" + commit-message: "Update gpuCI `CURRENT_RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`, `CURRENT_UCX_PY_VER` to `${{ env.NEW_UCX_PY_VER }}`" + title: "Start building images with RAPIDS `${{ env.NEW_RAPIDS_VER }}`, ucx-py `${{ env.NEW_UCX_PY_VER }}`" author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> branch: "upgrade-gpuci-rapids" body: | New cuDF and ucx-py nightly versions have been detected. - Updated `dask.yaml` and `run.sh` to use `CURRENT_RAPIDS_VER=${{ env.NEW_RAPIDS_VER }}` and `UCX_PY_VER=${{ env.NEW_UCX_PY_VER }}`. + Updated `.github/workflows/build.yml` to use `CURRENT_RAPIDS_VER=${{ env.NEW_RAPIDS_VER }}` and `CURRENT_UCX_PY_VER=${{ env.NEW_UCX_PY_VER }}`. From f5e395affbfa33e16cbebf5e10731d2971037f98 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:19:51 -0700 Subject: [PATCH 26/27] Typo in updating workflow --- .github/workflows/update-gpuci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml index d138e05..68aeb50 100644 --- a/.github/workflows/update-gpuci.yml +++ b/.github/workflows/update-gpuci.yml @@ -29,7 +29,7 @@ jobs: CURRENT_UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_CURRENT_UCX_PY_VER }} run: | echo OLD_RAPIDS_VER=$OLD_RAPIDS_VER >> $GITHUB_ENV - echo OLD_UCX_PY_VER=$OLD_UCX_PY_VER >> $GITHUB_E + echo OLD_UCX_PY_VER=$OLD_UCX_PY_VER >> $GITHUB_ENV echo CURRENT_RAPIDS_VER=$CURRENT_RAPIDS_VER >> $GITHUB_ENV echo CURRENT_UCX_PY_VER=$CURRENT_UCX_PY_VER >> $GITHUB_ENV From 2067ca909b67c51fdf16330bbc55f38a5bbb3819 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:31:06 -0700 Subject: [PATCH 27/27] Rename hardcoded matrix values --- .github/workflows/build.yml | 29 +++++++++++++---------- .github/workflows/update-gpuci.yml | 38 +++++++++++++++--------------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e70215..fefa50d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Docker build +name: Docker build & publish on: push: @@ -11,13 +11,14 @@ on: env: push: ${{ github.repository == 'rapidsai/dask-build-environment' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} - OLD_RAPIDS_VER: "24.10" - OLD_UCX_PY_VER: "0.40" - CURRENT_RAPIDS_VER: "24.12" - CURRENT_UCX_PY_VER: "0.41" + STABLE_RAPIDS_VER: "24.10" + STABLE_UCX_PY_VER: "0.40" + NIGHTLY_RAPIDS_VER: "24.12" + NIGHTLY_UCX_PY_VER: "0.41" jobs: docker: + name: ${{ matrix.image.tag }} (${{ matrix.rapids }} RAPIDS, python ${{ matrix.python }}, CTK ${{ matrix.cuda }}, ${{ matrix.linux }}, ${{ matrix.platform }}) runs-on: ubuntu-latest strategy: @@ -27,7 +28,7 @@ jobs: cuda: ["11.8.0"] python: ["3.10", "3.11", "3.12"] linux: ["ubuntu20.04"] - rapids: ["old", "current"] + rapids: ["stable", "nightly"] image: - tag: "rapidsai/dask" context: "./dask" @@ -39,7 +40,7 @@ jobs: exclude: - image: tag: "rapidsai/dask_image" - rapids: "old" + rapids: "stable" steps: - name: Checkout @@ -48,13 +49,17 @@ jobs: - name: Compute RAPIDS / UCX-Py versions run: | case ${{ matrix.rapids }} in - "old") - rapids_ver=${{ env.OLD_RAPIDS_VER }} - ucx_py_ver=${{ env.OLD_UCX_PY_VER }} + "stable") + rapids_ver=${{ env.STABLE_RAPIDS_VER }} + ucx_py_ver=${{ env.STABLE_UCX_PY_VER }} + ;; + "nightly") + rapids_ver=${{ env.NIGHTLY_RAPIDS_VER }} + ucx_py_ver=${{ env.NIGHTLY_UCX_PY_VER }} ;; *) - rapids_ver=${{ env.CURRENT_RAPIDS_VER }} - ucx_py_ver=${{ env.CURRENT_UCX_PY_VER }} + echo "Invalid RAPIDS version '${{ matrix.rapids }}'" + exit 1 ;; esac diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml index 68aeb50..91231f0 100644 --- a/.github/workflows/update-gpuci.yml +++ b/.github/workflows/update-gpuci.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Fetch current RAPIDS versions from build matrix + - name: Parse build workflow file id: parse_yaml uses: the-coding-turtle/ga-yaml-parser@v0.1.2 with: @@ -21,17 +21,17 @@ jobs: export_to_envs: false return_to_outputs: true - - name: Get old RAPIDS / UCX-Py versions + - name: Fetch RAPIDS/UCX-Py versions from parsed file env: - OLD_RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_OLD_RAPIDS_VER }} - OLD_UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_OLD_UCX_PY_VER }} - CURRENT_RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_CURRENT_RAPIDS_VER }} - CURRENT_UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_CURRENT_UCX_PY_VER }} + STABLE_RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_STABLE_RAPIDS_VER }} + STABLE_UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_STABLE_UCX_PY_VER }} + NIGHTLY_RAPIDS_VER: ${{ steps.parse_yaml.outputs.env_NIGHTLY_RAPIDS_VER }} + NIGHTLY_UCX_PY_VER: ${{ steps.parse_yaml.outputs.env_NIGHTLY_UCX_PY_VER }} run: | - echo OLD_RAPIDS_VER=$OLD_RAPIDS_VER >> $GITHUB_ENV - echo OLD_UCX_PY_VER=$OLD_UCX_PY_VER >> $GITHUB_ENV - echo CURRENT_RAPIDS_VER=$CURRENT_RAPIDS_VER >> $GITHUB_ENV - echo CURRENT_UCX_PY_VER=$CURRENT_UCX_PY_VER >> $GITHUB_ENV + echo STABLE_RAPIDS_VER=$STABLE_RAPIDS_VER >> $GITHUB_ENV + echo STABLE_UCX_PY_VER=$STABLE_UCX_PY_VER >> $GITHUB_ENV + echo NIGHTLY_RAPIDS_VER=$NIGHTLY_RAPIDS_VER >> $GITHUB_ENV + echo NIGHTLY_UCX_PY_VER=$NIGHTLY_UCX_PY_VER >> $GITHUB_ENV - name: Get latest cuDF nightly version id: cudf_latest @@ -60,7 +60,7 @@ jobs: - name: Update new RAPIDS versions uses: jacobtomlinson/gha-find-replace@v3 with: - find: "${{ env.CURRENT_RAPIDS_VER }}" + find: "${{ env.NIGHTLY_RAPIDS_VER }}" replace: "${{ env.NEW_RAPIDS_VER }}" include: ".github/workflows/build.yml" regex: false @@ -68,15 +68,15 @@ jobs: - name: Update old RAPIDS versions uses: jacobtomlinson/gha-find-replace@v3 with: - find: "${{ env.OLD_RAPIDS_VER }}" - replace: "${{ env.CURRENT_RAPIDS_VER }}" + find: "${{ env.STABLE_RAPIDS_VER }}" + replace: "${{ env.NIGHTLY_RAPIDS_VER }}" include: ".github/workflows/build.yml" regex: false - name: Update new UCX-Py versions uses: jacobtomlinson/gha-find-replace@v3 with: - find: "${{ env.CURRENT_UCX_PY_VER }}" + find: "${{ env.NIGHTLY_UCX_PY_VER }}" replace: "${{ env.NEW_UCX_PY_VER }}" include: ".github/workflows/build.yml" regex: false @@ -84,21 +84,21 @@ jobs: - name: Update old UCX-Py versions uses: jacobtomlinson/gha-find-replace@v3 with: - find: "${{ env.OLD_UCX_PY_VER }}" - replace: "${{ env.CURRENT_UCX_PY_VER }}" + find: "${{ env.STABLE_UCX_PY_VER }}" + replace: "${{ env.NIGHTLY_UCX_PY_VER }}" include: ".github/workflows/build.yml" regex: false - name: Create pull request - if: ${{ env.CURRENT_RAPIDS_VER != env.NEW_RAPIDS_VER && env.CURRENT_UCX_PY_VER != env.NEW_UCX_PY_VER }} + if: ${{ env.NIGHTLY_RAPIDS_VER != env.NEW_RAPIDS_VER && env.NIGHTLY_UCX_PY_VER != env.NEW_UCX_PY_VER }} uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "Update gpuCI `CURRENT_RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`, `CURRENT_UCX_PY_VER` to `${{ env.NEW_UCX_PY_VER }}`" + commit-message: "Update gpuCI `NIGHTLY_RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`, `NIGHTLY_UCX_PY_VER` to `${{ env.NEW_UCX_PY_VER }}`" title: "Start building images with RAPIDS `${{ env.NEW_RAPIDS_VER }}`, ucx-py `${{ env.NEW_UCX_PY_VER }}`" author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> branch: "upgrade-gpuci-rapids" body: | New cuDF and ucx-py nightly versions have been detected. - Updated `.github/workflows/build.yml` to use `CURRENT_RAPIDS_VER=${{ env.NEW_RAPIDS_VER }}` and `CURRENT_UCX_PY_VER=${{ env.NEW_UCX_PY_VER }}`. + Updated `.github/workflows/build.yml` to use `NIGHTLY_RAPIDS_VER=${{ env.NEW_RAPIDS_VER }}` and `NIGHTLY_UCX_PY_VER=${{ env.NEW_UCX_PY_VER }}`.