From e98d1879542981b02019831f1b7878e50e81199e Mon Sep 17 00:00:00 2001 From: Matt Bell Date: Fri, 15 Dec 2023 16:58:37 +1100 Subject: [PATCH 1/6] Refactor GitHub Action --- .github/workflows/build-images.yaml | 108 +++++++++++++++------------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 7f36782..6a31ca4 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -1,19 +1,24 @@ -name: Create and publish Docker images +name: Build and publish pipeline images on: workflow_dispatch: push: branches: [main] + pull_request: + branches: ['*'] release: types: [published] - jobs: - push_to_registry: - name: Build and push Docker image to GitHub Packages + prepare: + name: Prepare the environment runs-on: ubuntu-latest steps: - #https://saturncloud.io/blog/github-action-ecr-optimizing-disk-space/#handling-or-maximizing-github-runner-out-of-disk-space-error + # + # ---- Preparing the environment ---- + # + # Clean unnecessary files to save disk space + # ref: https://saturncloud.io/blog/github-action-ecr-optimizing-disk-space/#handling-or-maximizing-github-runner-out-of-disk-space-error - name: clean unncessary files to save space run: | docker rmi `docker images -q` @@ -38,10 +43,6 @@ jobs: with: submodules: 'recursive' - - name: update-base - run: ./update_base.sh - shell: bash - - name: Add SHORT_SHA env property with commit short sha run: echo "SHORT_SHA=`git rev-parse --short HEAD`" >> $GITHUB_ENV @@ -56,19 +57,19 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + + # Downloads the latest docker-stacks source + - name: update-base + run: ./update_base.sh + shell: bash + get-metadata: + name: Extract the metadata + runs-on: ubuntu-latest + steps: + # + # ---- Extract metadata for docker labels/tags --- + # - # Intermediate images used as base images - - name: Build minimal-notebook:gpu image - uses: docker/build-push-action@v5 - with: - build-args: | - ROOT_CONTAINER=nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04 - context: ./docker-stacks-main/images/minimal-notebook - push: false - tags: minimal-notebook:gpu - - # Published pipeline images - # CPU ONLY IMAGE - based on default jupyterhub docker stacks - name: Extract metadata for pipeline-base id: meta-base uses: docker/metadata-action@v5 @@ -85,17 +86,7 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=sha ${{ env.SHORT_SHA }}-${{ github.run_number }} - - - name: Build and push pipeline-base image - uses: docker/build-push-action@v5 - with: - context: ./base - push: true - tags: | - ghcr.io/auscalabledronecloud/pipeline-base:latest - ${{ steps.meta-base.outputs.tags }} - labels: ${{ steps.meta-base.outputs.labels }} - + - name: Extract metadata for pipeline-gpu id: meta-gpu uses: docker/metadata-action@v5 @@ -113,19 +104,6 @@ jobs: type=sha ${{ env.SHORT_SHA }}-${{ github.run_number }} - - name: Build and push pipeline-gpu image - uses: docker/build-push-action@v5 - with: - build-args: | - BASE_CONTAINER=minimal-notebook:gpu - GPU=true - context: ./base - push: true - tags: | - ghcr.io/auscalabledronecloud/pipeline-gpu:latest - ${{ steps.meta-gpu.outputs.tags }} - labels: ${{ steps.meta-base.outputs.labels }} - - name: Extract metadata for pipeline-ml id: meta-ml uses: docker/metadata-action@v5 @@ -140,13 +118,47 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=sha ${{ env.SHORT_SHA }}-${{ github.run_number }} + build: + name: Build and Publish + runs-on: ubuntu-latest + needs: get-metadata + steps: + # + # ---- Build and push images --- + # + + - name: Build and push pipeline-base image + uses: docker/build-push-action@v5 + with: + context: ./base + push: ${{ github.ref == 'refs/heads/main' }} # Publish only if the branch is main + # Use the metadata from the previous step to tag the image + tags: | + ghcr.io/auscalabledronecloud/pipeline-base:latest + ${{ needs.get-metadata.outputs['meta-base'].tags }} + # Use the metadata from the previous step to add labels to the image + + labels: ${{ needs.get-metadata.outputs['meta-base'].lables }} + + - name: Build and push pipeline-gpu image + uses: docker/build-push-action@v5 + with: + build-args: | + BASE_CONTAINER=minimal-notebook:gpu + GPU=true + context: ./base + push: ${{ github.ref == 'refs/heads/main' }} # Publish only if the branch is main + tags: | + ghcr.io/auscalabledronecloud/pipeline-gpu:latest + ${{ needs.get-metadata.outputs['meta-gpu'].tags }} + labels: ${{ needs.get-metadata.outputs['meta-gpu'].labels }} - name: Build and push pipeline-ml image uses: docker/build-push-action@v5 with: context: ./ml - push: true + push: ${{ github.ref == 'refs/heads/main' }} # Publish only if the branch is main tags: | ghcr.io/auscalabledronecloud/pipeline-ml:latest - ${{ steps.meta-ml.outputs.tags }} - labels: ${{ steps.meta-ml.outputs.labels }} + ${{ needs.get-metadata.outputs['meta-ml'].tags }} + labels: ${{ needs.get-metadata.outputs['meta-ml'].labels }} From e85a022fde5a966860d873e4d28867d130f853a7 Mon Sep 17 00:00:00 2001 From: Matt Bell Date: Fri, 15 Dec 2023 17:05:49 +1100 Subject: [PATCH 2/6] Add a need for the get-metadata job --- .github/workflows/build-images.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 6a31ca4..ac7a456 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -65,6 +65,7 @@ jobs: get-metadata: name: Extract the metadata runs-on: ubuntu-latest + needs: prepare steps: # # ---- Extract metadata for docker labels/tags --- From 1f49366007f8683bee845a540d0b820b73ae6bba Mon Sep 17 00:00:00 2001 From: Matt Bell Date: Fri, 15 Dec 2023 17:24:47 +1100 Subject: [PATCH 3/6] Rearrange steps --- .github/workflows/build-images.yaml | 89 ++++++++++++++++------------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index ac7a456..12927cd 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -10,63 +10,44 @@ on: types: [published] jobs: prepare: - name: Prepare the environment + name: Prepare the repository runs-on: ubuntu-latest steps: # - # ---- Preparing the environment ---- + # ---- Prepare the repository ---- # - - # Clean unnecessary files to save disk space - # ref: https://saturncloud.io/blog/github-action-ecr-optimizing-disk-space/#handling-or-maximizing-github-runner-out-of-disk-space-error - - name: clean unncessary files to save space - run: | - docker rmi `docker images -q` - sudo rm -rf /usr/share/dotnet /etc/mysql /etc/php /etc/sudo apt/sources.list.d - sudo apt -y autoremove --purge - sudo apt -y autoclean - sudo apt clean - rm --recursive --force "$AGENT_TOOLSDIRECTORY" - df -h - - # Free up disk space on Ubuntu - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # This might remove tools that are actually needed, if set to "true" but frees about 6 GB - tool-cache: false - large-packages: true - swap-storage: true - - name: Checkout repository uses: actions/checkout@v4 with: submodules: 'recursive' - - name: Add SHORT_SHA env property with commit short sha - run: echo "SHORT_SHA=`git rev-parse --short HEAD`" >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker # defaults to "docker-containerized" - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # Downloads the latest docker-stacks source - name: update-base run: ./update_base.sh shell: bash + + # Store the repository as an artifact + - name: Upload repository as artifact + uses: actions/upload-artifact@v2 + with: + name: repo + path: ./ + get-metadata: name: Extract the metadata runs-on: ubuntu-latest needs: prepare steps: + # Fetch repo + - name: Download repository from artifact + uses: actions/download-artifact@v2 + with: + name: repo + + # Get the SHA for the current commit + - name: Add SHORT_SHA env property with commit short sha + run: echo "SHORT_SHA=`git rev-parse --short HEAD`" >> $GITHUB_ENV + # # ---- Extract metadata for docker labels/tags --- # @@ -124,6 +105,36 @@ jobs: runs-on: ubuntu-latest needs: get-metadata steps: + # Fetch repo + - name: Download repository from artifact + uses: actions/download-artifact@v2 + with: + name: repo + + # + # -- Clean up unnecessary files to save disk space -- + # + + # ref: https://saturncloud.io/blog/github-action-ecr-optimizing-disk-space/#handling-or-maximizing-github-runner-out-of-disk-space-error + - name: clean unncessary files to save space + run: | + docker rmi `docker images -q` + sudo rm -rf /usr/share/dotnet /etc/mysql /etc/php /etc/sudo apt/sources.list.d + sudo apt -y autoremove --purge + sudo apt -y autoclean + sudo apt clean + rm --recursive --force "$AGENT_TOOLSDIRECTORY" + df -h + + # Free up disk space on Ubuntu + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # This might remove tools that are actually needed, if set to "true" but frees about 6 GB + tool-cache: false + large-packages: true + swap-storage: true + # # ---- Build and push images --- # From eb3d444777d0c8f0b23016ec30c6b044ce1b8442 Mon Sep 17 00:00:00 2001 From: Matt Bell Date: Fri, 15 Dec 2023 17:40:54 +1100 Subject: [PATCH 4/6] Add back some missing steps --- .github/workflows/build-images.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 12927cd..b39005c 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -135,6 +135,23 @@ jobs: large-packages: true swap-storage: true + # + # --- Prep for building and pushing images + # + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker # defaults to "docker-containerized" + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # # ---- Build and push images --- # From b8b1079c25c120fd5598c80fc2c391a2872480cb Mon Sep 17 00:00:00 2001 From: Matt Bell Date: Fri, 15 Dec 2023 17:55:50 +1100 Subject: [PATCH 5/6] Add gpu image --- .github/workflows/build-images.yaml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index b39005c..1c38bdd 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -150,13 +150,22 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + + # Build a minimal-notebook image from docker-stacks source, but use the nvidia/cuda image as the base + - name: Build minimal-notebook:gpu image + uses: docker/build-push-action@v5 + with: + build-args: | + ROOT_CONTAINER=nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04 + context: ./docker-stacks-main/images/minimal-notebook + push: false + tags: minimal-notebook:gpu # # ---- Build and push images --- # - - name: Build and push pipeline-base image + - name: Build and push (if merged) pipeline-base image uses: docker/build-push-action@v5 with: context: ./base @@ -169,7 +178,7 @@ jobs: labels: ${{ needs.get-metadata.outputs['meta-base'].lables }} - - name: Build and push pipeline-gpu image + - name: Build and push (if merged) pipeline-gpu image uses: docker/build-push-action@v5 with: build-args: | @@ -182,9 +191,11 @@ jobs: ${{ needs.get-metadata.outputs['meta-gpu'].tags }} labels: ${{ needs.get-metadata.outputs['meta-gpu'].labels }} - - name: Build and push pipeline-ml image + - name: Build and push (if merged) pipeline-ml image uses: docker/build-push-action@v5 with: + build-args: | + BASE_CONTAINER=ghcr.io/auscalabledronecloud/pipeline-gpu:latest context: ./ml push: ${{ github.ref == 'refs/heads/main' }} # Publish only if the branch is main tags: | From fb5db260e2018ef800d42d618de8e2001540d229 Mon Sep 17 00:00:00 2001 From: Matt Bell Date: Fri, 15 Dec 2023 17:57:30 +1100 Subject: [PATCH 6/6] Fix warnings --- .github/workflows/build-images.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 1c38bdd..595dba0 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -28,7 +28,7 @@ jobs: # Store the repository as an artifact - name: Upload repository as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: repo path: ./ @@ -40,7 +40,7 @@ jobs: steps: # Fetch repo - name: Download repository from artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: repo @@ -107,7 +107,7 @@ jobs: steps: # Fetch repo - name: Download repository from artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: repo