Refactor GitHub Action #173
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build and publish pipeline images | |
on: | |
workflow_dispatch: | |
push: | |
branches: [main] | |
pull_request: | |
branches: ['*'] | |
release: | |
types: [published] | |
jobs: | |
prepare: | |
name: Prepare the repository | |
runs-on: ubuntu-latest | |
steps: | |
# | |
# ---- Prepare the repository ---- | |
# | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
# Downloads the latest docker-stacks source | |
- name: update-base | |
run: ./update_base.sh | |
shell: bash | |
# Store the repository as an artifact | |
- name: Upload repository as artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: repo | |
path: ./ | |
get-metadata: | |
name: Extract the metadata | |
runs-on: ubuntu-latest | |
needs: prepare | |
steps: | |
# Fetch repo | |
- name: Download repository from artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: repo | |
# Get the SHA for the current commit | |
- name: Add SHORT_SHA env property with commit short sha | |
run: echo "SHORT_SHA=`git rev-parse --short HEAD`" >> $GITHUB_ENV | |
# | |
# ---- Extract metadata for docker labels/tags --- | |
# | |
- name: Extract metadata for pipeline-base | |
id: meta-base | |
uses: docker/metadata-action@v5 | |
with: | |
# list of Docker images to use as base name for tags | |
images: | | |
ghcr.io/auscalabledronecloud/pipeline-base | |
tags: | | |
type=schedule | |
type=ref,event=branch | |
type=ref,event=tag | |
type=ref,event=pr | |
type=semver,pattern={{version}} | |
type=semver,pattern={{major}}.{{minor}} | |
type=sha | |
${{ env.SHORT_SHA }}-${{ github.run_number }} | |
- name: Extract metadata for pipeline-gpu | |
id: meta-gpu | |
uses: docker/metadata-action@v5 | |
with: | |
# list of Docker images to use as base name for tags | |
images: | | |
ghcr.io/auscalabledronecloud/pipeline-gpu | |
tags: | | |
type=schedule | |
type=ref,event=branch | |
type=ref,event=tag | |
type=ref,event=pr | |
type=semver,pattern={{version}} | |
type=semver,pattern={{major}}.{{minor}} | |
type=sha | |
${{ env.SHORT_SHA }}-${{ github.run_number }} | |
- name: Extract metadata for pipeline-ml | |
id: meta-ml | |
uses: docker/metadata-action@v5 | |
with: | |
images: ghcr.io/auscalabledronecloud/pipeline-ml | |
tags: | | |
type=schedule | |
type=ref,event=branch | |
type=ref,event=tag | |
type=ref,event=pr | |
type=semver,pattern={{version}} | |
type=semver,pattern={{major}}.{{minor}} | |
type=sha | |
${{ env.SHORT_SHA }}-${{ github.run_number }} | |
build: | |
name: Build and Publish | |
runs-on: ubuntu-latest | |
needs: get-metadata | |
steps: | |
# Fetch repo | |
- name: Download repository from artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: repo | |
# | |
# -- Clean up unnecessary files to save disk space -- | |
# | |
# ref: https://saturncloud.io/blog/github-action-ecr-optimizing-disk-space/#handling-or-maximizing-github-runner-out-of-disk-space-error | |
- name: clean unncessary files to save space | |
run: | | |
docker rmi `docker images -q` | |
sudo rm -rf /usr/share/dotnet /etc/mysql /etc/php /etc/sudo apt/sources.list.d | |
sudo apt -y autoremove --purge | |
sudo apt -y autoclean | |
sudo apt clean | |
rm --recursive --force "$AGENT_TOOLSDIRECTORY" | |
df -h | |
# Free up disk space on Ubuntu | |
- name: Free Disk Space (Ubuntu) | |
uses: jlumbroso/free-disk-space@main | |
with: | |
# This might remove tools that are actually needed, if set to "true" but frees about 6 GB | |
tool-cache: false | |
large-packages: true | |
swap-storage: true | |
# | |
# --- Prep for building and pushing images | |
# | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
with: | |
driver: docker # defaults to "docker-containerized" | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v3 | |
with: | |
registry: ghcr.io | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
# Build a minimal-notebook image from docker-stacks source, but use the nvidia/cuda image as the base | |
- name: Build minimal-notebook:gpu image | |
uses: docker/build-push-action@v5 | |
with: | |
build-args: | | |
ROOT_CONTAINER=nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04 | |
context: ./docker-stacks-main/images/minimal-notebook | |
push: false | |
tags: minimal-notebook:gpu | |
# | |
# ---- Build and push images --- | |
# | |
- name: Build and push (if merged) pipeline-base image | |
uses: docker/build-push-action@v5 | |
with: | |
context: ./base | |
push: ${{ github.ref == 'refs/heads/main' }} # Publish only if the branch is main | |
# Use the metadata from the previous step to tag the image | |
tags: | | |
ghcr.io/auscalabledronecloud/pipeline-base:latest | |
${{ needs.get-metadata.outputs['meta-base'].tags }} | |
# Use the metadata from the previous step to add labels to the image | |
labels: ${{ needs.get-metadata.outputs['meta-base'].lables }} | |
- name: Build and push (if merged) pipeline-gpu image | |
uses: docker/build-push-action@v5 | |
with: | |
build-args: | | |
BASE_CONTAINER=minimal-notebook:gpu | |
GPU=true | |
context: ./base | |
push: ${{ github.ref == 'refs/heads/main' }} # Publish only if the branch is main | |
tags: | | |
ghcr.io/auscalabledronecloud/pipeline-gpu:latest | |
${{ needs.get-metadata.outputs['meta-gpu'].tags }} | |
labels: ${{ needs.get-metadata.outputs['meta-gpu'].labels }} | |
- name: Build and push (if merged) pipeline-ml image | |
uses: docker/build-push-action@v5 | |
with: | |
build-args: | | |
BASE_CONTAINER=ghcr.io/auscalabledronecloud/pipeline-gpu:latest | |
context: ./ml | |
push: ${{ github.ref == 'refs/heads/main' }} # Publish only if the branch is main | |
tags: | | |
ghcr.io/auscalabledronecloud/pipeline-ml:latest | |
${{ needs.get-metadata.outputs['meta-ml'].tags }} | |
labels: ${{ needs.get-metadata.outputs['meta-ml'].labels }} |