Skip to content

Commit

Permalink
feat(backend): backend arm builds, create dockerfile (#1831)
Browse files Browse the repository at this point in the history
- Make workflow names systematic (-test/-image)
- Create workflow that builds all arm images for a ref (for local testing)
  • Loading branch information
corneliusroemer authored May 11, 2024
1 parent 2d73e4c commit 1c814c3
Show file tree
Hide file tree
Showing 14 changed files with 268 additions and 98 deletions.
Original file line number Diff line number Diff line change
@@ -1,15 +1,36 @@
name: backend
name: backend-image

on:
pull_request:
push:
branches:
- main
workflow_dispatch:
inputs:
build_arm:
type: boolean
description: "Build for ARM as well"
default: false
required: false
workflow_call:
inputs:
build_arm:
type: string
description: "Build for ARM as well"
default: "false"
required: true

env:
DOCKER_IMAGE_NAME: ghcr.io/loculus-project/backend
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}

concurrency:
group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-backend
cancel-in-progress: true

jobs:
dockerImage:
backend-image:
name: Build Backend Docker Image # Don't change: Referenced by .github/workflows/update-argocd-metadata.yml
runs-on: ubuntu-latest
timeout-minutes: 15
Expand All @@ -25,8 +46,8 @@ jobs:
- name: Generate files hash
id: files-hash
run: |
DIR_HASH=$(echo -n ${{ hashFiles('backend/**', '.github/workflows/backend.yml') }})
echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
DIR_HASH=$(echo -n ${{ hashFiles('backend/**', '.github/workflows/backend-image.yml') }})
echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
- name: Setup Docker metadata
id: dockerMetadata
Expand All @@ -38,7 +59,7 @@ jobs:
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=ref,event=branch
type=sha,prefix=commit-
type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
Expand All @@ -52,6 +73,9 @@ jobs:
EXISTS=$(docker manifest inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Set up JDK
if: env.CACHE_HIT == 'false'
uses: actions/setup-java@v4
Expand All @@ -63,22 +87,24 @@ jobs:
if: env.CACHE_HIT == 'false'
uses: gradle/actions/setup-gradle@v3

- name: Build Docker Image For Branch
- name: Build Backend
if: env.CACHE_HIT == 'false'
run: ./gradlew bootBuildImage --imageName=${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }}
working-directory: ./backend
env:
USER: ${{ github.actor }}
TOKEN: "${{ secrets.GITHUB_TOKEN }}"
run: ./gradlew bootJar

- name: Push Docker Image
- name: Build and push image if input files changed
if: env.CACHE_HIT == 'false'
run: docker push ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }}

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/build-push-action@v5
with:
context: ./backend
push: true
tags: ${{ steps.dockerMetadata.outputs.tags }}
cache-from: type=gha,scope=backend-${{ github.ref }}
cache-to: type=gha,mode=max,scope=backend-${{ github.ref }}
platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}

- name: Tag and push existing images
- name: Retag and push existing image if cache hit
if: env.CACHE_HIT == 'true'
run: |
TAGS=(${{ steps.dockerMetadata.outputs.tags }})
for TAG in "${TAGS[@]}"; do
Expand Down
45 changes: 45 additions & 0 deletions .github/workflows/build-arm-images.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Trigger a build of all docker images including ARM images

on:
workflow_dispatch:

permissions:
contents: read
packages: write
checks: read

jobs:
trigger-backend:
uses: ./.github/workflows/backend.yml
with:
build_arm: true

trigger-config-preprocessor:
uses: ./.github/workflows/config-preprocessor-build.yml
with:
build_arm: true

trigger-dummy-preprocessing:
uses: ./.github/workflows/dummyPreprocessing.yml
with:
build_arm: true

trigger-ingest:
uses: ./.github/workflows/ingest.yml
with:
build_arm: true

trigger-keycloakify:
uses: ./.github/workflows/keycloakify-build.yml
with:
build_arm: true

trigger-preprocessing-nextclade:
uses: ./.github/workflows/preprocessing-nextclade.yml
with:
build_arm: true

trigger-website:
uses: ./.github/workflows/website.yml
with:
build_arm: true
Original file line number Diff line number Diff line change
@@ -1,29 +1,40 @@
name: config-processor-build
name: config-processor-image

on:
pull_request:
push:
branches:
- main
workflow_dispatch:
inputs:
build_arm:
type: boolean
description: "Build for ARM as well"
default: false
required: false
required: true
workflow_call:
inputs:
build_arm:
type: string
description: "Build for ARM as well"
default: "false"
required: true

env:
DOCKER_IMAGE_NAME: ghcr.io/loculus-project/config-processor
BUILD_ARM: ${{ github.ref == 'refs/heads/main' || github.event.inputs.build_arm }} # When to build for arm as well
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}

defaults:
run:
working-directory: ./kubernetes/config-processor

concurrency:
group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-config-processor
group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-config-processor-${{github.event.inputs.build_arm}}
cancel-in-progress: true

jobs:
dockerImage:
config-processor-image:
name: Build config-processor Docker Image # Don't change: Referenced by .github/workflows/update-argocd-metadata.yml
runs-on: ubuntu-latest
timeout-minutes: 15
Expand All @@ -50,6 +61,7 @@ jobs:
type=ref,event=branch
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=sha,prefix=commit-
type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
- name: Build and push image
uses: docker/build-push-action@v5
with:
Expand All @@ -58,4 +70,4 @@ jobs:
tags: ${{ steps.dockerMetadata.outputs.tags }}
cache-from: type=gha,scope=config-preprocessor-${{ github.ref }}
cache-to: type=gha,mode=max,scope=config-preprocessor-${{ github.ref }}
platforms: ${{ env.BUILD_ARM && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
name: Docs Build Check

on:
push:
pull_request:
paths:
- "docs/**"
- .github/workflows/docs-build-check.yml
push:
branches:
- main

workflow_dispatch:

concurrency:
Expand Down
74 changes: 39 additions & 35 deletions .github/workflows/e2e-k3d.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: E2E test (on kubernetes)

on:
workflow_dispatch:
push:
pull_request:
paths:
- "backend/**"
- "keycloak/**"
Expand All @@ -11,6 +11,9 @@ on:
- "deploy.py"
- ".github/scripts/**"
- ".github/workflows/**"
push:
branches:
- main

concurrency:
group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-e2e-k3d
Expand Down Expand Up @@ -93,40 +96,41 @@ jobs:
run: cd website && npx playwright install-deps
if: steps.playwright-cache.outputs.cache-hit == 'true'

# Waits are identical to the update-argocd-metadata.yml file
# Mirror changes to that file
- name: Wait for Config Processor Docker Image
uses: lewagon/[email protected]
with:
ref: ${{ github.sha }}
check-name: Build config-processor Docker Image
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 2

- name: Wait for Backend Docker Image
uses: lewagon/[email protected]
with:
ref: ${{ github.sha }}
check-name: Build Backend Docker Image
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 2

- name: Wait for Website Docker Image
uses: lewagon/[email protected]
with:
ref: ${{ github.sha }}
check-name: Build Website Docker Image
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 2

- name: Wait for Keycloakify Docker Image
uses: lewagon/[email protected]
with:
ref: ${{ github.sha }}
check-name: Build keycloakify Docker Image
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 2
# End of wait block
# Action misbehaved
# # Waits are identical to the update-argocd-metadata.yml file
# # Mirror changes to that file
# - name: Wait for Config Processor Docker Image
# uses: lewagon/[email protected]
# with:
# ref: ${{ github.sha }}
# check-name: Build config-processor Docker Image
# repo-token: ${{ secrets.GITHUB_TOKEN }}
# wait-interval: 2

# - name: Wait for Backend Docker Image
# uses: lewagon/[email protected]
# with:
# ref: ${{ github.sha }}
# check-name: Build Backend Docker Image
# repo-token: ${{ secrets.GITHUB_TOKEN }}
# wait-interval: 2

# - name: Wait for Website Docker Image
# uses: lewagon/[email protected]
# with:
# ref: ${{ github.sha }}
# check-name: Build Website Docker Image
# repo-token: ${{ secrets.GITHUB_TOKEN }}
# wait-interval: 2

# - name: Wait for Keycloakify Docker Image
# uses: lewagon/[email protected]
# with:
# ref: ${{ github.sha }}
# check-name: Build keycloakify Docker Image
# repo-token: ${{ secrets.GITHUB_TOKEN }}
# wait-interval: 2
# # End of wait block

- name: Wait for the pods to be ready (timeout 480s)
run: ./.github/scripts/wait_for_pods_to_be_ready.py
Expand Down
Original file line number Diff line number Diff line change
@@ -1,24 +1,35 @@
name: ingest
name: ingest-image
on:
pull_request:
push:
branches:
- main
workflow_dispatch:
inputs:
build_arm:
type: boolean
description: "Build for ARM as well"
default: false
required: false
workflow_call:
inputs:
build_arm:
type: boolean
description: "Build for ARM as well"
default: false
required: false

env:
DOCKER_IMAGE_NAME: ghcr.io/loculus-project/ingest
BUILD_ARM: ${{ github.ref == 'refs/heads/main' || github.event.inputs.build_arm }}
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}

concurrency:
group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-ingest
group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-ingest-${{github.event.inputs.build_arm}}
cancel-in-progress: true

jobs:
dockerImage:
ingest-image:
name: Build ingest Docker Image # Don't change: Referenced by .github/workflows/update-argocd-metadata.yml
runs-on: ubuntu-latest
timeout-minutes: 15
Expand All @@ -32,8 +43,8 @@ jobs:
- name: Generate files hash
id: files-hash
run: |
DIR_HASH=$(echo -n ${{ hashFiles('ingest/**', '.github/workflows/ingest.yml') }})
echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM && '-arm'|| '' }}" >> $GITHUB_ENV
DIR_HASH=$(echo -n ${{ hashFiles('ingest/**', '.github/workflows/ingest-image.yml') }})
echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
- name: Setup Docker metadata
id: dockerMetadata
Expand All @@ -45,6 +56,7 @@ jobs:
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=ref,event=branch
type=sha,prefix=commit-
type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
Expand All @@ -71,7 +83,7 @@ jobs:
tags: ${{ steps.dockerMetadata.outputs.tags }}
cache-from: type=gha,scope=ingest-${{ github.ref }}
cache-to: type=gha,mode=max,scope=ingest-${{ github.ref }}
platforms: ${{ env.BUILD_ARM && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}

- name: Retag and push existing image if cache hit
if: env.CACHE_HIT == 'true'
Expand Down
Loading

0 comments on commit 1c814c3

Please sign in to comment.