Merge pull request #52 from snh/container-redo

Bump to bookworm and parallelise container builds
projecthorus · Sep 12, 2024 · d4a194a · d4a194a
2 parents 6c9d163 + 0a3aead
commit d4a194a
Show file tree

Hide file tree

Showing 3 changed files with 122 additions and 29 deletions.
diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml
@@ -1,4 +1,4 @@
-name: Container Images
+name: Container image
 
 on:
   push:
@@ -10,38 +10,130 @@ on:
   workflow_dispatch:
 
 jobs:
-  main:
+  build:
+    name: Build container image
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [linux/amd64, linux/386, linux/arm64, linux/arm/v6, linux/arm/v7]
     steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v2
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: Calculate Container Metadata
-        id: meta
-        uses: crazy-max/ghaction-docker-meta@v1
+      - name: Declare platform
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Cache wheels
+        uses: actions/cache@v4
         with:
-          images: ghcr.io/${{ github.repository }}
+          path: ${{ github.workspace }}/wheels
+          key: wheels-${{ env.PLATFORM }}-${{ github.run_id }}
+          restore-keys: |
+            wheels-${{ env.PLATFORM }}-
+
+      - name: List wheels
+        run: ls -lR ${{ github.workspace }}/wheels || true
 
       - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Setup Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and Push Images
-        uses: docker/build-push-action@v3
+      - name: Calculate container metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+
+      - name: Build stage
+        id: build
+        uses: docker/build-push-action@v6
         with:
           context: .
-          platforms: linux/amd64, linux/386, linux/arm64, linux/arm/v6, linux/arm/v7
-          cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache
-          cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          platforms: ${{ matrix.platform }}
+          provenance: false
           labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=local,dest=/tmp/build-output
+          cache-to: type=local,dest=/tmp/build-cache,mode=max
+          target: build
+
+      - name: Final stage and push by digest
+        id: final
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          provenance: false
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=ghcr.io/${{ github.repository }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' && 'true' || 'false' }}
+          cache-from: type=local,src=/tmp/build-cache
+
+      - name: Export digest
+        if: ${{ github.event_name != 'pull_request' }}
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.final.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+      - name: Move and list wheels
+        run: |
+          mv -f /tmp/build-output/root/.cache/pip/wheels ${{ github.workspace }}/ || true
+          ls -lR ${{ github.workspace }}/wheels || true
+
+  merge:
+    name: Publish multi-platform image
+    if: ${{ github.event_name != 'pull_request' }}
+    runs-on: ubuntu-latest
+    needs: [build]
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Calculate container metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+
+      - name: Setup Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf 'ghcr.io/${{ github.repository }}@sha256:%s ' *)
+
+      - name: Inspect container image
+        run: |
+          docker buildx imagetools inspect ghcr.io/${{ github.repository }}:${{ steps.meta.outputs.version }}
diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
 # -------------------
 # The build container
 # -------------------
-FROM python:3.9-bullseye AS build
+FROM python:3.11-bookworm AS build
 
 # Upgrade base packages.
 RUN apt-get update && \
@@ -12,13 +12,18 @@ RUN apt-get update && \
   libatlas-base-dev && \
   rm -rf /var/lib/apt/lists/*
 
+# Copy in existing wheels.
+COPY wheel[s]/ /root/.cache/pip/wheels/
+
+# No wheels might exist.
+RUN mkdir -p /root/.cache/pip/wheels/
+
 # Copy in requirements.txt.
 COPY requirements.txt /root/chasemapper/requirements.txt
 
 # Install Python packages.
-RUN pip3 --no-cache-dir install --user --no-warn-script-location \
-  --ignore-installed --no-binary numpy \
-  -r /root/chasemapper/requirements.txt
+RUN pip3 install --user --break-system-packages  --no-warn-script-location \
+  --ignore-installed -r /root/chasemapper/requirements.txt
 
 # Copy in chasemapper.
 COPY . /root/chasemapper
@@ -36,15 +41,11 @@ RUN unzip /root/cusf_predictor_wrapper-master.zip -d /root && \
 # -------------------------
 # The application container
 # -------------------------
-FROM python:3.9-slim-bullseye
+FROM python:3.11-bookworm
 EXPOSE 5001/tcp
 
 # Upgrade base packages and install application dependencies.
-RUN case $(uname -m) in \
-    "armv6l") extra_packages="" ;; \
-    "armv7l") extra_packages="" ;; \
-  esac && \
-  apt-get update && \
+RUN apt-get update && \
   apt-get upgrade -y && \
   apt-get install -y \
   libeccodes0 \

diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
---no-binary eccodes
+eccodes
 cusfpredict
 flask
 flask-socketio