CI: release: enable reproducible builds

The binaries are now bit-for-bit reproducible (<https://reproducible-builds.org/>): ```bash docker buildx build \ -o /tmp/fuse-overlayfs-builds \ --build-arg SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) \ --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \ -f Containerfile.cross . ``` This ensures that the binaries are not built from any unpublished source. - Implementation is similar to <rootless-containers/slirp4netns@v1.2.1...v1.2.2> - `/etc/apt/sources.list` is modified to use <http://snapshot.ubuntu.com> using <https://github.com/reproducible-containers/repro-sources-list.sh>. - As <http://snapshot.ubuntu.com> is slow, `/var/cache/apt` is cached on GHA using <https://github.com/reproducible-containers/buildkit-cache-dance>. - For testing sake, `.github/workflows/release.yaml` is now executed for every push events. The release artifacts are published only on a push event with a `v*` tag. Signed-off-by: Akihiro Suda <[email protected]>
containers · Sep 13, 2023 · 5755b22 · 5755b22
1 parent ff6b14d
commit 5755b22
Show file tree

Hide file tree

Showing 2 changed files with 96 additions and 88 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -1,98 +1,82 @@
 name: Release
 on:
   push:
-    tags:
-    - 'test-cross-*'
-    - 'v*'
+  pull_request:
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Build binaries from Containerfile.cross"
-      run: docker buildx build -o out --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 -f Containerfile.cross .
-    - name: "Create out/artifact"
-      run: |
-        mkdir -p out/artifact
-        mv out/linux_amd64/fuse-overlayfs out/artifact/fuse-overlayfs-x86_64
-        mv out/linux_arm64/fuse-overlayfs out/artifact/fuse-overlayfs-aarch64
-        mv out/linux_arm_v7/fuse-overlayfs out/artifact/fuse-overlayfs-armv7l
-        mv out/linux_s390x/fuse-overlayfs out/artifact/fuse-overlayfs-s390x
-        mv out/linux_ppc64le/fuse-overlayfs out/artifact/fuse-overlayfs-ppc64le
-        mv out/linux_riscv64/fuse-overlayfs out/artifact/fuse-overlayfs-riscv64
-    - name: "SHA256SUMS"
-      run: (cd out/artifact; sha256sum *) | tee out/SHA256SUMS
-    - name: "Create release"
-      uses: actions/create-release@v1
-      id: create_release
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - uses: actions/checkout@v4
       with:
-        tag_name: ${{ github.ref }}
-        release_name: ${{ github.ref }}
-        draft: true
-    - name: "Upload fuse-overlayfs-x86_64"
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # Reproduce the exact commit hash value
+        fetch-depth: 0
+        ref: ${{ github.event.pull_request.head.sha }}
+    - run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV
+    - uses: docker/setup-buildx-action@v2
+    - name: Cache var-cache-apt
+      uses: actions/cache@v3
       with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: out/artifact/fuse-overlayfs-x86_64
-        asset_name: fuse-overlayfs-x86_64
-        asset_content_type: application/octet-stream
-    - name: "Upload fuse-overlayfs-aarch64"
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        path: var-cache-apt
+        key: var-cache-apt-${{ hashFiles('Containerfile.cross') }}
+    - name: Cache var-lib-apt
+      uses: actions/cache@v3
       with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: out/artifact/fuse-overlayfs-aarch64
-        asset_name: fuse-overlayfs-aarch64
-        asset_content_type: application/octet-stream
-    - name: "Upload fuse-overlayfs-armv7l"
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        path: var-lib-apt
+        key: var-lib-apt-${{ hashFiles('Containerfile.cross') }}
+    - name: inject var-cache-apt into docker
+      uses: reproducible-containers/[email protected]
       with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: out/artifact/fuse-overlayfs-armv7l
-        asset_name: fuse-overlayfs-armv7l
-        asset_content_type: application/octet-stream
-    - name: "Upload fuse-overlayfs-s390x"
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        cache-source: var-cache-apt
+        cache-target: /var/cache/apt
+    - name: inject var-lib-apt into docker
+      uses: reproducible-containers/[email protected]
       with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: out/artifact/fuse-overlayfs-s390x
-        asset_name: fuse-overlayfs-s390x
-        asset_content_type: application/octet-stream
-    - name: "Upload fuse-overlayfs-ppc64le"
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: out/artifact/fuse-overlayfs-ppc64le
-        asset_name: fuse-overlayfs-ppc64le
-        asset_content_type: application/octet-stream
-    - name: "Upload fuse-overlayfs-riscv64"
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: out/artifact/fuse-overlayfs-riscv64
-        asset_name: fuse-overlayfs-riscv64
-        asset_content_type: application/octet-stream
-    - name: "Upload SHA256SUMS"
-      uses: actions/[email protected]
+        cache-source: var-lib-apt
+        cache-target: /var/lib/apt
+    - name: "Build binaries from Containerfile.cross"
+      run: docker buildx build -o /tmp/fuse-overlayfs-builds --build-arg SOURCE_DATE_EPOCH --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 -f Containerfile.cross .
+    - name: "Create /tmp/artifact"
+      run: |
+        mkdir -p /tmp/artifact
+        mv /tmp/fuse-overlayfs-builds/linux_amd64/fuse-overlayfs   /tmp/artifact/fuse-overlayfs-x86_64
+        mv /tmp/fuse-overlayfs-builds/linux_arm64/fuse-overlayfs   /tmp/artifact/fuse-overlayfs-aarch64
+        mv /tmp/fuse-overlayfs-builds/linux_arm_v7/fuse-overlayfs  /tmp/artifact/fuse-overlayfs-armv7l
+        mv /tmp/fuse-overlayfs-builds/linux_s390x/fuse-overlayfs   /tmp/artifact/fuse-overlayfs-s390x
+        mv /tmp/fuse-overlayfs-builds/linux_ppc64le/fuse-overlayfs /tmp/artifact/fuse-overlayfs-ppc64le
+        mv /tmp/fuse-overlayfs-builds/linux_riscv64/fuse-overlayfs /tmp/artifact/fuse-overlayfs-riscv64
+        echo "${SOURCE_DATE_EPOCH}" >/tmp/artifact/SOURCE_DATE_EPOCH
+    - name: "SHA256SUMS"
+      run: (cd /tmp/artifact; sha256sum *) | tee /tmp/SHA256SUMS
+    - name: "The sha256sum of the SHA256SUMS file"
+      run: sha256sum /tmp/SHA256SUMS
+    - name: "Prepare the release note"
+      run: |
+        tag="${GITHUB_REF##*/}"
+        cat << EOF | tee /tmp/release-note.txt
+        ${tag}
+
+        #### About the binaries
+        The binaries were built automatically on GitHub Actions.
+        The build log is available for 90 days: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+        The binaries should be reproducible with the following command:
+        \`\`\`
+        docker buildx build \
+          -o /tmp/fuse-overlayfs-builds \
+          --build-arg SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} \
+          --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 \
+          --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \
+          -f Containerfile.cross \
+          "https://github.com/${{ github.repository }}.git#${tag}"
+        \`\`\`
+        EOF
+    - name: "Create release"
+      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: out/SHA256SUMS
-        asset_name: SHA256SUMS
-        asset_content_type: text/plain
+      run: |
+        tag="${GITHUB_REF##*/}"
+        asset_flags=()
+        for f in /tmp/artifact/* /tmp/SHA256SUMS; do asset_flags+=("-a" "$f"); done
+        hub release create "${asset_flags[@]}" -F /tmp/release-note.txt --draft "${tag}"
diff --git a/Containerfile.cross b/Containerfile.cross
@@ -1,14 +1,38 @@
-FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.1.0 AS xx
-FROM --platform=$BUILDPLATFORM ubuntu:22.04 AS fuse-overlayfs
-RUN apt-get update && \
+# Usage:
+# docker buildx build \
+#   -o /tmp/fuse-overlayfs-builds \
+#   --build-arg SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) \
+#   --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \
+#   -f Containerfile.cross .
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.2.1 AS xx
+FROM --platform=$BUILDPLATFORM ubuntu:jammy-20230804 AS fuse-overlayfs
+ADD --chmod=0755 \
+  https://raw.githubusercontent.com/reproducible-containers/repro-sources-list.sh/v0.1.0/repro-sources-list.sh \
+  /usr/local/bin/repro-sources-list.sh
+RUN \
+  --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  repro-sources-list.sh && \
+  apt-get update && \
   apt-get install --no-install-recommends -y \
   git make automake autoconf pkgconf file go-md2man
+# Set SOURCE_DATE_EPOCH after running repro-sources-list.sh, for cache efficiency
+ARG SOURCE_DATE_EPOCH
 COPY . /fuse-overlayfs
 WORKDIR /fuse-overlayfs
 COPY --from=xx / /
 ARG TARGETPLATFORM
 ENV DEBIAN_FRONTEND=noninteractive
-RUN xx-apt-get install -y gcc libfuse3-dev
+# xx-apt-get cannot be used, as it clobbers /etc/apt/sources.list created by repro-sources-list.sh
+RUN \
+  --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  darch="$(xx-info debian-arch)" && \
+  dpkg --add-architecture ${darch} && \
+  apt-get update && \
+  gcc="gcc" && \
+  if xx-info is-cross; then gcc="gcc-$(xx-info triple)"; fi; \
+  apt-get install -y "${gcc}" "libfuse3-dev:${darch}"
 RUN ./autogen.sh && \
   LIBS="-ldl" LDFLAGS="-static" ./configure --host=$(xx-info) && \
   make && mkdir /out && cp fuse-overlayfs /out && \