From 5755b22bcb78b85b372b26c303b3a4424c03ec88 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Wed, 13 Sep 2023 23:49:23 +0900 Subject: [PATCH] CI: release: enable reproducible builds The binaries are now bit-for-bit reproducible (): ```bash docker buildx build \ -o /tmp/fuse-overlayfs-builds \ --build-arg SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) \ --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \ -f Containerfile.cross . ``` This ensures that the binaries are not built from any unpublished source. - Implementation is similar to - `/etc/apt/sources.list` is modified to use using . - As is slow, `/var/cache/apt` is cached on GHA using . - For testing sake, `.github/workflows/release.yaml` is now executed for every push events. The release artifacts are published only on a push event with a `v*` tag. Signed-off-by: Akihiro Suda --- .github/workflows/release.yaml | 152 +++++++++++++++------------------ Containerfile.cross | 32 ++++++- 2 files changed, 96 insertions(+), 88 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 54f0a41..4399e2a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,98 +1,82 @@ name: Release on: push: - tags: - - 'test-cross-*' - - 'v*' + pull_request: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v2 - - uses: docker/setup-buildx-action@v1 - - name: "Build binaries from Containerfile.cross" - run: docker buildx build -o out --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 -f Containerfile.cross . - - name: "Create out/artifact" - run: | - mkdir -p out/artifact - mv out/linux_amd64/fuse-overlayfs out/artifact/fuse-overlayfs-x86_64 - mv out/linux_arm64/fuse-overlayfs out/artifact/fuse-overlayfs-aarch64 - mv out/linux_arm_v7/fuse-overlayfs out/artifact/fuse-overlayfs-armv7l - mv out/linux_s390x/fuse-overlayfs out/artifact/fuse-overlayfs-s390x - mv out/linux_ppc64le/fuse-overlayfs out/artifact/fuse-overlayfs-ppc64le - mv out/linux_riscv64/fuse-overlayfs out/artifact/fuse-overlayfs-riscv64 - - name: "SHA256SUMS" - run: (cd out/artifact; sha256sum *) | tee out/SHA256SUMS - - name: "Create release" - uses: actions/create-release@v1 - id: create_release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 with: - tag_name: ${{ github.ref }} - release_name: ${{ github.ref }} - draft: true - - name: "Upload fuse-overlayfs-x86_64" - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Reproduce the exact commit hash value + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV + - uses: docker/setup-buildx-action@v2 + - name: Cache var-cache-apt + uses: actions/cache@v3 with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: out/artifact/fuse-overlayfs-x86_64 - asset_name: fuse-overlayfs-x86_64 - asset_content_type: application/octet-stream - - name: "Upload fuse-overlayfs-aarch64" - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + path: var-cache-apt + key: var-cache-apt-${{ hashFiles('Containerfile.cross') }} + - name: Cache var-lib-apt + uses: actions/cache@v3 with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: out/artifact/fuse-overlayfs-aarch64 - asset_name: fuse-overlayfs-aarch64 - asset_content_type: application/octet-stream - - name: "Upload fuse-overlayfs-armv7l" - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + path: var-lib-apt + key: var-lib-apt-${{ hashFiles('Containerfile.cross') }} + - name: inject var-cache-apt into docker + uses: reproducible-containers/buildkit-cache-dance@v2.1.2 with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: out/artifact/fuse-overlayfs-armv7l - asset_name: fuse-overlayfs-armv7l - asset_content_type: application/octet-stream - - name: "Upload fuse-overlayfs-s390x" - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + cache-source: var-cache-apt + cache-target: /var/cache/apt + - name: inject var-lib-apt into docker + uses: reproducible-containers/buildkit-cache-dance@v2.1.2 with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: out/artifact/fuse-overlayfs-s390x - asset_name: fuse-overlayfs-s390x - asset_content_type: application/octet-stream - - name: "Upload fuse-overlayfs-ppc64le" - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: out/artifact/fuse-overlayfs-ppc64le - asset_name: fuse-overlayfs-ppc64le - asset_content_type: application/octet-stream - - name: "Upload fuse-overlayfs-riscv64" - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: out/artifact/fuse-overlayfs-riscv64 - asset_name: fuse-overlayfs-riscv64 - asset_content_type: application/octet-stream - - name: "Upload SHA256SUMS" - uses: actions/upload-release-asset@v1.0.2 + cache-source: var-lib-apt + cache-target: /var/lib/apt + - name: "Build binaries from Containerfile.cross" + run: docker buildx build -o /tmp/fuse-overlayfs-builds --build-arg SOURCE_DATE_EPOCH --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 -f Containerfile.cross . + - name: "Create /tmp/artifact" + run: | + mkdir -p /tmp/artifact + mv /tmp/fuse-overlayfs-builds/linux_amd64/fuse-overlayfs /tmp/artifact/fuse-overlayfs-x86_64 + mv /tmp/fuse-overlayfs-builds/linux_arm64/fuse-overlayfs /tmp/artifact/fuse-overlayfs-aarch64 + mv /tmp/fuse-overlayfs-builds/linux_arm_v7/fuse-overlayfs /tmp/artifact/fuse-overlayfs-armv7l + mv /tmp/fuse-overlayfs-builds/linux_s390x/fuse-overlayfs /tmp/artifact/fuse-overlayfs-s390x + mv /tmp/fuse-overlayfs-builds/linux_ppc64le/fuse-overlayfs /tmp/artifact/fuse-overlayfs-ppc64le + mv /tmp/fuse-overlayfs-builds/linux_riscv64/fuse-overlayfs /tmp/artifact/fuse-overlayfs-riscv64 + echo "${SOURCE_DATE_EPOCH}" >/tmp/artifact/SOURCE_DATE_EPOCH + - name: "SHA256SUMS" + run: (cd /tmp/artifact; sha256sum *) | tee /tmp/SHA256SUMS + - name: "The sha256sum of the SHA256SUMS file" + run: sha256sum /tmp/SHA256SUMS + - name: "Prepare the release note" + run: | + tag="${GITHUB_REF##*/}" + cat << EOF | tee /tmp/release-note.txt + ${tag} + + #### About the binaries + The binaries were built automatically on GitHub Actions. + The build log is available for 90 days: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + + The binaries should be reproducible with the following command: + \`\`\` + docker buildx build \ + -o /tmp/fuse-overlayfs-builds \ + --build-arg SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} \ + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 \ + --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \ + -f Containerfile.cross \ + "https://github.com/${{ github.repository }}.git#${tag}" + \`\`\` + EOF + - name: "Create release" + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: out/SHA256SUMS - asset_name: SHA256SUMS - asset_content_type: text/plain + run: | + tag="${GITHUB_REF##*/}" + asset_flags=() + for f in /tmp/artifact/* /tmp/SHA256SUMS; do asset_flags+=("-a" "$f"); done + hub release create "${asset_flags[@]}" -F /tmp/release-note.txt --draft "${tag}" diff --git a/Containerfile.cross b/Containerfile.cross index 61d6722..0610827 100644 --- a/Containerfile.cross +++ b/Containerfile.cross @@ -1,14 +1,38 @@ -FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.1.0 AS xx -FROM --platform=$BUILDPLATFORM ubuntu:22.04 AS fuse-overlayfs -RUN apt-get update && \ +# Usage: +# docker buildx build \ +# -o /tmp/fuse-overlayfs-builds \ +# --build-arg SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) \ +# --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \ +# -f Containerfile.cross . +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.2.1 AS xx +FROM --platform=$BUILDPLATFORM ubuntu:jammy-20230804 AS fuse-overlayfs +ADD --chmod=0755 \ + https://raw.githubusercontent.com/reproducible-containers/repro-sources-list.sh/v0.1.0/repro-sources-list.sh \ + /usr/local/bin/repro-sources-list.sh +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + repro-sources-list.sh && \ + apt-get update && \ apt-get install --no-install-recommends -y \ git make automake autoconf pkgconf file go-md2man +# Set SOURCE_DATE_EPOCH after running repro-sources-list.sh, for cache efficiency +ARG SOURCE_DATE_EPOCH COPY . /fuse-overlayfs WORKDIR /fuse-overlayfs COPY --from=xx / / ARG TARGETPLATFORM ENV DEBIAN_FRONTEND=noninteractive -RUN xx-apt-get install -y gcc libfuse3-dev +# xx-apt-get cannot be used, as it clobbers /etc/apt/sources.list created by repro-sources-list.sh +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + darch="$(xx-info debian-arch)" && \ + dpkg --add-architecture ${darch} && \ + apt-get update && \ + gcc="gcc" && \ + if xx-info is-cross; then gcc="gcc-$(xx-info triple)"; fi; \ + apt-get install -y "${gcc}" "libfuse3-dev:${darch}" RUN ./autogen.sh && \ LIBS="-ldl" LDFLAGS="-static" ./configure --host=$(xx-info) && \ make && mkdir /out && cp fuse-overlayfs /out && \