Skip to content

Commit

Permalink
CI: release: enable reproducible builds
Browse files Browse the repository at this point in the history
The binaries are now bit-for-bit reproducible (<https://reproducible-builds.org/>):
```bash
docker buildx build \
   -o /tmp/fuse-overlayfs-builds \
   --build-arg SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) \
   --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \
   -f Containerfile.cross .
```

This ensures that the binaries are not built from any unpublished source.

- Implementation is similar to <rootless-containers/slirp4netns@v1.2.1...v1.2.2>
- `/etc/apt/sources.list` is modified to use <http://snapshot.ubuntu.com> using
  <https://github.com/reproducible-containers/repro-sources-list.sh>.
- As <http://snapshot.ubuntu.com> is slow, `/var/cache/apt` is cached on GHA using
  <https://github.com/overmindtech/buildkit-cache-dance>.
- For testing sake, `.github/workflows/release.yaml` is now executed for every push events.
  The release artifacts are published only on a push event with a `v*` tag.

Signed-off-by: Akihiro Suda <[email protected]>
  • Loading branch information
AkihiroSuda committed Sep 13, 2023
1 parent ff6b14d commit bb64c54
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 86 deletions.
158 changes: 76 additions & 82 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
@@ -1,98 +1,92 @@
name: Release
on:
push:
tags:
- 'test-cross-*'
- 'v*'
pull_request:

jobs:
release:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- uses: docker/setup-buildx-action@v1
- name: "Build binaries from Containerfile.cross"
run: docker buildx build -o out --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 -f Containerfile.cross .
- name: "Create out/artifact"
run: |
mkdir -p out/artifact
mv out/linux_amd64/fuse-overlayfs out/artifact/fuse-overlayfs-x86_64
mv out/linux_arm64/fuse-overlayfs out/artifact/fuse-overlayfs-aarch64
mv out/linux_arm_v7/fuse-overlayfs out/artifact/fuse-overlayfs-armv7l
mv out/linux_s390x/fuse-overlayfs out/artifact/fuse-overlayfs-s390x
mv out/linux_ppc64le/fuse-overlayfs out/artifact/fuse-overlayfs-ppc64le
mv out/linux_riscv64/fuse-overlayfs out/artifact/fuse-overlayfs-riscv64
- name: "SHA256SUMS"
run: (cd out/artifact; sha256sum *) | tee out/SHA256SUMS
- name: "Create release"
uses: actions/create-release@v1
id: create_release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/checkout@v4
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
draft: true
- name: "Upload fuse-overlayfs-x86_64"
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Reproduce the exact commit hash value
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV
- uses: docker/setup-buildx-action@v2
- name: Cache var-cache-apt
uses: actions/cache@v3
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/artifact/fuse-overlayfs-x86_64
asset_name: fuse-overlayfs-x86_64
asset_content_type: application/octet-stream
- name: "Upload fuse-overlayfs-aarch64"
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
path: var-cache-apt
key: var-cache-apt-${{ hashFiles('Dockerfile.artifact') }}
- name: Cache var-lib-apt
uses: actions/cache@v3
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/artifact/fuse-overlayfs-aarch64
asset_name: fuse-overlayfs-aarch64
asset_content_type: application/octet-stream
- name: "Upload fuse-overlayfs-armv7l"
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
path: var-lib-apt
key: var-lib-apt-${{ hashFiles('Dockerfile.artifact') }}
- name: inject var-cache-apt into docker
uses: overmindtech/buildkit-cache-dance/inject@306d31a77191f643c0c4a95083f36c6ddccb4a16
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/artifact/fuse-overlayfs-armv7l
asset_name: fuse-overlayfs-armv7l
asset_content_type: application/octet-stream
- name: "Upload fuse-overlayfs-s390x"
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
cache-source: var-cache-apt
cache-target: /var/cache/apt
- name: inject var-lib-apt into docker
uses: overmindtech/buildkit-cache-dance/inject@306d31a77191f643c0c4a95083f36c6ddccb4a16
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/artifact/fuse-overlayfs-s390x
asset_name: fuse-overlayfs-s390x
asset_content_type: application/octet-stream
- name: "Upload fuse-overlayfs-ppc64le"
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
cache-source: var-lib-apt
cache-target: /var/lib/apt
- name: "Build binaries from Dockerfile.artifact"
run: docker buildx build -o /tmp/fuse-overlayfs-builds --build-arg SOURCE_DATE_EPOCH --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 -f Dockerfile.artifact .
- name: extract var-cache-apt into docker
uses: overmindtech/buildkit-cache-dance/extract@306d31a77191f643c0c4a95083f36c6ddccb4a16
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/artifact/fuse-overlayfs-ppc64le
asset_name: fuse-overlayfs-ppc64le
asset_content_type: application/octet-stream
- name: "Upload fuse-overlayfs-riscv64"
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
cache-source: var-cache-apt
cache-target: /var/cache/apt
- name: extract var-lib-apt into docker
uses: overmindtech/buildkit-cache-dance/extract@306d31a77191f643c0c4a95083f36c6ddccb4a16
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/artifact/fuse-overlayfs-riscv64
asset_name: fuse-overlayfs-riscv64
asset_content_type: application/octet-stream
- name: "Upload SHA256SUMS"
uses: actions/[email protected]
cache-source: var-lib-apt
cache-target: /var/lib/apt
- name: "Create /tmp/artifact"
run: |
mkdir -p /tmp/artifact
mv /tmp/fuse-overlayfs-builds/linux_amd64/fuse-overlayfs /tmp/artifact/fuse-overlayfs-x86_64
mv /tmp/fuse-overlayfs-builds/linux_arm64/fuse-overlayfs /tmp/artifact/fuse-overlayfs-aarch64
mv /tmp/fuse-overlayfs-builds/linux_arm_v7/fuse-overlayfs /tmp/artifact/fuse-overlayfs-armv7l
mv /tmp/fuse-overlayfs-builds/linux_s390x/fuse-overlayfs /tmp/artifact/fuse-overlayfs-s390x
mv /tmp/fuse-overlayfs-builds/linux_ppc64le/fuse-overlayfs /tmp/artifact/fuse-overlayfs-ppc64le
mv /tmp/fuse-overlayfs-builds/linux_riscv64/fuse-overlayfs /tmp/artifact/fuse-overlayfs-riscv64
echo "${SOURCE_DATE_EPOCH}" >/tmp/artifact/SOURCE_DATE_EPOCH
- name: "SHA256SUMS"
run: (cd /tmp/artifact; sha256sum *) | tee /tmp/SHA256SUMS
- name: "The sha256sum of the SHA256SUMS file"
run: sha256sum /tmp/SHA256SUMS
- name: "Prepare the release note"
run: |
tag="${GITHUB_REF##*/}"
cat << EOF | tee /tmp/release-note.txt
${tag}
#### About the binaries
The binaries were built automatically on GitHub Actions.
The build log is available for 90 days: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
The binaries should be reproducible with the following command:
\`\`\`
docker buildx build \
-o /tmp/fuse-overlayfs-builds \
--build-arg SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} \
--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 \
--platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \
-f Dockerfile.artifact \
"https://github.com/${{ github.repository }}.git#${tag}"
\`\`\`
EOF
- name: "Create release"
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/SHA256SUMS
asset_name: SHA256SUMS
asset_content_type: text/plain
run: |
tag="${GITHUB_REF##*/}"
asset_flags=()
for f in /tmp/artifact/* /tmp/SHA256SUMS; do asset_flags+=("-a" "$f"); done
hub release create "${asset_flags[@]}" -F /tmp/release-note.txt --draft "${tag}"
32 changes: 28 additions & 4 deletions Containerfile.cross
Original file line number Diff line number Diff line change
@@ -1,14 +1,38 @@
FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.1.0 AS xx
FROM --platform=$BUILDPLATFORM ubuntu:22.04 AS fuse-overlayfs
RUN apt-get update && \
# Usage:
# docker buildx build \
# -o /tmp/fuse-overlayfs-builds \
# --build-arg SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) \
# --platform=amd64,arm64,arm,s390x,ppc64le,riscv64 \
# -f Dockerfile.artifact .
FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.2.1 AS xx
FROM --platform=$BUILDPLATFORM ubuntu:jammy-20230804 AS fuse-overlayfs
ADD --chmod=0755 \
https://raw.githubusercontent.com/reproducible-containers/repro-sources-list.sh/v0.1.0/repro-sources-list.sh \
/usr/local/bin/repro-sources-list.sh
RUN \
--mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
repro-sources-list.sh && \
apt-get update && \
apt-get install --no-install-recommends -y \
git make automake autoconf pkgconf file go-md2man
# Set SOURCE_DATE_EPOCH after running repro-sources-list.sh, for cache efficiency
ARG SOURCE_DATE_EPOCH
COPY . /fuse-overlayfs
WORKDIR /fuse-overlayfs
COPY --from=xx / /
ARG TARGETPLATFORM
ENV DEBIAN_FRONTEND=noninteractive
RUN xx-apt-get install -y gcc libfuse3-dev
# xx-apt-get cannot be used, as it clobbers /etc/apt/sources.list created by repro-sources-list.sh
RUN \
--mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
darch="$(xx-info debian-arch)" && \
dpkg --add-architecture ${darch} && \
apt-get update && \
gcc="gcc" && \
if xx-info is-cross; then gcc="gcc-$(xx-info triple)"; fi; \
apt-get install -y "${gcc}" "libfuse3-dev:${darch}"
RUN ./autogen.sh && \
LIBS="-ldl" LDFLAGS="-static" ./configure --host=$(xx-info) && \
make && mkdir /out && cp fuse-overlayfs /out && \
Expand Down

0 comments on commit bb64c54

Please sign in to comment.