From e19a5decc89c8ed579806c6807bb03cf8efccfb7 Mon Sep 17 00:00:00 2001 From: Yousaf Nabi Date: Tue, 30 Apr 2024 15:05:41 +0100 Subject: [PATCH] feat: linux musl static bins / windows aarch64 - create linux executables linked statically to musl - drop gnu (glibc) target, executable will work on alpine & glibc based distros (debian etc) - create windows aarch64 target - refactor release script & workflow to match pact-reference - builds on pull request to master/main branch, in debug mode Related epics - musl all the things - https://github.com/pact-foundation/devrel/issues/30 --- .github/workflows/release.yml | 54 +++++++++-- .gitignore | 3 +- Cross.toml | 10 +- release.sh | 168 ++++++++++++++++++++++++---------- scripts/gzip-and-sum.sh | 18 ++++ 5 files changed, 194 insertions(+), 59 deletions(-) create mode 100755 scripts/gzip-and-sum.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b42489..724456e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,13 @@ on: release: types: [published] + pull_request: + branches: + - main + push: + branches: + - feat/musl_linux_win_aarch64 + concurrency: group: ci-${{ github.ref }} cancel-in-progress: false @@ -13,18 +20,43 @@ jobs: runs-on: ${{ matrix.operating-system }} strategy: matrix: - operating-system: [ubuntu-20.04, windows-2019, macos-12] - env: - pact_do_not_track: true + include: + - operating-system: ubuntu-20.04 + targets: x86_64-unknown-linux-gnu,x86_64-unknown-linux-musl,aarch64-unknown-linux-gnu,aarch64-unknown-linux-musl + - operating-system: windows-2019 + targets: aarch64-pc-windows-msvc,x86_64-pc-windows-msvc + - operating-system: macos-12 + targets: aarch64-apple-darwin,x86_64-apple-darwin + fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install stable Rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable with: - profile: minimal toolchain: stable - override: true - target: aarch64-apple-darwin + targets: ${{ matrix.targets }} + + - name: Rust caching + uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + if: runner.os == 'Linux' + uses: docker/setup-buildx-action@v3 + - name: Cargo flags + id: cargo-flags + shell: bash + run: | + if [[ "${{ github.event_name }}" = "release" ]]; then + echo "flags=--release" >> "$GITHUB_OUTPUT" + else + echo "flags=" >> "$GITHUB_OUTPUT" + fi - name: Install Protoc uses: arduino/setup-protoc@v1 with: @@ -35,10 +67,12 @@ jobs: - run: ./release.sh ${{ runner.os }} ${{ github.ref }} shell: bash - name: Upload Release Assets + if: | + startsWith(github.ref, 'refs/tags/v') id: upload-release-asset uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: target/artifacts/* + file: release_artifacts/* file_glob: true - tag: ${{ github.ref }} + tag: ${{ github.ref }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2ac0f59..839670f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # will have compiled files and executables target/ - +# Generated Release Artifacts +release_artifacts # These are backup files generated by rustfmt **/*.rs.bk diff --git a/Cross.toml b/Cross.toml index 3f3b6ef..5874bc7 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,6 +1,12 @@ -[target.aarch64-unknown-linux-gnu] +[target.x86_64-unknown-linux-musl] pre-build=[ - "dpkg --add-architecture arm64 && apt-get update && apt-get install --assume-yes wget unzip", + "apt-get update && apt-get install --assume-yes wget unzip", "wget https://github.com/protocolbuffers/protobuf/releases/download/v21.5/protoc-21.5-linux-x86_64.zip", "unzip protoc-21.5-linux-x86_64.zip -d /usr/local/" ] +[target.aarch64-unknown-linux-musl] +pre-build=[ + "dpkg --add-architecture arm64 && apt-get update && apt-get install --assume-yes wget unzip", + "wget https://github.com/protocolbuffers/protobuf/releases/download/v21.5/protoc-21.5-linux-x86_64.zip", + "unzip protoc-21.5-linux-x86_64.zip -d /usr/local/" +] \ No newline at end of file diff --git a/release.sh b/release.sh index 049054f..9899d15 100755 --- a/release.sh +++ b/release.sh @@ -1,61 +1,137 @@ #!/bin/bash -if [ $# -lt 1 ] +set -e +set -x + +RUST_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd )" + +source "$RUST_DIR/scripts/gzip-and-sum.sh" +ARTIFACTS_DIR=${ARTIFACTS_DIR:-"$RUST_DIR/release_artifacts"} +mkdir -p "$ARTIFACTS_DIR" +export CARGO_TARGET_DIR=${CARO_TARGET_DIR:-"$RUST_DIR/target"} + +if [ $# -lt 2 ] then - echo "Usage : $0 " + echo "Usage : $0 " exit fi -set -e +APP=pact-protobuf-plugin +OS=$1 +shift; +VERSION=$1 +shift; +echo Building Release for "$OS" +# All flags passed to this script are passed to cargo. +cargo_flags=( "$@" ) +build_manifest() { + cp pact-plugin.json release_artifacts + NEXT=$(echo "$VERSION" | sed 's/^refs\/tags\/v-//') + # get latest release tag, if NEXT still contains refs + if [[ "${NEXT}" =~ "refs"* ]]; then + CRATE_VERSION=$(cat Cargo.toml| grep 'version = ".*"' -m1 | cut -d '"' -f 2) + echo "defaulting NEXT=$VERSION to version from Cargo.toml $CRATE_VERSION" + NEXT=$CRATE_VERSION + # LATEST_RELEASE=$(echo $(curl -s https://api.github.com/repos/pact-foundation/pact-stub-server/releases/latest | jq -r '.name') | sed 's/v//') + # echo "defaulting NEXT=$VERSION to latest release $LATEST_RELEASE" + # NEXT=$LATEST_RELEASE + fi + sed -e 's/VERSION=\"0.1.5\"/VERSION=\"'${NEXT}'\"/' "$RUST_DIR/scripts/install-plugin.sh" > "$ARTIFACTS_DIR/install-plugin.sh" + openssl dgst -sha256 -r $ARTIFACTS_DIR/install-plugin.sh > "$ARTIFACTS_DIR/install-plugin.sh.sha256" +} +install_cross() { + cargo install cross@0.2.5 +} + +build_linux_x86_64() { + install_cross + cargo clean + cross build --target=x86_64-unknown-linux-musl "${cargo_flags[@]}" + if [[ "${cargo_flags[*]}" =~ "--release" ]]; then + gzip_and_sum \ + "$CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/${APP}" \ + "$ARTIFACTS_DIR/${APP}-linux-x86_64.gz" + # cargo clean + fi +} + +build_linux_aarch64() { + install_cross + cargo clean + cross build --target=aarch64-unknown-linux-musl "${cargo_flags[@]}" + + if [[ "${cargo_flags[*]}" =~ "--release" ]]; then + gzip_and_sum \ + "$CARGO_TARGET_DIR/aarch64-unknown-linux-musl/release/${APP}" \ + "$ARTIFACTS_DIR/${APP}-linux-aarch64.gz" + fi +} +# Build the x86_64 darwin release +build_macos_x86_64() { + cargo build --target x86_64-apple-darwin "${cargo_flags[@]}" + + if [[ "${cargo_flags[*]}" =~ "--release" ]]; then + gzip_and_sum \ + "$CARGO_TARGET_DIR/x86_64-apple-darwin/release/${APP}" \ + "$ARTIFACTS_DIR/${APP}-osx-x86_64.gz" + gzip_and_sum \ + "$CARGO_TARGET_DIR/x86_64-apple-darwin/release/${APP}" \ + "$ARTIFACTS_DIR/${APP}-macos-x86_64.gz" + fi +} + +# Build the aarch64 darwin release +build_macos_aarch64() { + cargo build --target aarch64-apple-darwin "${cargo_flags[@]}" + + if [[ "${cargo_flags[*]}" =~ "--release" ]]; then + gzip_and_sum \ + "$CARGO_TARGET_DIR/aarch64-apple-darwin/release/${APP}" \ + "$ARTIFACTS_DIR/${APP}-osx-aarch64.gz" + gzip_and_sum \ + "$CARGO_TARGET_DIR/aarch64-apple-darwin/release/${APP}" \ + "$ARTIFACTS_DIR/${APP}-macos-aarch64.gz" + fi +} + +# Build the x86_64 windows release +build_windows_x86_64() { + cargo build --target x86_64-pc-windows-msvc "${cargo_flags[@]}" + + # If --release in cargo flags, then gzip and sum the release artifacts + if [[ "${cargo_flags[*]}" =~ "--release" ]]; then + gzip_and_sum \ + "$CARGO_TARGET_DIR/x86_64-pc-windows-msvc/release/${APP}.exe" \ + "$ARTIFACTS_DIR/${APP}-windows-x86_64.exe.gz" + fi +} -echo Building Release for "$1" - "$2" +# Build the aarch64 windows release +build_windows_aarch64() { + cargo build --target aarch64-pc-windows-msvc "${cargo_flags[@]}" -cargo clean -mkdir -p target/artifacts + if [[ "${cargo_flags[*]}" =~ "--release" ]]; then + gzip_and_sum \ + "$CARGO_TARGET_DIR/aarch64-pc-windows-msvc/release/${APP}.exe" \ + "$ARTIFACTS_DIR/${APP}-windows-aarch64.exe.gz" + fi +} -case "$1" in +case "$OS" in Linux) echo "Building for Linux" - docker run --rm --user "$(id -u)":"$(id -g)" -v "$(pwd):/workspace" -w /workspace -t \ - pactfoundation/rust-musl-build -c 'LIBZ_SYS_STATIC=1 cargo build --release' - gzip -c target/release/pact-protobuf-plugin > target/artifacts/pact-protobuf-plugin-linux-x86_64.gz - openssl dgst -sha256 -r target/artifacts/pact-protobuf-plugin-linux-x86_64.gz > target/artifacts/pact-protobuf-plugin-linux-x86_64.gz.sha256 - cp pact-plugin.json target/artifacts - NEXT=$(echo "$2" | cut -d\- -f2) - sed -e 's/VERSION=\"0.1.5\"/VERSION=\"'${NEXT}'\"/' scripts/install-plugin.sh > target/artifacts/install-plugin.sh - openssl dgst -sha256 -r target/artifacts/install-plugin.sh > target/artifacts/install-plugin.sh.sha256 - - # Build aarch64 - cargo install cross --git https://github.com/cross-rs/cross - cross build --target aarch64-unknown-linux-gnu --release - gzip -c target/aarch64-unknown-linux-gnu/release/pact-protobuf-plugin > target/artifacts/pact-protobuf-plugin-linux-aarch64.gz - openssl dgst -sha256 -r target/artifacts/pact-protobuf-plugin-linux-aarch64.gz > target/artifacts/pact-protobuf-plugin-linux-aarch64.gz.sha256 + # build_linux_x86_64 + # build_linux_aarch64 + build_manifest ;; - Windows) echo "Building for Windows" - cargo build --release - gzip -c target/release/pact-protobuf-plugin.exe > target/artifacts/pact-protobuf-plugin-windows-x86_64.exe.gz - openssl dgst -sha256 -r target/artifacts/pact-protobuf-plugin-windows-x86_64.exe.gz > target/artifacts/pact-protobuf-plugin-windows-x86_64.exe.gz.sha256 + Windows) echo "Building for windows" + build_windows_x86_64 + build_windows_aarch64 ;; - macOS) echo "Building for OSX" - cargo build --release - gzip -c target/release/pact-protobuf-plugin > target/artifacts/pact-protobuf-plugin-osx-x86_64.gz - openssl dgst -sha256 -r target/artifacts/pact-protobuf-plugin-osx-x86_64.gz > target/artifacts/pact-protobuf-plugin-osx-x86_64.gz.sha256 - # macos - gzip -c target/release/pact-protobuf-plugin > target/artifacts/pact-protobuf-plugin-macos-x86_64.gz - openssl dgst -sha256 -r target/artifacts/pact-protobuf-plugin-macos-x86_64.gz > target/artifacts/pact-protobuf-plugin-macos-x86_64.gz.sha256 - - # M1 - # export SDKROOT=$(xcrun -sdk macosx11.1 --show-sdk-path) - # export MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx11.1 --show-sdk-platform-version) - export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET:-12} - cargo build --target aarch64-apple-darwin --release - - gzip -c target/aarch64-apple-darwin/release/pact-protobuf-plugin > target/artifacts/pact-protobuf-plugin-osx-aarch64.gz - openssl dgst -sha256 -r target/artifacts/pact-protobuf-plugin-osx-aarch64.gz > target/artifacts/pact-protobuf-plugin-osx-aarch64.gz.sha256 - # macos - gzip -c target/aarch64-apple-darwin/release/pact-protobuf-plugin > target/artifacts/pact-protobuf-plugin-macos-aarch64.gz - openssl dgst -sha256 -r target/artifacts/pact-protobuf-plugin-macos-aarch64.gz > target/artifacts/pact-protobuf-plugin-macos-aarch64.gz.sha256 + macOS) echo "Building for macos" + build_macos_x86_64 + build_macos_aarch64 ;; - *) echo "$1 is not a recognised OS" + *) echo "$OS is not a recognised OS" exit 1 ;; -esac +esac \ No newline at end of file diff --git a/scripts/gzip-and-sum.sh b/scripts/gzip-and-sum.sh new file mode 100755 index 0000000..8b20c52 --- /dev/null +++ b/scripts/gzip-and-sum.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Gzip and sum a file. +# +# Usage: gzip_and_sum [] +# +# - orig_file: the file to gzip and sum +# - target_file: the file to write the gzipped file to +# - digest_file: the file to write the digest to. If not provided, defaults to +# .sha256 +gzip_and_sum() { + orig_file=$1 + target_file=$2 + digest_file=${3:-$target_file.sha256} + + gzip --stdout --best "$orig_file" > "$target_file" + openssl dgst -sha256 -r "$target_file" > "$digest_file" +} \ No newline at end of file