From 0bef88c0d889df525105ed4fca48ad5664fe5419 Mon Sep 17 00:00:00 2001 From: Albin Date: Mon, 18 Sep 2023 13:01:45 +0200 Subject: [PATCH 1/4] Use unique cargo volumes for android builds --- ci/buildserver-build-android.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/buildserver-build-android.sh b/ci/buildserver-build-android.sh index 61bf556d8305..9786b784b1e4 100755 --- a/ci/buildserver-build-android.sh +++ b/ci/buildserver-build-android.sh @@ -61,6 +61,8 @@ function build_ref { echo "Building Android app" ANDROID_CREDENTIALS_DIR=$ANDROID_CREDENTIALS_DIR \ + CARGO_TARGET_VOLUME_NAME="cargo-target-android" \ + CARGO_REGISTRY_VOLUME_NAME="cargo-registry-android" \ USE_MOLD=false \ ./building/containerized-build.sh android --app-bundle || return 0 From 9ba1bd01efddb6a146eeeea59e177b614095c350 Mon Sep 17 00:00:00 2001 From: Albin Date: Mon, 18 Sep 2023 13:30:56 +0200 Subject: [PATCH 2/4] Unify android build server script with the desktop equivalent The scripts deviated some time ago so this commits aims to unify the android script with the desktop equivalent in terms of its structure as well as how it handles versioning and artifacts. --- ci/buildserver-build-android.sh | 92 ++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 29 deletions(-) diff --git a/ci/buildserver-build-android.sh b/ci/buildserver-build-android.sh index 9786b784b1e4..ff788a445469 100755 --- a/ci/buildserver-build-android.sh +++ b/ci/buildserver-build-android.sh @@ -19,37 +19,41 @@ BRANCHES_TO_BUILD=("origin/main") TAG_PATTERN_TO_BUILD="^android/" function upload { - for f in MullvadVPN-*.{apk,aab}; do - sha256sum "$f" > "$f.sha256" - mv "$f" "$f.sha256" "$UPLOAD_DIR/" - done -} + version=$1 -function build_ref { - ref=$1 - tag=${2:-""} + files=( * ) + checksums_path="$version+$(hostname).sha256" + sha256sum "${files[@]}" > "$checksums_path" - current_hash="$(git rev-parse "$ref^{commit}")" - if [ -f "$LAST_BUILT_DIR/$current_hash" ]; then - # This commit has already been built - return 0 - fi + mv "${files[@]}" "$checksums_path" "$UPLOAD_DIR/" +} - echo "" - echo "[#] $ref: $current_hash, building new packages." +# Builds the app artifacts and move them to the passed in `artifact_dir`. +# Must pass `artifact_dir` to show where to move the built artifacts. +function build { + ANDROID_CREDENTIALS_DIR=$ANDROID_CREDENTIALS_DIR \ + CARGO_TARGET_VOLUME_NAME="cargo-target-android" \ + CARGO_REGISTRY_VOLUME_NAME="cargo-registry-android" \ + USE_MOLD=false \ + ./building/containerized-build.sh android --app-bundle || return 0 + + mv dist/*.{aab,apk} "$artifact_dir" || return 1 +} +# Checks out the passed git reference passed to the working directory. +# Returns an error code if the commit/tag at `ref` is not properly signed. +function checkout_ref { + ref=$1 if [[ $ref == "refs/tags/"* ]] && ! git verify-tag "$ref"; then echo "!!!" echo "[#] $ref is a tag, but it failed GPG verification!" echo "!!!" - sleep 60 - return 0 + return 1 elif [[ $ref == "refs/remotes/"* ]] && ! git verify-commit "$current_hash"; then echo "!!!" echo "[#] $ref is a branch, but it failed GPG verification!" echo "!!!" - sleep 60 - return 0 + return 1 fi # Clean our working dir and check out the code we want to build @@ -58,13 +62,33 @@ function build_ref { git checkout "$ref" git submodule update git clean -df +} + +function build_ref { + ref=$1 + tag=${2:-""} + + current_hash="$(git rev-parse "$ref^{commit}")" + if [ -f "$LAST_BUILT_DIR/$current_hash" ]; then + # This commit has already been built + return 0 + fi + + echo "" + echo "[#] $ref: $current_hash, building new packages." + echo "" + + checkout_ref "$ref" || return 1 + + # podman appends a trailing carriage return to the output. So we use `tr` to strip it + local version="" + version="$(run_in_build_env cargo run -q --bin mullvad-version | tr -d "\r" || return 1)" + + local artifact_dir="dist/$version" + mkdir -p "$artifact_dir" echo "Building Android app" - ANDROID_CREDENTIALS_DIR=$ANDROID_CREDENTIALS_DIR \ - CARGO_TARGET_VOLUME_NAME="cargo-target-android" \ - CARGO_REGISTRY_VOLUME_NAME="cargo-registry-android" \ - USE_MOLD=false \ - ./building/containerized-build.sh android --app-bundle || return 0 + artifact_dir=$artifact_dir build || return 1 # If there is a tag for this commit then we append that to the produced artifacts # A version suffix should only be created if there is a tag for this commit and it is not a release build @@ -73,18 +97,27 @@ function build_ref { version_suffix="+${tag//[^0-9a-z_-]/}" # Will only match paths that include *-dev-* which means release builds will not be included # Pipes all matching names and their new name to mv - pushd dist + pushd "$artifact_dir" for original_file in MullvadVPN-*-dev-*{.apk,.aab}; do new_file=$(echo "$original_file" | sed -nE "s/^(MullvadVPN-.*-dev-.*)(\.apk|\.aab)$/\1$version_suffix\2/p") mv "$original_file" "$new_file" done popd + + if [[ $version == *"-dev-"* ]]; then + version="$version$version_suffix" + fi fi - (cd dist/ && upload) || return 0 + (cd "$artifact_dir" && upload "$version") || return 1 + # shellcheck disable=SC2216 + yes | rm -r "$artifact_dir" touch "$LAST_BUILT_DIR/$current_hash" - echo "Successfully finished Android build at $(date)" + + echo "" + echo "Successfully finished building $version at $(date)" + echo "" } cd "$BUILD_DIR" @@ -95,16 +128,17 @@ while true; do git fetch --prune --tags 2> /dev/null || continue + # Only build android/* tags. # Tags can't include spaces so SC2207 isn't a problem here # shellcheck disable=SC2207 tags=( $(git tag | grep "$TAG_PATTERN_TO_BUILD") ) for tag in "${tags[@]}"; do - build_ref "refs/tags/$tag" "$tag" + build_ref "refs/tags/$tag" "$tag" || echo "Failed to build tag $tag" done for branch in "${BRANCHES_TO_BUILD[@]}"; do - build_ref "refs/remotes/$branch" + build_ref "refs/remotes/$branch" || echo "Failed to build branch $tag" done sleep 240 From 9bf9ae8c6ce32c6f049700eaa79da410ef807a37 Mon Sep 17 00:00:00 2001 From: Albin Date: Tue, 19 Sep 2023 10:42:01 +0200 Subject: [PATCH 3/4] Fix call to generate version --- ci/buildserver-build-android.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/buildserver-build-android.sh b/ci/buildserver-build-android.sh index ff788a445469..04a2d05e2629 100755 --- a/ci/buildserver-build-android.sh +++ b/ci/buildserver-build-android.sh @@ -28,6 +28,10 @@ function upload { mv "${files[@]}" "$checksums_path" "$UPLOAD_DIR/" } +function run_in_linux_container { + USE_MOLD=false ./building/container-run.sh linux "$@" +} + # Builds the app artifacts and move them to the passed in `artifact_dir`. # Must pass `artifact_dir` to show where to move the built artifacts. function build { @@ -82,7 +86,7 @@ function build_ref { # podman appends a trailing carriage return to the output. So we use `tr` to strip it local version="" - version="$(run_in_build_env cargo run -q --bin mullvad-version | tr -d "\r" || return 1)" + version="$(run_in_linux_container cargo run -q --bin mullvad-version | tr -d "\r" || return 1)" local artifact_dir="dist/$version" mkdir -p "$artifact_dir" From ca7d8ec043ee2fcf5e408f7487fdf72fb5a14d4d Mon Sep 17 00:00:00 2001 From: Albin Date: Tue, 19 Sep 2023 15:33:43 +0200 Subject: [PATCH 4/4] Fix containerized build return code --- ci/buildserver-build-android.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/buildserver-build-android.sh b/ci/buildserver-build-android.sh index 04a2d05e2629..1e04a9879309 100755 --- a/ci/buildserver-build-android.sh +++ b/ci/buildserver-build-android.sh @@ -39,7 +39,7 @@ function build { CARGO_TARGET_VOLUME_NAME="cargo-target-android" \ CARGO_REGISTRY_VOLUME_NAME="cargo-registry-android" \ USE_MOLD=false \ - ./building/containerized-build.sh android --app-bundle || return 0 + ./building/containerized-build.sh android --app-bundle || return 1 mv dist/*.{aab,apk} "$artifact_dir" || return 1 }