From 6da62d211b5cbf9608f49c2932d41608b475eadb Mon Sep 17 00:00:00 2001 From: Martin Habovstiak Date: Mon, 18 Dec 2023 12:08:02 +0100 Subject: [PATCH] Parallelize CI build CI was building packages one after another in a single VM. This is slow, has the risk of one package affecting another and makes it harder to figure out which job failed. This change adds a preparation job which builds the image and gets the list of packages and changes build job into matrix depending on the prepare job. --- .github/workflows/ci.yml | 86 +++++++++++++++++++++++++--------------- Dockerfile | 3 +- Makefile | 2 +- build_rules/remir.pin | 1 + build_template.mustache | 6 ++- ci_build.sh | 11 +++++ debcrafter-version | 1 + 7 files changed, 74 insertions(+), 36 deletions(-) create mode 100644 build_rules/remir.pin create mode 100755 ci_build.sh create mode 100644 debcrafter-version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63a1a26..cb73389 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,49 +18,69 @@ env: EXECUTE_CMD: "$PODMAN_CMD exec -u $USER_NAME -i $CONTAINER_NAME" jobs: - build: + prepare: runs-on: ubuntu-latest + outputs: + sources: ${{ steps.list-sources.outputs.sources }} steps: - uses: actions/checkout@v2 - - name: Setup CI Build Script - run: | - tee -a $CI_SCRIPT < CADR_image.tar - - name: Upload the Running Environment Image to Artifact - uses: actions/upload-artifact@v2 - if: ${{ always() }} + - name: Install jq + run: sudo apt-get install -y jq + - name: List sources + id: list-sources + run: echo "::set-output name=sources::$(ls build_rules/*.yaml | grep -v remir | sed -e 's:^build_rules/::' -e 's/\.yaml$//' | jq -R -s -c 'split("\n")[:-1]')" + build: + needs: prepare + if: ${{ success() }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + source: ${{ fromJson(needs.prepare.outputs.sources) }} + steps: + - uses: actions/checkout@v2 + - name: Fetch CADR image cache + id: cache-cadr-image + uses: actions/cache/restore@v3 + env: + cache-name: cache-cadr-image with: - name: CADR_image + # npm cache files are stored in `~/.npm` on Linux/macOS path: CADR_image.tar + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }} + - name: Load Running Environment Image + run: | + $PODMAN_CMD load < CADR_image.tar + mkdir build - name: Spawn Podman Container to Prepare Running Environment run: | eval $SPAWN_CONTAINER - name: Build CADR run: | - eval $EXECUTE_CMD ${BUILD_DIR}/$CI_SCRIPT + eval $EXECUTE_CMD bash "${BUILD_DIR}/$CI_SCRIPT" "${{ matrix.source }}" - name: Upload Debian Packages Just Built to Artifact uses: actions/upload-artifact@v2 - if: ${{ always() }} with: - name: CADR_debs + name: ${{ matrix.source }} path: build/*.deb - name: Check SHA256 - if: ${{ always() }} run: | sudo chown -R $USER build cd build @@ -71,17 +91,15 @@ jobs: done - name: Upload SHA256 of Debian Packages Just Built to Artifact uses: actions/upload-artifact@v2 - if: ${{ always() }} with: name: CADR_debs_sha256sum path: build/*.deb.sha256sum - name: Fix the Dir Permission for Post checkout - if: ${{ always() }} run: | sudo chown -R $USER $PWD test: - needs: build + needs: [build, prepare] if: ${{ success() }} runs-on: ubuntu-latest strategy: @@ -110,22 +128,27 @@ jobs: run: | sudo apt-get update sudo apt-get upgrade podman - - name: Download Pre-built Container Image - uses: actions/download-artifact@v3 + - name: Fetch CADR image cache + id: cache-cadr-image + uses: actions/cache/restore@v3 + env: + cache-name: cache-cadr-image with: - name: CADR_image - path: . + # npm cache files are stored in `~/.npm` on Linux/macOS + path: CADR_image.tar + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }} - name: Load Running Environment Image run: | $PODMAN_CMD load < CADR_image.tar mkdir build - name: Download Pre-built Debian Packages - uses: actions/download-artifact@v3 + uses: thecodenebula/download-artifact@e9e49e9bbce8ff2b901957ee034714cab099644a with: - name: CADR_debs + #name: ${{ join(fromJson(needs.prepare.outputs.sources), '\n') }} path: build - name: Test CADR Basic run: | + mv build/*/*.deb build/ eval $SPAWN_CONTAINER eval $EXECUTE_CMD bash -c "\"$TEST_PREFIX test-here-basic-${{ matrix.package }}\"" $PODMAN_CMD rm -f $CONTAINER_NAME @@ -134,6 +157,5 @@ jobs: eval $SPAWN_CONTAINER eval $EXECUTE_CMD bash -c "\"$TEST_PREFIX SPLIT_STRATEGY=upgrade test-here-upgrade-${{ matrix.package }}\"" - name: Fix the Dir Permission for Post checkout - if: ${{ always() }} run: | sudo chown -R $USER $PWD diff --git a/Dockerfile b/Dockerfile index 474dade..a74d864 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ RUN echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf.d/90assumeyes COPY tests/data/microsoft_key.gpg /tmp/ COPY tests/data/microsoft_apt.list /tmp/ +COPY debcrafter-version /tmp/ RUN apt-get update && apt-get dist-upgrade && \ apt-get install apt-utils ca-certificates && \ @@ -18,7 +19,7 @@ RUN apt-get update && apt-get dist-upgrade && \ mv /tmp/microsoft_apt.list /etc/apt/sources.list.d/microsoft.list && \ apt-key add < /tmp/microsoft_key.gpg && \ apt-get update && \ - cargo install --root /usr/local --locked --git https://github.com/Kixunil/debcrafter && \ + cargo install --root /usr/local --locked --git https://github.com/Kixunil/debcrafter --rev "`cat /tmp/debcrafter-version`" && \ cargo install --root /usr/local --locked cfg_me && \ apt-get autoremove && apt-get clean && \ rm -rf /root/.cargo \ diff --git a/Makefile b/Makefile index e7453e5..f6c83e7 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ update-pin: include common_rules.mk build-dep: - sudo apt-get build-dep -a $(DEB_ARCH) $(realpath $(BITCOIN_DIR) $(BITCOIN_RPC_PROXY_BUILD_DIR) $(ELECTRS_BUILD_DIR) $(ELECTRUM_BUILD_DIR) $(TOR_EXTRAS_BUILD_DIR) $(LND_BUILD_DIR) $(NBXPLORER_BUILD_DIR) $(BTCPAYSERVER_BUILD_DIR) $(SELFHOST_BUILD_DIR) $(RIDETHELN_BUILD_DIR) $(LNPBP_TESTKIT_BUILD_DIR) $(REMIR_BUILD_DIR)) + sudo apt-get build-dep -a $(DEB_ARCH) $(BUILD_DIRS) test: $(TEST_DEPS) diff --git a/build_rules/remir.pin b/build_rules/remir.pin new file mode 100644 index 0000000..477f8b1 --- /dev/null +++ b/build_rules/remir.pin @@ -0,0 +1 @@ +3f9fc4bef627fd4f8a7c7bd01202c90ceac2e39f diff --git a/build_template.mustache b/build_template.mustache index 4213067..7b8949d 100644 --- a/build_template.mustache +++ b/build_template.mustache @@ -3,6 +3,7 @@ include $(SOURCE_DIR){{{include}}} {{/include}} {{pkg_name_upper}}_BUILD_DIR=$(BUILD_DIR)/{{{source_name}}}-$({{{pkg_name_upper}}}_VERSION) +BUILD_DIRS+=$({{pkg_name_upper}}_BUILD_DIR) {{pkg_name_upper}}_DEPS=$({{{pkg_name_upper}}}_BUILD_DIR) {{pkg_name_upper}}_ASSETS={{#copy_assets}}$({{{pkg_name_upper}}}_BUILD_DIR)/{{{to}}} {{/copy_assets}}{{#get_assets}}$({{{pkg_name_upper}}}_BUILD_DIR)/{{{file_name}}} {{/get_assets}} {{#shasums}} @@ -112,13 +113,14 @@ $(BUILD_DIR)/verify-signature-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{ {{/verify_commit}} touch $@ -$(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{{{file_name}}}{{/unpack}}{{#clone_url}}$(BUILD_DIR)/fetch-{{{source_name}}}.stamp{{/clone_url}} $({{{pkg_name_upper}}}_FILTERED_SHASUMS) +$(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{{{file_name}}}{{/unpack}}{{#clone_url}}$(BUILD_DIR)/fetch-{{{source_name}}}.stamp{{/clone_url}} {{#unpack}} cd $(BUILD_DIR) && sha256sum -c $(SOURCE_DIR)/build_rules/{{{source_name}}}.pin {{/unpack}} {{#clone_url}} test "`cat $(SOURCE_DIR)/build_rules/{{{source_name}}}.pin`" = "`cd "$({{{pkg_name_upper}}}_BUILD_DIR)" && git rev-parse HEAD`" {{/clone_url}} + touch $@ {{#unpinned}} $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-signature-{{{source_name}}}.stamp @@ -127,7 +129,7 @@ $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-signature-{{{so $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp {{/unpinned}} {{#unpack}} - tar -C $(BUILD_DIR) -x{{#compression}}{{{compression}}}{{/compression}}{{^compression}}z{{/compression}}mf $< + tar -C $(BUILD_DIR) -x{{#compression}}{{{compression}}}{{/compression}}{{^compression}}z{{/compression}}mf $(BUILD_DIR)/{{{file_name}}} {{#rename}} mv -T {{{rename}}} $({{{pkg_name_upper}}}_BUILD_DIR) {{/rename}} diff --git a/ci_build.sh b/ci_build.sh new file mode 100755 index 0000000..8c7f12c --- /dev/null +++ b/ci_build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ex + +BUILD_DIR=/home/user/cadr-build +sudo apt-get update +sudo chown -R user $BUILD_DIR +cd $BUILD_DIR +MKCMD="make SOURCES="$1" BUILD_DIR=${BUILD_DIR}/build" +$MKCMD build-dep +$MKCMD all diff --git a/debcrafter-version b/debcrafter-version new file mode 100644 index 0000000..9641975 --- /dev/null +++ b/debcrafter-version @@ -0,0 +1 @@ +b42f2fad79f23a03108ac69ae3e0e6987f64a60f