Parallelize CI build

CI was building packages one after another in a single VM. This is slow, has the risk of one package affecting another and makes it harder to figure out which job failed. This change adds a preparation job which builds the image and gets the list of packages and changes build job into matrix depending on the prepare job.
debian-cryptoanarchy · Dec 18, 2023 · 6da62d2 · 6da62d2
1 parent e09a6bf
commit 6da62d2
Show file tree

Hide file tree

Showing 7 changed files with 74 additions and 36 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,49 +18,69 @@ env:
   EXECUTE_CMD: "$PODMAN_CMD exec -u $USER_NAME -i $CONTAINER_NAME"
 
 jobs:
-  build:
+  prepare:
     runs-on: ubuntu-latest
+    outputs:
+            sources: ${{ steps.list-sources.outputs.sources }}
     steps:
       - uses: actions/checkout@v2
-      - name: Setup CI Build Script
-        run: |
-          tee -a $CI_SCRIPT <<EOF
-          #!/bin/bash
-          set -ex
-          sudo apt-get update
-          sudo chown -R $USER_NAME $BUILD_DIR
-          cd $BUILD_DIR
-          MKCMD="make BUILD_DIR=${BUILD_DIR}/build"
-          \$MKCMD build-dep
-          \$MKCMD all
-          EOF
-          chmod +x $CI_SCRIPT
+      - name: Check CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-cadr-image
+        with:
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
+          lookup-only: true
       - name: Build CADR Running Environment Image
+        if: steps.cache-cadr-image.outputs.cache-hit != 'true'
         run: |
           sudo apt-get update
           sudo apt-get upgrade podman
           $PODMAN_CMD build -t $IMAGE_NAME .
           $PODMAN_CMD save $IMAGE_NAME > CADR_image.tar
-      - name: Upload the Running Environment Image to Artifact
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
+      - name: Install jq
+        run: sudo apt-get install -y jq
+      - name: List sources
+        id: list-sources
+        run: echo "::set-output name=sources::$(ls build_rules/*.yaml | grep -v remir | sed -e 's:^build_rules/::' -e 's/\.yaml$//' | jq -R -s -c 'split("\n")[:-1]')"
+  build:
+    needs: prepare
+    if: ${{ success() }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        source: ${{ fromJson(needs.prepare.outputs.sources) }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Fetch CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache/restore@v3
+        env:
+          cache-name: cache-cadr-image
         with:
-          name: CADR_image
+          # npm cache files are stored in `~/.npm` on Linux/macOS
           path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
+      - name: Load Running Environment Image
+        run: |
+          $PODMAN_CMD load < CADR_image.tar
+          mkdir build
       - name: Spawn Podman Container to Prepare Running Environment
         run: |
           eval $SPAWN_CONTAINER
       - name: Build CADR
         run: |
-          eval $EXECUTE_CMD ${BUILD_DIR}/$CI_SCRIPT
+          eval $EXECUTE_CMD bash "${BUILD_DIR}/$CI_SCRIPT" "${{ matrix.source }}"
       - name: Upload Debian Packages Just Built to Artifact
         uses: actions/upload-artifact@v2
-        if: ${{ always() }}
         with:
-          name: CADR_debs
+          name: ${{ matrix.source }}
           path: build/*.deb
       - name: Check SHA256
-        if: ${{ always() }}
         run: |
           sudo chown -R $USER build
           cd build
@@ -71,17 +91,15 @@ jobs:
           done
       - name: Upload SHA256 of Debian Packages Just Built to Artifact
         uses: actions/upload-artifact@v2
-        if: ${{ always() }}
         with:
           name: CADR_debs_sha256sum
           path: build/*.deb.sha256sum
       - name: Fix the Dir Permission for Post checkout
-        if: ${{ always() }}
         run: |
           sudo chown -R $USER $PWD
 
   test:
-    needs: build
+    needs: [build, prepare]
     if: ${{ success() }}
     runs-on: ubuntu-latest
     strategy:
@@ -110,22 +128,27 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get upgrade podman
-      - name: Download Pre-built Container Image
-        uses: actions/download-artifact@v3
+      - name: Fetch CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache/restore@v3
+        env:
+          cache-name: cache-cadr-image
         with:
-          name: CADR_image
-          path: .
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
       - name: Load Running Environment Image
         run: |
           $PODMAN_CMD load < CADR_image.tar
           mkdir build
       - name: Download Pre-built Debian Packages
-        uses: actions/download-artifact@v3
+        uses: thecodenebula/download-artifact@e9e49e9bbce8ff2b901957ee034714cab099644a
         with:
-          name: CADR_debs
+          #name: ${{ join(fromJson(needs.prepare.outputs.sources), '\n') }}
           path: build
       - name: Test CADR Basic
         run: |
+          mv build/*/*.deb build/
           eval $SPAWN_CONTAINER
           eval $EXECUTE_CMD bash -c "\"$TEST_PREFIX test-here-basic-${{ matrix.package }}\""
           $PODMAN_CMD rm -f $CONTAINER_NAME
@@ -134,6 +157,5 @@ jobs:
           eval $SPAWN_CONTAINER
           eval $EXECUTE_CMD bash -c "\"$TEST_PREFIX SPLIT_STRATEGY=upgrade test-here-upgrade-${{ matrix.package }}\""
       - name: Fix the Dir Permission for Post checkout
-        if: ${{ always() }}
         run: |
           sudo chown -R $USER $PWD
diff --git a/Dockerfile b/Dockerfile
@@ -8,6 +8,7 @@ RUN echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf.d/90assumeyes
 
 COPY tests/data/microsoft_key.gpg /tmp/
 COPY tests/data/microsoft_apt.list /tmp/
+COPY debcrafter-version /tmp/
 
 RUN apt-get update && apt-get dist-upgrade && \
     apt-get install apt-utils ca-certificates && \
@@ -18,7 +19,7 @@ RUN apt-get update && apt-get dist-upgrade && \
     mv /tmp/microsoft_apt.list /etc/apt/sources.list.d/microsoft.list && \
     apt-key add < /tmp/microsoft_key.gpg && \
     apt-get update && \
-    cargo install --root /usr/local --locked --git https://github.com/Kixunil/debcrafter && \
+    cargo install --root /usr/local --locked --git https://github.com/Kixunil/debcrafter --rev "`cat /tmp/debcrafter-version`" && \
     cargo install --root /usr/local --locked cfg_me && \
     apt-get autoremove && apt-get clean && \
     rm -rf /root/.cargo \

diff --git a/Makefile b/Makefile
@@ -42,7 +42,7 @@ update-pin:
 include common_rules.mk
 
 build-dep:
-	sudo apt-get build-dep -a $(DEB_ARCH) $(realpath $(BITCOIN_DIR) $(BITCOIN_RPC_PROXY_BUILD_DIR) $(ELECTRS_BUILD_DIR) $(ELECTRUM_BUILD_DIR) $(TOR_EXTRAS_BUILD_DIR) $(LND_BUILD_DIR) $(NBXPLORER_BUILD_DIR) $(BTCPAYSERVER_BUILD_DIR) $(SELFHOST_BUILD_DIR) $(RIDETHELN_BUILD_DIR) $(LNPBP_TESTKIT_BUILD_DIR) $(REMIR_BUILD_DIR))
+	sudo apt-get build-dep -a $(DEB_ARCH) $(BUILD_DIRS)
 
 test: $(TEST_DEPS)
 

diff --git a/build_rules/remir.pin b/build_rules/remir.pin
@@ -0,0 +1 @@
+3f9fc4bef627fd4f8a7c7bd01202c90ceac2e39f
diff --git a/build_template.mustache b/build_template.mustache
@@ -3,6 +3,7 @@ include $(SOURCE_DIR){{{include}}}
 {{/include}}
 
 {{pkg_name_upper}}_BUILD_DIR=$(BUILD_DIR)/{{{source_name}}}-$({{{pkg_name_upper}}}_VERSION)
+BUILD_DIRS+=$({{pkg_name_upper}}_BUILD_DIR)
 {{pkg_name_upper}}_DEPS=$({{{pkg_name_upper}}}_BUILD_DIR)
 {{pkg_name_upper}}_ASSETS={{#copy_assets}}$({{{pkg_name_upper}}}_BUILD_DIR)/{{{to}}} {{/copy_assets}}{{#get_assets}}$({{{pkg_name_upper}}}_BUILD_DIR)/{{{file_name}}} {{/get_assets}}
 {{#shasums}}
@@ -112,13 +113,14 @@ $(BUILD_DIR)/verify-signature-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{
 	{{/verify_commit}}
 	touch $@
 
-$(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{{{file_name}}}{{/unpack}}{{#clone_url}}$(BUILD_DIR)/fetch-{{{source_name}}}.stamp{{/clone_url}} $({{{pkg_name_upper}}}_FILTERED_SHASUMS)
+$(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{{{file_name}}}{{/unpack}}{{#clone_url}}$(BUILD_DIR)/fetch-{{{source_name}}}.stamp{{/clone_url}}
 	{{#unpack}}
 	cd $(BUILD_DIR) && sha256sum -c $(SOURCE_DIR)/build_rules/{{{source_name}}}.pin
 	{{/unpack}}
 	{{#clone_url}}
 	test "`cat $(SOURCE_DIR)/build_rules/{{{source_name}}}.pin`" = "`cd "$({{{pkg_name_upper}}}_BUILD_DIR)" && git rev-parse HEAD`"
 	{{/clone_url}}
+	touch $@
 
 {{#unpinned}}
 $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-signature-{{{source_name}}}.stamp
@@ -127,7 +129,7 @@ $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-signature-{{{so
 $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp
 {{/unpinned}}
 {{#unpack}}
-	tar -C $(BUILD_DIR) -x{{#compression}}{{{compression}}}{{/compression}}{{^compression}}z{{/compression}}mf $<
+	tar -C $(BUILD_DIR) -x{{#compression}}{{{compression}}}{{/compression}}{{^compression}}z{{/compression}}mf $(BUILD_DIR)/{{{file_name}}}
 	{{#rename}}
 	mv -T {{{rename}}} $({{{pkg_name_upper}}}_BUILD_DIR)
 	{{/rename}}

diff --git a/ci_build.sh b/ci_build.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ex
+
+BUILD_DIR=/home/user/cadr-build
+sudo apt-get update
+sudo chown -R user $BUILD_DIR
+cd $BUILD_DIR
+MKCMD="make SOURCES="$1" BUILD_DIR=${BUILD_DIR}/build"
+$MKCMD build-dep
+$MKCMD all
diff --git a/debcrafter-version b/debcrafter-version
@@ -0,0 +1 @@
+b42f2fad79f23a03108ac69ae3e0e6987f64a60f