Parallelize CI build

CI was building packages one after another in a single VM. This is slow, has the risk of one package affecting another and makes it harder to figure out which job failed. This change adds a preparation job which builds the image and gets the list of packages and changes build job into matrix depending on the prepare job.
debian-cryptoanarchy · Dec 18, 2023 · b3a4da8 · b3a4da8
1 parent e09a6bf
commit b3a4da8
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 21 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,8 +18,47 @@ env:
   EXECUTE_CMD: "$PODMAN_CMD exec -u $USER_NAME -i $CONTAINER_NAME"
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+            sources: ${{ steps.list-sources.outputs.sources }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-cadr-image
+        with:
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
+          lookup-only: true
+      - name: Build CADR Running Environment Image
+        if: steps.cache-cadr-image.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get upgrade podman
+          $PODMAN_CMD build -t $IMAGE_NAME .
+          $PODMAN_CMD save $IMAGE_NAME > CADR_image.tar
+      - name: Upload the Running Environment Image to Artifact
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: CADR_image
+          path: CADR_image.tar
+      - name: Install jq
+        run: sudo apt-get install -y jq
+      - name: List sources
+        id: list-sources
+        run: echo "::set-output name=sources::$(ls pkg_specs/*.sss | sed -e 's:^pkg_specs/::' -e 's/\.sss$//' | jq -R -s -c 'split("\n")[:-1]')"
   build:
+    needs: prepare
+    if: ${{ success() }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        source: ${{ fromJson(needs.prepare.outputs.sources) }}
     steps:
       - uses: actions/checkout@v2
       - name: Setup CI Build Script
@@ -35,29 +74,30 @@ jobs:
           \$MKCMD all
           EOF
           chmod +x $CI_SCRIPT
-      - name: Build CADR Running Environment Image
-        run: |
-          sudo apt-get update
-          sudo apt-get upgrade podman
-          $PODMAN_CMD build -t $IMAGE_NAME .
-          $PODMAN_CMD save $IMAGE_NAME > CADR_image.tar
-      - name: Upload the Running Environment Image to Artifact
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
+      - name: Fetch CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache/restore@v3
+        env:
+          cache-name: cache-cadr-image
         with:
-          name: CADR_image
+          # npm cache files are stored in `~/.npm` on Linux/macOS
           path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
+      - name: Load Running Environment Image
+        run: |
+          $PODMAN_CMD load < CADR_image.tar
+          mkdir build
       - name: Spawn Podman Container to Prepare Running Environment
         run: |
           eval $SPAWN_CONTAINER
       - name: Build CADR
         run: |
-          eval $EXECUTE_CMD ${BUILD_DIR}/$CI_SCRIPT
+          eval SOURCES=${{ matrix.source }} $EXECUTE_CMD ${BUILD_DIR}/$CI_SCRIPT
       - name: Upload Debian Packages Just Built to Artifact
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: CADR_debs
+          name: ${{ matrix.source }}
           path: build/*.deb
       - name: Check SHA256
         if: ${{ always() }}
@@ -81,7 +121,7 @@ jobs:
           sudo chown -R $USER $PWD
 
   test:
-    needs: build
+    needs: [build, prepare]
     if: ${{ success() }}
     runs-on: ubuntu-latest
     strategy:
@@ -110,20 +150,24 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get upgrade podman
-      - name: Download Pre-built Container Image
-        uses: actions/download-artifact@v3
+      - name: Fetch CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache/restore@v3
+        env:
+          cache-name: cache-cadr-image
         with:
-          name: CADR_image
-          path: .
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
       - name: Load Running Environment Image
         run: |
           $PODMAN_CMD load < CADR_image.tar
           mkdir build
       - name: Download Pre-built Debian Packages
-        uses: actions/download-artifact@v3
+        uses: thecodenebula/download-artifact@e9e49e9bbce8ff2b901957ee034714cab099644a
         with:
-          name: CADR_debs
-          path: build
+          names: ${{ fromJson(needs.prepare.outputs.sources) }}
+          paths: build
       - name: Test CADR Basic
         run: |
           eval $SPAWN_CONTAINER

diff --git a/Dockerfile b/Dockerfile
@@ -8,6 +8,7 @@ RUN echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf.d/90assumeyes
 
 COPY tests/data/microsoft_key.gpg /tmp/
 COPY tests/data/microsoft_apt.list /tmp/
+COPY debcrafter-version /tmp/
 
 RUN apt-get update && apt-get dist-upgrade && \
     apt-get install apt-utils ca-certificates && \
@@ -18,7 +19,7 @@ RUN apt-get update && apt-get dist-upgrade && \
     mv /tmp/microsoft_apt.list /etc/apt/sources.list.d/microsoft.list && \
     apt-key add < /tmp/microsoft_key.gpg && \
     apt-get update && \
-    cargo install --root /usr/local --locked --git https://github.com/Kixunil/debcrafter && \
+    cargo install --root /usr/local --locked --git https://github.com/Kixunil/debcrafter --rev "`cat /tmp/debcrafter-version`" && \
     cargo install --root /usr/local --locked cfg_me && \
     apt-get autoremove && apt-get clean && \
     rm -rf /root/.cargo \

diff --git a/debcrafter-version b/debcrafter-version
@@ -0,0 +1 @@
+b42f2fad79f23a03108ac69ae3e0e6987f64a60f