wip

.github/generate-matrix/action.yml

@@ -18,6 +18,19 @@ inputs:
   substituters:
     description: Substituters
     required: true
+  flake_pre:
+    description: Prefix for the flake to evaluate
+    required: false
+    default: ""
+  flake_post:
+    description: Postfix for the flake to evaluate
+    required: false
+    default: ""
+  post_comment:
+    description: Whether to post a comment or not
+    required: false
+    default: true
+
 outputs:
   matrix:
     description: 'Generated Matrix'
@@ -43,14 +56,15 @@ runs:
       run: |
         mkdir -p /home/runner/.config/nix
         echo "machine ${{inputs.cachix-cache}}.cachix.org password ${{inputs.cachix-auth-token}}" >> /home/runner/.config/nix/netrc
+
     - name: Generate CI Matrix
       id: generate-matrix
       shell: bash
       env:
         IS_INITIAL: ${{ inputs.is-initial }}
         CACHIX_CACHE: ${{ inputs.cachix-cache }}
         CACHIX_AUTH_TOKEN: ${{ inputs.cachix-auth-token }}
-      run: nix run github:metacraft-labs/nixos-modules/feat/CD#ci-matrix
+      run: nix run github:metacraft-labs/nixos-modules/feat/CD#ci-matrix ${{ inputs.prefix }} ${{ inputs.postfix }}
 
     - name: Upload CI Matrix
       uses: actions/upload-artifact@v4
@@ -59,6 +73,7 @@ runs:
         path: matrix-${{ inputs.is-initial == 'true' && 'pre' || 'post' }}.json
 
     - name: Update GitHub Comment
+      if: ${{ inputs.post_comment }}
       uses: marocchino/[email protected]
       with:
         path: comment.md

.github/workflows/ci.yml

@@ -28,12 +28,35 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  generate-matrix-matrix:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v9
+        with:
+          extra-conf: accept-flake-config = true
+          nix-installer-tag: v0.15.1
+
+      - name: Generate Matrix for Matrix
+        id: generate-matrix
+        run: |
+          nix run github:metacraft-labs/nixos-modules/feat/CD#shard-matrix
+    outputs:
+      gen_matrix: ${{ steps.generate-matrix.outputs.gen_matrix }}
   generate-matrix:
+    needs: generate-matrix-matrix
     runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJSON(needs.generate-matrix-matrix.outputs.gen_matrix)}}
+
+    name: Generate Matrix ${{ matrix.digit != -1 && matrix.digit || '' }}
     steps:
       - uses: actions/checkout@v4
 
       - name: 'Post initial package status comment'
+        if: ${{ matrix.digit < 1 }}
         uses: marocchino/[email protected]
         with:
           recreate: true
@@ -44,13 +67,17 @@ jobs:
 
       - name: Generate CI Matrix
         id: generate-matrix
+        if: ${{ github.repository == 'metacraft-labs/nixos-machine-config' }}
         uses:  metacraft-labs/nixos-modules/.github/generate-matrix@feat/CD
         with:
           is-initial: 'true'
           cachix-cache: ${{ vars.CACHIX_CACHE }}
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
           trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
           substituters: ${{ vars.SUBSTITUTERS }}
+          flake_pre: ${{ matrix.prefix }}
+          flake_post: ${{ matrix.postfix }}
+          post_comment: ${{ matrix.digit < 1}}
     outputs:
       matrix: ${{ steps.generate-matrix.outputs.matrix }}
       comment: ${{ steps.generate-matrix.outputs.comment }}

packages/default.nix

@@ -11,10 +11,11 @@
         lido-withdrawals-automation = pkgs.callPackage ./lido-withdrawals-automation {};
         pyroscope = pkgs.callPackage ./pyroscope {};
         grafana-agent = import ./grafana-agent {inherit inputs';};
-        deploy-spec = pkgs.callPackage ./deploy-spec {};
       }
       // pkgs.lib.optionalAttrs pkgs.hostPlatform.isLinux rec {
         ci-matrix = pkgs.callPackage ./ci-matrix {inherit unstablePkgs;};
+        deploy-spec = pkgs.callPackage ./deploy-spec {};
+        shard-matrix = pkgs.callPackage ./shard-matrix {inherit ci-matrix;};
         validator-ejector = inputs'.validator-ejector.packages.validator-ejector;
       };
     checks = packages;

packages/shard-matrix/default.nix

@@ -0,0 +1,15 @@
+{
+  pkgs,
+  ci-matrix,
+}: let
+  jqBin = "${pkgs.jq}/bin/jq";
+  nixBin = "${pkgs.nix}/bin/nix";
+in
+  pkgs.substituteAll {
+    name = "shard-matrix";
+    inherit jqBin nixBin;
+    dir = "bin";
+    isExecutable = true;
+    src = ./shard-matrix.sh;
+    meta.mainProgram = "shard-matrix";
+  }

packages/shard-matrix/shard-matrix.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+root_dir="$(git rev-parse --show-toplevel)"
+result_dir="$root_dir/.result"
+
+save_gh_ci_matrix() {
+
+  echo "gen_matrix=$1" >> "${GITHUB_OUTPUT:-${result_dir}/gh-output.env}"
+
+}
+
+hasShards="$(@nixBin@ eval .#legacyPackages.x86_64-linux.checks.shardCount && echo "true" || echo "false")";
+if [ "$hasShards" = "false" ]; then
+  echo "No shards found, exiting"
+  save_gh_ci_matrix '{"include":[{prefix: "", postfix: "", "digit": -1}]}'
+  exit 0
+fi
+
+numShards=$(( $(@nixBin@ eval .#legacyPackages.x86_64-linux.checks.shardCount) - 1))
+
+shards=$(for i in $(seq 0 $numShards); do echo '{"prefix" : ".#legacyPackages", "postfix" : "checks.shards.'"$i"'", "digit": '"$i"' }'; done | paste -sd, -)
+
+gen_matrix='{"include":'["$shards"]'}'
+
+save_gh_ci_matrix "$(echo "$gen_matrix" | @jqBin@ -c .)"