Move all CI to GitHub Actions (#923)

* .buildkite: rm As awesome as Buildkite is, it is unfortunately not useful for this setup: * We use Buildkite to build installer binaries * We then use GitHub Actions to run the installer binaries on a host without Nix already installed We don't use Buildkite for both steps because setting up agents to install Nix on would be a pain, as erasing everything takes a while. This is preparation for using GitHub Actions for everything in our CI pipeline, as it is much faster and we don't need to worry about keeping the runners running ourselves (there have been times where our Buildkite agents were unreachable for some reason or another). * ci: build test Nix binaries in GHA * ci: use same runner version as build steps * ci: factor out build steps into composable workflows So that when I go to fix the releases action, it will be easier to reuse. * release-prs: fix and build twice :/ There are no GitHub Actions that make it easy to wait for another workflow to complete :/ * release-branches: fix * release-tags: fix * release-prs: do the label check everywhere * release-*: fixup cache key * release-*: re-enable * fixup: add i686-linux builds
DeterminateSystems · Apr 15, 2024 · 60458ab · 60458ab
1 parent 1c4976a
commit 60458ab
Show file tree

Hide file tree

Showing 9 changed files with 408 additions and 109 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
diff --git a/.github/workflows/build-aarch64-darwin.yml b/.github/workflows/build-aarch64-darwin.yml
@@ -0,0 +1,29 @@
+name: Build aarch64 Darwin
+
+on: 
+  workflow_call:
+    inputs:
+      cache-key:
+        type: string
+        required: false
+        default: aarch64-darwin-artifacts-${{ github.sha }}
+
+jobs:
+  build-aarch64-darwin:
+    name: Build aarch64 Darwin
+    runs-on: macos-latest-xlarge
+    concurrency: ${{ inputs.cache-key }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build the installer
+        run: |
+          nix build .#packages.aarch64-darwin.nix-installer -L
+          cp result/bin/nix-installer .
+      - name: Create GitHub cache from build artifacts
+        uses: actions/cache/save@v3
+        with:
+          path: nix-installer
+          key: ${{ inputs.cache-key }}
diff --git a/.github/workflows/build-aarch64-linux.yml b/.github/workflows/build-aarch64-linux.yml
@@ -0,0 +1,29 @@
+name: Build aarch64 Linux (static)
+
+on: 
+  workflow_call:
+    inputs:
+      cache-key:
+        type: string
+        required: false
+        default: aarch64-linux-artifacts-${{ github.sha }}
+
+jobs:
+  build-aarch64-linux:
+    name: Build aarch64 Linux (static)
+    runs-on: namespace-profile-default-arm64
+    concurrency: ${{ inputs.cache-key }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build the installer
+        run: |
+          nix build .#packages.aarch64-linux.nix-installer-static -L
+          cp result/bin/nix-installer .
+      - name: Create GitHub cache from build artifacts
+        uses: actions/cache/save@v3
+        with:
+          path: nix-installer
+          key: ${{ inputs.cache-key }}
diff --git a/.github/workflows/build-x86_64-darwin.yml b/.github/workflows/build-x86_64-darwin.yml
@@ -0,0 +1,29 @@
+name: Build x86_64 Darwin
+
+on: 
+  workflow_call:
+    inputs:
+      cache-key:
+        type: string
+        required: false
+        default: x86_64-darwin-artifacts-${{ github.sha }}
+
+jobs:
+  build-x86_64-darwin:
+    name: Build x86_64 Darwin
+    runs-on: macos-latest-large
+    concurrency: ${{ inputs.cache-key }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build the installer
+        run: |
+          nix build .#packages.x86_64-darwin.nix-installer -L
+          cp result/bin/nix-installer .
+      - name: Create GitHub cache from build artifacts
+        uses: actions/cache/save@v3
+        with:
+          path: nix-installer
+          key: ${{ inputs.cache-key }}
diff --git a/.github/workflows/build-x86_64-linux.yml b/.github/workflows/build-x86_64-linux.yml
@@ -0,0 +1,29 @@
+name: Build x86_64 Linux (static)
+
+on: 
+  workflow_call:
+    inputs:
+      cache-key:
+        type: string
+        required: false
+        default: x86_64-linux-artifacts-${{ github.sha }}
+
+jobs:
+  build-x86_64-linux:
+    name: Build x86_64 Linux (static)
+    runs-on: UbuntuLatest32Cores128G
+    concurrency: ${{ inputs.cache-key }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build the installer
+        run: |
+          nix build .#packages.x86_64-linux.nix-installer-static -L
+          cp result/bin/nix-installer .
+      - name: Create GitHub cache from build artifacts
+        uses: actions/cache/save@v3
+        with:
+          path: nix-installer
+          key: ${{ inputs.cache-key }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,9 +6,15 @@ on:
     branches: [main]
 
 jobs:
+  build-x86_64-linux:
+    uses: ./.github/workflows/build-x86_64-linux.yml
+
+  build-x86_64-darwin:
+    uses: ./.github/workflows/build-x86_64-darwin.yml
+
   lints:
     name: Lints
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Check Nixpkgs input
@@ -29,42 +35,25 @@ jobs:
         run: nix develop --command check-nixpkgs-fmt
       - name: Check EditorConfig conformance
         run: nix develop --command check-editorconfig
-      - name: Download Buildkite Artifacts
-        uses: EnricoMi/[email protected]
-        with:
-          buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
-          output_path: artifacts
-      - name: Output list of Buildkite artifacts
-        run: |
-          ls -lah artifacts/
-          ls -lah artifacts/**/*
-      # Mac's can't run this action, so we're forced to do this.
-      - name: Create Github cache from Buildkite artifacts
-        id: cache-buildkite-artifacts
-        uses: actions/cache/save@v3
-        with:
-          path: artifacts
-          key: buildkite-artifacts-${{ github.sha }}
 
   run-x86_64-linux:
     name: Run x86_64 Linux
-    runs-on: ubuntu-22.04
-    needs: [lints]
+    runs-on: ubuntu-latest
+    needs: [lints, build-x86_64-linux]
     steps:
       - uses: actions/checkout@v3
       - name: Restore Github cache of Buildkite artifacts
-        id: cache-buildkite-artifacts
         uses: actions/cache/restore@v3
         with:
-          path: artifacts
-          key: buildkite-artifacts-${{ github.sha }}
-      - run: sudo apt install fish zsh
+          path: nix-installer
+          key: x86_64-linux-artifacts-${{ github.sha }}
       - name: Move & set executable
         run: |
           mkdir install-root
           cp nix-installer.sh install-root/nix-installer.sh
-          mv ./artifacts/nix-installer-x86_64-linux-*/* install-root/nix-installer-x86_64-linux
+          mv ./nix-installer install-root/nix-installer-x86_64-linux
           chmod +x install-root/nix-installer-x86_64-linux install-root/nix-installer.sh
+      - run: sudo apt install fish zsh
       - name: Initial install
         uses: DeterminateSystems/nix-installer-action@main
         with:
@@ -152,23 +141,22 @@ jobs:
 
   run-x86_64-linux-no-init:
     name: Run x86_64 Linux (No init)
-    runs-on: ubuntu-22.04
-    needs: [lints]
+    runs-on: ubuntu-latest
+    needs: [lints, build-x86_64-linux]
     steps:
       - uses: actions/checkout@v3
       - name: Restore Github cache of Buildkite artifacts
-        id: cache-buildkite-artifacts
         uses: actions/cache/restore@v3
         with:
-          path: artifacts
-          key: buildkite-artifacts-${{ github.sha }}
-      - run: sudo apt install fish zsh
+          path: nix-installer
+          key: x86_64-linux-artifacts-${{ github.sha }}
       - name: Move & set executable
         run: |
           mkdir install-root
           cp nix-installer.sh install-root/nix-installer.sh
-          mv ./artifacts/nix-installer-x86_64-linux-*/* install-root/nix-installer-x86_64-linux
+          mv ./nix-installer install-root/nix-installer-x86_64-linux
           chmod +x install-root/nix-installer-x86_64-linux install-root/nix-installer.sh
+      - run: sudo apt install fish zsh
       - name: Initial install
         uses: DeterminateSystems/nix-installer-action@main
         with:
@@ -262,23 +250,22 @@ jobs:
 
   run-x86_64-darwin:
     name: Run x86_64 Darwin
-    runs-on: macos-12
-    needs: [lints]
+    runs-on: macos-latest
+    needs: [lints, build-x86_64-darwin]
     steps:
       - uses: actions/checkout@v3
       - name: Restore Github cache of Buildkite artifacts
-        id: cache-buildkite-artifacts
         uses: actions/cache/restore@v3
         with:
-          path: artifacts
-          key: buildkite-artifacts-${{ github.sha }}
-      - run: brew install fish coreutils
+          path: nix-installer
+          key: x86_64-darwin-artifacts-${{ github.sha }}
       - name: Move & set executable
         run: |
           mkdir install-root
           cp nix-installer.sh install-root/nix-installer.sh
-          mv ./artifacts/nix-installer-x86_64-darwin-*/* install-root/nix-installer-x86_64-darwin
+          mv ./nix-installer install-root/nix-installer-x86_64-darwin
           chmod +x install-root/nix-installer-x86_64-darwin install-root/nix-installer.sh
+      - run: brew install fish coreutils
       - name: Initial install
         uses: DeterminateSystems/nix-installer-action@main
         with:

diff --git a/.github/workflows/release-branches.yml b/.github/workflows/release-branches.yml
@@ -1,5 +1,8 @@
 name: Release Branch
 
+concurrency:
+  group: release
+
 on:
   push:
     branches:
@@ -8,19 +11,83 @@ on:
       - 'main'
 
 jobs:
+  build-x86_64-linux:
+    uses: ./.github/workflows/build-x86_64-linux.yml
+    with:
+      cache-key: release-x86_64-linux-artifacts-${{ github.sha }}
+  build-i686-linux:
+    uses: ./.github/workflows/build-i686-linux.yml
+    with:
+      cache-key: release-i686-linux-artifacts-${{ github.sha }}
+  build-aarch64-linux:
+    uses: ./.github/workflows/build-aarch64-linux.yml
+    with:
+      cache-key: release-aarch64-linux-artifacts-${{ github.sha }}
+  build-x86_64-darwin:
+    uses: ./.github/workflows/build-x86_64-darwin.yml
+    with:
+      cache-key: release-x86_64-darwin-artifacts-${{ github.sha }}
+  build-aarch64-darwin:
+    uses: ./.github/workflows/build-aarch64-darwin.yml
+    with:
+      cache-key: release-aarch64-darwin-artifacts-${{ github.sha }}
+
   release:
-    concurrency: release
     runs-on: ubuntu-latest
     permissions:
       id-token: write # In order to request a JWT for AWS auth
+    needs:
+      - build-x86_64-linux
+      - build-i686-linux
+      - build-aarch64-linux
+      - build-x86_64-darwin
+      - build-aarch64-darwin
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Download Buildkite Artifacts
-        uses: EnricoMi/[email protected]
+      - name: Create artifacts directory
+        run: mkdir -p ./artifacts
+
+      - name: Fetch cached x86_64-linux binary
+        uses: actions/cache/restore@v3
+        with:
+          path: nix-installer
+          key: release-x86_64-linux-artifacts-${{ github.sha }}
+      - name: Move artifact to artifacts directory
+        run: mv ./nix-installer ./artifacts/nix-installer-x86_64-linux
+
+      - name: Fetch cached i686-linux binary
+        uses: actions/cache/restore@v3
+        with:
+          path: nix-installer
+          key: release-i686-linux-artifacts-${{ github.sha }}
+      - name: Move artifact to artifacts directory
+        run: mv ./nix-installer ./artifacts/nix-installer-i686-linux
+
+      - name: Fetch cached aarch64-linux binary
+        uses: actions/cache/restore@v3
+        with:
+          path: nix-installer
+          key: release-aarch64-linux-artifacts-${{ github.sha }}
+      - name: Move artifact to artifacts directory
+        run: mv ./nix-installer ./artifacts/nix-installer-aarch64-linux
+
+      - name: Fetch cached x86_64-darwin binary
+        uses: actions/cache/restore@v3
         with:
-          buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
-          output_path: artifacts
+          path: nix-installer
+          key: release-x86_64-darwin-artifacts-${{ github.sha }}
+      - name: Move artifact to artifacts directory
+        run: mv ./nix-installer ./artifacts/nix-installer-x86_64-darwin
+
+      - name: Fetch cached aarch64-darwin binary
+        uses: actions/cache/restore@v3
+        with:
+          path: nix-installer
+          key: release-aarch64-darwin-artifacts-${{ github.sha }}
+      - name: Move artifact to artifacts directory
+        run: mv ./nix-installer ./artifacts/nix-installer-aarch64-darwin
+
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
         with: