From a487747708567130ac5d9d434a9b3605432761f1 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Thu, 28 Sep 2023 01:20:16 +0200
Subject: [PATCH 1/2] workflows/check-by-name: Better error when base branch
 also fails

Previously, even if the check also failed on the base branch, it looked
like the PR introduced the failure.

We can easily have a better error message for such cases.

Meanwhile this also paves the road for something like
https://github.com/NixOS/nixpkgs/issues/256788
---
 .github/workflows/check-by-name.yml | 91 ++++++++++++++++++++++++++++-
 1 file changed, 90 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/check-by-name.yml b/.github/workflows/check-by-name.yml
index 7a3598dbe2a4b..e8f7aa9071022 100644
--- a/.github/workflows/check-by-name.yml
+++ b/.github/workflows/check-by-name.yml
@@ -21,6 +21,16 @@ jobs:
         with:
           # pull_request_target checks out the base branch by default
           ref: refs/pull/${{ github.event.pull_request.number }}/merge
+          # Fetches the merge commit and its parents
+          fetch-depth: 2
+      - name: Determining PR git hashes
+        run: |
+          echo "mergedSha=$(git rev-parse HEAD)" >> "$GITHUB_ENV"
+
+          # For pull_request_target this is the same as $GITHUB_SHA
+          echo "baseSha=$(git rev-parse HEAD^1)" >> "$GITHUB_ENV"
+
+          echo "headSha=$(git rev-parse HEAD^2)" >> "$GITHUB_ENV"
       - uses: cachix/install-nix-action@v23
       - name: Determining channel to use for dependencies
         run: |
@@ -51,4 +61,83 @@ jobs:
           # Passing --max-jobs 0 makes sure that we won't build anything
           nix-build "$nixpkgs" -A tests.nixpkgs-check-by-name --max-jobs 0
       - name: Running nixpkgs-check-by-name
-        run: result/bin/nixpkgs-check-by-name .
+        run: |
+          echo "Checking whether the check succeeds on the base branch $GITHUB_BASE_REF"
+          git checkout -q "$baseSha"
+          if baseOutput=$(result/bin/nixpkgs-check-by-name . 2>&1); then
+            baseSuccess=1
+          else
+            baseSuccess=
+          fi
+          printf "%s\n" "$baseOutput"
+
+          echo "Checking whether the check would succeed after merging this pull request"
+          git checkout -q "$mergedSha"
+          if mergedOutput=$(result/bin/nixpkgs-check-by-name . 2>&1); then
+            mergedSuccess=1
+            exitCode=0
+          else
+            mergedSuccess=
+            exitCode=1
+          fi
+          printf "%s\n" "$mergedOutput"
+
+          resultToEmoji() {
+            if [[ -n "$1" ]]; then
+              echo ":heavy_check_mark:"
+            else
+              echo ":x:"
+            fi
+          }
+
+          # Print a markdown summary in GitHub actions
+          {
+            echo "| Nixpkgs version | Check result |"
+            echo "| --- | --- |"
+            echo "| Latest base commit | $(resultToEmoji "$baseSuccess") |"
+            echo "| After merging this PR | $(resultToEmoji "$mergedSuccess") |"
+            echo ""
+
+            if [[ -n "$baseSuccess" ]]; then
+              if [[ -n "$mergedSuccess" ]]; then
+                echo "The check succeeds both the base branch and after merging this PR"
+              else
+                echo "The check succeeds on the base branch, but would fail after merging this PR:"
+                echo "\`\`\`"
+                echo "$mergedOutput"
+                echo "\`\`\`"
+                echo ""
+              fi
+            else
+              if [[ -n "$mergedSuccess" ]]; then
+                echo "The check fails on the base branch, but this PR fixes it, nicely done!"
+              else
+                echo "The check fails both on the base branch and after merging this PR, unknown if only this PRs changes would satisfy the check, the base branch needs to be fixed first."
+                echo ""
+                echo "Failure on the base branch:"
+                echo "\`\`\`"
+                echo "$baseOutput"
+                echo "\`\`\`"
+                echo ""
+                echo "Failure after merging this PR:"
+                echo "\`\`\`"
+                echo "$mergedOutput"
+                echo "\`\`\`"
+                echo ""
+              fi
+            fi
+
+            echo "### Details"
+            echo "- nixpkgs-check-by-name tool:"
+            echo "  - Channel: $channel"
+            echo "  - Nixpkgs commit: [$rev](https://github.com/${GITHUB_REPOSITORY}/commit/$rev)"
+            echo "  - Store path: $(realpath result)"
+            echo "- Tested Nixpkgs:"
+            echo "  - Base branch $GITHUB_BASE_REF"
+            echo "  - Latest base branch commit: [$baseSha](https://github.com/${GITHUB_REPOSITORY}/commit/$baseSha)"
+            echo "  - Latest PR commit: [$headSha](https://github.com/${GITHUB_REPOSITORY}/commit/$headSha)"
+            echo "  - Merge commit: [$mergedSha](https://github.com/${GITHUB_REPOSITORY}/commit/$mergedSha)"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          exit "$exitCode"
+

From 8807975dd433c4b27758fd4ba56045b628c56c25 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Thu, 28 Sep 2023 01:57:15 +0200
Subject: [PATCH 2/2] This is okay

---
 pkgs/by-name/fo/foo/package.nix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 pkgs/by-name/fo/foo/package.nix

diff --git a/pkgs/by-name/fo/foo/package.nix b/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 0000000000000..f86d0f699385d
--- /dev/null
+++ b/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ emptyFile }: emptyFile