Tweak benchmark CI (#501)

[only benchmarks]
JuliaGPU · Dec 18, 2024 · ba62282 · ba62282
1 parent d37e9dd
commit ba62282
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 48 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -129,33 +129,29 @@ steps:
         timeout_in_minutes: 60
 
   - wait: ~
+    continue_on_failure: true
 
-  - group: ":racehorse: Benchmarks"
-    steps:
-      - label: "Benchmarks"
-        plugins:
-          - JuliaCI/julia#v1:
-              version: "1.11"
-        command: |
-          julia --project=perf -e '
-            using Pkg
+  - label: ":racehorse: Benchmarks"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.11"
+    command: |
+      julia --project=perf -e '
+        using Pkg
 
-            println("--- :julia: Instantiating project")
-            Pkg.develop([PackageSpec(path=pwd())])
-            Pkg.instantiate()
-            push!(LOAD_PATH, @__DIR__)
+        println("--- :julia: Instantiating project")
+        Pkg.develop([PackageSpec(path=pwd())])
 
-            println("+++ :julia: Benchmarking")
-            include("perf/runbenchmarks.jl")'
-        artifact_paths:
-          - "benchmarkresults.json"
-        agents:
-          queue: "juliaecosystem"
-          os: "macos"
-          arch: "aarch64"
-        if: |
-          !build.pull_request.repository.fork &&
-          (build.message =~ /\[only benchmarks\]/ ||
-           build.message !~ /\[only/ && !build.pull_request.draft &&
-             build.message !~ /\[skip benchmarks\]/)
-        timeout_in_minutes: 30
+        println("+++ :julia: Benchmarking")
+        include("perf/runbenchmarks.jl")'
+    artifact_paths:
+      - "benchmarkresults.json"
+    agents:
+      queue: "juliaecosystem"
+      os: "macos"
+      arch: "aarch64"
+    if: |
+      build.message =~ /\[only benchmarks\]/ ||
+      build.message !~ /\[only/ && !build.pull_request.draft &&
+        build.message !~ /\[skip benchmarks\]/
+    timeout_in_minutes: 30
diff --git a/.buildkite/secure_pipeline.yml b/.buildkite/secure_pipeline.yml
@@ -25,5 +25,8 @@ steps:
       queue: "juliaecosystem"
       os: "macos"
       arch: "aarch64"
-    if: build.message !~ /\[skip docs\]/ && !build.pull_request.draft
+    if: |
+      build.message =~ /\[only docs\]/ ||
+      build.message !~ /\[only/ && !build.pull_request.draft &&
+        build.message !~ /\[skip docs\]/
     timeout_in_minutes: 30
diff --git a/.buildkite/secure_pipeline.yml.signature b/.buildkite/secure_pipeline.yml.signature
diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml
@@ -1,12 +1,12 @@
 name: Benchmarks
 permissions:
-  contents: write # contents permission to update benchmark contents in gh-pages branch
-  statuses: read
-  deployments: write # deployments permission to deploy GitHub pages website
-  pull-requests: write
+  statuses: read        # find Buildkite URL from PR status
+  contents: write       # update benchmark contents in gh-pages branch
+  pull-requests: write  # comment on PR with benchmark results
+  deployments: write    # deploy GitHub pages website
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
       - main
     paths:
@@ -16,7 +16,6 @@ on:
       - "perf/**/*"
       - ".buildkite/**/*"
       - "Project.toml"
-      - ".github/workflows/Benchmark.yml"
   push:
     branches:
       - main
@@ -27,26 +26,17 @@ on:
       - "perf/**/*"
       - ".buildkite/**/*"
       - "Project.toml"
-      - ".github/workflows/Benchmark.yml"
 
 jobs:
   benchmark:
-    if: |
-      (github.event_name != 'pull_request' ||
-       github.event.pull_request.head.repo.full_name == github.repository) &&
-      (contains(github.event.head_commit.message, '[only benchmarks]') ||
-       !contains(github.event.head_commit.message, '[only') &&
-       !contains(github.event.head_commit.message, '[skip benchmarks]') &&
-       github.event.pull_request.draft == false)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
       - name: Download Buildkite Artifacts
         id: download
         uses: EnricoMi/download-buildkite-artifact-action@v1
         with:
           buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
-          ignore_build_states: blocked,canceled,skipped,not_run,failed
+          ignore_build_states: blocked,canceled,skipped,not_run
           ignore_job_states: timed_out,failed
           output_path: artifacts
 
@@ -66,6 +56,6 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           comment-always: ${{ github.event_name == 'pull_request' }}
           summary-always: true
-          alert-threshold: "150%"
+          alert-threshold: "125%"
           fail-on-alert: false
           auto-push: ${{ github.event_name != 'pull_request' }}
diff --git a/perf/runbenchmarks.jl b/perf/runbenchmarks.jl
@@ -7,9 +7,6 @@ using BenchmarkTools
 using StableRNGs
 rng = StableRNG(123)
 
-# to find untuned benchmarks
-BenchmarkTools.DEFAULT_PARAMETERS.evals = 0
-
 # print system information
 @info "System information:\n" * sprint(io->Metal.versioninfo(io))