diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 255f3e27..1a4e416a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -129,33 +129,29 @@ steps: timeout_in_minutes: 60 - wait: ~ + continue_on_failure: true - - group: ":racehorse: Benchmarks" - steps: - - label: "Benchmarks" - plugins: - - JuliaCI/julia#v1: - version: "1.11" - command: | - julia --project=perf -e ' - using Pkg + - label: ":racehorse: Benchmarks" + plugins: + - JuliaCI/julia#v1: + version: "1.11" + command: | + julia --project=perf -e ' + using Pkg - println("--- :julia: Instantiating project") - Pkg.develop([PackageSpec(path=pwd())]) - Pkg.instantiate() - push!(LOAD_PATH, @__DIR__) + println("--- :julia: Instantiating project") + Pkg.develop([PackageSpec(path=pwd())]) - println("+++ :julia: Benchmarking") - include("perf/runbenchmarks.jl")' - artifact_paths: - - "benchmarkresults.json" - agents: - queue: "juliaecosystem" - os: "macos" - arch: "aarch64" - if: | - !build.pull_request.repository.fork && - (build.message =~ /\[only benchmarks\]/ || - build.message !~ /\[only/ && !build.pull_request.draft && - build.message !~ /\[skip benchmarks\]/) - timeout_in_minutes: 30 + println("+++ :julia: Benchmarking") + include("perf/runbenchmarks.jl")' + artifact_paths: + - "benchmarkresults.json" + agents: + queue: "juliaecosystem" + os: "macos" + arch: "aarch64" + if: | + build.message =~ /\[only benchmarks\]/ || + build.message !~ /\[only/ && !build.pull_request.draft && + build.message !~ /\[skip benchmarks\]/ + timeout_in_minutes: 30 diff --git a/.buildkite/secure_pipeline.yml b/.buildkite/secure_pipeline.yml index 5040254c..3e853b78 100644 --- a/.buildkite/secure_pipeline.yml +++ b/.buildkite/secure_pipeline.yml @@ -25,5 +25,8 @@ steps: queue: "juliaecosystem" os: "macos" arch: "aarch64" - if: build.message !~ /\[skip docs\]/ && !build.pull_request.draft + if: | + build.message =~ /\[only docs\]/ || + build.message !~ /\[only/ && !build.pull_request.draft && + build.message !~ /\[skip docs\]/ timeout_in_minutes: 30 diff --git a/.buildkite/secure_pipeline.yml.signature b/.buildkite/secure_pipeline.yml.signature index 30cee603..a04dd890 100644 Binary files a/.buildkite/secure_pipeline.yml.signature and b/.buildkite/secure_pipeline.yml.signature differ diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 1871214a..fdb0d50e 100644 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -1,12 +1,12 @@ name: Benchmarks permissions: - contents: write # contents permission to update benchmark contents in gh-pages branch - statuses: read - deployments: write # deployments permission to deploy GitHub pages website - pull-requests: write + statuses: read # find Buildkite URL from PR status + contents: write # update benchmark contents in gh-pages branch + pull-requests: write # comment on PR with benchmark results + deployments: write # deploy GitHub pages website on: - pull_request: + pull_request_target: branches: - main paths: @@ -16,7 +16,6 @@ on: - "perf/**/*" - ".buildkite/**/*" - "Project.toml" - - ".github/workflows/Benchmark.yml" push: branches: - main @@ -27,26 +26,17 @@ on: - "perf/**/*" - ".buildkite/**/*" - "Project.toml" - - ".github/workflows/Benchmark.yml" jobs: benchmark: - if: | - (github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository) && - (contains(github.event.head_commit.message, '[only benchmarks]') || - !contains(github.event.head_commit.message, '[only') && - !contains(github.event.head_commit.message, '[skip benchmarks]') && - github.event.pull_request.draft == false) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - name: Download Buildkite Artifacts id: download uses: EnricoMi/download-buildkite-artifact-action@v1 with: buildkite_token: ${{ secrets.BUILDKITE_TOKEN }} - ignore_build_states: blocked,canceled,skipped,not_run,failed + ignore_build_states: blocked,canceled,skipped,not_run ignore_job_states: timed_out,failed output_path: artifacts @@ -66,6 +56,6 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} comment-always: ${{ github.event_name == 'pull_request' }} summary-always: true - alert-threshold: "150%" + alert-threshold: "125%" fail-on-alert: false auto-push: ${{ github.event_name != 'pull_request' }} diff --git a/perf/runbenchmarks.jl b/perf/runbenchmarks.jl index 6888700a..5e80945d 100644 --- a/perf/runbenchmarks.jl +++ b/perf/runbenchmarks.jl @@ -7,9 +7,6 @@ using BenchmarkTools using StableRNGs rng = StableRNG(123) -# to find untuned benchmarks -BenchmarkTools.DEFAULT_PARAMETERS.evals = 0 - # print system information @info "System information:\n" * sprint(io->Metal.versioninfo(io))