Add Benchmarking CI

JuliaHealth · Jul 3, 2024 · 704d2a0 · 704d2a0
1 parent 17ef89d
commit 704d2a0
Show file tree

Hide file tree

Showing 10 changed files with 605 additions and 159 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,166 +1,28 @@
 steps:
-  - label: "CPU: Run tests on v{{matrix.version}}"
-    matrix:
-      setup:
-        version:
-          - "1.9"
-          - "1"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "{{matrix.version}}"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-          dirs:
-            - KomaMRICore/src
-            - KomaMRICore/ext
-    command: |
-      julia -e 'println("--- :julia: Instantiating project")
-          using Pkg
-          Pkg.develop([
-              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
-              PackageSpec(path=pwd(), subdir="KomaMRICore"),
-          ])'
-
-      julia -e 'println("--- :julia: Running tests")
-          using Pkg
-          Pkg.test("KomaMRICore"; coverage=true, julia_args=`--threads=auto`)'
+  - label: ":pipeline: Launch Tests"
+    command: buildkite-agent pipeline upload .buildkite/runtests.yml
     agents:
       queue: "juliagpu"
-    timeout_in_minutes: 120
-
-  - label: "AMDGPU: Run tests on v{{matrix.version}}"
-    matrix:
-      setup:
-        version:
-          - "1"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "{{matrix.version}}"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-          dirs:
-            - KomaMRICore/src
-            - KomaMRICore/ext
-    command: |
-      julia -e 'println("--- :julia: Instantiating project")
-          using Pkg
-          Pkg.develop([
-              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
-              PackageSpec(path=pwd(), subdir="KomaMRICore"),
-          ])'
-      
-      julia --project=KomaMRICore/test -e 'println("--- :julia: Add AMDGPU to test environment")
-          using Pkg
-          Pkg.add("AMDGPU")'
-      
-      julia -e 'println("--- :julia: Running tests")
-          using Pkg
-          Pkg.test("KomaMRICore"; coverage=true, test_args=["AMDGPU"])'
-    agents:
-      queue: "juliagpu"
-      rocm: "*"
-    timeout_in_minutes: 120
 
-  - label: "CUDA: Run tests on v{{matrix.version}}"
-    matrix:
-      setup:
-        version:
-          - "1.9"
-          - "1"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "{{matrix.version}}"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-          dirs:
-            - KomaMRICore/src
-            - KomaMRICore/ext
-    command: |
-      julia -e 'println("--- :julia: Instantiating project")
-          using Pkg
-          Pkg.develop([
-              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
-              PackageSpec(path=pwd(), subdir="KomaMRICore"),
-          ])'
-      
-      julia --project=KomaMRICore/test -e 'println("--- :julia: Add CUDA to test environment")
-          using Pkg
-          Pkg.add("CUDA")'
-      
-      julia -e 'println("--- :julia: Running tests")
-          using Pkg
-          Pkg.test("KomaMRICore"; coverage=true, test_args=["CUDA"])'
+  - label: ":pipeline: Launch Benchmarks"
+    if: build.message !~ /skip benchmarks/
     agents:
       queue: "juliagpu"
-      cuda: "*"
-    timeout_in_minutes: 120
-
-  - label: "Metal: Run tests on v{{matrix.version}}"
-    matrix:
-      setup:
-        version:
-          - "1.9"
-          - "1"
     plugins:
-      - JuliaCI/julia#v1:
-          version: "{{matrix.version}}"
-    command: |
-      julia -e 'println("--- :julia: Instantiating project")
-          using Pkg
-          Pkg.develop([
-              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
-              PackageSpec(path=pwd(), subdir="KomaMRICore"),
-          ])'
-      
-      julia --project=KomaMRICore/test -e 'println("--- :julia: Add Metal to test environment")
-          using Pkg
-          Pkg.add("Metal")'
-
-      julia -e 'println("--- :julia: Running tests")
-          using Pkg
-          Pkg.test("KomaMRICore"; test_args=["Metal"])'
-    agents:
-      queue: "juliaecosystem"
-      os: "macos"
-      arch: "aarch64"
-    timeout_in_minutes: 120
-
-  - label: "oneAPI: Run tests on v{{matrix.version}}"
-    matrix:
-      setup:
-        version:
-          - "1.9"
-          - "1"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "{{matrix.version}}"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-          dirs:
-            - KomaMRICore/src
-            - KomaMRICore/ext
-    command: |
-      julia -e 'println("--- :julia: Instantiating project")
-          using Pkg
-          Pkg.develop([
-              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
-              PackageSpec(path=pwd(), subdir="KomaMRICore"),
-          ])'
-      
-      julia --project=KomaMRICore/test -e 'println("--- :julia: Add oneAPI to test environment")
-          using Pkg
-          Pkg.add("oneAPI")'
-      
-      julia -e 'println("--- :julia: Running tests")
-          using Pkg
-          Pkg.test("KomaMRICore"; coverage=true, test_args=["oneAPI"])'
-    agents:
-      queue: "juliagpu"
-      intel: "*"
-    timeout_in_minutes: 120
-
-env:
-  CI: BUILDKITE
-  CODECOV_FLAGS: core
-  JULIA_PKG_SERVER: ""
-  SECRET_CODECOV_TOKEN: "lUmUVYkTlE8u0mR/ymv5rtE1A59wXZmQ3miRkmKciC/4+xHPNjpeIJ03FupuC1ElGBeX0m6DDFavZ9burLosGxbBYIPziBQZ5P9NdPDZjBdo7NM3QSSBeUfDsDYbHsYglfJZ35UL6Pd2YTAkJG0ePrTpfUaBb9rcll926NdqUP0vE2hbR2leKFFgBVNtK9Zf+NE7hO3meZQEZ+sN5tA7xGr24A3Ay7ckPg5HbPPD3KII2/fLtW+w0fQUfJdXNFrajJ0FyNE0kNLDlIzoYN6XM4yUeLBXoyXFHnqT5dSu7pqrByEk/ptYpjUzKFoRSnDQy+p8vrx9e/iAVf3lwwgwWA==;U2FsdGVkX1+IZLGTj2FNEy4XvUQbzkZFEwUpwIdRDCKrEea5O/OKGV5vYJufty3m9yniE+av4937HabS9dO1RA=="
+      - monorepo-diff#v1.0.1:
+          diff: ".buildkite/scripts/diff.sh $BUILDKITE_COMMIT"
+          interpolation: false
+          watch:
+            - path:
+                - "KomaMRICore/src/**/*"
+                - "KomaMRICore/ext/**/*"
+                - "KomaMRICore/Project.toml"
+                - "KomaMRIBase/src/**/*"
+                - "KomaMRIBase/Project.toml"
+                - "benchmarks/**/*"
+                - ".buildkite/**/*"
+                - "Project.toml"
+              config:
+                command: "buildkite-agent pipeline upload .buildkite/runbenchmarks.yml"
+                agents:
+                  queue: "juliagpu"
diff --git a/.buildkite/runbenchmarks.yml b/.buildkite/runbenchmarks.yml
@@ -0,0 +1,164 @@
+steps:
+  - group: ":racehorse: Benchmarks"
+    steps:
+      - label: "CPU: Run Benchmarks with {{matrix.threads}} thread(s)"
+        matrix:
+          setup:
+            threads:
+              - "1"
+              - "2"
+              - "4"
+              - "8"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "1"
+        command: |
+          julia --project=benchmarks -e 'println("--- :julia: Instantiating project")
+              using Pkg
+              Pkg.develop([
+                  PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+                  PackageSpec(path=pwd(), subdir="KomaMRICore"),
+              ])'
+          
+          julia --project=benchmarks -e 'println("--- :julia: Run Benchmarks")
+              include("benchmarks/runbenchmarks.jl")'
+        artifact_paths: 
+          - "benchmarks/results/*"
+        agents:
+          queue: "benchmark"
+        env:
+          BENCHMARK_GROUP: CPU
+          JULIA_NUM_THREADS: "{{matrix.threads}}"
+        timeout_in_minutes: 120
+
+      - label: "AMDGPU: Run Benchmarks"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "1"
+        command: |
+          julia --project=benchmarks -e 'println("--- :julia: Instantiating project")
+              using Pkg
+              Pkg.develop([
+                  PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+                  PackageSpec(path=pwd(), subdir="KomaMRICore"),
+              ])'
+
+          julia --project=benchmarks -e 'println("---:julia: Add AMDGPU to benchmarks environment")
+              using Pkg
+              Pkg.add("AMDGPU")'
+          
+          julia --project=benchmarks -e 'println("--- :julia: Run Benchmarks")
+              include("benchmarks/runbenchmarks.jl")'
+        artifact_paths: 
+          - "benchmarks/results/*"
+        agents:
+          queue: "juliagpu"
+          rocm: "*"
+          rocmgpu: "gfx1101"
+        env:
+          BENCHMARK_GROUP: AMDGPU
+        timeout_in_minutes: 120
+
+      - label: "CUDA: Run Benchmarks"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "1"
+        command: |
+          julia --project=benchmarks -e 'println("--- :julia: Instantiating project")
+              using Pkg
+              Pkg.develop([
+                  PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+                  PackageSpec(path=pwd(), subdir="KomaMRICore"),
+              ])'
+
+          julia --project=benchmarks -e 'println("---:julia: Add CUDA to benchmarks environment")
+              using Pkg
+              Pkg.add("CUDA")'
+          
+          julia --project=benchmarks -e 'println("--- :julia: Run Benchmarks")
+              include("benchmarks/runbenchmarks.jl")'
+        artifact_paths: 
+          - "benchmarks/results/*"
+        agents:
+          queue: "benchmark"
+          gpu: "rtx2070"
+          cuda: "*"
+        env:
+          BENCHMARK_GROUP: CUDA
+        timeout_in_minutes: 120
+
+      - label: "Metal: Run Benchmarks"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "1"
+        command: |
+          julia --project=benchmarks -e 'println("--- :julia: Instantiating project")
+              using Pkg
+              Pkg.develop([
+                  PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+                  PackageSpec(path=pwd(), subdir="KomaMRICore"),
+              ])'
+
+          julia --project=benchmarks -e 'println("---:julia: Add Metal to benchmarks environment")
+              using Pkg
+              Pkg.add("Metal")'
+          
+          julia --project=benchmarks -e 'println("--- :julia: Run Benchmarks")
+              include("benchmarks/runbenchmarks.jl")'
+        artifact_paths: 
+          - "benchmarks/results/*"
+        agents:
+          queue: "juliaecosystem"
+          os: "macos"
+          arch: "aarch64"
+        env:
+          BENCHMARK_GROUP: Metal
+        timeout_in_minutes: 120
+
+      - label: "oneAPI: Run Benchmarks"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "1"
+        command: |
+          julia --project=benchmarks -e 'println("--- :julia: Instantiating project")
+              using Pkg
+              Pkg.develop([
+                  PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+                  PackageSpec(path=pwd(), subdir="KomaMRICore"),
+              ])'
+
+          julia --project=benchmarks -e 'println("---:julia: Add CUDA to benchmarks environment")
+              using Pkg
+              Pkg.add("oneAPI")'
+          
+          julia --project=benchmarks -e 'println("--- :julia: Run Benchmarks")
+              include("benchmarks/runbenchmarks.jl")'
+        artifact_paths: 
+          - "benchmarks/results/*"
+        agents:
+          queue: "juliagpu"
+          intel: "*"
+        env:
+          BENCHMARK_GROUP: oneAPI
+        timeout_in_minutes: 120
+
+      - wait: ~
+
+      - label: "Combine benchmarks"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "1"
+        command: |
+          buildkite-agent artifact download "benchmarks/results/*" .
+
+          julia --project=benchmarks -e 'println("--- :julia: Instantiating project")
+              using Pkg
+              Pkg.instantiate()'
+
+          julia --project=benchmarks -e 'println("--- :julia: Combining Benchmarks")
+              include("benchmarks/aggregate.jl")'
+        artifact_paths:
+          - "benchmarks/results/combinedbenchmarks.json"
+        agents:
+          queue: "juliagpu"
+        timeout_in_minutes: 10