Merge pull request #1583 from helmholtz-analytics/fix/remove-trigger-…

…benchmark-status-failure

feat: CB Grafana Dashboard and PR comments

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,6 +6,8 @@
 - Implementation:
     - [ ] unit tests: all split configurations tested
     - [ ] unit tests: multiple dtypes tested
+    - [ ] benchmarks: created for new functionality
+    - [ ] benchmarks: performance improved or maintained
     - [ ] documentation updated where needed
 
 ## Description

.github/workflows/bench_trigger.yml

@@ -28,7 +28,7 @@ jobs:
           SHA: ${{ github.event.pull_request.head.sha }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
         run: |
-          SHORT_SHA=$(git rev-parse --short ${{ github.event.pull_request.head.sha }})
+          SHORT_SHA=$(git rev-parse --short $SHA)
           curl -s -X POST \
             --fail-with-body \
             -F "token=$PIPE_TRIGGER_TOKEN" \
@@ -45,7 +45,7 @@ jobs:
         env:
           AUTHOR: ${{ github.event.pull_request.assignee.login }}
           PIPE_TRIGGER_TOKEN: ${{ secrets.BENCH_PIPE_TRIGGER }}
-          SHA: ${{ github.event.pull_request.head.sha }}
+          SHA: ${{ github.sha }}
         run: |
           SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)
           curl -s -X POST \
@@ -57,16 +57,3 @@ jobs:
             -F "variables[BRANCH]=main" \
             -F "variables[AUTHOR]=${AUTHOR:-heat_team}" \
             https://codebase.helmholtz.cloud/api/v4/projects/7930/trigger/pipeline
-      - name: Create status
-        if: ${{ steps.setup_pr.outcome == 'success' || steps.setup_push.outcome == 'success'}}
-        env:
-          REPO: ${{ github.repository }}
-          SHA: ${{ github.event.pull_request.head.sha }}
-        run: |
-          curl -L -X POST \
-            --fail-with-body \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/$REPO/statuses/$SHA \
-            -d '{ "state":"pending", "target_url":"https://codebase.helmholtz.cloud/helmholtz-analytics/cb/-/pipelines", "description":"Waiting for benchmarks to execute.", "context":"cb/report" }'

.perun.ini

@@ -5,3 +5,12 @@ data_out = ./bench_data
 [benchmarking]
 rounds = 10
 warmup_rounds = 1
+metrics=runtime
+region_metrics=runtime
+
+[benchmarking.units]
+joule = k
+second =
+percent =
+watt =
+byte = G

README.md

@@ -19,7 +19,7 @@ Heat is a distributed tensor framework for high performance data analytics.
 [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/helmholtz-analytics/heat/badge)](https://securityscorecards.dev/viewer/?uri=github.com/helmholtz-analytics/heat)
 [![OpenSSF Best Practices](https://bestpractices.coreinfrastructure.org/projects/7688/badge)](https://bestpractices.coreinfrastructure.org/projects/7688)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2531472.svg)](https://doi.org/10.5281/zenodo.2531472)
-[![Benchmarks](https://img.shields.io/badge/Github--Pages-Benchmarks-2ea44f)](https://helmholtz-analytics.github.io/heat/dev/bench)
+[![Benchmarks](https://img.shields.io/badge/Grafana-Benchmarks-2ea44f)](https://57bc8d92-72f2-4869-accd-435ec06365cb.ka.bw-cloud-instance.org:3000/d/adjpqduq9r7k0a/heat-cb?orgId=1)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 # Table of Contents

benchmarks/cb/manipulations.py

@@ -1,5 +1,6 @@
 # flake8: noqa
 import heat as ht
+from typing import List
 from perun import monitor
 
 
@@ -15,6 +16,13 @@ def reshape(arrays):
         a = ht.reshape(array, (10000000, -1), new_split=1)
 
 
+@monitor()
+def resplit(array, new_split: List[int | None]):
+    for new_split in new_split:
+        a = ht.resplit(array, axis=new_split)
+        del a
+
+
 def run_manipulation_benchmarks():
     sizes = [10000, 20000, 40000]
     arrays = []
@@ -30,3 +38,13 @@ def run_manipulation_benchmarks():
             split = 1
         arrays.append(ht.zeros((1000, size), split=split))
     concatenate(arrays)
+
+    if ht.comm.size > 1:
+        shape = [100, 50, 50, 20, 86]
+        n_elements = ht.array(shape).prod().item()
+        mem = n_elements * 4 / 1e9
+        array = ht.reshape(ht.arange(0, n_elements, split=0, dtype=ht.float32), shape) * (
+            ht.comm.rank + 1
+        )
+
+        resplit(array, [None, 2, 4])