chore: Add benchmarking dashboard

.github/workflows/reports.yml

@@ -428,6 +428,10 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
+      # deployments permission to deploy GitHub pages website
+      deployments: write
+      # contents permission to update benchmark contents in gh-pages branch
+      contents: write
 
     steps:
       - uses: actions/checkout@v4
@@ -464,6 +468,27 @@ jobs:
           header: compilation
           message: ${{ steps.compilation_report.outputs.markdown }}
 
+      - name: Convert to `benchmark-action` format
+        run: |
+          jq ".compilation_reports | map({name: .artifact_name, value: (.time[:-1] | tonumber), unit: \"s\"}) " ./compilation_report.json > time_bench.json
+
+      - name: Store benchmark result
+        # We always push new benchmarks so we want this to only run on master to avoid garbage data.``
+        # if: github.ref == 'refs/heads/master'
+        continue-on-error: true
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+        with:
+          name: "Compilation Time"
+          tool: "customSmallerIsBetter"
+          output-file-path: ./time_bench.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: "110%"
+          comment-on-alert: true
+          fail-on-alert: false
+          alert-comment-cc-users: "@TomAFrench"
+          max-items-in-chart: 50
+
   external_repo_memory_report:
     needs: [build-nargo]
     runs-on: ubuntu-latest
@@ -507,6 +532,7 @@ jobs:
           path: scripts
           sparse-checkout: |
             test_programs/memory_report.sh
+            test_programs/parse_memory.sh
           sparse-checkout-cone-mode: false
 
       - name: Checkout
@@ -520,6 +546,7 @@ jobs:
         working-directory: ./test-repo/${{ matrix.project.path }}
         run: |
           mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh
+          mv /home/runner/work/noir/noir/scripts/test_programs/parse_memory.sh ./parse_memory.sh
           ./memory_report.sh 1
           # Rename the memory report as the execution report is about to write to the same file
           cp memory_report.json compilation_memory_report.json
@@ -611,6 +638,27 @@ jobs:
           header: compilation_memory
           message: ${{ steps.compilation_mem_report.outputs.markdown }}
 
+      - name: Convert to `benchmark-action` format
+        run: |
+          jq ".memory_reports | map({name: .artifact_name, value: (.peak_memory | tonumber), unit: \"MB\"}) " ./memory_report.json > memory_bench.json
+
+      - name: Store benchmark result
+        # We always push new benchmarks so we want this to only run on master to avoid garbage data.``
+         # if: github.ref == 'refs/heads/master'
+        continue-on-error: true
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+        with:
+          name: "Compilation Memory"
+          tool: "customSmallerIsBetter"
+          output-file-path: ./memory_bench.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: "110%"
+          comment-on-alert: true
+          fail-on-alert: false
+          alert-comment-cc-users: "@TomAFrench"
+          max-items-in-chart: 50
+
   upload_execution_memory_report:
     name: Upload execution memory report 
     needs: [generate_memory_report, external_repo_memory_report]
@@ -657,6 +705,28 @@ jobs:
           header: execution_memory
           message: ${{ steps.execution_mem_report.outputs.markdown }}
 
+      - name: Convert to `benchmark-action` format
+        run: |
+          jq ".memory_reports | map({name: .artifact_name, value: (.peak_memory | tonumber), unit: \"MB\"}) " ./execution_memory_report.json > memory_bench.json
+
+      - name: Store benchmark result
+        # We always push new benchmarks so we want this to only run on master to avoid garbage data.``
+         # if: github.ref == 'refs/heads/master'
+        continue-on-error: true
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+        with:
+          name: "Execution Memory"
+          tool: "customSmallerIsBetter"
+          output-file-path: ./memory_bench.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: "110%"
+          comment-on-alert: true
+          fail-on-alert: false
+          alert-comment-cc-users: "@TomAFrench"
+          max-items-in-chart: 50
+
+
   upload_execution_report:
     name: Upload execution report 
     needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report]
@@ -665,6 +735,10 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
+      # deployments permission to deploy GitHub pages website
+      deployments: write
+      # contents permission to update benchmark contents in gh-pages branch
+      contents: write
 
     steps:
       - uses: actions/checkout@v4
@@ -701,3 +775,23 @@ jobs:
           header: execution_time
           message: ${{ steps.execution_report.outputs.markdown }}
 
+      - name: Convert to `benchmark-action` format
+        run: |
+          jq ".execution_reports | map({name: .artifact_name, value: (.time[:-1] | tonumber), unit: \"s\"}) " ./execution_report.json > time_bench.json
+
+      - name: Store benchmark result
+        # We always push new benchmarks so we want this to only run on master to avoid garbage data.``
+        # if: github.ref == 'refs/heads/master'
+        continue-on-error: true
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+        with:
+          name: "Execution Time"
+          tool: "customSmallerIsBetter"
+          output-file-path: ./time_bench.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: "110%"
+          comment-on-alert: true
+          fail-on-alert: false
+          alert-comment-cc-users: "@TomAFrench"
+          max-items-in-chart: 50

test_programs/memory_report.sh

@@ -4,6 +4,8 @@ set -e
 sudo apt-get install heaptrack
 
 NARGO="nargo"
+PARSE_MEMORY=$(realpath "$(dirname "$0")/parse_memory.sh")
+
 
 # Tests to be profiled for memory report
 tests_to_profile=("keccak256" "workspace" "regression_4709" "ram_blowup_regression")
@@ -54,7 +56,8 @@ for test_name in ${tests_to_profile[@]}; do
         len=${#consumption}-30
         peak=${consumption:30:len}
         rm $current_dir/$test_name"_heap_analysis.txt"
-        echo -e " {\n    \"artifact_name\":\"$test_name\",\n    \"peak_memory\":\"$peak\"\n }" >> $current_dir"/memory_report.json"
+        peak_memory=$($PARSE_MEMORY $peak)
+        echo -e " {\n    \"artifact_name\":\"$test_name\",\n    \"peak_memory\":\"$peak_memory\"\n }" >> $current_dir"/memory_report.json"
 done
 
 echo "]}" >> $current_dir"/memory_report.json"

test_programs/parse_memory.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+
+# This script accepts a string representing the amount of memory allocatedas outputted by `heaptrack`
+# and standardizes it to be in terms of megabytes as `heaptrack` will report different units depending on the duration.
+
+DIGITS='([0-9]+(\.[0-9]+)?)'
+KILOBYTES_REGEX=^${DIGITS}K$
+MEGABYTES_REGEX=^${DIGITS}M$
+GIGABYTES_REGEX=^${DIGITS}G$
+
+if [[ $1 =~ $KILOBYTES_REGEX ]]; then
+  echo ${BASH_REMATCH[1]} 1000 | awk '{printf "%.3f\n", $1/$2}'
+elif [[ $1 =~ $MEGABYTES_REGEX ]]; then
+  echo ${BASH_REMATCH[1]} | awk '{printf "%.3f\n", $1}'
+elif [[ $1 =~ $GIGABYTES_REGEX ]]; then
+  echo ${BASH_REMATCH[1]} 1000 | awk '{printf "%.3f\n", $1*$2}'
+else 
+  echo "Could not parse memory: unrecognized format" 1>&2
+  exit 1
+fi