diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index e14f116d105..ff2e32e5641 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -428,6 +428,10 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write steps: - uses: actions/checkout@v4 @@ -464,6 +468,26 @@ jobs: header: compilation message: ${{ steps.compilation_report.outputs.markdown }} + - name: Convert to `benchmark-action` format + run: | + jq ".compilation_reports | map({name: .artifact_name, value: (.time[:-1] | tonumber), unit: \"s\"}) " ./compilation_report.json > time_bench.json + + - name: Store benchmark result + continue-on-error: true + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 + with: + name: "Compilation Time" + tool: "customSmallerIsBetter" + output-file-path: ./time_bench.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # We want this to only run on master to avoid garbage data from PRs being added. + auto-push: ${{ github.ref == 'refs/heads/master' }} + alert-threshold: "110%" + comment-on-alert: true + fail-on-alert: false + alert-comment-cc-users: "@TomAFrench" + max-items-in-chart: 50 + external_repo_memory_report: needs: [build-nargo] runs-on: ubuntu-latest @@ -507,6 +531,7 @@ jobs: path: scripts sparse-checkout: | test_programs/memory_report.sh + test_programs/parse_memory.sh sparse-checkout-cone-mode: false - name: Checkout @@ -520,6 +545,7 @@ jobs: working-directory: ./test-repo/${{ matrix.project.path }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh + mv /home/runner/work/noir/noir/scripts/test_programs/parse_memory.sh ./parse_memory.sh ./memory_report.sh 1 # Rename the memory report as the execution report is about to write to the same file cp memory_report.json compilation_memory_report.json @@ -575,6 +601,10 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write steps: - uses: actions/checkout@v4 @@ -611,6 +641,26 @@ jobs: header: compilation_memory message: ${{ steps.compilation_mem_report.outputs.markdown }} + - name: Convert to `benchmark-action` format + run: | + jq ".memory_reports | map({name: .artifact_name, value: (.peak_memory | tonumber), unit: \"MB\"}) " ./memory_report.json > memory_bench.json + + - name: Store benchmark result + continue-on-error: true + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 + with: + name: "Compilation Memory" + tool: "customSmallerIsBetter" + output-file-path: ./memory_bench.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # We want this to only run on master to avoid garbage data from PRs being added. + auto-push: ${{ github.ref == 'refs/heads/master' }} + alert-threshold: "110%" + comment-on-alert: true + fail-on-alert: false + alert-comment-cc-users: "@TomAFrench" + max-items-in-chart: 50 + upload_execution_memory_report: name: Upload execution memory report needs: [generate_memory_report, external_repo_memory_report] @@ -619,6 +669,10 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write steps: - uses: actions/checkout@v4 @@ -657,6 +711,27 @@ jobs: header: execution_memory message: ${{ steps.execution_mem_report.outputs.markdown }} + - name: Convert to `benchmark-action` format + run: | + jq ".memory_reports | map({name: .artifact_name, value: (.peak_memory | tonumber), unit: \"MB\"}) " ./execution_memory_report.json > memory_bench.json + + - name: Store benchmark result + continue-on-error: true + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 + with: + name: "Execution Memory" + tool: "customSmallerIsBetter" + output-file-path: ./memory_bench.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # We want this to only run on master to avoid garbage data from PRs being added. + auto-push: ${{ github.ref == 'refs/heads/master' }} + alert-threshold: "110%" + comment-on-alert: true + fail-on-alert: false + alert-comment-cc-users: "@TomAFrench" + max-items-in-chart: 50 + + upload_execution_report: name: Upload execution report needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report] @@ -665,6 +740,10 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write steps: - uses: actions/checkout@v4 @@ -701,3 +780,22 @@ jobs: header: execution_time message: ${{ steps.execution_report.outputs.markdown }} + - name: Convert to `benchmark-action` format + run: | + jq ".execution_reports | map({name: .artifact_name, value: (.time[:-1] | tonumber), unit: \"s\"}) " ./execution_report.json > time_bench.json + + - name: Store benchmark result + continue-on-error: true + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 + with: + name: "Execution Time" + tool: "customSmallerIsBetter" + output-file-path: ./time_bench.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # We want this to only run on master to avoid garbage data from PRs being added. + auto-push: ${{ github.ref == 'refs/heads/master' }} + alert-threshold: "110%" + comment-on-alert: true + fail-on-alert: false + alert-comment-cc-users: "@TomAFrench" + max-items-in-chart: 50 diff --git a/test_programs/memory_report.sh b/test_programs/memory_report.sh index a5c13f58b0f..4ba7579b390 100755 --- a/test_programs/memory_report.sh +++ b/test_programs/memory_report.sh @@ -4,6 +4,8 @@ set -e sudo apt-get install heaptrack NARGO="nargo" +PARSE_MEMORY=$(realpath "$(dirname "$0")/parse_memory.sh") + # Tests to be profiled for memory report tests_to_profile=("keccak256" "workspace" "regression_4709" "ram_blowup_regression") @@ -54,7 +56,8 @@ for test_name in ${tests_to_profile[@]}; do len=${#consumption}-30 peak=${consumption:30:len} rm $current_dir/$test_name"_heap_analysis.txt" - echo -e " {\n \"artifact_name\":\"$test_name\",\n \"peak_memory\":\"$peak\"\n }" >> $current_dir"/memory_report.json" + peak_memory=$($PARSE_MEMORY $peak) + echo -e " {\n \"artifact_name\":\"$test_name\",\n \"peak_memory\":\"$peak_memory\"\n }" >> $current_dir"/memory_report.json" done echo "]}" >> $current_dir"/memory_report.json" diff --git a/test_programs/parse_memory.sh b/test_programs/parse_memory.sh new file mode 100755 index 00000000000..8e125b13b23 --- /dev/null +++ b/test_programs/parse_memory.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -e + +# This script accepts a string representing the amount of memory allocated as outputted by `heaptrack` +# and standardizes it to be in terms of megabytes as `heaptrack` will report different units depending on the duration. + +DIGITS='([0-9]+(\.[0-9]+)?)' +KILOBYTES_REGEX=^${DIGITS}K$ +MEGABYTES_REGEX=^${DIGITS}M$ +GIGABYTES_REGEX=^${DIGITS}G$ + +if [[ $1 =~ $KILOBYTES_REGEX ]]; then + echo ${BASH_REMATCH[1]} 1000 | awk '{printf "%.3f\n", $1/$2}' +elif [[ $1 =~ $MEGABYTES_REGEX ]]; then + echo ${BASH_REMATCH[1]} | awk '{printf "%.3f\n", $1}' +elif [[ $1 =~ $GIGABYTES_REGEX ]]; then + echo ${BASH_REMATCH[1]} 1000 | awk '{printf "%.3f\n", $1*$2}' +else + echo "Could not parse memory: unrecognized format" 1>&2 + exit 1 +fi