Implement benchmark report

.github/workflows/clang-benchmarks-linux-nix-check.yml

@@ -0,0 +1,44 @@
+name: Build and Test benchmark tests on Linux with clang
+
+on:
+  workflow_call:
+
+jobs:
+  build-and-test:
+    name: "Build and test benchmark tests on Linux with clang"
+    runs-on: [self-hosted, Linux, X64, aws_autoscaling]
+    continue-on-error: true
+    steps:
+      # https://github.com/actions/checkout/issues/1552
+      - name: Clean up after previous checkout
+        run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*;
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Run checks
+        run: |
+          nix build -L .?#checks.x86_64-linux.all-clang-benchmarks
+          mkdir results
+          ls -l -a
+          ls -l -a ./result
+          ls -l -a ./result/test-logs/
+          cat ./result/test-logs/test_errors.txt
+          cp result/test-logs/*_test.xml results/
+          cp result/test-logs/*_benchmark.xml results/
+          ls -l -a results/
+        continue-on-error: true
+        env:
+          NIX_CONFIG: |
+            cores = 8
+
+      - name: Publish Benchmarks Test Results
+        uses: EnricoMi/publish-unit-test-result-action/linux@v2
+        with:
+          check_name: "Benchmarks Test Results"
+          files: "results/*.xml"
+          comment_mode: ${{ github.event.pull_request.head.repo.fork && 'off' || 'always' }} # Don't create PR comment from fork runs
+          action_fail_on_inconclusive: true # fail, if no reports
+

.github/workflows/nightly.yml

@@ -1,8 +1,14 @@
 name: Nightly Testing
 
+#on:
+#  schedule:
+#    - cron: '0 0 * * *'
 on:
-  schedule:
-    - cron: '0 0 * * *'
+  pull_request:
+  merge_group:
+  push:
+    branches:
+      - master
 
 jobs:
   test-linux-sanitizers:
@@ -12,11 +18,19 @@ jobs:
       always() && !cancelled()
     secrets: inherit
 
+  test-linux-benchmarks:
+    name: Linux placeholder benchmark testing with clang
+    uses: ./.github/workflows/clang-benchmarks-linux-nix-check.yml
+    if: |
+      always() && !cancelled()
+    secrets: inherit
+
   post-telemetry:
     name: Post test results in Open Telemetry format
     runs-on: [self-hosted, Linux, X64, aws_autoscaling]
     needs:
       - test-linux-sanitizers
+      - test-linux-benchmarks
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -27,7 +41,6 @@ jobs:
         run: |
           ls -l -a
           nix build -L .?#checks.x86_64-linux.all-clang-sanitize
-          cat ./result/test-logs/test_errors.txt
           export UndefinedBehaviorSanitizer=$(grep UndefinedBehaviorSanitizer result/test-logs/test_errors.txt | wc -l)
           export AddressSanitizer=$(grep AddressSanitizer result/test-logs/test_errors.txt | wc -l)
           export LeakSanitizer=$(grep LeakSanitizer result/test-logs/test_errors.txt | wc -l)
@@ -43,3 +56,11 @@ jobs:
             --service_name nightly-build \
             python3 ./parse_tests.py
 
+          nix build -L .?#checks.x86_64-linux.all-clang-benchmarks
+          /home/ec2-user/.local/bin/opentelemetry-instrument \
+            --traces_exporter console,otlp \
+            --metrics_exporter console,otlp \
+            --logs_exporter console,otlp \
+            --service_name nightly-build \
+            python3 ./parse_tests.py
+

crypto3.nix

@@ -39,7 +39,7 @@ in stdenv.mkDerivation {
     ];
 
   cmakeBuildType = if enableDebug then "Debug" else "Release";
-  doCheck = runTests; # tests are inside crypto3-tests derivation
+  doCheck = runTests || benchmarkTests;
 
   checkPhase = ''
     # JUNIT file without explicit file name is generated after the name of the master test suite inside `CMAKE_CURRENT_SOURCE_DIR`
@@ -50,6 +50,7 @@ in stdenv.mkDerivation {
     cd ..
     mkdir -p ${placeholder "out"}/test-logs
     find .. -type f -name '*_test.xml' -exec cp {} ${placeholder "out"}/test-logs \;
+    find .. -type f -name '*_benchmark.xml' -exec cp {} ${placeholder "out"}/test-logs \;
     cp crypto3/test_errors.txt ${placeholder "out"}/test-logs \
   '';
 

crypto3/benchmarks/CMakeLists.txt

@@ -60,7 +60,7 @@ set(BENCHMARK_NAMES
 
     "multiprecision/modular_adaptor_fixed"
 
-    "zk/lpc"
+    #"zk/lpc"
     "zk/pedersen"
 )
 

crypto3/benchmarks/algebra/fields.cpp

@@ -25,7 +25,7 @@
 // SOFTWARE.
 //---------------------------------------------------------------------------//
 
-#define BOOST_TEST_MODULE algebra_fields_bench_test
+#define BOOST_TEST_MODULE algebra_fields_bench_benchmark
 
 #include <ostream>
 #include <fstream>

crypto3/benchmarks/algebra/multiexp.cpp

@@ -23,7 +23,7 @@
 // SOFTWARE.
 //---------------------------------------------------------------------------//
 
-#define BOOST_TEST_MODULE multiexpr_test
+#define BOOST_TEST_MODULE multiexpr_benchmark
 
 #include <boost/test/unit_test.hpp>
 #include <boost/test/data/test_case.hpp>

crypto3/benchmarks/algebra_benchmark.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite tests="6" skipped="0" errors="0" failures="0" id="0" name="algebra_benchmark" time="1268.28">
+<testcase assertions="0" classname="curves_benchmark" name="pallas" time="40.1403">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case curves_benchmark/pallas did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="curves_benchmark" name="vesta" time="40.2609">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case curves_benchmark/vesta did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="curves_benchmark" name="bls12_381" time="250.289">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case curves_benchmark/bls12_381 did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="curves_benchmark" name="bls12_377" time="224.689">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case curves_benchmark/bls12_377 did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="curves_benchmark" name="mnt4_298" time="248.203">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case curves_benchmark/mnt4_298 did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="curves_benchmark" name="mnt6_298" time="464.699">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case curves_benchmark/mnt6_298 did not check any assertions
+
+]]></system-out>
+</testcase>
+</testsuite>

crypto3/benchmarks/math/polynomial_dfs.cpp

@@ -22,7 +22,7 @@
 // SOFTWARE.
 //---------------------------------------------------------------------------//
 
-#define BOOST_TEST_MODULE polynomial_dfs_benchmark_test
+#define BOOST_TEST_MODULE polynomial_dfs_benchmark_benchmark
 
 #include <algorithm>
 #include <cctype>

crypto3/benchmarks/modular_fixed_multiprecision_benchmark.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite tests="5" skipped="0" errors="0" failures="0" id="0" name="modular_fixed_multiprecision_benchmark" time="16.1268">
+<testcase assertions="0" classname="runtime_tests" name="modular_adaptor_montgomery_mult_perf_test" time="3.2846">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case runtime_tests/modular_adaptor_montgomery_mult_perf_test did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="runtime_tests" name="modular_adaptor_backend_sub_perf_test" time="3.15055">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case runtime_tests/modular_adaptor_backend_sub_perf_test did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="runtime_tests" name="modular_adaptor_backend_add_perf_test" time="3.13206">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case runtime_tests/modular_adaptor_backend_add_perf_test did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="runtime_tests" name="modular_adaptor_backend_mult_perf_test" time="3.26774">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case runtime_tests/modular_adaptor_backend_mult_perf_test did not check any assertions
+
+]]></system-out>
+</testcase>
+<testcase assertions="0" classname="runtime_tests" name="modular_adaptor_number_mult_perf_test" time="3.29176">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case runtime_tests/modular_adaptor_number_mult_perf_test did not check any assertions
+
+]]></system-out>
+</testcase>
+</testsuite>

crypto3/benchmarks/multiexpr_benchmark.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite tests="1" skipped="0" errors="0" failures="0" id="0" name="multiexpr_benchmark" time="179.731">
+<testcase assertions="0" classname="multiexp_test_suite" name="multiexp_test_case" time="179.731">
+<system-out><![CDATA[MESSAGE:
+- file   : boost.test framework
+- line   : 0
+- message: Test case multiexp_test_suite/multiexp_test_case did not check any assertions
+
+]]></system-out>
+</testcase>
+</testsuite>

crypto3/benchmarks/multiprecision/modular_adaptor_fixed.cpp

@@ -7,7 +7,7 @@
 // http://www.boost.org/LICENSE_1_0.txt
 //---------------------------------------------------------------------------//
 
-#define BOOST_TEST_MODULE modular_fixed_multiprecision_test
+#define BOOST_TEST_MODULE modular_fixed_multiprecision_benchmark
 
 #define TEST_CPP_INT
 

crypto3/benchmarks/zk/lpc.cpp

@@ -24,7 +24,7 @@
 // SOFTWARE.
 //---------------------------------------------------------------------------//
 
-#define BOOST_TEST_MODULE lpc_test
+#define BOOST_TEST_MODULE lpc_benchmark
 
 // Do it manually for all performance tests
 #define PROFILING_ENABLED

crypto3/benchmarks/zk/pedersen.cpp

@@ -25,7 +25,7 @@
 // SOFTWARE.
 //---------------------------------------------------------------------------//
 
-#define BOOST_TEST_MODULE pedersen_test
+#define BOOST_TEST_MODULE pedersen_benchmark
 
 #include <vector>
 #include <iostream>