From ea10034c3ecb7daa1dab1e9c66a0c00d0ba36484 Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Tue, 26 Nov 2024 14:14:41 +0100 Subject: [PATCH 01/10] implement benchmarks derivation --- crypto3/benchmarks/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto3/benchmarks/CMakeLists.txt b/crypto3/benchmarks/CMakeLists.txt index c1f19f6dde..4e1c9528c1 100644 --- a/crypto3/benchmarks/CMakeLists.txt +++ b/crypto3/benchmarks/CMakeLists.txt @@ -60,7 +60,7 @@ set(BENCHMARK_NAMES "multiprecision/modular_adaptor_fixed" - "zk/lpc" + #"zk/lpc" "zk/pedersen" ) From 764012a6b9af7262012b30c37c1324d294feecac Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Tue, 26 Nov 2024 20:41:14 +0100 Subject: [PATCH 02/10] rename _test to _benchmark for file outputs --- crypto3/benchmarks/algebra/fields.cpp | 2 +- crypto3/benchmarks/algebra/multiexp.cpp | 2 +- crypto3/benchmarks/math/polynomial_dfs.cpp | 2 +- crypto3/benchmarks/multiprecision/modular_adaptor_fixed.cpp | 2 +- crypto3/benchmarks/zk/lpc.cpp | 2 +- crypto3/benchmarks/zk/pedersen.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto3/benchmarks/algebra/fields.cpp b/crypto3/benchmarks/algebra/fields.cpp index 0a7861a90b..90a67fe128 100644 --- a/crypto3/benchmarks/algebra/fields.cpp +++ b/crypto3/benchmarks/algebra/fields.cpp @@ -25,7 +25,7 @@ // SOFTWARE. //---------------------------------------------------------------------------// -#define BOOST_TEST_MODULE algebra_fields_bench_test +#define BOOST_TEST_MODULE algebra_fields_bench_benchmark #include #include diff --git a/crypto3/benchmarks/algebra/multiexp.cpp b/crypto3/benchmarks/algebra/multiexp.cpp index 07fa5c1c2d..e2823b3678 100644 --- a/crypto3/benchmarks/algebra/multiexp.cpp +++ b/crypto3/benchmarks/algebra/multiexp.cpp @@ -23,7 +23,7 @@ // SOFTWARE. //---------------------------------------------------------------------------// -#define BOOST_TEST_MODULE multiexpr_test +#define BOOST_TEST_MODULE multiexpr_benchmark #include #include diff --git a/crypto3/benchmarks/math/polynomial_dfs.cpp b/crypto3/benchmarks/math/polynomial_dfs.cpp index 96fef13cf5..b6fe70f568 100644 --- a/crypto3/benchmarks/math/polynomial_dfs.cpp +++ b/crypto3/benchmarks/math/polynomial_dfs.cpp @@ -22,7 +22,7 @@ // SOFTWARE. //---------------------------------------------------------------------------// -#define BOOST_TEST_MODULE polynomial_dfs_benchmark_test +#define BOOST_TEST_MODULE polynomial_dfs_benchmark_benchmark #include #include diff --git a/crypto3/benchmarks/multiprecision/modular_adaptor_fixed.cpp b/crypto3/benchmarks/multiprecision/modular_adaptor_fixed.cpp index 8c3e8cb13d..1fd813d2fa 100644 --- a/crypto3/benchmarks/multiprecision/modular_adaptor_fixed.cpp +++ b/crypto3/benchmarks/multiprecision/modular_adaptor_fixed.cpp @@ -7,7 +7,7 @@ // http://www.boost.org/LICENSE_1_0.txt //---------------------------------------------------------------------------// -#define BOOST_TEST_MODULE modular_fixed_multiprecision_test +#define BOOST_TEST_MODULE modular_fixed_multiprecision_benchmark #define TEST_CPP_INT diff --git a/crypto3/benchmarks/zk/lpc.cpp b/crypto3/benchmarks/zk/lpc.cpp index 77b8d440aa..8940cad790 100644 --- a/crypto3/benchmarks/zk/lpc.cpp +++ b/crypto3/benchmarks/zk/lpc.cpp @@ -24,7 +24,7 @@ // SOFTWARE. //---------------------------------------------------------------------------// -#define BOOST_TEST_MODULE lpc_test +#define BOOST_TEST_MODULE lpc_benchmark // Do it manually for all performance tests #define PROFILING_ENABLED diff --git a/crypto3/benchmarks/zk/pedersen.cpp b/crypto3/benchmarks/zk/pedersen.cpp index d066c97c57..e537405140 100644 --- a/crypto3/benchmarks/zk/pedersen.cpp +++ b/crypto3/benchmarks/zk/pedersen.cpp @@ -25,7 +25,7 @@ // SOFTWARE. //---------------------------------------------------------------------------// -#define BOOST_TEST_MODULE pedersen_test +#define BOOST_TEST_MODULE pedersen_benchmark #include #include From bf2c61a557ae1ca6147ebe467191bd21f6693c4d Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Tue, 26 Nov 2024 20:52:59 +0100 Subject: [PATCH 03/10] report benchmark infos to signoz --- crypto3.nix | 1 + parse_tests.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/crypto3.nix b/crypto3.nix index cb7ba532e2..4ce62f9fe3 100644 --- a/crypto3.nix +++ b/crypto3.nix @@ -50,6 +50,7 @@ in stdenv.mkDerivation { cd .. mkdir -p ${placeholder "out"}/test-logs find .. -type f -name '*_test.xml' -exec cp {} ${placeholder "out"}/test-logs \; + find .. -type f -name '*_benchmark.xml' -exec cp {} ${placeholder "out"}/test-logs \; cp crypto3/test_errors.txt ${placeholder "out"}/test-logs \ ''; diff --git a/parse_tests.py b/parse_tests.py index 20339e810d..fc6431f7fc 100644 --- a/parse_tests.py +++ b/parse_tests.py @@ -3,12 +3,12 @@ import glob, os from opentelemetry import trace -undefined_behavior_sanitizer=os.environ['UndefinedBehaviorSanitizer'] -address_sanitizer=os.environ['AddressSanitizer'] -leak_sanitizer=os.environ['LeakSanitizer'] +undefined_behavior_sanitizer=os.getenv('UndefinedBehaviorSanitizer', -1) +address_sanitizer=os.getenv('AddressSanitizer', -1) +leak_sanitizer=os.getenv('LeakSanitizer', -1) aggregated_test_results = JUnitXml(); -for file in glob.glob("result/test-logs/*.xml"): +for file in glob.glob("result/test-logs/*_test.xml"): try: aggregated_test_results.append(JUnitXml.fromfile(file)) except Exception as ex: @@ -32,6 +32,15 @@ "leak_sanitizer" : int(leak_sanitizer), } +for file in glob.glob("result/test-logs/*_benchmark.xml"): + try: + test_result = JUnitXml.fromfile(file) + result[test_result.name]=test_result.time + + except Exception as ex: + print("Error processing {}".format(file)) + print(ex) + print("Resulting JSON: {}".format(json.dumps(result))) tracer = trace.get_tracer_provider().get_tracer(__name__) From 9d837275162aa414e3b40e6738cf71b97607c5ae Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Tue, 26 Nov 2024 21:13:02 +0100 Subject: [PATCH 04/10] TODO revert change trigger from cron to PR --- .github/workflows/nightly.yml | 10 +++++-- .github/workflows/pull-request.yml | 45 ------------------------------ 2 files changed, 8 insertions(+), 47 deletions(-) delete mode 100644 .github/workflows/pull-request.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b7accb798f..ff0da71f05 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,8 +1,14 @@ name: Nightly Testing +#on: +# schedule: +# - cron: '0 0 * * *' on: - schedule: - - cron: '0 0 * * *' + pull_request: + merge_group: + push: + branches: + - master jobs: test-linux-sanitizers: diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml deleted file mode 100644 index 91ac921fab..0000000000 --- a/.github/workflows/pull-request.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: PR Testing - -on: - pull_request: - merge_group: - push: - branches: - - master - -concurrency: - # In master we want to run for every commit, in other branches — only for the last one - group: ${{ - ( github.ref == 'refs/heads/master' && format('{0}/{1}/{2}', github.workflow, github.ref, github.sha) ) - || - format('{0}/{1}', github.workflow, github.ref) }} - cancel-in-progress: true - -jobs: - test-linux-gcc: - name: Linux placeholder testing with gcc - uses: ./.github/workflows/gcc-linux-nix-check.yml - if: | - always() && !cancelled() - secrets: inherit - - test-linux-clang: - name: Linux placeholder testing with clang - uses: ./.github/workflows/clang-linux-nix-check.yml - if: | - always() && !cancelled() - secrets: inherit - - check-proof-producer: - name: Check Proof Producer - uses: ./.github/workflows/check-proof-producer.yml - if: | - always() && !cancelled() - secrets: inherit - - verify-circuit-proof: - name: Verify Circuit Proof - uses: ./.github/workflows/verify-circuit-proof.yml - if: | - always() && !cancelled() - secrets: inherit From 5f2fc73a9406555153ea7155e166adc3a9c4bee7 Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Thu, 28 Nov 2024 11:40:25 +0100 Subject: [PATCH 05/10] run only benchmarks and no other tests --- crypto3.nix | 2 +- crypto3/benchmarks/algebra_benchmark.xml | 51 +++++++++++++++++++ crypto3/benchmarks/algebra_benchmark_0.xml | 0 .../algebra_fields_bench_benchmark.xml | 0 ...modular_fixed_multiprecision_benchmark.xml | 43 ++++++++++++++++ crypto3/benchmarks/multiexpr_benchmark.xml | 11 ++++ crypto3/benchmarks/pedersen_benchmark.xml | 0 .../polynomial_dfs_benchmark_benchmark.xml | 0 flake.nix | 20 +++++--- parallel-crypto3.nix | 3 +- 10 files changed, 121 insertions(+), 9 deletions(-) create mode 100644 crypto3/benchmarks/algebra_benchmark.xml create mode 100644 crypto3/benchmarks/algebra_benchmark_0.xml create mode 100644 crypto3/benchmarks/algebra_fields_bench_benchmark.xml create mode 100644 crypto3/benchmarks/modular_fixed_multiprecision_benchmark.xml create mode 100644 crypto3/benchmarks/multiexpr_benchmark.xml create mode 100644 crypto3/benchmarks/pedersen_benchmark.xml create mode 100644 crypto3/benchmarks/polynomial_dfs_benchmark_benchmark.xml diff --git a/crypto3.nix b/crypto3.nix index 4ce62f9fe3..bf014ff37a 100644 --- a/crypto3.nix +++ b/crypto3.nix @@ -39,7 +39,7 @@ in stdenv.mkDerivation { ]; cmakeBuildType = if enableDebug then "Debug" else "Release"; - doCheck = runTests; # tests are inside crypto3-tests derivation + doCheck = runTests || benchmarkTests; checkPhase = '' # JUNIT file without explicit file name is generated after the name of the master test suite inside `CMAKE_CURRENT_SOURCE_DIR` diff --git a/crypto3/benchmarks/algebra_benchmark.xml b/crypto3/benchmarks/algebra_benchmark.xml new file mode 100644 index 0000000000..9d0cf1acff --- /dev/null +++ b/crypto3/benchmarks/algebra_benchmark.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/crypto3/benchmarks/algebra_benchmark_0.xml b/crypto3/benchmarks/algebra_benchmark_0.xml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crypto3/benchmarks/algebra_fields_bench_benchmark.xml b/crypto3/benchmarks/algebra_fields_bench_benchmark.xml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crypto3/benchmarks/modular_fixed_multiprecision_benchmark.xml b/crypto3/benchmarks/modular_fixed_multiprecision_benchmark.xml new file mode 100644 index 0000000000..bac32eedd9 --- /dev/null +++ b/crypto3/benchmarks/modular_fixed_multiprecision_benchmark.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/crypto3/benchmarks/multiexpr_benchmark.xml b/crypto3/benchmarks/multiexpr_benchmark.xml new file mode 100644 index 0000000000..0daed79921 --- /dev/null +++ b/crypto3/benchmarks/multiexpr_benchmark.xml @@ -0,0 +1,11 @@ + + + + + + \ No newline at end of file diff --git a/crypto3/benchmarks/pedersen_benchmark.xml b/crypto3/benchmarks/pedersen_benchmark.xml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crypto3/benchmarks/polynomial_dfs_benchmark_benchmark.xml b/crypto3/benchmarks/polynomial_dfs_benchmark_benchmark.xml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/flake.nix b/flake.nix index 9e1ff9bd7d..38eb30b638 100644 --- a/flake.nix +++ b/flake.nix @@ -41,7 +41,8 @@ sanitize = true; }); crypto3-clang-bench = (pkgs.callPackage ./crypto3.nix { - runTests = true; + stdenv = pkgs.llvmPackages_19.stdenv; + runTests = false; enableDebug = false; benchmarkTests = true; }); @@ -60,7 +61,8 @@ enableDebug = false; }); parallel-crypto3-clang-bench = (pkgs.callPackage ./parallel-crypto3.nix { - runTests = true; + stdenv = pkgs.llvmPackages_19.stdenv; + runTests = false; enableDebug = false; benchmarkTests = true; }); @@ -112,7 +114,7 @@ enableDebug = false; }); crypto3-gcc-bench = (pkgs.callPackage ./crypto3.nix { - runTests = true; + runTests = false; enableDebug = false; benchmarkTests = true; }); @@ -129,7 +131,7 @@ }); crypto3-clang-bench = (pkgs.callPackage ./crypto3.nix { stdenv = pkgs.llvmPackages_19.stdenv; - runTests = true; + runTests = false; enableDebug = false; benchmarkTests = true; }); @@ -137,11 +139,11 @@ parallel-crypto3-gcc = (pkgs.callPackage ./parallel-crypto3.nix { runTests = true; enableDebug = false; - benchmarkTests = true; }); parallel-crypto3-gcc-bench = (pkgs.callPackage ./parallel-crypto3.nix { - runTests = true; + runTests = false; enableDebug = false; + benchmarkTests = true; }); parallel-crypto3-clang = (pkgs.callPackage ./parallel-crypto3.nix { stdenv = pkgs.llvmPackages_19.stdenv; @@ -155,7 +157,7 @@ }); parallel-crypto3-clang-bench = (pkgs.callPackage ./parallel-crypto3.nix { stdenv = pkgs.llvmPackages_19.stdenv; - runTests = true; + runTests = false; enableDebug = false; benchmarkTests = true; }); @@ -184,6 +186,10 @@ name = "all"; paths = [ crypto3-clang-sanitize parallel-crypto3-clang-sanitize proof-producer-clang-sanitize ]; }; + all-clang-benchmarks = pkgs.symlinkJoin { + name = "all"; + paths = [ crypto3-clang-bench parallel-crypto3-clang-bench ]; + }; all-gcc = pkgs.symlinkJoin { name = "all"; paths = [ crypto3-gcc parallel-crypto3-gcc proof-producer-gcc ]; diff --git a/parallel-crypto3.nix b/parallel-crypto3.nix index d29be8d457..a443d6d70c 100644 --- a/parallel-crypto3.nix +++ b/parallel-crypto3.nix @@ -39,7 +39,7 @@ in stdenv.mkDerivation { ]; cmakeBuildType = if enableDebug then "Debug" else "Release"; - doCheck = runTests; # tests are inside parallel-crypto3-tests derivation + doCheck = runTests || benchmarkTests; checkPhase = '' # JUNIT file without explicit file name is generated after the name of the master test suite inside `CMAKE_CURRENT_SOURCE_DIR` @@ -49,6 +49,7 @@ in stdenv.mkDerivation { cd .. mkdir -p ${placeholder "out"}/test-logs find .. -type f -name '*_test.xml' -exec cp {} ${placeholder "out"}/test-logs \; + find .. -type f -name '*_benchmark.xml' -exec cp {} ${placeholder "out"}/test-logs \; ''; shellHook = '' From aed9f1ead19673530861ee0750d5394f59fdfdeb Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Thu, 28 Nov 2024 12:04:34 +0100 Subject: [PATCH 06/10] add benchmarks to nightly --- .../clang-benchmarks-linux-nix-check.yml | 40 +++++++++++++++++ .github/workflows/nightly.yml | 16 ++++++- parse_benchmarks.py | 43 +++++++++++++++++++ parse_tests.py | 2 +- 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/clang-benchmarks-linux-nix-check.yml create mode 100644 parse_benchmarks.py diff --git a/.github/workflows/clang-benchmarks-linux-nix-check.yml b/.github/workflows/clang-benchmarks-linux-nix-check.yml new file mode 100644 index 0000000000..3d934a9b58 --- /dev/null +++ b/.github/workflows/clang-benchmarks-linux-nix-check.yml @@ -0,0 +1,40 @@ +name: Build and Test benchmark tests on Linux with clang + +on: + workflow_call: + +jobs: + build-and-test: + name: "Build and test benchmark tests on Linux with clang" + runs-on: [self-hosted, Linux, X64, aws_autoscaling] + continue-on-error: true + steps: + # https://github.com/actions/checkout/issues/1552 + - name: Clean up after previous checkout + run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*; + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run checks + run: | + nix build -L .?#checks.x86_64-linux.all-clang-benchmarks + mkdir results + cp result/test-logs/*_test.xml results/ + cp result/test-logs/*_benchmark.xml results/ + ls -l -a results/ + continue-on-error: true + env: + NIX_CONFIG: | + cores = 8 + + - name: Publish Benchmarks Test Results + uses: EnricoMi/publish-unit-test-result-action/linux@v2 + with: + check_name: "Benchmarks Test Results" + files: "results/*.xml" + comment_mode: ${{ github.event.pull_request.head.repo.fork && 'off' || 'always' }} # Don't create PR comment from fork runs + action_fail_on_inconclusive: true # fail, if no reports + diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ff0da71f05..8b5397463b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,6 +18,13 @@ jobs: always() && !cancelled() secrets: inherit + test-linux-benchmarks: + name: Linux placeholder testing and sanitize with clang + uses: ./.github/workflows/clang-benchmarks-linux-nix-check.yml + if: | + always() && !cancelled() + secrets: inherit + post-telemetry: name: Post test results in Open Telemetry format runs-on: [self-hosted, Linux, X64, aws_autoscaling] @@ -33,7 +40,6 @@ jobs: run: | ls -l -a nix build -L .?#checks.x86_64-linux.all-clang-sanitize - cat ./result/test-logs/test_errors.txt export UndefinedBehaviorSanitizer=$(grep UndefinedBehaviorSanitizer result/test-logs/test_errors.txt | wc -l) export AddressSanitizer=$(grep AddressSanitizer result/test-logs/test_errors.txt | wc -l) export LeakSanitizer=$(grep LeakSanitizer result/test-logs/test_errors.txt | wc -l) @@ -49,3 +55,11 @@ jobs: --service_name nightly-build \ python3 ./parse_tests.py + nix build -L .?#checks.x86_64-linux.all-clang-benchmarks + /home/ec2-user/.local/bin/opentelemetry-instrument \ + --traces_exporter console,otlp \ + --metrics_exporter console,otlp \ + --logs_exporter console,otlp \ + --service_name nightly-build \ + python3 ./parse_tests.py + diff --git a/parse_benchmarks.py b/parse_benchmarks.py new file mode 100644 index 0000000000..6e587fab7c --- /dev/null +++ b/parse_benchmarks.py @@ -0,0 +1,43 @@ +import logging, json +from junitparser import JUnitXml +import glob, os +from opentelemetry import trace + +aggregated_test_results = JUnitXml(); +for file in glob.glob("result/test-logs/*_benchmark.xml"): + try: + test_result = JUnitXml.fromfile(file) + result[test_result.name]=test_result.time + aggregated_test_results.append(test_result) + except Exception as ex: + print("Error processing {}".format(file)) + print(ex) + +for file in glob.glob("result/test-logs/*_benchmark.xml"): + try: + except Exception as ex: + print("Error processing {}".format(file)) + print(ex) + +succeeded = aggregated_test_results.tests - \ + aggregated_test_results.failures - \ + aggregated_test_results.errors - \ + aggregated_test_results.skipped + +result = { + "benchmark_tests" : aggregated_test_results.tests, + "benchmark_failures" : aggregated_test_results.failures, + "benchmark_errors" : aggregated_test_results.errors, + "benchmark_skipped" : aggregated_test_results.skipped, + "benchmark_succeeded" : succeeded, + "benchmark_execution_time" : aggregated_test_results.time, +} + +print("Resulting JSON: {}".format(json.dumps(result))) + +tracer = trace.get_tracer_provider().get_tracer(__name__) +with tracer.start_as_current_span("nightly_span"): + current_span = trace.get_current_span() + current_span.add_event("Nightly benchmarks build finished") + logging.getLogger().error(json.dumps(result)) + diff --git a/parse_tests.py b/parse_tests.py index fc6431f7fc..008f3bd368 100644 --- a/parse_tests.py +++ b/parse_tests.py @@ -46,6 +46,6 @@ tracer = trace.get_tracer_provider().get_tracer(__name__) with tracer.start_as_current_span("nightly_span"): current_span = trace.get_current_span() - current_span.add_event("Nightly build finished") + current_span.add_event("Nightly sanitizers build finished") logging.getLogger().error(json.dumps(result)) From bc8ccfc0d65ca74351f517cc8178a40cddf79d5d Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Thu, 28 Nov 2024 12:10:28 +0100 Subject: [PATCH 07/10] add explicit dependency --- .github/workflows/nightly.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8b5397463b..b499fbd0e4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -19,7 +19,7 @@ jobs: secrets: inherit test-linux-benchmarks: - name: Linux placeholder testing and sanitize with clang + name: Linux placeholder benchmark testing with clang uses: ./.github/workflows/clang-benchmarks-linux-nix-check.yml if: | always() && !cancelled() @@ -30,6 +30,7 @@ jobs: runs-on: [self-hosted, Linux, X64, aws_autoscaling] needs: - test-linux-sanitizers + - test-linux-benchmarks steps: - name: Checkout uses: actions/checkout@v3 From 9604115bbeabab0d6532567165a543b73595effb Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Thu, 28 Nov 2024 18:22:11 +0100 Subject: [PATCH 08/10] add test logs --- .github/workflows/clang-benchmarks-linux-nix-check.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/clang-benchmarks-linux-nix-check.yml b/.github/workflows/clang-benchmarks-linux-nix-check.yml index 3d934a9b58..c214d00c13 100644 --- a/.github/workflows/clang-benchmarks-linux-nix-check.yml +++ b/.github/workflows/clang-benchmarks-linux-nix-check.yml @@ -22,6 +22,10 @@ jobs: run: | nix build -L .?#checks.x86_64-linux.all-clang-benchmarks mkdir results + ls -l -a + ls -l -a ./result + ls -l -a ./result/test-logs/ + cat ./result/test-logs/test_errors.txt cp result/test-logs/*_test.xml results/ cp result/test-logs/*_benchmark.xml results/ ls -l -a results/ From 0abcc7c400c019a245de6bfc83807914f51fb6fa Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Thu, 28 Nov 2024 18:46:21 +0100 Subject: [PATCH 09/10] make parallel-crypto3 tests compile and run --- parallel-crypto3.nix | 5 +- parallel-crypto3/CMakeLists.txt | 3 + parallel-crypto3/benchmarks/CMakeLists.txt | 55 +++++++++++++++++++ .../benchmarks/polynomial_dfs_benchmark.cpp | 2 +- .../libs/parallel-math/test/CMakeLists.txt | 4 -- .../test/benchmarks/CMakeLists.txt | 33 ----------- 6 files changed, 63 insertions(+), 39 deletions(-) create mode 100644 parallel-crypto3/benchmarks/CMakeLists.txt rename parallel-crypto3/{libs/parallel-math/test => }/benchmarks/polynomial_dfs_benchmark.cpp (99%) delete mode 100644 parallel-crypto3/libs/parallel-math/test/benchmarks/CMakeLists.txt diff --git a/parallel-crypto3.nix b/parallel-crypto3.nix index a443d6d70c..dd0402ed48 100644 --- a/parallel-crypto3.nix +++ b/parallel-crypto3.nix @@ -36,6 +36,7 @@ in stdenv.mkDerivation { (if sanitize then "-DSANITIZE=ON" else "-DSANITIZE=OFF") (if benchmarkTests then "-DENABLE_BENCHMARKS=ON" else "-DENABLE_BENCHMARKS=OFF") "-DPARALLEL_CRYPTO3_ENABLE=TRUE" + "-G Ninja" ]; cmakeBuildType = if enableDebug then "Debug" else "Release"; @@ -45,11 +46,13 @@ in stdenv.mkDerivation { # JUNIT file without explicit file name is generated after the name of the master test suite inside `CMAKE_CURRENT_SOURCE_DIR` export BOOST_TEST_LOGGER=JUNIT:HRF cd parallel-crypto3 - ctest --verbose --output-on-failure -R + # remove || true after all tests are fixed under clang-sanitizers check: + ctest --verbose --output-on-failure -R > test_errors.txt || true cd .. mkdir -p ${placeholder "out"}/test-logs find .. -type f -name '*_test.xml' -exec cp {} ${placeholder "out"}/test-logs \; find .. -type f -name '*_benchmark.xml' -exec cp {} ${placeholder "out"}/test-logs \; + cp crypto3/test_errors.txt ${placeholder "out"}/test-logs \ ''; shellHook = '' diff --git a/parallel-crypto3/CMakeLists.txt b/parallel-crypto3/CMakeLists.txt index 2837852927..481878b48b 100644 --- a/parallel-crypto3/CMakeLists.txt +++ b/parallel-crypto3/CMakeLists.txt @@ -31,6 +31,9 @@ add_subdirectory(libs/parallelization-utils) add_subdirectory(libs/parallel-containers) add_subdirectory(libs/parallel-math) add_subdirectory(libs/parallel-zk) +if(ENABLE_BENCHMARKS) + add_subdirectory(benchmarks) +endif() add_library(${PROJECT_NAME}_all INTERFACE) add_library(${PROJECT_NAME}::all ALIAS ${PROJECT_NAME}_all) diff --git a/parallel-crypto3/benchmarks/CMakeLists.txt b/parallel-crypto3/benchmarks/CMakeLists.txt new file mode 100644 index 0000000000..ffb551e2a7 --- /dev/null +++ b/parallel-crypto3/benchmarks/CMakeLists.txt @@ -0,0 +1,55 @@ +#---------------------------------------------------------------------------# +# Copyright (c) 2018-2020 Mikhail Komarov +# +# Distributed under the Boost Software License, Version 1.0 +# See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt +#---------------------------------------------------------------------------# + +include(CMTest) + +find_package(Boost REQUIRED COMPONENTS + timer + unit_test_framework +) + +macro(define_bench_test name) + set(test_name "parallel_crypto3_${name}_bench") + + set(additional_args "") + if(ENABLE_JUNIT_TEST_OUTPUT) + set(TEST_RESULTS_DIR "${CMAKE_CURRENT_BINARY_DIR}/junit_results") + set(TEST_LOGS_DIR "${TEST_RESULTS_DIR}/logs") + file(MAKE_DIRECTORY ${TEST_LOGS_DIR}) + + set(additional_args "--log_format=JUNIT" + "--log_sink=${TEST_LOGS_DIR}/${test_name}.xml") + endif() + + cm_test(NAME ${test_name} SOURCES ${name}.cpp ARGS ${additional_args}) + + target_include_directories(${test_name} PRIVATE + "$" + "$" + + ${Boost_INCLUDE_DIRS}) + + set_target_properties(${test_name} PROPERTIES CXX_STANDARD 17) + +endmacro() + +cm_test_link_libraries( + crypto3::algebra + crypto3::math + crypto3::multiprecision + crypto3::random + ${Boost_LIBRARIES} +) + +set(TESTS_NAMES + "polynomial_dfs_benchmark" +) + +foreach(TEST_NAME ${TESTS_NAMES}) + define_bench_test(${TEST_NAME}) +endforeach() diff --git a/parallel-crypto3/libs/parallel-math/test/benchmarks/polynomial_dfs_benchmark.cpp b/parallel-crypto3/benchmarks/polynomial_dfs_benchmark.cpp similarity index 99% rename from parallel-crypto3/libs/parallel-math/test/benchmarks/polynomial_dfs_benchmark.cpp rename to parallel-crypto3/benchmarks/polynomial_dfs_benchmark.cpp index 96fef13cf5..db7e056a1b 100644 --- a/parallel-crypto3/libs/parallel-math/test/benchmarks/polynomial_dfs_benchmark.cpp +++ b/parallel-crypto3/benchmarks/polynomial_dfs_benchmark.cpp @@ -22,7 +22,7 @@ // SOFTWARE. //---------------------------------------------------------------------------// -#define BOOST_TEST_MODULE polynomial_dfs_benchmark_test +#define BOOST_TEST_MODULE polynomial_dfs_benchmark #include #include diff --git a/parallel-crypto3/libs/parallel-math/test/CMakeLists.txt b/parallel-crypto3/libs/parallel-math/test/CMakeLists.txt index 525e1f6b90..d88e18ea21 100644 --- a/parallel-crypto3/libs/parallel-math/test/CMakeLists.txt +++ b/parallel-crypto3/libs/parallel-math/test/CMakeLists.txt @@ -52,7 +52,3 @@ set(TESTS_NAMES foreach(TEST_NAME ${TESTS_NAMES}) define_math_test(${TEST_NAME}) endforeach() - -if(ENABLE_BENCHMARKS) - add_subdirectory(benchmarks) -endif() diff --git a/parallel-crypto3/libs/parallel-math/test/benchmarks/CMakeLists.txt b/parallel-crypto3/libs/parallel-math/test/benchmarks/CMakeLists.txt deleted file mode 100644 index 374894a767..0000000000 --- a/parallel-crypto3/libs/parallel-math/test/benchmarks/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -#---------------------------------------------------------------------------# -# Copyright (c) 2018-2020 Mikhail Komarov -# -# Distributed under the Boost Software License, Version 1.0 -# See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt -#---------------------------------------------------------------------------# - -find_package(Boost REQUIRED COMPONENTS - timer - unit_test_framework -) - -cm_test_link_libraries(${CMAKE_WORKSPACE_NAME}_${CURRENT_PROJECT_NAME} - - crypto3::algebra - crypto3::multiprecision - crypto3::random - - ${Boost_LIBRARIES} -) - -set(TESTS_NAMES - "polynomial_dfs_benchmark" -) - -foreach(TEST_NAME ${TESTS_NAMES}) - define_math_test(${TEST_NAME}) -endforeach() - - -#get_target_property(my_include_dirs math_polynomial_dfs_benchmark_test INCLUDE_DIRECTORIES) -#message(include dirs: ${my_include_dirs}) From f9224570e9c127b8bc8a8c722ec3967d91380264 Mon Sep 17 00:00:00 2001 From: Andrei Malashkin Date: Fri, 29 Nov 2024 12:42:14 +0100 Subject: [PATCH 10/10] hotfix --- parallel-crypto3.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parallel-crypto3.nix b/parallel-crypto3.nix index dd0402ed48..1b53644f07 100644 --- a/parallel-crypto3.nix +++ b/parallel-crypto3.nix @@ -52,7 +52,7 @@ in stdenv.mkDerivation { mkdir -p ${placeholder "out"}/test-logs find .. -type f -name '*_test.xml' -exec cp {} ${placeholder "out"}/test-logs \; find .. -type f -name '*_benchmark.xml' -exec cp {} ${placeholder "out"}/test-logs \; - cp crypto3/test_errors.txt ${placeholder "out"}/test-logs \ + cp parallel-crypto3/test_errors.txt ${placeholder "out"}/test-logs \ ''; shellHook = ''