147 keccak round bbf

.github/workflows/check-proof-producer.yml

@@ -233,8 +233,8 @@ jobs:
       always() && !cancelled() &&
       (needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') &&
       (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped')
-    uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
+    uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@a68465f641680eccc43ebdae02810eeab0537954
     with:
       artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }}
-      evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
+      evm-placeholder-verification-ref: a68465f641680eccc43ebdae02810eeab0537954
       test-names: ${{ needs.prepare-targets.outputs.evm-targets }}

.github/workflows/clang-linux-nix-check.yml

@@ -22,7 +22,7 @@ jobs:
           nix build -L .?#checks.x86_64-linux.crypto3-clang
         env:
           NIX_CONFIG: |
-            cores = 4
+            cores = 16
 
       - name: Run all checks
         # This includes cached crypto3 check from previous step. We don't limit cores in this part

.github/workflows/clang-sanitizers-linux-nix-check.yml

@@ -0,0 +1,37 @@
+name: Build and Test sanitizers on Linux with clang
+
+on:
+  workflow_call:
+
+jobs:
+  build-and-test:
+    name: "Build and test Linux with clang"
+    runs-on: [self-hosted, Linux, X64, aws_autoscaling]
+    continue-on-error: true
+    steps:
+      # https://github.com/actions/checkout/issues/1552
+      - name: Clean up after previous checkout
+        run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*;
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Run checks
+        run: |
+          nix build -L .?#checks.x86_64-linux.all-clang-sanitize
+          mkdir results
+          cp result/test-logs/*_test.xml results/
+        env:
+          NIX_CONFIG: |
+            cores = 16
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action/linux@v2
+        with:
+          check_name: "Sanitizers Test Results"
+          files: "results/*.xml"
+          comment_mode: ${{ github.event.pull_request.head.repo.fork && 'off' || 'always' }} # Don't create PR comment from fork runs
+          action_fail_on_inconclusive: true # fail, if no reports
+

.github/workflows/gcc-debug-proof-producer-nix-check.yml

@@ -0,0 +1,23 @@
+name: Build and run on Linux Platforms
+
+on:
+  workflow_call:
+
+jobs:
+  build-and-run:
+    name: "Build proof-producer binary in debug mode"
+    runs-on: [self-hosted, Linux, X64, aws_autoscaling]
+    steps:
+      # https://github.com/actions/checkout/issues/1552
+      - name: Clean up after previous checkout
+        run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*;
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and run proof-producer in debug mode
+        run: |
+          nix run .#multi-threaded  -- --help
+          nix run .#single-threaded -- --help

.github/workflows/gcc-linux-nix-check.yml

@@ -22,17 +22,13 @@ jobs:
           nix build -L .?#checks.x86_64-linux.crypto3-gcc
         env:
           NIX_CONFIG: |
-            cores = 4
-
+            cores = 16
       - name: Run all checks
         # This includes cached crypto3 check from previous step. We don't limit cores in this part
         run: |
           nix build -L .?#checks.x86_64-linux.all-gcc
           mkdir results
           cp result/test-logs/*_test.xml results/
-        env:
-          NIX_CONFIG: |
-            cores = 16
 
       - name: Publish Test Results
         uses: EnricoMi/publish-unit-test-result-action/linux@v2

.github/workflows/nightly.yml

@@ -0,0 +1,52 @@
+name: Nightly Testing
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  test-linux-sanitizers:
+    name: Linux placeholder testing and sanitize with clang
+    uses: ./.github/workflows/clang-sanitizers-linux-nix-check.yml
+    if: |
+      always() && !cancelled()
+    secrets: inherit
+
+  test-linux-proof-producer-debug-gcc:
+    name: Gcc debug proof-producer Linux testing
+    uses: ./.github/workflows/gcc-debug-proof-producer-nix-check.yml
+    if: |
+      always() && !cancelled()
+    secrets: inherit
+
+  post-telemetry:
+    name: Post test results in Open Telemetry format
+    runs-on: [self-hosted, Linux, X64, aws_autoscaling]
+    needs:
+      - test-linux-sanitizers
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Post logs to Sig Noz
+        run: |
+          ls -l -a
+          nix build -L .?#checks.x86_64-linux.all-clang-sanitize
+          cat ./result/test-logs/test_errors.txt
+          export UndefinedBehaviorSanitizer=$(grep UndefinedBehaviorSanitizer result/test-logs/test_errors.txt | wc -l)
+          export AddressSanitizer=$(grep AddressSanitizer result/test-logs/test_errors.txt | wc -l)
+          export LeakSanitizer=$(grep LeakSanitizer result/test-logs/test_errors.txt | wc -l)
+          export OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true
+          python3 --version
+          sudo yum update -y
+          sudo yum install -y python3-pip
+          pip3 install -r requirements.txt
+          /home/ec2-user/.local/bin/opentelemetry-instrument \
+            --traces_exporter console,otlp \
+            --metrics_exporter console,otlp \
+            --logs_exporter console,otlp \
+            --service_name nightly-build \
+            python3 ./parse_tests.py
+

.github/workflows/pull-request.yml

@@ -2,6 +2,7 @@ name: PR Testing
 
 on:
   pull_request:
+  merge_group:
   push:
     branches:
       - master
@@ -16,14 +17,14 @@ concurrency:
 
 jobs:
   test-linux-gcc:
-    name: Linux placeholder testing with gcc
+    name: Gcc release full Linux testing
     uses: ./.github/workflows/gcc-linux-nix-check.yml
     if: |
       always() && !cancelled()
     secrets: inherit
 
   test-linux-clang:
-    name: Linux placeholder testing with clang
+    name: Clang release full Linux testing
     uses: ./.github/workflows/clang-linux-nix-check.yml
     if: |
       always() && !cancelled()
@@ -42,4 +43,3 @@ jobs:
     if: |
       always() && !cancelled()
     secrets: inherit
-

.github/workflows/verify-circuit-proof.yml

@@ -11,30 +11,27 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Configure
-        run: cd crypto3 && mkdir build && cd build && cmake ../ -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE -DBUILD_CRYPTO3_TESTS=TRUE -G Ninja
+        run: mkdir build && cd build && cmake ../ -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE -DBUILD_CRYPTO3_TESTS=TRUE -G Ninja
         shell: nix develop .#crypto3-tests --command bash -e {0}
       - name: Build
-        run: cd crypto3/build && ninja transpiler_evm_test
+        run: cd build && ninja transpiler_evm_test
         shell: nix develop .#crypto3-tests --command bash -e {0}
       - name: Execute
-        run: cd crypto3/build && ./libs/transpiler/test/transpiler_evm_test -- --save-proof-data
+        run: cd build && ./crypto3/libs/transpiler/test/transpiler_evm_test -- --save-proof-data
         shell: nix develop .#crypto3-tests --command bash -e {0}
-          cd crypto3 && eval "$configurePhase"
-          nix develop -c ninja transpiler_evm_test
-          nix develop -c ./libs/transpiler/test/transpiler_evm_test -- --save-proof-data
       - name: Publish Proofs
         uses: actions/upload-artifact@v4
         with:
           name: proofs
-          path: crypto3/build/circuit*
+          path: build/circuit*
           if-no-files-found: error
 
   verify-proofs:
     name: Verify proofs
     needs:
       - produce-proofs
     if: needs.produce-proofs.result == 'success'
-    uses: NilFoundation/evm-placeholder-verification/.github/workflows/verify-proof.yml@e5a8dffd6b00d3c036c9b5c09c06b10515e4785c
+    uses: NilFoundation/evm-placeholder-verification/.github/workflows/verify-proof.yml@a68465f641680eccc43ebdae02810eeab0537954
     with:
-      evm-placeholder-verification-ref: e5a8dffd6b00d3c036c9b5c09c06b10515e4785c
+      evm-placeholder-verification-ref: a68465f641680eccc43ebdae02810eeab0537954
 

.gitignore

@@ -13,6 +13,10 @@
 .*.lb
 *.csv
 
+#protobuf generated files
+*pb.h
+*pb.cc
+
 ## Intermediate documents:
 *.dvi
 *.xdv
@@ -626,7 +630,6 @@ callgrind.*
 /*.c
 /*.cpp
 /*.h
-/*.py
 /*.key
 /*.pem
 /*.der

CMakeLists.txt

@@ -8,9 +8,11 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
   add_compile_options (-fcolor-diagnostics)
 endif ()
 
-if(DEFINED CMAKE_BUILD_TYPE AND ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+if(DEFINED CMAKE_BUILD_TYPE)
+  if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -ggdb")
     set(BOOST_FORCEINLINE "OFF") # improves debugging traces
+  endif()
 endif()
 
 option(SANITIZE "Build sanitizers" FALSE)
@@ -19,12 +21,20 @@ if(${SANITIZE})
   add_link_options(-fsanitize=undefined,address,leak)
 endif()
 
+# Add dummy target for the more efficient reusing of precompiled headers
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/generated-dummy.cpp)
+add_library(crypto3_precompiled_headers STATIC ${CMAKE_CURRENT_BINARY_DIR}/generated-dummy.cpp)
+set_target_properties(crypto3_precompiled_headers PROPERTIES
+    LINKER_LANGUAGE CXX
+    CXX_STANDARD 20
+    CXX_STANDARD_REQUIRED TRUE
+)
 
 add_subdirectory("${CMAKE_CURRENT_LIST_DIR}/crypto3")
-if ((${PARALLEL_CRYPTO3_ENABLE}) OR (${PROOF_PRODUCER_ENABLE}) OR (${ZKEVM_FRAMEWORK_ENABLE}))
+if ((${PARALLEL_CRYPTO3_ENABLE}) OR (${PROOF_PRODUCER_ENABLE}))
   add_subdirectory("${CMAKE_CURRENT_LIST_DIR}/parallel-crypto3")
 endif()
-if ((${PROOF_PRODUCER_ENABLE}) OR (${ZKEVM_FRAMEWORK_ENABLE}))
+if (${PROOF_PRODUCER_ENABLE})
   add_subdirectory("${CMAKE_CURRENT_LIST_DIR}/proof-producer")
 endif()
 if (${DEBUG_TOOLS_ENABLE})

crypto3.nix

@@ -6,11 +6,13 @@
   boost,
   gdb,
   lldb,
+  mold,
   cmake_modules,
   enableDebugging,
   enableDebug ? false,
   runTests ? false,
   sanitize ? false,
+  benchmarkTests ? false,
   }:
 let
   inherit (lib) optional;
@@ -32,19 +34,24 @@ in stdenv.mkDerivation {
   cmakeFlags =
     [
       (if runTests then "-DBUILD_CRYPTO3_TESTS=TRUE" else "-DBUILD_CRYPTO3_TESTS=False")
-      (if enableDebug then "-DCMAKE_BUILD_TYPE=Debug" else "-DCMAKE_BUILD_TYPE=Release")
       (if sanitize then "-DSANITIZE=ON" else "-DSANITIZE=OFF")
+      (if benchmarkTests then "-DBUILD_CRYPTO3_BENCH_TESTS=ON" else "-DBUILD_CRYPTO3_BENCH_TESTS=OFF")
       "-G Ninja"
     ];
 
+  cmakeBuildType = if enableDebug then "Debug" else "Release";
   doCheck = runTests; # tests are inside crypto3-tests derivation
 
   checkPhase = ''
     # JUNIT file without explicit file name is generated after the name of the master test suite inside `CMAKE_CURRENT_SOURCE_DIR`
     export BOOST_TEST_LOGGER=JUNIT:HRF
-    cd crypto3 && ctest --verbose --output-on-failure -R && cd ..
+    cd crypto3
+    # remove || true after all tests are fixed under clang-sanitizers check:
+    ctest --verbose --output-on-failure -R > test_errors.txt || true
+    cd ..
     mkdir -p ${placeholder "out"}/test-logs
     find .. -type f -name '*_test.xml' -exec cp {} ${placeholder "out"}/test-logs \;
+    cp crypto3/test_errors.txt ${placeholder "out"}/test-logs \
   '';
 
   shellHook = ''

crypto3/CMakeLists.txt

@@ -6,7 +6,7 @@
 # http://www.boost.org/LICENSE_1_0.txt
 #---------------------------------------------------------------------------#
 
-cmake_minimum_required(VERSION 3.21)
+cmake_minimum_required(VERSION 3.22 FATAL_ERROR)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
 

crypto3/benchmarks/CMakeLists.txt

@@ -40,7 +40,7 @@ macro(define_benchmark benchmark)
 
     set_target_properties(${full_name}
         PROPERTIES
-        CXX_STANDARD 17
+        CXX_STANDARD 20
         CXX_STANDARD_REQUIRED TRUE)
 
     if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
@@ -49,6 +49,7 @@ macro(define_benchmark benchmark)
         target_compile_options(${full_name} PRIVATE "-fconstexpr-ops-limit=4294967295")
     endif()
 
+    target_precompile_headers(${full_name} REUSE_FROM crypto3_precompiled_headers)
 endmacro()
 
 set(BENCHMARK_NAMES
@@ -59,9 +60,9 @@ set(BENCHMARK_NAMES
     "math/polynomial_dfs"
 
     "multiprecision/modular_adaptor_fixed"
+    "multiprecision/big_int"
 
     "zk/lpc"
-    "zk/pedersen"
 )
 
 foreach(BENCHMARK_NAME ${BENCHMARK_NAMES})

crypto3/benchmarks/algebra/curves.cpp

@@ -113,7 +113,7 @@ void benchmark_curve_operations(std::string const& curve_name)
             return A *= B;
             });
 
-    if constexpr (has_template_g2_type<curve_type>::value) {
+    if constexpr (has_type_g2_type<curve_type>::value) {
         using g2_type = typename curve_type::template g2_type<>;
 
         using g2_field = typename g2_type::field_type;