Merge commit 'd056f23bd79bf099c12c916b50e21b3ae296c27a' into feature/…

…backing_tests Signed-off-by: iceseer <[email protected]> # Conflicts: # core/network/impl/protocols/protocol_fetch_chunk.hpp # core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp # core/parachain/validator/impl/parachain_processor.cpp
qdrvm · Dec 18, 2024 · c5bf92b · c5bf92b
2 parents 4f259e0 + d056f23
commit c5bf92b
Show file tree

Hide file tree

Showing 213 changed files with 7,560 additions and 1,608 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -69,15 +69,20 @@ env:
 jobs:
   MacOS:
     runs-on: macos-15
-    timeout-minutes: 120
+    timeout-minutes: 240
     strategy:
       fail-fast: false
       matrix:
         options:
           - name: "MacOS: Build Debug"
             build_type: "Debug"
+            wasm_compiler: "WasmEdge"
           - name: "MacOS: Build Release"
             build_type: "Release"
+            wasm_compiler: "WasmEdge"
+          - name: "MacOS WAVM: Build Debug"
+            build_type: "Debug"
+            wasm_compiler: "WAVM"
     name: "${{ matrix.options.name }}"
     steps:
     - uses: actions/checkout@v4
@@ -91,12 +96,11 @@ jobs:
       env:
         KAGOME_MAC_CI: 1
     - name: build
-      run: ./housekeeping/make_build.sh -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.options.build_type }} -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake
+      run: ./housekeeping/make_build.sh -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.options.build_type }} -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=${{ matrix.options.wasm_compiler }} -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake
       env:
         KAGOME_IN_DOCKER: 0
         KAGOME_MAC_CI: 1
 
-
   Linux:
     if: false # Need to fix
       # ${{
@@ -106,12 +110,12 @@ jobs:
       fail-fast: false
       matrix:
         options:
-          - name: "Linux: clang-16 External Project"
-            run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake
+          - name: "Linux: clang-19 External Project"
+            run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-19_cxx20.cmake
     name: "${{ matrix.options.name }}"
     runs-on: ubuntu-24.04
     timeout-minutes: 120
-    container: qdrvm/kagome-dev:9-minideb
+    container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19
     steps:
       - uses: actions/checkout@v4
       - uses: actions/cache@v4
@@ -133,44 +137,12 @@ jobs:
           df -m || true
           du -hd1 /__w /github || true
 
-  Linux-self-hosted:
-    strategy:
-      fail-fast: false
-      matrix:
-        options:
-          - name: "Self-hosted: Linux: gcc-13 ASAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON
-          - name: "Self-hosted: Linux: clang-16 TSAN WAVM"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM
-          - name: "Self-hosted: Linux: clang-16 UBSAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON -DUBSAN_TRAP=OFF -DUBSAN_ABORT=ON
-            env: UBSAN_OPTIONS=print_stacktrace=1
-        # Need to fix
-        #  - name: "Self-hosted: Linux: clang-16 External Project"
-        #    run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake
-
-    name: "${{ matrix.options.name }}"
-    runs-on: [ actions-runner-controller ]
-    timeout-minutes: 120
-    container: qdrvm/kagome-dev:9-minideb
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/cache@v4
-        if: ${{ env.USE_CACHE == 'true' }}
-        with:
-          path: ${{ env.CACHE_PATHS }}
-          key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }}
-      - name: Install mold
-        run: ./housekeeping/ci_install_mold.sh --make-default
-      - name: "${{ matrix.options.name }}"
-        run: "${{ matrix.options.run }}"
-
   coverage-self-hosted:
     if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted')  }}
     name: "Self-hosted: Linux: gcc-13 coverage/sonar"
     runs-on: [ actions-runner-controller ]
     timeout-minutes: 120
-    container: qdrvm/kagome-dev:9-minideb
+    container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19
     steps:
       - uses: actions/checkout@v4
       - uses: actions/cache@v4
@@ -202,6 +174,73 @@ jobs:
           BRANCH_NAME: ${{ github.ref }}
         run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi
 
+  kagome_dev_docker_build_sanitizers:
+    strategy:
+      fail-fast: false
+      matrix:
+        options:
+          - name: "Self-hosted: Linux: gcc-13 ASAN"
+            params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON
+          - name: "Self-hosted: Linux: clang-19 TSAN WAVM"
+            params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-19_cxx20.cmake -DTSAN=ON
+          - name: "Self-hosted: Linux: clang-19 UBSAN"
+            params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-19_cxx20.cmake -DUBSAN=ON -DUBSAN_TRAP=OFF -DUBSAN_ABORT=ON
+
+    name: "${{ matrix.options.name }}"
+    runs-on: [ actions-runner-controller ]
+    timeout-minutes: 180
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Get master branch"
+        if: github.ref != 'refs/heads/master'
+        run: git fetch origin master:master || true
+
+      - name: "Get commit version"
+        working-directory: ./housekeeping/docker/kagome-dev
+        run: make get_versions
+
+      - name: "Check version"
+        working-directory: ./housekeeping/docker/kagome-dev
+        run: |
+          SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2)
+          echo "short_commit_hash=${SHORT_COMMIT_HASH}" | tee $GITHUB_ENV
+
+      - name: "Cache dependencies"
+        id: cache-restore
+        if: ${{ env.USE_CACHE == 'true' }}
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ github.job }}-${{ env.CACHE_VERSION }}-sanitizers-${{ env.short_commit_hash }}
+          restore-keys: |
+            ${{ github.job }}-${{ env.CACHE_VERSION }}-sanitizers-
+
+      - name: "Build target"
+        working-directory: ./housekeeping/docker/kagome-dev
+        run:
+          make kagome_dev_docker_build_sanitizers \
+          GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
+          GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
+          BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \
+          CI="true"
+          SAN_PARAMS="${{ matrix.options.params }}"
+
+      - name: "Cleaning cache"
+        run: |
+          find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \;
+
+      - name: "Always Save Cache"
+        id: cache-save
+        if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True')
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ steps.cache-restore.outputs.cache-primary-key }}
+
   kagome_dev_docker_build_tidy:
     runs-on: ubuntu-24.04
     timeout-minutes: 600 # TODO(xDimon): revert after merge PR#2208

diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake
@@ -25,13 +25,6 @@ hunter_config(
     KEEP_PACKAGE_SOURCES
 )
 
-hunter_config(
-    benchmark
-    URL https://github.com/google/benchmark/archive/refs/tags/v1.8.3.zip
-    SHA1 bf9870756ee3f8d2d3b346b24ee3600a41c74d3d
-    CMAKE_ARGS BENCHMARK_ENABLE_TESTING=OFF
-)
-
 hunter_config(
     rocksdb
     VERSION 9.6.1
@@ -99,8 +92,8 @@ endif ()
 
 hunter_config(
     kagome-crates
-    URL  https://github.com/qdrvm/kagome-crates/archive/refs/tags/1.0.2.tar.gz
-    SHA1 946c48508545380e155ab831be54228b916544d3
+    URL  https://github.com/qdrvm/kagome-crates/archive/refs/tags/v1.0.3.tar.gz
+    SHA1 4207446a0e45764b814805821aa6860924b03cb7
 )
 
 hunter_config(
@@ -112,29 +105,23 @@ hunter_config(
 
 hunter_config(
     libp2p
-    URL https://github.com/libp2p/cpp-libp2p/archive/c96d45f792fafd6970a7e37ec816b02a9167e2b6.zip
-    SHA1 884932112bc75996eeecd4a7bbcb932565fe9859
+    VERSION 0.1.28
 )
 
 hunter_config(
     scale
-    URL  https://github.com/qdrvm/scale-codec-cpp/archive/e1a3c7afafc2eeda0c8e2daed08da6b7789f44b3.zip
-    SHA1 b56bcda34fb0d293c88d8b642b2f3fdc2d16a3e5
+    VERSION 1.1.4
 )
 
 hunter_config(
     erasure_coding_crust
-#    VERSION 0.0.8
-    URL  https://github.com/qdrvm/erasure-coding-crust/archive/refs/tags/v0.0.8.tar.gz
-    SHA1 6bcdb6327f5da2dcec5c70f2fa63b95a44925af0
+    VERSION 0.0.8
     KEEP_PACKAGE_SOURCES
 )
 
 hunter_config(
     soralog
-#    VERSION 0.2.4
-    URL  https://github.com/qdrvm/soralog/archive/refs/tags/v0.2.4.tar.gz
-    SHA1 1de495d8a3a73c1e940be3fdddf263a2d673aec1
+    VERSION 0.2.4
     KEEP_PACKAGE_SOURCES
 )
 
diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake
@@ -1,5 +1,5 @@
 HunterGate(
-  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm23.zip
-  SHA1 3fb58496e599fa3d35e94d1810324c6b379029f1
+  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm25.zip
+  SHA1 59c66ff04ebd2cbdf86c3b996d38d4be6eaaa78b
   LOCAL
 )
diff --git a/cmake/toolchain/clang-16_cxx20.cmake b/cmake/toolchain/clang-16_cxx20.cmake
diff --git a/cmake/toolchain/clang-19_cxx20.cmake b/cmake/toolchain/clang-19_cxx20.cmake
@@ -0,0 +1,2 @@
+include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-19.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake)
diff --git a/cmake/toolchain/clang-16_mold_cxx20.cmake → cmake/toolchain/clang-19_mold_cxx20.cmake b/cmake/toolchain/clang-16_mold_cxx20.cmake → cmake/toolchain/clang-19_mold_cxx20.cmake
@@ -1,3 +1,3 @@
-include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-16.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-19.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/linker/mold.cmake)
diff --git a/cmake/toolchain/compiler/clang-16.cmake → cmake/toolchain/compiler/clang-19.cmake b/cmake/toolchain/compiler/clang-16.cmake → cmake/toolchain/compiler/clang-19.cmake
@@ -1,7 +1,7 @@
-if(DEFINED POLLY_COMPILER_CLANG_16_CMAKE)
+if(DEFINED POLLY_COMPILER_CLANG_19_CMAKE)
   return()
 else()
-  set(POLLY_COMPILER_CLANG_16_CMAKE 1)
+  set(POLLY_COMPILER_CLANG_19_CMAKE 1)
 endif()
 
 include(${CMAKE_CURRENT_LIST_DIR}/../../print.cmake)
@@ -13,29 +13,29 @@ if(XCODE_VERSION)
   fatal_error(${_err})
 endif()
 
-find_program(CMAKE_C_COMPILER clang-16)
-find_program(CMAKE_CXX_COMPILER clang++-16)
+find_program(CMAKE_C_COMPILER clang-19)
+find_program(CMAKE_CXX_COMPILER clang++-19)
 
 if (CMAKE_CXX_COMPILER STREQUAL "CMAKE_CXX_COMPILER-NOTFOUND")
-  message(STATUS "clang++-16 not found, checking clang++")
+  message(STATUS "clang++-19 not found, checking clang++")
   cmake_path(GET CMAKE_C_COMPILER PARENT_PATH compiler_path)
   message(STATUS "Assumed compiler path: ${compiler_path}")
-  # clang++-16 doesn't always exist
+  # clang++-19 doesn't always exist
   find_program(CMAKE_CXX_COMPILER clang++ PATHS "${compiler_path}" NO_DEFAULT_PATH REQUIRED)
 
   execute_process(COMMAND "${CMAKE_CXX_COMPILER}" --version OUTPUT_VARIABLE compiler_version_output)
   string(REGEX MATCH "clang version ([0-9]+)\\.[0-9]+\\.[0-9]+" compiler_version "${compiler_version_output}")
-  if (NOT CMAKE_MATCH_1 STREQUAL "16")
-    message(FATAL_ERROR "Found clang++ version ${CMAKE_MATCH_1}, 16 is required")
+  if (NOT CMAKE_MATCH_1 STREQUAL "19")
+    message(FATAL_ERROR "Found clang++ version ${CMAKE_MATCH_1}, 19 is required")
   endif()
 endif()
 
 if(NOT CMAKE_C_COMPILER)
-  fatal_error("clang-16 not found")
+  fatal_error("clang-19 not found")
 endif()
 
 if(NOT CMAKE_CXX_COMPILER)
-  fatal_error("clang++-16 not found")
+  fatal_error("clang++-19 not found")
 endif()
 
 set(

diff --git a/core/application/CMakeLists.txt b/core/application/CMakeLists.txt
@@ -83,6 +83,14 @@ target_link_libraries(recovery_mode
     logger
     )
 
+add_library(kagome_key
+    modes/key_main.cpp
+)
+
+target_link_libraries(kagome_key
+    application_injector
+)
+
 add_library(kagome_application
     impl/kagome_application_impl.cpp
     )

diff --git a/core/application/chain_spec.hpp b/core/application/chain_spec.hpp
@@ -38,8 +38,8 @@ namespace kagome::application {
     virtual const std::vector<libp2p::multi::Multiaddress> &bootNodes()
         const = 0;
 
-    virtual const std::vector<std::pair<std::string, size_t>>
-        &telemetryEndpoints() const = 0;
+    virtual const std::vector<std::pair<std::string, size_t>> &
+    telemetryEndpoints() const = 0;
 
     virtual const std::string &protocolId() const = 0;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-19.cmake)
		include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake)