diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml new file mode 100644 index 000000000..2a68272cc --- /dev/null +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -0,0 +1,25 @@ +name: MacOS TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-macos-14 + path: artifacts diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 50d2661ba..367dd6638 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -35,6 +35,14 @@ jobs: workflow_conclusion: success skip_unpack: true + - name: Download Mac arm64 artifacts + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ton-arm64-macos.yml + path: artifacts + workflow_conclusion: success + skip_unpack: true + - name: Download and unzip Mac x86-64 artifacts uses: dawidd6/action-download-artifact@v2 with: @@ -43,6 +51,14 @@ jobs: workflow_conclusion: success skip_unpack: false + - name: Download and unzip arm64 artifacts + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ton-arm64-macos.yml + path: artifacts + workflow_conclusion: success + skip_unpack: false + - name: Download Windows artifacts uses: dawidd6/action-download-artifact@v2 with: @@ -297,6 +313,97 @@ jobs: asset_name: tonlib-cli-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} + + # mac arm64 + + - name: Upload Mac arm64 artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries.zip + asset_name: ton-mac-arm64.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - fift + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/fift + asset_name: fift-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - func + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/func + asset_name: func-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - lite-client + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/lite-client + asset_name: lite-client-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - rldp-http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/rldp-http-proxy + asset_name: rldp-http-proxy-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/http-proxy + asset_name: http-proxy-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - storage-daemon-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/storage-daemon-cli + asset_name: storage-daemon-cli-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/storage-daemon + asset_name: storage-daemon-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/libtonlibjson.dylib + asset_name: tonlibjson-mac-arm64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/libemulator.dylib + asset_name: libemulator-mac-arm64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/tonlib-cli + asset_name: tonlib-cli-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + # linux x86-64 - name: Upload Linux x86-64 artifacts diff --git a/.github/workflows/ton-arm64-macos.yml b/.github/workflows/ton-arm64-macos.yml new file mode 100644 index 000000000..9e8302e80 --- /dev/null +++ b/.github/workflows/ton-arm64-macos.yml @@ -0,0 +1,37 @@ +name: MacOS TON build (portable, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - uses: cachix/install-nix-action@v23 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + + - name: Build TON + run: | + cp assembly/nix/build-macos-nix.sh . + chmod +x build-macos-nix.sh + ./build-macos-nix.sh -t + + - name: Simple binaries test + run: | + sudo mv /nix/store /nix/store2 + artifacts/validator-engine -V + artifacts/lite-client -V + artifacts/fift -V + artifacts/func -V + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-arm64-macos-binaries + path: artifacts diff --git a/Changelog.md b/Changelog.md index a02410557..effe339f4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,16 @@ +## 2024.06 Update + +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only. + + ## 2024.04 Update 1. Emulator: Single call optimized runGetMethod added diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh index 8e5c367c9..7e85a8712 100644 --- a/assembly/nix/build-linux-arm64-nix.sh +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -15,8 +15,6 @@ while getopts 't' flag; do done cp assembly/nix/linux-arm64* . -cp assembly/nix/microhttpd.nix . -cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz if [ "$with_tests" = true ]; then @@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result + nix-build linux-arm64-tonlib.nix + cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh index 38431ca43..c1f1dcf37 100644 --- a/assembly/nix/build-linux-x86-64-nix.sh +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -15,8 +15,6 @@ while getopts 't' flag; do done cp assembly/nix/linux-x86-64* . -cp assembly/nix/microhttpd.nix . -cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz if [ "$with_tests" = true ]; then @@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result + nix-build linux-x86-64-tonlib.nix + cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh index 12977745b..c75ca0428 100644 --- a/assembly/nix/build-macos-nix.sh +++ b/assembly/nix/build-macos-nix.sh @@ -28,7 +28,9 @@ cp ./result-bin/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result-bin + nix-build macos-tonlib.nix + cp ./result/lib/libtonlibjson.dylib artifacts/ cp ./result/lib/libemulator.dylib artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix index 8c2749b07..536152d26 100644 --- a/assembly/nix/linux-arm64-static.nix +++ b/assembly/nix/linux-arm64-static.nix @@ -6,9 +6,23 @@ , testing ? false }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + in -with import microhttpdmy; stdenv.mkDerivation { pname = "ton"; version = "dev-bin"; @@ -16,31 +30,33 @@ stdenv.mkDerivation { src = ./.; nativeBuildInputs = with pkgs; - [ - cmake ninja git pkg-config - ]; + [ cmake ninja git pkg-config ]; buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static ]; - makeStatic = true; - doCheck = testing; - cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" + "-DTON_USE_JEMALLOC=ON" ]; + makeStatic = true; + doCheck = testing; + LDFLAGS = [ - "-static-libgcc" "-static-libstdc++" "-static" + "-static-libgcc" "-static-libstdc++" "-static" ]; } diff --git a/assembly/nix/linux-arm64-tonlib.nix b/assembly/nix/linux-arm64-tonlib.nix index ae62ca263..a051e34cd 100644 --- a/assembly/nix/linux-arm64-tonlib.nix +++ b/assembly/nix/linux-arm64-tonlib.nix @@ -5,9 +5,21 @@ , stdenv ? pkgs.stdenv }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); in -with import microhttpdmy; pkgs.llvmPackages_16.stdenv.mkDerivation { pname = "ton"; version = "dev-lib"; @@ -21,7 +33,12 @@ pkgs.llvmPackages_16.stdenv.mkDerivation { buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev ]; dontAddStaticConfigureFlags = false; @@ -29,9 +46,6 @@ pkgs.llvmPackages_16.stdenv.mkDerivation { cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" ]; LDFLAGS = [ diff --git a/assembly/nix/linux-x86-64-static.nix b/assembly/nix/linux-x86-64-static.nix index 8c2749b07..a96a9e867 100644 --- a/assembly/nix/linux-x86-64-static.nix +++ b/assembly/nix/linux-x86-64-static.nix @@ -6,9 +6,23 @@ , testing ? false }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + in -with import microhttpdmy; stdenv.mkDerivation { pname = "ton"; version = "dev-bin"; @@ -16,31 +30,33 @@ stdenv.mkDerivation { src = ./.; nativeBuildInputs = with pkgs; - [ - cmake ninja git pkg-config - ]; + [ cmake ninja git pkg-config ]; buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static ]; - makeStatic = true; - doCheck = testing; - cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" + "-DTON_USE_JEMALLOC=ON" ]; + makeStatic = true; + doCheck = testing; + LDFLAGS = [ - "-static-libgcc" "-static-libstdc++" "-static" + "-static-libgcc" "-static-libstdc++" "-fPIC" ]; } diff --git a/assembly/nix/linux-x86-64-tonlib.nix b/assembly/nix/linux-x86-64-tonlib.nix index 5a6e43e8f..afcbe3ba6 100644 --- a/assembly/nix/linux-x86-64-tonlib.nix +++ b/assembly/nix/linux-x86-64-tonlib.nix @@ -7,20 +7,35 @@ , stdenv ? pkgs.stdenv }: let - system = builtins.currentSystem; - - nixos1909 = (import (builtins.fetchTarball { - url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; - sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; - }) { inherit system; }); - glibc227 = nixos1909.glibc // { pname = "glibc"; }; - stdenv227 = let - cc = pkgs.wrapCCWith { - cc = nixos1909.buildPackages.gcc-unwrapped; - libc = glibc227; - bintools = pkgs.binutils.override { libc = glibc227; }; - }; - in (pkgs.overrideCC pkgs.stdenv cc); + system = builtins.currentSystem; + + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + + nixos1909 = (import (builtins.fetchTarball { + url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; + sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; + }) { inherit system; }); + glibc227 = nixos1909.glibc // { pname = "glibc"; }; + stdenv227 = let + cc = pkgs.wrapCCWith { + cc = nixos1909.buildPackages.gcc-unwrapped; + libc = glibc227; + bintools = pkgs.binutils.override { libc = glibc227; }; + }; + in (pkgs.overrideCC pkgs.stdenv cc); in stdenv227.mkDerivation { @@ -34,7 +49,12 @@ stdenv227.mkDerivation { buildInputs = with pkgs; [ - pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev ]; dontAddStaticConfigureFlags = false; diff --git a/assembly/nix/macos-static.nix b/assembly/nix/macos-static.nix index be15579ca..2fd0b3a66 100644 --- a/assembly/nix/macos-static.nix +++ b/assembly/nix/macos-static.nix @@ -17,7 +17,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { buildInputs = with pkgs; lib.forEach [ - secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext (gnutls.override { withP11-kit = false; }).dev + secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext jemalloc (gnutls.override { withP11-kit = false; }).dev ] (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" "--disable-tests" ]; dontDisableStatic = true; })) ++ [ @@ -38,13 +38,13 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" + "-DTON_USE_JEMALLOC=ON" "-DCMAKE_CROSSCOMPILING=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_CXX_FLAGS=-stdlib=libc++" "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.3" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" ]; LDFLAGS = [ diff --git a/assembly/nix/microhttpd.nix b/assembly/nix/microhttpd.nix deleted file mode 100644 index 4f871425a..000000000 --- a/assembly/nix/microhttpd.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ pkgs ? import { system = builtins.currentSystem; } -, stdenv ? pkgs.stdenv -, fetchgit ? pkgs.fetchgit -}: - -stdenv.mkDerivation rec { - name = "microhttpdmy"; - - - src = fetchgit { - url = "https://git.gnunet.org/libmicrohttpd.git"; - rev = "refs/tags/v0.9.77"; - sha256 = "sha256-x+nfB07PbZwBlFc6kZZFYiRpk0a3QN/ByHB+hC8na/o="; - }; - - nativeBuildInputs = with pkgs; [ automake libtool autoconf texinfo ]; - - buildInputs = with pkgs; [ ]; - - configurePhase = '' - ./autogen.sh - ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - ''; - - installPhase = '' - make install DESTDIR=$out - ''; -} diff --git a/assembly/nix/openssl.nix b/assembly/nix/openssl.nix deleted file mode 100644 index 8d30aa504..000000000 --- a/assembly/nix/openssl.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ pkgs ? import { system = builtins.currentSystem; } -, stdenv ? pkgs.stdenv -, fetchFromGitHub ? pkgs.fetchFromGitHub -}: - -stdenv.mkDerivation rec { - name = "opensslmy"; - - src = fetchFromGitHub { - owner = "openssl"; - repo = "openssl"; - rev = "refs/tags/openssl-3.1.4"; - sha256 = "sha256-Vvf1wiNb4ikg1lIS9U137aodZ2JzM711tSWMJFYWtWI="; - }; - - nativeBuildInputs = with pkgs; [ perl ]; - - buildInputs = with pkgs; [ ]; - - postPatch = '' - patchShebangs Configure - ''; - - configurePhase = '' - ./Configure no-shared - ''; - installPhase = '' - make install DESTDIR=$out - ''; -} diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 29b954664..306194408 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -479,10 +479,17 @@ if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN) OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_FIF} ) set(ARG_DEST_CPP "${ARG_DEST}.cpp") + + if (WIN32) + set(ARG_LIB_DIR "fift/lib@smartcont") + else() + set(ARG_LIB_DIR "fift/lib:smartcont") + endif() + add_custom_command( COMMENT "Generate ${ARG_DEST_CPP}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND fift -Ifift/lib:smartcont -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} + COMMAND fift -I${ARG_LIB_DIR} -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} MAIN_DEPENDENCY ${ARG_SOURCE} DEPENDS fift ${ARG_DEST_FIF} smartcont/asm-to-cpp.fif fift/lib/Asm.fif OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_CPP} diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 183da0a73..348377e94 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -814,11 +814,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } diff --git a/crypto/fift/fift-main.cpp b/crypto/fift/fift-main.cpp index ef833f43e..fd424e8cf 100644 --- a/crypto/fift/fift-main.cpp +++ b/crypto/fift/fift-main.cpp @@ -62,7 +62,7 @@ void usage(const char* progname) { << " [-i] [-n] [-I ] {-L } ...\n"; std::cerr << "\t-n\tDo not preload standard preamble file `Fift.fif`\n" "\t-i\tForce interactive mode even if explicit source file names are indicated\n" - "\t-I\tSets colon-separated library source include path. If not indicated, " + "\t-I\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, " "$FIFTPATH is used instead.\n" "\t-L\tPre-loads a library source file\n" "\t-d\tUse a ton database\n" @@ -75,11 +75,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 09ae1b661..a5f87774d 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/utils/CancellationToken.h" + #include #include #include "vm/db/DynamicBagOfCellsDb.h" @@ -331,7 +333,7 @@ td::Result>> std_boc_deserialize_multi(td::Slice data, int max_roots = BagOfCells::default_max_roots); td::Result std_boc_serialize_multi(std::vector> root, int mode = 0); -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode = 0); +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index fbd065dc9..a7dae1b08 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -33,7 +33,8 @@ class LargeBocSerializer { public: using Hash = Cell::Hash; - explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { + explicit LargeBocSerializer(std::shared_ptr reader, td::CancellationToken cancellation_token = {}) + : reader(std::move(reader)), cancellation_token(std::move(cancellation_token)) { } void add_root(Hash root); @@ -84,6 +85,7 @@ class LargeBocSerializer { int revisit(int cell_idx, int force = 0); td::uint64 compute_sizes(int mode, int& r_size, int& o_size); + td::CancellationToken cancellation_token; td::Timestamp log_speed_at_; size_t processed_cells_ = 0; static constexpr double LOG_SPEED_PERIOD = 120.0; @@ -112,6 +114,9 @@ td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token.check()); + } if (log_speed_at_.is_in_past()) { log_speed_at_ += LOG_SPEED_PERIOD; LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; @@ -408,6 +413,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { store_ref(k); } ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token.check()); + } if (log_speed_at_.is_in_past()) { log_speed_at_ += LOG_SPEED_PERIOD; LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; @@ -428,10 +436,10 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } // namespace td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, - int mode) { + int mode, td::CancellationToken cancellation_token) { td::Timer timer; CHECK(reader != nullptr) - LargeBocSerializer serializer(reader); + LargeBocSerializer serializer(reader, std::move(cancellation_token)); serializer.add_root(root_hash); TRY_STATUS(serializer.import_cells()); TRY_STATUS(serializer.serialize(fd, mode)); diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 37a158ebb..025cf7d51 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -170,7 +170,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 409f09935..e81fecc70 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -229,7 +229,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { continue; } - if (X->get_version() <= td::Clocks::system() - 600) { + if (public_ && X->get_version() <= td::Clocks::system() - 600) { if (X->is_neighbour()) { bool found = false; for (auto &n : neighbours_) { @@ -301,7 +301,7 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, auto t = td::Clocks::system(); while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) { auto P = peers_.get_random(); - if (P->get_version() + 3600 < t) { + if (public_ && P->get_version() + 3600 < t) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); } else if (P->is_alive()) { diff --git a/recent_changelog.md b/recent_changelog.md index f0b029eea..25a93c189 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,13 +1,11 @@ ## 2024.04 Update -1. Emulator: Single call optimized runGetMethod added -2. Tonlib: a series of proof improvements, also breaking Change in `liteServer.getAllShardsInfo` method (see below) -3. DB: usage statistics now collected, outdated persistent states are not serialized -4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests -5. Network: lz4 compression of block candidates (disabled by default). -6. Overlays: add custom overlays -7. Transaction Executor: fixed issue with due_payment collection - -* `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block. - -Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue). +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only. diff --git a/tdutils/td/utils/CancellationToken.h b/tdutils/td/utils/CancellationToken.h index 9f30d204c..7ef304979 100644 --- a/tdutils/td/utils/CancellationToken.h +++ b/tdutils/td/utils/CancellationToken.h @@ -20,6 +20,7 @@ #include #include +#include "Status.h" namespace td { @@ -38,6 +39,12 @@ class CancellationToken { } return token_->is_cancelled_.load(std::memory_order_acquire); } + Status check() const { + if (*this) { + return Status::Error(653, "cancelled"); // cancelled = 653 + } + return Status::OK(); + } CancellationToken() = default; explicit CancellationToken(std::shared_ptr token) : token_(std::move(token)) { } diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index 7697f3176..6bc1eb655 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -99,7 +99,7 @@ liteServer.getLibrariesWithProof id:tonNode.blockIdExt mode:# library_list:(vect liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof; liteServer.getOutMsgQueueSizes mode:# wc:mode.0?int shard:mode.0?long = liteServer.OutMsgQueueSizes; -liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.1?long = liteServer.nonfinal.ValidatorGroups; +liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.ValidatorGroups; liteServer.nonfinal.getCandidate id:liteServer.nonfinal.candidateId = liteServer.nonfinal.Candidate; liteServer.queryPrefix = Object; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index d7d852893..e72be25ff 100644 Binary files a/tl/generate/scheme/lite_api.tlo and b/tl/generate/scheme/lite_api.tlo differ diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5d1e5b504..58691ffd7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -399,6 +399,8 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; // signature may be empty, at least for now tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; +tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; @@ -590,11 +592,13 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; -engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) +engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig; +engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) fullnodeconfig:engine.validator.fullNodeConfig + extraconfig:engine.validator.extraConfig liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; @@ -709,6 +713,8 @@ engine.validator.addCustomOverlay overlay:engine.validator.customOverlay = engin engine.validator.delCustomOverlay name:string = engine.validator.Success; engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig; +engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 89948bb4a..ad6bb0c99 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 38f524fc0..5385d2e6c 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1180,3 +1180,26 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { } return td::Status::OK(); } + +td::Status SetStateSerializerEnabledQuery::run() { + TRY_RESULT(value, tokenizer_.get_token()); + if (value != 0 && value != 1) { + return td::Status::Error("expected 0 or 1"); + } + TRY_STATUS(tokenizer_.check_endl()); + enabled_ = value; + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::send() { + auto b = ton::create_serialize_tl_object(enabled_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 34b516d6c..3047350fe 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1207,3 +1207,25 @@ class ShowCustomOverlaysQuery : public Query { return get_name(); } }; + +class SetStateSerializerEnabledQuery : public Query { + public: + SetStateSerializerEnabledQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "setstateserializerenabled"; + } + static std::string get_help() { + return "setstateserializerenabled \tdisable or enable persistent state serializer; value is 0 or 1"; + } + std::string name() const override { + return get_name(); + } + + private: + bool enabled_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 263bca3c0..4878a292f 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -146,6 +146,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 66c75827d..897e3c53b 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -73,6 +73,7 @@ #include "block-parse.h" #include "common/delay.h" #include "block/precompiled-smc/PrecompiledSmartContract.h" +#include "interfaces/validator-manager.h" Config::Config() { out_port = 3278; @@ -155,6 +156,11 @@ Config::Config(ton::ton_api::engine_validator_config &config) { if (config.fullnodeconfig_) { full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_); } + if (config.extraconfig_) { + state_serializer_enabled = config.extraconfig_->state_serializer_enabled_; + } else { + state_serializer_enabled = true; + } for (auto &serv : config.liteservers_) { config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure(); @@ -231,6 +237,12 @@ ton::tl_object_ptr Config::tl() const { full_node_config_obj = full_node_config.tl(); } + ton::tl_object_ptr extra_config_obj = {}; + if (!state_serializer_enabled) { + // Non-default values + extra_config_obj = ton::create_tl_object(state_serializer_enabled); + } + std::vector> liteserver_vec; for (auto &x : liteservers) { liteserver_vec.push_back(ton::create_tl_object(x.second.tl(), x.first)); @@ -253,7 +265,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(full_node_config_obj), - std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1399,6 +1411,7 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); + validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled); return td::Status::OK(); } @@ -3642,6 +3655,34 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto custom_overlays_config_, true)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (query.enabled_ == validator_options_->get_state_serializer_enabled()) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + validator_options_.write().set_state_serializer_enabled(query.enabled_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + config_.state_serializer_enabled = query.enabled_; + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::create_serialize_tl_object()); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3856,7 +3897,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); - p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", + p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -3866,7 +3907,7 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", + 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -3979,7 +4020,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); p.add_checked_option( - '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 50G)", + '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); if (v == 0) { @@ -3989,12 +4030,12 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_option( - '\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb (forced when celldb cache is < 30G)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); }); + '\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); }); p.add_option( - '\0', "celldb-no-preload-all", - "disable preloading all cells from CellDb on startup (enabled by default)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, false); }); }); + '\0', "celldb-preload-all", + "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 44fb1c27c..8adc8d9a7 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -90,6 +90,8 @@ struct Config { std::map controls; std::set gc; + bool state_serializer_enabled = true; + void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { keys_refcnt[key]++; @@ -209,9 +211,9 @@ class ValidatorEngine : public td::actor::Actor { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; - td::optional celldb_cache_size_ = 50LL << 30; - bool celldb_direct_io_ = true; - bool celldb_preload_all_ = true; + td::optional celldb_cache_size_ = 1LL << 30; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; bool read_config_ = false; bool started_keyring_ = false; @@ -473,6 +475,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/fabric.h b/validator/fabric.h index 326b17aec..6bb668452 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -49,8 +49,8 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); -void run_check_external_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, td::Promise> promise); +void run_check_external_message(td::Ref message, td::actor::ActorId manager, + td::Promise> promise); void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index ffe5468d4..e5ea1f0be 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -52,18 +52,37 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { - if (query.data_.size() > FullNode::max_block_size()) { + process_block_candidate_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, + ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; return; } - BlockIdExt block_id = create_block_id(query.id_); - if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; return; } VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, - query.validator_set_hash_, std::move(query.data_)); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); } void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { @@ -99,12 +118,15 @@ void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, Catc if (!inited_) { return; } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_block_id(block_id), cc_seqno, validator_set_hash, - create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { @@ -230,25 +252,42 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod } void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ << "\" from unauthorized sender " << src; return; } - if (query.data_.size() > FullNode::max_block_size()) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; return; } - BlockIdExt block_id = create_block_id(query.id_); - if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; return; } - // ignore cc_seqno and validator_hash for now VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, - query.validator_set_hash_, std::move(query.data_)); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); } void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { @@ -298,12 +337,15 @@ void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSe if (!inited_) { return; } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_block_id(block_id), cc_seqno, validator_set_hash, - create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodeCustomOverlay::start_up() { diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index e310824f0..a0022fa03 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -27,7 +27,11 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -43,15 +47,11 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { config_ = std::move(config); } - void set_enable_compression(bool value) { - enable_compression_ = value; - } - void start_up() override; void tear_down() override; FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - FileHash zero_state_file_hash, FullNodeConfig config, bool enable_compression, + FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, @@ -61,7 +61,6 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { , nodes_(std::move(nodes)) , zero_state_file_hash_(zero_state_file_hash) , config_(config) - , enable_compression_(enable_compression) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -76,7 +75,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { std::vector nodes_; FileHash zero_state_file_hash_; FullNodeConfig config_; - bool enable_compression_; + bool enable_compression_ = true; td::actor::ActorId keyring_; td::actor::ActorId adnl_; @@ -101,7 +100,11 @@ class FullNodeCustomOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 42e682864..94dc2155e 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -152,4 +152,63 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id return S; } +td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Slice data, + bool compression_enabled) { + if (!compression_enabled) { + return create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), td::BufferSlice(data)); + } + TRY_RESULT(root, vm::std_boc_deserialize(data)); + TRY_RESULT(data_new, vm::std_boc_serialize(root, 2)); + td::BufferSlice compressed = td::lz4_compress(data_new); + VLOG(FULL_NODE_DEBUG) << "Compressing block candidate broadcast: " << data.size() << " -> " << compressed.size(); + return create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), 0, std::move(compressed)); +} + +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcast& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + data = std::move(obj.data_); + return td::Status::OK(); +} + +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressed& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data, + int max_decompressed_data_size) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + TRY_RESULT(decompressed, td::lz4_decompress(obj.compressed_, max_decompressed_data_size)); + TRY_RESULT(root, vm::std_boc_deserialize(decompressed)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast: " << obj.compressed_.size() << " -> " + << data.size(); + return td::Status::OK(); +} + +td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, + CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, + td::BufferSlice& data, int max_decompressed_data_size) { + td::Status S; + ton_api::downcast_call(obj, td::overloaded( + [&](ton_api::tonNode_newBlockCandidateBroadcast& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data); + }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressed& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size); + }, + [&](auto&) { S = td::Status::Error("unknown data type"); })); + return S; +} + } // namespace ton::validator::fullnode diff --git a/validator/full-node-serializer.hpp b/validator/full-node-serializer.hpp index a5c73cbc2..f6751689c 100644 --- a/validator/full-node-serializer.hpp +++ b/validator/full-node-serializer.hpp @@ -28,4 +28,11 @@ td::Result serialize_block_full(const BlockIdExt& id, td::Slice td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size); +td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Slice data, + bool compression_enabled); +td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, + CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, + td::BufferSlice& data, int max_decompressed_data_size); + } // namespace ton::validator::fullnode diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 9b2b321e5..fa0c3e62a 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -649,19 +649,32 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { - if (query.data_.size() > FullNode::max_block_size()) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (data.size() > FullNode::max_block_size()) { VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; return; } - BlockIdExt block_id = create_block_id(query.id_); - if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; return; } - // ignore cc_seqno and validator_hash for now VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, - query.validator_set_hash_, std::move(query.data_)); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { @@ -762,12 +775,15 @@ void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno UNREACHABLE(); return; } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate: " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_block_id(block_id), cc_seqno, validator_set_hash, - create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, - overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 08f48f862..cec7c6494 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -153,7 +153,11 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index ff8f1eca0..a72be3ff4 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -369,14 +369,12 @@ td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull ds return get_shard(shard_prefix(dst, 60)); } -void FullNodeImpl::got_key_block_state(td::Ref state) { - auto m = td::Ref{std::move(state)}; - +void FullNodeImpl::got_key_block_config(td::Ref config) { PublicKeyHash l = PublicKeyHash::zero(); std::vector keys; std::map current_validators; for (td::int32 i = -1; i <= 1; i++) { - auto r = m->get_total_validator_set(i < 0 ? i : 1 - i); + auto r = config->get_total_validator_set(i < 0 ? i : 1 - i); if (r.not_null()) { auto vec = r->export_vector(); for (auto &el : vec) { @@ -392,16 +390,15 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } } - set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3); - if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); update_private_overlays(); } - if (keys == all_validators_) { - return; - } + // Let's turn off this optimization, since keyblocks are rare enough to update on each keyblock + // if (keys == all_validators_) { + // return; + // } all_validators_ = keys; sign_cert_by_ = l; @@ -413,15 +410,31 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } void FullNodeImpl::new_key_block(BlockHandle handle) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to get key block state: " << R.move_as_error(); - } else { - td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_state, R.move_as_ok()); - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, - std::move(P)); + if (handle->id().seqno() == 0) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to get zero state: " << R.move_as_error(); + } else { + auto s = td::Ref{R.move_as_ok()}; + CHECK(s.not_null()); + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, s->get_config_holder().move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, + std::move(P)); + } else { + CHECK(handle->is_key_block()); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to get key block proof: " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, + R.ok()->get_key_block_config().move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_proof_link_from_db, handle, + std::move(P)); + } } void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { @@ -549,16 +562,6 @@ void FullNodeImpl::update_private_overlays() { } } -void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) { - if (private_block_overlays_enable_compression_ == value) { - return; - } - private_block_overlays_enable_compression_ = true; - for (auto &p : private_block_overlays_) { - td::actor::send_closure(p.second, &FullNodePrivateBlockOverlay::set_enable_compression, value); - } -} - void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { @@ -567,9 +570,8 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { nodes.push_back(p.second); } private_block_overlays_[key] = td::actor::create_actor( - "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, - private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, - actor_id(this)); + "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, + adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); } } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 9b78b3e01..3dfa17fdd 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -82,7 +82,7 @@ class FullNodeImpl : public FullNode { void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); - void got_key_block_state(td::Ref state); + void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); void process_block_broadcast(BlockBroadcast broadcast) override; @@ -127,7 +127,6 @@ class FullNodeImpl : public FullNode { FullNodeConfig config_; std::map> private_block_overlays_; - bool private_block_overlays_enable_compression_ = false; bool broadcast_block_candidates_in_public_overlay_ = false; struct CustomOverlayInfo { @@ -139,7 +138,6 @@ class FullNodeImpl : public FullNode { std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); - void set_private_block_overlays_enable_compression(bool value); void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 073e7360e..2fdb491bc 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -86,24 +86,18 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; } -void ExtMessageQ::run_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, +void ExtMessageQ::run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise) { - auto R = create_ext_message(std::move(data), limits); - if (R.is_error()) { - return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); - } - auto M = R.move_as_ok(); - auto root = M->root_cell(); + auto root = message->root_cell(); block::gen::CommonMsgInfo::Record_ext_in_msg_info info; tlb::unpack_cell_inexact(root, info); // checked in create message - ton::StdSmcAddress addr = M->addr(); - ton::WorkchainId wc = M->wc(); + ton::StdSmcAddress addr = message->addr(); + ton::WorkchainId wc = message->wc(); run_fetch_account_state( wc, addr, manager, - [promise = std::move(promise), msg_root = root, wc, addr, - M](td::Result, UnixTime, LogicalTime, std::unique_ptr>> + [promise = std::move(promise), msg_root = root, wc, addr, message]( + td::Result, UnixTime, LogicalTime, std::unique_ptr>> res) mutable { if (res.is_error()) { promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); @@ -120,7 +114,7 @@ void ExtMessageQ::run_message(td::BufferSlice data, block::SizeLimitsConfig::Ext } else { auto status = run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config)); if (status.is_ok()) { - promise.set_value(std::move(M)); + promise.set_value(std::move(message)); } else { promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted\n" << status.message())); diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index d50847617..ad7ecc74e 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -61,8 +61,7 @@ class ExtMessageQ : public ExtMessage { ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits); - static void run_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, + static void run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise); static td::Status run_message_on_account(ton::WorkchainId wc, block::Account* acc, diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index e3478594b..997fa9a18 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -119,10 +119,9 @@ td::Result> create_ext_message(td::BufferSlice data, return std::move(res); } -void run_check_external_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, +void run_check_external_message(Ref message, td::actor::ActorId manager, td::Promise> promise) { - ExtMessageQ::run_message(std::move(data), limits, std::move(manager), std::move(promise)); + ExtMessageQ::run_message(std::move(message), std::move(manager), std::move(promise)); } td::Result> create_ihr_message(td::BufferSlice data) { diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 12bd68ef2..389c7c0de 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -422,6 +422,10 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_result(td::Status::Error("not implemented")); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } + private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 3a124f827..7bf95b3f7 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -483,6 +483,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { promise.set_result(td::Status::Error("not implemented")); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index f91f04a17..2af818e6f 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -412,14 +412,42 @@ void ValidatorManagerImpl::add_external_message(td::Ref msg, int pri ext_messages_hashes_[id.hash] = {priority, id}; } void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise> promise) { - ++ls_stats_check_ext_messages_; auto state = do_get_last_liteserver_state(); if (state.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - run_check_external_message(std::move(data), state->get_ext_msg_limits(), actor_id(this), - std::move(promise)); + auto R = create_ext_message(std::move(data), state->get_ext_msg_limits()); + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to parse external message: ")); + return; + } + auto message = R.move_as_ok(); + WorkchainId wc = message->wc(); + StdSmcAddress addr = message->addr(); + if (checked_ext_msg_counter_.get_msg_count(wc, addr) >= max_ext_msg_per_addr()) { + promise.set_error( + td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); + return; + } + + promise = [self = this, wc, addr, promise = std::move(promise), + SelfId = actor_id(this)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_lambda(SelfId, [=, promise = std::move(promise), message = R.move_as_ok()]() mutable { + if (self->checked_ext_msg_counter_.inc_msg_count(wc, addr) > max_ext_msg_per_addr()) { + promise.set_error( + td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); + return; + } + promise.set_result(std::move(message)); + }); + }; + ++ls_stats_check_ext_messages_; + run_check_external_message(std::move(message), actor_id(this), std::move(promise)); } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { @@ -2592,6 +2620,16 @@ void ValidatorManagerImpl::alarm() { log_ls_stats_at_ = td::Timestamp::in(60.0); } alarm_timestamp().relax(log_ls_stats_at_); + if (cleanup_mempool_at_.is_in_past()) { + if (is_validator()) { + get_external_messages(ShardIdFull{masterchainId, shardIdAll}, + [](td::Result, int>>>) {}); + get_external_messages(ShardIdFull{basechainId, shardIdAll}, + [](td::Result, int>>>) {}); + } + cleanup_mempool_at_ = td::Timestamp::in(250.0); + } + alarm_timestamp().relax(cleanup_mempool_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { @@ -3094,6 +3132,14 @@ void ValidatorManagerImpl::get_validator_groups_info_for_litequery( td::actor::create_actor("get-validator-groups-info", std::move(groups), std::move(promise)).release(); } +void ValidatorManagerImpl::update_options(td::Ref opts) { + // Currently options can be updated only to change state_serializer_enabled flag + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); + } + opts_ = std::move(opts); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, @@ -3102,6 +3148,29 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } +size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it1 = counter_cur_.find({wc, addr}); + auto it2 = counter_prev_.find({wc, addr}); + return (it1 == counter_cur_.end() ? 0 : it1->second) + (it2 == counter_prev_.end() ? 0 : it2->second); +} +size_t ValidatorManagerImpl::CheckedExtMsgCounter::inc_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it2 = counter_prev_.find({wc, addr}); + return (it2 == counter_prev_.end() ? 0 : it2->second) + ++counter_cur_[{wc, addr}]; +} +void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { + while (cleanup_at_.is_in_past()) { + counter_prev_ = std::move(counter_cur_); + counter_cur_.clear(); + if (counter_prev_.empty()) { + cleanup_at_ = td::Timestamp::in(max_ext_msg_per_addr_time_window() / 2.0); + break; + } + cleanup_at_ += max_ext_msg_per_addr_time_window() / 2.0; + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index be7f18db9..f76900a9e 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -241,10 +241,20 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map ext_msgs_; // priority -> messages std::map>> ext_messages_hashes_; // hash -> priority + td::Timestamp cleanup_mempool_at_; // IHR ? std::map, std::unique_ptr>> ihr_messages_; std::map> ihr_messages_hashes_; + struct CheckedExtMsgCounter { + std::map, size_t> counter_cur_, counter_prev_; + td::Timestamp cleanup_at_ = td::Timestamp::now(); + + size_t get_msg_count(WorkchainId wc, StdSmcAddress addr); + size_t inc_msg_count(WorkchainId wc, StdSmcAddress addr); + void before_query(); + } checked_ext_msg_counter_; + private: // VALIDATOR GROUPS ValidatorSessionId get_validator_set_id(ShardIdFull shard, td::Ref val_set, td::Bits256 opts_hash, @@ -578,6 +588,8 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; + void update_options(td::Ref opts) override; + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { if (last_masterchain_state_.is_null()) { @@ -678,6 +690,12 @@ class ValidatorManagerImpl : public ValidatorManager { size_t max_cached_candidates() const { return 128; } + static double max_ext_msg_per_addr_time_window() { + return 10.0; + } + static size_t max_ext_msg_per_addr() { + return 3 * 10; + } private: std::map> shard_client_waiters_; diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 93363d3b2..4f10d959e 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -27,6 +27,9 @@ namespace ton { namespace validator { void AsyncStateSerializer::start_up() { + if (!opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "Persistent state serializer is disabled"; + } alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 10) * 1.0); running_ = true; @@ -130,7 +133,7 @@ void AsyncStateSerializer::next_iteration() { } CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && - need_serialize(masterchain_handle_)) { + need_serialize(masterchain_handle_) && opts_->get_state_serializer_enabled()) { if (!have_masterchain_state_) { LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str(); // block next attempts immediately, but send actual request later @@ -174,6 +177,9 @@ void AsyncStateSerializer::next_iteration() { return; } if (masterchain_handle_->inited_next_left()) { + if (need_serialize(masterchain_handle_) && !opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str(); + } last_block_id_ = masterchain_handle_->one_next(true); have_masterchain_state_ = false; masterchain_handle_ = nullptr; @@ -200,6 +206,10 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { + if (!opts_->get_state_serializer_enabled()) { + stored_masterchain_state(); + return; + } LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); have_masterchain_state_ = true; CHECK(next_idx_ == 0); @@ -210,11 +220,16 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state shards_.push_back(v->top_block_id()); } - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader, + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state); }); @@ -253,13 +268,22 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { + if (!opts_->get_state_serializer_enabled()) { + success_handler(); + return; + } LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader, + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { - R.ensure(); - LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), @@ -285,6 +309,14 @@ void AsyncStateSerializer::success_handler() { next_iteration(); } +void AsyncStateSerializer::update_options(td::Ref opts) { + opts_ = std::move(opts); + if (!opts_->get_state_serializer_enabled()) { + cancellation_token_source_.cancel(); + } +} + + bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { return opts_->need_monitor(shard); } diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index ee2aace01..0bee70315 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -37,6 +37,7 @@ class AsyncStateSerializer : public td::actor::Actor { bool saved_to_db_ = true; td::Ref opts_; + td::CancellationTokenSource cancellation_token_source_; td::actor::ActorId manager_; @@ -89,6 +90,8 @@ class AsyncStateSerializer : public td::actor::Actor { void fail_handler(td::Status reason); void fail_handler_cont(); void success_handler(); + + void update_options(td::Ref opts); }; } // namespace validator diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 593c75bf6..37006bdad 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -141,6 +141,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } + bool get_state_serializer_enabled() const override { + return state_serializer_enabled_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -221,6 +224,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } + void set_state_serializer_enabled(bool value) override { + state_serializer_enabled_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -272,6 +278,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; + bool state_serializer_enabled_ = true; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 4e3390cc5..9082fd882 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -90,6 +90,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_celldb_direct_io() const = 0; virtual bool get_celldb_preload_all() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; + virtual bool get_state_serializer_enabled() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -118,13 +119,14 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; + virtual void set_state_serializer_enabled(bool value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, - double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, + double state_ttl = 3600, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); }; @@ -249,6 +251,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_perf_timer_stat(std::string name, double duration) = 0; virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) = 0; + virtual void update_options(td::Ref opts) = 0; }; } // namespace validator