Merge pull request #1016 from ton-blockchain/testnet

Merge developer branch
ton-blockchain · Jun 9, 2024 · 5c392e0 · 5c392e0
2 parents db505f4 + 7841d75
commit 5c392e0
Show file tree

Hide file tree

Showing 89 changed files with 2,311 additions and 704 deletions.
diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml
@@ -0,0 +1,25 @@
+name: MacOS TON build (shared, arm64)
+
+on: [push,workflow_dispatch,workflow_call]
+
+jobs:
+  build:
+    runs-on: macos-14
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Build TON
+        run: |
+          cp assembly/native/build-macos-shared.sh .
+          chmod +x build-macos-shared.sh
+          ./build-macos-shared.sh -t -a
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@master
+        with:
+          name: ton-binaries-macos-14
+          path: artifacts
diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
@@ -35,6 +35,14 @@ jobs:
           workflow_conclusion: success
           skip_unpack: true
 
+      - name: Download Mac arm64 artifacts
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: ton-arm64-macos.yml
+          path: artifacts
+          workflow_conclusion: success
+          skip_unpack: true
+
       - name: Download and unzip Mac x86-64 artifacts
         uses: dawidd6/action-download-artifact@v2
         with:
@@ -43,6 +51,14 @@ jobs:
           workflow_conclusion: success
           skip_unpack: false
 
+      - name: Download and unzip arm64 artifacts
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: ton-arm64-macos.yml
+          path: artifacts
+          workflow_conclusion: success
+          skip_unpack: false
+
       - name: Download Windows artifacts
         uses: dawidd6/action-download-artifact@v2
         with:
@@ -297,6 +313,97 @@ jobs:
           asset_name: tonlib-cli-mac-x86-64
           tag: ${{ steps.tag.outputs.TAG }}
 
+
+      # mac arm64
+
+      - name: Upload Mac arm64 artifacts
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries.zip
+          asset_name: ton-mac-arm64.zip
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - fift
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/fift
+          asset_name: fift-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - func
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/func
+          asset_name: func-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - lite-client
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/lite-client
+          asset_name: lite-client-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - rldp-http-proxy
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/rldp-http-proxy
+          asset_name: rldp-http-proxy-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - http-proxy
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/http-proxy
+          asset_name: http-proxy-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - storage-daemon-cli
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/storage-daemon-cli
+          asset_name: storage-daemon-cli-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - storage-daemon
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/storage-daemon
+          asset_name: storage-daemon-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - tonlibjson
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/libtonlibjson.dylib
+          asset_name: tonlibjson-mac-arm64.dylib
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - libemulator
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/libemulator.dylib
+          asset_name: libemulator-mac-arm64.dylib
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - tonlib-cli
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/tonlib-cli
+          asset_name: tonlib-cli-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
       # linux x86-64
 
       - name: Upload Linux x86-64 artifacts

diff --git a/.github/workflows/ton-arm64-macos.yml b/.github/workflows/ton-arm64-macos.yml
@@ -0,0 +1,37 @@
+name: MacOS TON build (portable, arm64)
+
+on: [push,workflow_dispatch,workflow_call]
+
+jobs:
+  build:
+    runs-on: macos-14
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: 'recursive'
+
+    - uses: cachix/install-nix-action@v23
+      with:
+        extra_nix_config: |
+          access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build TON
+      run: |
+        cp assembly/nix/build-macos-nix.sh .
+        chmod +x build-macos-nix.sh
+        ./build-macos-nix.sh -t
+
+    - name: Simple binaries test
+      run: |
+        sudo mv /nix/store /nix/store2
+        artifacts/validator-engine -V
+        artifacts/lite-client -V
+        artifacts/fift -V
+        artifacts/func -V
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@master
+      with:
+        name: ton-arm64-macos-binaries
+        path: artifacts
diff --git a/Changelog.md b/Changelog.md
@@ -1,3 +1,16 @@
+## 2024.06 Update
+
+1. Make Jemalloc default allocator
+2. Add candidate broadcasting and caching
+3. Limit per address speed for external messages broadcast by reasonably large number 
+4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks
+5. Extended statistics and logs for celldb usage, session stats, persistent state serialization
+6. Tonlib and explorer fixes
+7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all`
+8. Add valiator-console command to stop persistent state serialization
+9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only.
+
+
 ## 2024.04 Update
 
 1. Emulator: Single call optimized runGetMethod added

diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh
@@ -15,8 +15,6 @@ while getopts 't' flag; do
 done
 
 cp assembly/nix/linux-arm64* .
-cp assembly/nix/microhttpd.nix .
-cp assembly/nix/openssl.nix .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
 
 if [ "$with_tests" = true ]; then
@@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result
+
 nix-build linux-arm64-tonlib.nix
+
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
 cp ./result/lib/fift/* artifacts/lib/

diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh
@@ -15,8 +15,6 @@ while getopts 't' flag; do
 done
 
 cp assembly/nix/linux-x86-64* .
-cp assembly/nix/microhttpd.nix .
-cp assembly/nix/openssl.nix .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
 
 if [ "$with_tests" = true ]; then
@@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result
+
 nix-build linux-x86-64-tonlib.nix
+
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
 cp ./result/lib/fift/* artifacts/lib/

diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh
@@ -28,7 +28,9 @@ cp ./result-bin/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result-bin
+
 nix-build macos-tonlib.nix
+
 cp ./result/lib/libtonlibjson.dylib artifacts/
 cp ./result/lib/libemulator.dylib artifacts/
 cp ./result/lib/fift/* artifacts/lib/

diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix
@@ -6,41 +6,57 @@
 , testing ? false
 }:
 let
-  microhttpdmy = (import ./microhttpd.nix) {};
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+  jemallocStatic = (staticOptions pkgs.jemalloc);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
+
 in
-with import microhttpdmy;
 stdenv.mkDerivation {
   pname = "ton";
   version = "dev-bin";
 
   src = ./.;
 
   nativeBuildInputs = with pkgs;
-    [
-      cmake ninja git pkg-config
-    ];
+    [  cmake ninja git pkg-config ];
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      jemallocStatic
+      secp256k1Static
+      libsodiumStatic.dev
+      glibc.static
     ];
 
-  makeStatic = true;
-  doCheck = testing;
-
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
     "-DBUILD_SHARED_LIBS=OFF"
     "-DCMAKE_LINK_SEARCH_START_STATIC=ON"
     "-DCMAKE_LINK_SEARCH_END_STATIC=ON"
-    "-DMHD_FOUND=1"
-    "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include"
-    "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a"
-    "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800"
+    "-DTON_USE_JEMALLOC=ON"
   ];
 
+  makeStatic = true;
+  doCheck = testing;
+
   LDFLAGS = [
-     "-static-libgcc" "-static-libstdc++" "-static"
+    "-static-libgcc" "-static-libstdc++" "-static"
   ];
 }
diff --git a/assembly/nix/linux-arm64-tonlib.nix b/assembly/nix/linux-arm64-tonlib.nix
@@ -5,9 +5,21 @@
 , stdenv ? pkgs.stdenv
 }:
 let
-     microhttpdmy = (import ./microhttpd.nix) {};
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
 in
-with import microhttpdmy;
 pkgs.llvmPackages_16.stdenv.mkDerivation {
   pname = "ton";
   version = "dev-lib";
@@ -21,17 +33,19 @@ pkgs.llvmPackages_16.stdenv.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      secp256k1Static
+      libsodiumStatic.dev
     ];
 
   dontAddStaticConfigureFlags = false;
 
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
-    "-DMHD_FOUND=1"
-    "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include"
-    "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a"
   ];
 
   LDFLAGS = [