lz4 support in multiple builds (#946)

* Compress block candidates in validator-session * Compress blocks in full-node (disabled for now) * test pipeline with lz4 * tonlib compilation required lz4; try win compile; * install lz4 on mac. * wip, test builds * remove FindLZ4.cmake * fix typo * fix wasm lz4 path * increase groovy timeout to 120 sec * add lz4 for android and emscripten builds * add lz4 for android and emscripten builds * fix win build include path for lz4 * add precompiled lz4 for android * cleanup * adjust android include dir for lz4 * fix path for android arm of lz4 * cleanup * minor fix --------- Co-authored-by: SpyCheese <[email protected]>
ton-blockchain · Mar 28, 2024 · b784924 · b784924
1 parent 10487b1
commit b784924
Show file tree

Hide file tree

Showing 38 changed files with 11,004 additions and 68 deletions.
diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml
@@ -17,7 +17,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libgflags-dev \
         zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev \
-        libtool autoconf libsodium-dev libsecp256k1-dev
+        libtool autoconf libsodium-dev libsecp256k1-dev liblz4-dev
 
     - name: Build TON
       run: |
@@ -29,4 +29,4 @@ jobs:
       uses: actions/upload-artifact@master
       with:
         name: tonlib-android
-        path: artifacts
+        path: artifacts
diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml
@@ -19,7 +19,7 @@ jobs:
     - name: Install system libraries
       run: |
         sudo apt-get update
-        sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev
+        sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev
 
     - name: Install clang-16
       run: |

diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml
@@ -15,7 +15,7 @@ jobs:
     - name: Install system libraries
       run: |
         sudo apt-get update
-        sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev
+        sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev
 
     - name: Build TON WASM artifacts
       run: |
@@ -27,4 +27,4 @@ jobs:
       uses: actions/upload-artifact@master
       with:
         name: ton-wasm-binaries
-        path: artifacts
+        path: artifacts
diff --git a/CMake/FindLZ4.cmake b/CMake/FindLZ4.cmake
diff --git a/assembly/cicd/jenkins/test-builds.groovy b/assembly/cicd/jenkins/test-builds.groovy
@@ -8,7 +8,7 @@ pipeline {
                         label 'Ubuntu_x86-64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/native/build-ubuntu-shared.sh .
                                 chmod +x build-ubuntu-shared.sh
@@ -27,7 +27,7 @@ pipeline {
                         label 'Ubuntu_x86-64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/nix/build-linux-x86-64-nix.sh .
                                 chmod +x build-linux-x86-64-nix.sh
@@ -46,7 +46,7 @@ pipeline {
                         label 'Ubuntu_arm64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/native/build-ubuntu-shared.sh .
                                 chmod +x build-ubuntu-shared.sh
@@ -65,7 +65,7 @@ pipeline {
                         label 'Ubuntu_arm64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/nix/build-linux-arm64-nix.sh .
                                 chmod +x build-linux-arm64-nix.sh
@@ -84,7 +84,7 @@ pipeline {
                         label 'macOS_12.7_x86-64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
@@ -103,7 +103,7 @@ pipeline {
                         label 'macOS_12.7_x86-64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/nix/build-macos-nix.sh .
                                 chmod +x build-macos-nix.sh
@@ -122,7 +122,7 @@ pipeline {
                         label 'macOS_12.6-arm64-m1'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
@@ -141,7 +141,7 @@ pipeline {
                         label 'macOS_12.6-arm64-m1'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/nix/build-macos-nix.sh .
                                 chmod +x build-macos-nix.sh
@@ -160,7 +160,7 @@ pipeline {
                         label 'macOS_13.2-arm64-m2'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
@@ -179,7 +179,7 @@ pipeline {
                         label 'Windows_x86-64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             bat '''
                                 copy assembly\\native\\build-windows.bat .
                                 build-windows.bat -t
@@ -197,7 +197,7 @@ pipeline {
                         label 'Ubuntu_x86-64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cp assembly/android/build-android-tonlib.sh .
                                 chmod +x build-android-tonlib.sh
@@ -216,7 +216,7 @@ pipeline {
                         label 'Ubuntu_x86-64'
                     }
                     steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                             sh '''
                                 cd assembly/wasm
                                 chmod +x fift-func-wasm-build-ubuntu.sh

diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh
@@ -37,6 +37,21 @@ else
 fi
 export CCACHE_DISABLE=1
 
+if [ ! -d "lz4" ]; then
+git clone https://github.com/lz4/lz4.git
+cd lz4
+lz4Path=`pwd`
+git checkout v1.9.4
+make -j12
+test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+cd ..
+# ./lib/liblz4.a
+# ./lib
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi
+
 if [ ! -d "secp256k1" ]; then
 git clone https://github.com/bitcoin-core/secp256k1.git
 cd secp256k1
@@ -128,7 +143,10 @@ cmake -GNinja .. \
 -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \
 -DMHD_FOUND=1 \
 -DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \
--DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a
+-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \
+-DLZ4_FOUND=1 \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a
 
 
 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }

diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh
@@ -51,6 +51,19 @@ else
   echo "Using compiled secp256k1"
 fi
 
+if [ ! -d "lz4" ]; then
+  git clone https://github.com/lz4/lz4
+  cd lz4
+  lz4Path=`pwd`
+  git checkout v1.9.4
+  make -j12
+  test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+  cd ..
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi
+
 brew unlink [email protected]
 brew install openssl@3
 brew unlink openssl@3 &&  brew link --overwrite openssl@3
@@ -59,7 +72,10 @@ cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
 -DCMAKE_CXX_FLAGS="-stdlib=libc++" \
 -DSECP256K1_FOUND=1 \
 -DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \
--DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a
+-DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \
+-DLZ4_FOUND=1 \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib
 
 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 

diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh
@@ -28,6 +28,20 @@ export CC=$(which clang-16)
 export CXX=$(which clang++-16)
 export CCACHE_DISABLE=1
 
+if [ ! -d "lz4" ]; then
+git clone https://github.com/lz4/lz4.git
+cd lz4
+lz4Path=`pwd`
+git checkout v1.9.4
+make -j12
+test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+cd ..
+# ./lib/liblz4.a
+# ./lib
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi
 
 if [ ! -d "secp256k1" ]; then
 git clone https://github.com/bitcoin-core/secp256k1.git
@@ -120,7 +134,11 @@ cmake -GNinja .. \
 -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \
 -DMHD_FOUND=1 \
 -DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \
--DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a
+-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \
+-DLZ4_FOUND=1 \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a
+
 
 
 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }

diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh
@@ -1,7 +1,7 @@
 #/bin/bash
 
 #sudo apt-get update
-#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev
+#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev
 
 with_tests=false
 with_artifacts=false
@@ -119,4 +119,4 @@ if [ "$with_tests" = true ]; then
   cd build
 #  ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp"
   ctest --output-on-failure --timeout 1800
-fi
+fi
diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat
@@ -42,6 +42,22 @@ cd ..\..\..\..
 echo Using zlib...
 )
 
+if not exist "lz4" (
+git clone https://github.com/lz4/lz4.git
+cd lz4
+git checkout v1.9.4
+cd build\VS2017\liblz4
+msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143
+dir /s
+IF %errorlevel% NEQ 0 (
+  echo Can't install lz4
+  exit /b %errorlevel%
+)
+cd ..\..\..\..
+) else (
+echo Using lz4...
+)
+
 if not exist "secp256k1" (
 git clone https://github.com/bitcoin-core/secp256k1.git
 cd secp256k1
@@ -119,6 +135,9 @@ cmake -GNinja  -DCMAKE_BUILD_TYPE=Release ^
 -DSECP256K1_FOUND=1 ^
 -DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include ^
 -DSECP256K1_LIBRARY=%root%\secp256k1\build\src\Release\libsecp256k1.lib ^
+-DLZ4_FOUND=1 ^
+-DLZ4_INCLUDE_DIRS=%root%\lz4\lib ^
+-DLZ4_LIBRARIES=%root%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^
 -DMHD_FOUND=1 ^
 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib ^
 -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static ^

diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix
@@ -22,7 +22,7 @@ stdenv.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static
+      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
     ];
 
   makeStatic = true;

diff --git a/assembly/nix/linux-arm64-tonlib.nix b/assembly/nix/linux-arm64-tonlib.nix
@@ -21,7 +21,7 @@ pkgs.llvmPackages_16.stdenv.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1
+      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
     ];
 
   dontAddStaticConfigureFlags = false;

diff --git a/assembly/nix/linux-x86-64-static.nix b/assembly/nix/linux-x86-64-static.nix
@@ -22,7 +22,7 @@ stdenv.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static
+      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
     ];
 
   makeStatic = true;

diff --git a/assembly/nix/linux-x86-64-tonlib.nix b/assembly/nix/linux-x86-64-tonlib.nix
@@ -34,7 +34,7 @@ stdenv227.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1
+      pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
     ];
 
   dontAddStaticConfigureFlags = false;

diff --git a/assembly/nix/macos-static.nix b/assembly/nix/macos-static.nix
@@ -25,6 +25,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
       (openssl.override { static = true; }).dev
       (zlib.override { shared = false; }).dev
       (libiconv.override { enableStatic = true; enableShared = false; })
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
    ];
 
 

diff --git a/assembly/nix/macos-tonlib.nix b/assembly/nix/macos-tonlib.nix
@@ -23,6 +23,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
       (openssl.override { static = true; }).dev
       (zlib.override { shared = false; }).dev
       (libiconv.override { enableStatic = true; enableShared = false; })
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
     ];
 
   dontAddStaticConfigureFlags = true;
@@ -52,4 +53,4 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
         install_name_tool -change "$(otool -L "$fn" | grep libc++abi.1 | cut -d' ' -f1 | xargs)" libc++abi.dylib "$fn"
      done
   '';
-}
+}