Compress block candidates

CMake/FindLZ4.cmake

@@ -0,0 +1,39 @@
+###############################################################################
+# Find LZ4
+#
+# This sets the following variables:
+# LZ4_FOUND        - True if LZ4 was found.
+# LZ4_INCLUDE_DIRS - Directories containing the LZ4 include files.
+# LZ4_LIBRARIES    - Libraries needed to use LZ4.
+# LZ4_LIBRARY      - Library needed to use LZ4.
+# LZ4_LIBRARY_DIRS - Library needed to use LZ4.
+
+find_package(PkgConfig REQUIRED)
+
+# If found, LZ$_* variables will be defined
+pkg_check_modules(LZ4 REQUIRED liblz4)
+
+if(NOT LZ4_FOUND)
+  find_path(LZ4_INCLUDE_DIR lz4.h
+    HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+    PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+    PATH_SUFFIXES include)
+
+  find_library(LZ4_LIBRARY
+    NAMES lz4 lz4_static
+    HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+    PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+    PATH_SUFFIXES lib)
+
+  if(LZ4_LIBRARY)
+    set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+    get_filename_component(LZ4_LIBRARY_DIRS ${LZ4_LIBRARY} DIRECTORY)
+  endif()
+else()
+  find_library(LZ4_LIBRARY
+    NAMES lz4 lz4_static
+    PATHS ${LZ4_LIBRARY_DIRS}
+    NO_DEFAULT_PATH)
+endif()
+
+mark_as_advanced(LZ4_LIBRARY LZ4_INCLUDE_DIRS LZ4_LIBRARY_DIRS LZ4_LIBRARIES)

tdutils/CMakeLists.txt

@@ -15,6 +15,7 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
 endif()
 
 find_package(PkgConfig REQUIRED)
+find_package(LZ4)
 if (NOT ZLIB_FOUND)
   pkg_check_modules(ZLIB zlib)
 endif()
@@ -280,6 +281,15 @@ if (TDUTILS_MIME_TYPE)
   )
 endif()
 
+if (LZ4_FOUND)
+  set(TD_HAVE_LZ4 1)
+  set(TDUTILS_SOURCE
+    ${TDUTILS_SOURCE}
+    td/utils/lz4.cpp
+    td/utils/lz4.h
+    )
+endif()
+
 set(TDUTILS_TEST_SOURCE
   ${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
@@ -338,6 +348,11 @@ endif()
 if (CRC32C_FOUND)
   target_link_libraries(tdutils PRIVATE crc32c)
 endif()
+
+if (LZ4_FOUND)
+  target_link_libraries(tdutils PRIVATE ${LZ4_LIBRARIES})
+endif()
+
 if (ABSL_FOUND)
   target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash)
 endif()

tdutils/td/utils/config.h.in

@@ -3,6 +3,7 @@
 #cmakedefine01 TD_HAVE_OPENSSL
 #cmakedefine01 TD_HAVE_ZLIB
 #cmakedefine01 TD_HAVE_CRC32C
+#cmakedefine01 TD_HAVE_LZ4
 #cmakedefine01 TD_HAVE_COROUTINES
 #cmakedefine01 TD_HAVE_ABSL
 #cmakedefine01 TD_FD_DEBUG

tdutils/td/utils/lz4.cpp

@@ -0,0 +1,48 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "td/utils/buffer.h"
+#include "td/utils/misc.h"
+#include <lz4.h>
+
+namespace td {
+
+td::BufferSlice lz4_compress(td::Slice data) {
+  int size = narrow_cast<int>(data.size());
+  int buf_size = LZ4_compressBound(size);
+  td::BufferSlice compressed(buf_size);
+  int compressed_size = LZ4_compress_default(data.data(), compressed.data(), size, buf_size);
+  CHECK(compressed_size > 0);
+  return td::BufferSlice{compressed.as_slice().substr(0, compressed_size)};
+}
+
+td::Result<td::BufferSlice> lz4_decompress(td::Slice data, int max_decompressed_size) {
+  TRY_RESULT(size, narrow_cast_safe<int>(data.size()));
+  if (max_decompressed_size < 0) {
+    return td::Status::Error("invalid max_decompressed_size");
+  }
+  td::BufferSlice decompressed(max_decompressed_size);
+  int result = LZ4_decompress_safe(data.data(), decompressed.data(), size, max_decompressed_size);
+  if (result < 0) {
+    return td::Status::Error(PSTRING() << "lz4 decompression failed, error code: " << result);
+  }
+  if (result == max_decompressed_size) {
+    return decompressed;
+  }
+  return td::BufferSlice{decompressed.as_slice().substr(0, result)};
+}
+
+}  // namespace td

tdutils/td/utils/lz4.h

@@ -0,0 +1,27 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+
+#include "td/utils/buffer.h"
+#include "td/utils/Status.h"
+
+namespace td {
+
+td::BufferSlice lz4_compress(td::Slice data);
+td::Result<td::BufferSlice> lz4_decompress(td::Slice data, int max_decompressed_size);
+
+}  // namespace td

tl/generate/scheme/ton_api.tl

@@ -311,6 +311,7 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat
 
 validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate;
 validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate;
+validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate;
 
 validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int
         max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config;
@@ -784,7 +785,9 @@ validatorSession.stats id:tonNode.blockId timestamp:long self:int256 creator:int
     signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long
     first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats;
 
-collatorNode.generateBlockSuccess candidate:db.Candidate = collatorNode.GenerateBlockResult;
+collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate;
+collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate;
+collatorNode.generateBlockSuccess candidate:collatorNode.Candidate = collatorNode.GenerateBlockResult;
 collatorNode.generateBlockError code:int message:string = collatorNode.GenerateBlockResult;
 
 ---functions---

validator-session/CMakeLists.txt

@@ -5,12 +5,14 @@ if (NOT OPENSSL_FOUND)
 endif()
 
 set(VALIDATOR_SESSION_SOURCE
+  candidate-serializer.cpp
   persistent-vector.cpp
   validator-session-description.cpp
   validator-session-state.cpp
   validator-session.cpp
   validator-session-round-attempt-state.cpp
 
+  candidate-serializer.h
   persistent-vector.h
   validator-session-description.h
   validator-session-description.hpp

validator-session/candidate-serializer.cpp

@@ -0,0 +1,77 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+#include "candidate-serializer.h"
+#include "tl-utils/tl-utils.hpp"
+#include "vm/boc.h"
+#include "td/utils/lz4.h"
+
+namespace ton {
+
+namespace validatorsession {
+
+td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate> &block,
+                                                bool compression_enabled) {
+  if (!compression_enabled) {
+    return serialize_tl_object(block, true);
+  }
+  vm::BagOfCells boc1, boc2;
+  TRY_STATUS(boc1.deserialize(block->data_));
+  if (boc1.get_root_count() != 1) {
+    return td::Status::Error("block candidate should have exactly one root");
+  }
+  std::vector<td::Ref<vm::Cell>> roots = {boc1.get_root_cell()};
+  TRY_STATUS(boc2.deserialize(block->collated_data_));
+  for (int i = 0; i < boc2.get_root_count(); ++i) {
+    roots.push_back(boc2.get_root_cell(i));
+  }
+  TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2));
+  td::BufferSlice compressed = td::lz4_compress(data);
+  LOG(DEBUG) << "Compressing candidate: " << block->data_.size() + block->collated_data_.size() << " -> "
+             << compressed.size();
+  return create_serialize_tl_object<ton_api::validatorSession_compressedCandidate>(
+      0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed));
+}
+
+td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
+                                                                                     int max_decompressed_data_size) {
+  auto R = fetch_tl_object<ton_api::validatorSession_candidate>(data, true);
+  if (R.is_ok()) {
+    return R;
+  }
+  TRY_RESULT(f, fetch_tl_object<ton_api::validatorSession_compressedCandidate>(data, true));
+  if (f->decompressed_size_ > max_decompressed_data_size) {
+    return td::Status::Error("decompressed size is too big");
+  }
+  TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_));
+  if (decompressed.size() != (size_t)f->decompressed_size_) {
+    return td::Status::Error("decompressed size mismatch");
+  }
+  TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed));
+  if (roots.empty()) {
+    return td::Status::Error("boc is empty");
+  }
+  TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31));
+  roots.erase(roots.begin());
+  TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31));
+  return create_tl_object<ton_api::validatorSession_candidate>(f->src_, f->round_, f->root_hash_, std::move(block_data),
+                                                               std::move(collated_data));
+}
+
+}  // namespace validatorsession
+
+}  // namespace ton

validator-session/candidate-serializer.h

@@ -0,0 +1,32 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+#include "ton/ton-types.h"
+#include "auto/tl/ton_api.h"
+
+namespace ton {
+
+namespace validatorsession {
+
+td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate> &block,
+                                                bool compression_enabled);
+td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
+                                                                                     int max_decompressed_data_size);
+
+}  // namespace validatorsession
+
+}  // namespace ton