Merge branch 'testnet' into custom-overlays

Changelog.md

@@ -1,3 +1,17 @@
+## 2024.04 Update
+
+1. Emulator: Single call optimized runGetMethod added
+2. Tonlib: a series of proof improvements, also breaking Change in `liteServer.getAllShardsInfo` method (see below)
+3. DB: usage statistics now collected, outdated persistent states are not serialized
+4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests
+5. Network: lz4 compression of block candidates (disabled by default).
+6. Overlays: add custom overlays
+7. Transaction Executor: fixed issue with due_payment collection
+
+* `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block.
+
+Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue).
+
 ## 2024.03 Update
 
 1. Preparatory (not enabled yet) code for pre-compiled smart-contract.

Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:22.04 as builder
 RUN apt-get update && \
-	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev pkg-config autoconf automake libtool && \
+	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool && \
 	rm -rf /var/lib/apt/lists/*
 ENV CC clang
 ENV CXX clang++
@@ -19,7 +19,7 @@ RUN mkdir build && \
 
 FROM ubuntu:22.04
 RUN apt-get update && \
-    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev && \
+    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev && \
     rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /var/ton-work/db && \
@@ -36,4 +36,4 @@ WORKDIR /var/ton-work/db
 COPY ./docker/init.sh ./docker/control.template ./
 RUN chmod +x init.sh
 
-ENTRYPOINT ["/var/ton-work/db/init.sh"]
+ENTRYPOINT ["/var/ton-work/db/init.sh"]

common/global-version.h

@@ -19,6 +19,6 @@
 namespace ton {
 
 // See doc/GlobalVersions.md
-const int SUPPORTED_VERSION = 6;
+const int SUPPORTED_VERSION = 7;
 
 }

crypto/block/transaction.cpp

@@ -898,6 +898,9 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc
     res->fees_collected = to_pay;
     res->fees_due = td::zero_refint();
     balance -= std::move(to_pay);
+    if (cfg.global_version >= 7) {
+      due_payment = td::zero_refint();
+    }
   } else if (acc_status == Account::acc_frozen && !force_collect && to_pay < cfg.delete_due_limit) {
     // do not collect fee
     res->last_paid_updated = (res->is_special ? 0 : account.last_paid);
@@ -1318,7 +1321,7 @@ Ref<vm::Tuple> Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const {
       vm::StackEntry::maybe(cfg.global_config)    //   global_config:(Maybe Cell) ] = SmartContractInfo;
   };
   if (cfg.global_version >= 4) {
-    tuple.push_back(new_code);                            // code:Cell
+    tuple.push_back(vm::StackEntry::maybe(new_code));  // code:Cell
     if (msg_balance_remaining.is_valid()) {
       tuple.push_back(msg_balance_remaining.as_vm_tuple());  // in_msg_value:[Integer (Maybe Cell)]
     } else {
@@ -1335,11 +1338,10 @@ Ref<vm::Tuple> Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const {
     // Inside validator, collator and liteserver checking external message  contexts
     // prev_blocks_info is always not null, since get_prev_blocks_info()  
     // may only return tuple or raise Error (See crypto/block/mc-config.cpp#2223)
-    tuple.push_back(cfg.prev_blocks_info.not_null() ? vm::StackEntry(cfg.prev_blocks_info) : vm::StackEntry());
+    tuple.push_back(vm::StackEntry::maybe(cfg.prev_blocks_info));
   }
   if (cfg.global_version >= 6) {
-    tuple.push_back(cfg.unpacked_config_tuple.not_null() ? vm::StackEntry(cfg.unpacked_config_tuple)
-                                                         : vm::StackEntry());   // unpacked_config_tuple:[...]
+    tuple.push_back(vm::StackEntry::maybe(cfg.unpacked_config_tuple));          // unpacked_config_tuple:[...]
     tuple.push_back(due_payment.not_null() ? due_payment : td::zero_refint());  // due_payment:Integer
     tuple.push_back(compute_phase->precompiled_gas_usage
                         ? vm::StackEntry(td::make_refint(compute_phase->precompiled_gas_usage.value()))
@@ -3668,6 +3670,7 @@ td::Status FetchConfigParams::fetch_config_params(
     compute_phase_cfg->mc_gas_prices = std::move(mc_gas_prices);
     compute_phase_cfg->special_gas_full = config.get_global_version() >= 5;
     storage_phase_cfg->enable_due_payment = config.get_global_version() >= 4;
+    storage_phase_cfg->global_version = config.get_global_version();
     compute_phase_cfg->block_rand_seed = *rand_seed;
     compute_phase_cfg->max_vm_data_depth = size_limits.max_vm_data_depth;
     compute_phase_cfg->global_config = config.get_root_cell();

crypto/block/transaction.h

@@ -82,6 +82,7 @@ struct StoragePhaseConfig {
   td::RefInt256 freeze_due_limit;
   td::RefInt256 delete_due_limit;
   bool enable_due_payment{false};
+  int global_version = 0;
   StoragePhaseConfig() = default;
   StoragePhaseConfig(const std::vector<block::StoragePrices>* _pricing, td::RefInt256 freeze_limit = {},
                      td::RefInt256 delete_limit = {})

crypto/smc-envelope/SmartContract.cpp

@@ -157,11 +157,11 @@ td::Ref<vm::Tuple> prepare_vm_c7(SmartContract::Args args, td::Ref<vm::Cell> cod
       td::make_refint(0),              //TODO:                //   trans_lt:Integer
       std::move(rand_seed_int),                               //   rand_seed:Integer
       block::CurrencyCollection(args.balance).as_vm_tuple(),  //   balance_remaining:[Integer (Maybe Cell)]
-      vm::load_cell_slice_ref(address),                       //  myself:MsgAddressInt
-      vm::StackEntry::maybe(config)                           //vm::StackEntry::maybe(td::Ref<vm::Cell>())
+      vm::load_cell_slice_ref(address),                       //   myself:MsgAddressInt
+      vm::StackEntry::maybe(config)                           //   vm::StackEntry::maybe(td::Ref<vm::Cell>())
   };
   if (args.config && args.config.value()->get_global_version() >= 4) {
-    tuple.push_back(code.not_null() ? code : vm::StackEntry{});        // code:Cell
+    tuple.push_back(vm::StackEntry::maybe(code));                      // code:Cell
     tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple());  // in_msg_value:[Integer (Maybe Cell)]
     tuple.push_back(td::zero_refint());                                // storage_fees:Integer
 
@@ -175,7 +175,10 @@ td::Ref<vm::Tuple> prepare_vm_c7(SmartContract::Args args, td::Ref<vm::Cell> cod
     tuple.push_back(args.config.value()->get_unpacked_config_tuple(now));  // unpacked_config_tuple
     tuple.push_back(td::zero_refint());                                    // due_payment
     // precomiled_gas_usage:(Maybe Integer)
-    auto precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits());
+    td::optional<block::PrecompiledContractsConfig::Contract> precompiled;
+    if (code.not_null()) {
+      precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits());
+    }
     tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry());
   }
   auto tuple_ref = td::make_cnt_ref<std::vector<vm::StackEntry>>(std::move(tuple));

crypto/vm/large-boc-serializer.cpp

@@ -14,6 +14,9 @@
     You should have received a copy of the GNU Lesser General Public License
     along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
 */
+#include "td/utils/Time.h"
+#include "td/utils/Timer.h"
+
 #include <map>
 #include "vm/boc.h"
 #include "vm/boc-writers.h"
@@ -30,7 +33,8 @@ class LargeBocSerializer {
  public:
   using Hash = Cell::Hash;
 
-  explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader) : reader(std::move(reader)) {}
+  explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader) : reader(std::move(reader)) {
+  }
 
   void add_root(Hash root);
   td::Status import_cells();
@@ -65,7 +69,8 @@ class LargeBocSerializer {
   std::map<Hash, CellInfo> cells;
   std::vector<std::pair<const Hash, CellInfo>*> cell_list;
   struct RootInfo {
-    RootInfo(Hash hash, int idx) : hash(hash), idx(idx) {}
+    RootInfo(Hash hash, int idx) : hash(hash), idx(idx) {
+    }
     Hash hash;
     int idx;
   };
@@ -78,26 +83,40 @@ class LargeBocSerializer {
   void reorder_cells();
   int revisit(int cell_idx, int force = 0);
   td::uint64 compute_sizes(int mode, int& r_size, int& o_size);
+
+  td::Timestamp log_speed_at_;
+  size_t processed_cells_ = 0;
+  static constexpr double LOG_SPEED_PERIOD = 120.0;
 };
 
 void LargeBocSerializer::add_root(Hash root) {
   roots.emplace_back(root, -1);
 }
 
 td::Status LargeBocSerializer::import_cells() {
+  td::Timer timer;
+  log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD);
+  processed_cells_ = 0;
   for (auto& root : roots) {
     TRY_RESULT(idx, import_cell(root.hash));
     root.idx = idx;
   }
   reorder_cells();
   CHECK(!cell_list.empty());
+  LOG(ERROR) << "serializer: import_cells took " << timer.elapsed() << "s, " << cell_count << " cells";
   return td::Status::OK();
 }
 
 td::Result<int> LargeBocSerializer::import_cell(Hash hash, int depth) {
   if (depth > Cell::max_depth) {
     return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large");
   }
+  ++processed_cells_;
+  if (log_speed_at_.is_in_past()) {
+    log_speed_at_ += LOG_SPEED_PERIOD;
+    LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
+    processed_cells_ = 0;
+  }
   auto it = cells.find(hash);
   if (it != cells.end()) {
     it->second.should_cache = true;
@@ -282,6 +301,7 @@ td::uint64 LargeBocSerializer::compute_sizes(int mode, int& r_size, int& o_size)
 }
 
 td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
+  td::Timer timer;
   using Mode = BagOfCells::Mode;
   BagOfCells::Info info;
   if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) {
@@ -313,13 +333,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
     return td::Status::Error("bag of cells is too large");
   }
 
-  boc_writers::FileWriter writer{fd, (size_t) info.total_size};
-  auto store_ref = [&](unsigned long long value) {
-    writer.store_uint(value, info.ref_byte_size);
-  };
-  auto store_offset = [&](unsigned long long value) {
-    writer.store_uint(value, info.offset_byte_size);
-  };
+  boc_writers::FileWriter writer{fd, (size_t)info.total_size};
+  auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); };
+  auto store_offset = [&](unsigned long long value) { writer.store_uint(value, info.offset_byte_size); };
 
   writer.store_uint(info.magic, 4);
 
@@ -371,6 +387,8 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
   }
   DCHECK(writer.position() == info.data_offset);
   size_t keep_position = writer.position();
+  log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD);
+  processed_cells_ = 0;
   for (int i = 0; i < cell_count; ++i) {
     auto hash = cell_list[cell_count - 1 - i]->first;
     const auto& dc_info = cell_list[cell_count - 1 - i]->second;
@@ -389,24 +407,36 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
       DCHECK(k > i && k < cell_count);
       store_ref(k);
     }
+    ++processed_cells_;
+    if (log_speed_at_.is_in_past()) {
+      log_speed_at_ += LOG_SPEED_PERIOD;
+      LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
+      processed_cells_ = 0;
+    }
   }
   DCHECK(writer.position() - keep_position == info.data_size);
   if (info.has_crc32c) {
     unsigned crc = writer.get_crc32();
     writer.store_uint(td::bswap32(crc), 4);
   }
   DCHECK(writer.empty());
-  return writer.finalize();
-}
+  TRY_STATUS(writer.finalize());
+  LOG(ERROR) << "serializer: serialize took " << timer.elapsed() << "s, " << cell_count << " cells, "
+             << writer.position() << " bytes";
+  return td::Status::OK();
 }
+}  // namespace
 
-td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash,
-                                           td::FileFd& fd, int mode) {
+td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
+                                           int mode) {
+  td::Timer timer;
   CHECK(reader != nullptr)
   LargeBocSerializer serializer(reader);
   serializer.add_root(root_hash);
   TRY_STATUS(serializer.import_cells());
-  return serializer.serialize(fd, mode);
+  TRY_STATUS(serializer.serialize(fd, mode));
+  LOG(ERROR) << "serialization took " << timer.elapsed() << "s";
+  return td::Status::OK();
 }
 
-}
+}  // namespace vm

recent_changelog.md

@@ -5,11 +5,9 @@
 3. DB: usage statistics now collected, outdated persistent states are not serialized
 4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests
 5. Network: lz4 compression of block candidates (disabled by default).
-
-
-
----
+6. Overlays: add custom overlays
+7. Transaction Executor: fixed issue with due_payment collection
 
 * `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block.
 
-
+Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue).

tdutils/td/utils/Timer.cpp

@@ -87,4 +87,8 @@ void PerfWarningTimer::reset() {
   start_at_ = 0;
 }
 
+double PerfWarningTimer::elapsed() const {
+  return Time::now() - start_at_;
+}
+
 }  // namespace td

tdutils/td/utils/Timer.h

@@ -53,6 +53,7 @@ class PerfWarningTimer {
   PerfWarningTimer &operator=(PerfWarningTimer &&) = delete;
   ~PerfWarningTimer();
   void reset();
+  double elapsed() const;
 
  private:
   string name_;

tl/generate/scheme/ton_api.tl

@@ -752,15 +752,26 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se
 
 ---types---
 
-validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int block_timestamp:long comment:string = validatorSession.StatsProducer;
-
-validatorSession.statsRound timestamp:long producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound;
-
-validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:long self:int256 session_id:int256 cc_seqno:int
+validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string
+  block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double
+  collation_time:double collated_at:double collation_cached:Bool
+  validation_time:double validated_at:double validation_cached:Bool
+  gen_utime:double
+  approved_weight:long approved_33pct_at:double approved_66pct_at:double
+  signed_weight:long signed_33pct_at:double signed_66pct_at:double
+  serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer;
+
+validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound;
+
+validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int
     creator:int256 total_validators:int total_weight:long
     signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long
     first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats;
 
+validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node;
+validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double
+    self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats;
+
 ---functions---
 
 ---types---