Merge pull request ton-blockchain#908 from ton-blockchain/testnet

Merge developer branch

Changelog.md

@@ -1,3 +1,19 @@
+## 2024.02 Update
+
+1. Improvement of validator synchronisation:
+   * Better handling of block broadcasts -> faster sync
+   * Additional separate overlay among validators as second option for synchronisation
+2. Improvements in LS:
+   * c7 and library context is fully filled up for server-side rungetmethod
+   * Cache for runmethods and successfull external messages
+   * Logging of LS requests statistic
+3. Precise control of open files:
+   * almost instantaneous validator start
+   * `--max-archive-fd` option
+   * autoremoval of not used temp archive files
+   * `--archive-preload-period` option
+4. Preparatory (not enabled yet) code for addition on new TVM instructions for cheaper fee calculation onchain.
+
 ## 2024.01 Update
 
 1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to caunt gas on special accounts separately. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `version >= 5` in `ConfigParam 8;`.

adnl/adnl-peer-table.hpp

@@ -77,6 +77,10 @@ class AdnlPeerTableImpl : public AdnlPeerTable {
                         td::actor::ActorId<AdnlChannel> channel) override;
   void unregister_channel(AdnlChannelIdShort id) override;
 
+  void check_id_exists(AdnlNodeIdShort id, td::Promise<bool> promise) override {
+    promise.set_value(local_ids_.count(id));
+  }
+
   void write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node,
                                  td::Promise<td::Unit> promise) override;
   void get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id,

adnl/adnl-peer.cpp

@@ -212,7 +212,9 @@ void AdnlPeerPairImpl::receive_packet_from_channel(AdnlChannelIdShort id, AdnlPa
     VLOG(ADNL_NOTICE) << this << ": dropping IN message: outdated channel id" << id;
     return;
   }
-  channel_ready_ = true;
+  if (channel_inited_) {
+    channel_ready_ = true;
+  }
   receive_packet_checked(std::move(packet));
 }
 

adnl/adnl.h

@@ -97,6 +97,8 @@ class Adnl : public AdnlSenderInterface {
   virtual void add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) = 0;
   virtual void del_id(AdnlNodeIdShort id, td::Promise<td::Unit> promise) = 0;
 
+  virtual void check_id_exists(AdnlNodeIdShort id, td::Promise<bool> promise) = 0;
+
   // subscribe to (some) messages(+queries) to this local id
   virtual void subscribe(AdnlNodeIdShort dst, std::string prefix, std::unique_ptr<Callback> callback) = 0;
   virtual void unsubscribe(AdnlNodeIdShort dst, std::string prefix) = 0;

assembly/nix/build-linux-arm64-nix.sh

@@ -24,12 +24,14 @@ if [ "$with_tests" = true ]; then
 else
   nix-build linux-arm64-static.nix
 fi
-mkdir artifacts
+
+mkdir -p artifacts/lib
 cp ./result/bin/* artifacts/
+test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result
 nix-build linux-arm64-tonlib.nix
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
-cp -r crypto/fift/lib artifacts/
-cp -r crypto/smartcont artifacts/
+cp ./result/lib/fift/* artifacts/lib/
+cp -r ./result/share/ton/smartcont artifacts/

assembly/nix/build-linux-x86-64-nix.sh

@@ -25,12 +25,13 @@ else
   nix-build linux-x86-64-static.nix
 fi
 
-mkdir artifacts
+mkdir -p artifacts/lib
 cp ./result/bin/* artifacts/
+test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result
 nix-build linux-x86-64-tonlib.nix
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
-cp -r crypto/fift/lib artifacts/
-cp -r crypto/smartcont artifacts/
+cp ./result/lib/fift/* artifacts/lib/
+cp -r ./result/share/ton/smartcont artifacts/

assembly/nix/build-macos-nix.sh

@@ -22,12 +22,14 @@ if [ "$with_tests" = true ]; then
 else
   nix-build macos-static.nix
 fi
-mkdir artifacts
+
+mkdir -p artifacts/lib
 cp ./result-bin/bin/* artifacts/
+test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result-bin
 nix-build macos-tonlib.nix
 cp ./result/lib/libtonlibjson.dylib artifacts/
 cp ./result/lib/libemulator.dylib artifacts/
-cp -r crypto/fift/lib artifacts/
-cp -r crypto/smartcont artifacts/
+cp ./result/lib/fift/* artifacts/lib/
+cp -r ./result/share/ton/smartcont artifacts/

common/global-version.h

@@ -19,6 +19,6 @@
 namespace ton {
 
 // See doc/GlobalVersions.md
-const int SUPPORTED_VERSION = 5;
+const int SUPPORTED_VERSION = 6;
 
 }

crypto/CMakeLists.txt

@@ -401,6 +401,7 @@ if (USE_EMSCRIPTEN)
   target_link_options(funcfiftlib PRIVATE -sALLOW_MEMORY_GROWTH=1)
   target_link_options(funcfiftlib PRIVATE -sALLOW_TABLE_GROWTH=1)
   target_link_options(funcfiftlib PRIVATE --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/fift/lib@/fiftlib)
+  target_link_options(funcfiftlib PRIVATE --pre-js ${CMAKE_CURRENT_SOURCE_DIR}/funcfiftlib/funcfiftlib-prejs.js)
   target_link_options(funcfiftlib PRIVATE -fexceptions)
   target_compile_options(funcfiftlib PRIVATE -fexceptions -fno-stack-protector)
 endif()

crypto/block/mc-config.cpp

@@ -621,29 +621,40 @@ td::Result<std::vector<StoragePrices>> Config::get_storage_prices() const {
   }
   vm::Dictionary dict{std::move(cell), 32};
   if (!dict.check_for_each([&res](Ref<vm::CellSlice> cs_ref, td::ConstBitPtr key, int n) -> bool {
-        block::gen::StoragePrices::Record data;
-        if (!tlb::csr_unpack(std::move(cs_ref), data) || data.utime_since != key.get_uint(n)) {
+        auto r_prices = do_get_one_storage_prices(*cs_ref);
+        if (r_prices.is_error()) {
+          return false;
+        }
+        res.push_back(r_prices.move_as_ok());
+        if (res.back().valid_since != key.get_uint(n)) {
           return false;
         }
-        res.emplace_back(data.utime_since, data.bit_price_ps, data.cell_price_ps, data.mc_bit_price_ps,
-                         data.mc_cell_price_ps);
         return true;
       })) {
     return td::Status::Error("invalid storage prices dictionary in configuration parameter 18");
   }
   return std::move(res);
 }
 
-td::Result<GasLimitsPrices> Config::do_get_gas_limits_prices(td::Ref<vm::Cell> cell, int id) {
+td::Result<StoragePrices> Config::do_get_one_storage_prices(vm::CellSlice cs) {
+  block::gen::StoragePrices::Record data;
+  if (!tlb::unpack(cs, data)) {
+    return td::Status::Error("invalid storage prices dictionary in configuration parameter 18");
+  }
+  return StoragePrices{data.utime_since, data.bit_price_ps, data.cell_price_ps, data.mc_bit_price_ps,
+                       data.mc_cell_price_ps};
+}
+
+td::Result<GasLimitsPrices> Config::do_get_gas_limits_prices(vm::CellSlice cs, int id) {
   GasLimitsPrices res;
-  auto cs = vm::load_cell_slice(cell);
+  vm::CellSlice cs0 = cs;
   block::gen::GasLimitsPrices::Record_gas_flat_pfx flat;
   if (tlb::unpack(cs, flat)) {
     cs = *flat.other;
     res.flat_gas_limit = flat.flat_gas_limit;
     res.flat_gas_price = flat.flat_gas_price;
   } else {
-    cs = vm::load_cell_slice(cell);
+    cs = cs0;
   }
   auto f = [&](const auto& r, td::uint64 spec_limit) {
     res.gas_limit = r.gas_limit;
@@ -654,7 +665,6 @@ td::Result<GasLimitsPrices> Config::do_get_gas_limits_prices(td::Ref<vm::Cell> c
     res.delete_due_limit = r.delete_due_limit;
   };
   block::gen::GasLimitsPrices::Record_gas_prices_ext rec;
-  vm::CellSlice cs0 = cs;
   if (tlb::unpack(cs, rec)) {
     f(rec, rec.special_gas_limit);
   } else {
@@ -689,7 +699,7 @@ td::Result<GasLimitsPrices> Config::get_gas_limits_prices(bool is_masterchain) c
   if (cell.is_null()) {
     return td::Status::Error(PSLICE() << "configuration parameter " << id << " with gas prices is absent");
   }
-  return do_get_gas_limits_prices(std::move(cell), id);
+  return do_get_gas_limits_prices(vm::load_cell_slice(cell), id);
 }
 
 td::Result<MsgPrices> Config::get_msg_prices(bool is_masterchain) const {
@@ -698,7 +708,10 @@ td::Result<MsgPrices> Config::get_msg_prices(bool is_masterchain) const {
   if (cell.is_null()) {
     return td::Status::Error(PSLICE() << "configuration parameter " << id << " with msg prices is absent");
   }
-  auto cs = vm::load_cell_slice(std::move(cell));
+  return do_get_msg_prices(vm::load_cell_slice(cell), id);
+}
+
+td::Result<MsgPrices> Config::do_get_msg_prices(vm::CellSlice cs, int id) {
   block::gen::MsgForwardPrices::Record rec;
   if (!tlb::unpack(cs, rec)) {
     return td::Status::Error(PSLICE() << "configuration parameter " << id
@@ -1917,10 +1930,17 @@ std::vector<ton::ValidatorDescr> Config::compute_total_validator_set(int next) c
 }
 
 td::Result<SizeLimitsConfig> Config::get_size_limits_config() const {
-  SizeLimitsConfig limits;
   td::Ref<vm::Cell> param = get_config_param(43);
   if (param.is_null()) {
-    return limits;
+    return do_get_size_limits_config({});
+  }
+  return do_get_size_limits_config(vm::load_cell_slice_ref(param));
+}
+
+td::Result<SizeLimitsConfig> Config::do_get_size_limits_config(td::Ref<vm::CellSlice> cs) {
+  SizeLimitsConfig limits;
+  if (cs.is_null()) {
+    return limits; // default values
   }
   auto unpack_v1 = [&](auto& rec) {
     limits.max_msg_bits = rec.max_msg_bits;
@@ -1939,9 +1959,9 @@ td::Result<SizeLimitsConfig> Config::get_size_limits_config() const {
   };
   gen::SizeLimitsConfig::Record_size_limits_config rec_v1;
   gen::SizeLimitsConfig::Record_size_limits_config_v2 rec_v2;
-  if (tlb::unpack_cell(param, rec_v1)) {
+  if (tlb::csr_unpack(cs, rec_v1)) {
     unpack_v1(rec_v1);
-  } else if (tlb::unpack_cell(param, rec_v2)) {
+  } else if (tlb::csr_unpack(cs, rec_v2)) {
     unpack_v2(rec_v2);
   } else {
     return td::Status::Error("configuration parameter 43 is invalid");
@@ -1976,6 +1996,42 @@ BurningConfig Config::get_burning_config() const {
   return c;
 }
 
+td::Ref<vm::Tuple> Config::get_unpacked_config_tuple(ton::UnixTime now) const {
+  auto get_param = [&](td::int32 idx) -> vm::StackEntry {
+    auto cell = get_config_param(idx);
+    if (cell.is_null()) {
+      return {};
+    }
+    return vm::load_cell_slice_ref(cell);
+  };
+  auto get_current_storage_prices = [&]() -> vm::StackEntry {
+    auto cell = get_config_param(18);
+    if (cell.is_null()) {
+      return {};
+    }
+    vm::StackEntry res;
+    vm::Dictionary dict{std::move(cell), 32};
+    dict.check_for_each([&](Ref<vm::CellSlice> cs_ref, td::ConstBitPtr key, int n) -> bool {
+      auto utime_since = key.get_uint(n);
+      if (now >= utime_since) {
+        res = std::move(cs_ref);
+        return true;
+      }
+      return false;
+    });
+    return res;
+  };
+  std::vector<vm::StackEntry> tuple;
+  tuple.push_back(get_current_storage_prices());  // storage_prices
+  tuple.push_back(get_param(19));                 // global_id
+  tuple.push_back(get_param(20));                 // config_mc_gas_prices
+  tuple.push_back(get_param(21));                 // config_gas_prices
+  tuple.push_back(get_param(24));                 // config_mc_fwd_prices
+  tuple.push_back(get_param(25));                 // config_fwd_prices
+  tuple.push_back(get_param(43));                 // size_limits_config
+  return td::make_cnt_ref<std::vector<vm::StackEntry>>(std::move(tuple));
+}
+
 td::Result<std::pair<ton::UnixTime, ton::UnixTime>> Config::unpack_validator_set_start_stop(Ref<vm::Cell> vset_root) {
   if (vset_root.is_null()) {
     return td::Status::Error("validator set absent");

crypto/block/mc-config.h

@@ -350,7 +350,11 @@ struct GasLimitsPrices {
   td::uint64 freeze_due_limit{0};
   td::uint64 delete_due_limit{0};
 
-  td::RefInt256 compute_gas_price(td::uint64 gas_used) const;
+  td::RefInt256 compute_gas_price(td::uint64 gas_used) const {
+    return gas_used <= flat_gas_limit
+               ? td::make_refint(flat_gas_price)
+               : td::rshift(td::make_refint(gas_price) * (gas_used - flat_gas_limit), 16, 1) + flat_gas_price;
+  }
 };
 
 // msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms
@@ -365,6 +369,7 @@ struct MsgPrices {
   td::uint32 first_frac;
   td::uint32 next_frac;
   td::uint64 compute_fwd_fees(td::uint64 cells, td::uint64 bits) const;
+  td::RefInt256 compute_fwd_fees256(td::uint64 cells, td::uint64 bits) const;
   std::pair<td::uint64, td::uint64> compute_fwd_ihr_fees(td::uint64 cells, td::uint64 bits,
                                                          bool ihr_disabled = false) const;
   MsgPrices() = default;
@@ -604,9 +609,11 @@ class Config {
   bool is_special_smartcontract(const ton::StdSmcAddress& addr) const;
   static td::Result<std::unique_ptr<ValidatorSet>> unpack_validator_set(Ref<vm::Cell> valset_root);
   td::Result<std::vector<StoragePrices>> get_storage_prices() const;
+  static td::Result<StoragePrices> do_get_one_storage_prices(vm::CellSlice cs);
   td::Result<GasLimitsPrices> get_gas_limits_prices(bool is_masterchain = false) const;
-  static td::Result<GasLimitsPrices> do_get_gas_limits_prices(td::Ref<vm::Cell> cell, int id);
+  static td::Result<GasLimitsPrices> do_get_gas_limits_prices(vm::CellSlice cs, int id);
   td::Result<MsgPrices> get_msg_prices(bool is_masterchain = false) const;
+  static td::Result<MsgPrices> do_get_msg_prices(vm::CellSlice cs, int id);
   static CatchainValidatorsConfig unpack_catchain_validators_config(Ref<vm::Cell> cell);
   CatchainValidatorsConfig get_catchain_validators_config() const;
   td::Status visit_validator_params() const;
@@ -633,8 +640,10 @@ class Config {
                                                          ton::CatchainSeqno cc_seqno) const;
   std::vector<ton::ValidatorDescr> compute_total_validator_set(int next) const;
   td::Result<SizeLimitsConfig> get_size_limits_config() const;
+  static td::Result<SizeLimitsConfig> do_get_size_limits_config(td::Ref<vm::CellSlice> cs);
   std::unique_ptr<vm::Dictionary> get_suspended_addresses(ton::UnixTime now) const;
   BurningConfig get_burning_config() const;
+  td::Ref<vm::Tuple> get_unpacked_config_tuple(ton::UnixTime now) const;
   static std::vector<ton::ValidatorDescr> do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf,
                                                                    ton::ShardIdFull shard,
                                                                    const block::ValidatorSet& vset, ton::UnixTime time,

crypto/block/transaction.cpp

@@ -1337,6 +1337,11 @@ Ref<vm::Tuple> Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const {
     // may only return tuple or raise Error (See crypto/block/mc-config.cpp#2223)
     tuple.push_back(cfg.prev_blocks_info.not_null() ? vm::StackEntry(cfg.prev_blocks_info) : vm::StackEntry());
   }
+  if (cfg.global_version >= 6) {
+    tuple.push_back(cfg.unpacked_config_tuple.not_null() ? vm::StackEntry(cfg.unpacked_config_tuple)
+                                                         : vm::StackEntry());   // unpacked_config_tuple:[...]
+    tuple.push_back(due_payment.not_null() ? due_payment : td::zero_refint());  // due_payment:Integer
+  }
   auto tuple_ref = td::make_cnt_ref<std::vector<vm::StackEntry>>(std::move(tuple));
   LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string();
   return vm::make_tuple_ref(std::move(tuple_ref));
@@ -1920,6 +1925,25 @@ td::uint64 MsgPrices::compute_fwd_fees(td::uint64 cells, td::uint64 bits) const
                           .lo();
 }
 
+/**
+ * Computes the forward fees for a message based on the number of cells and bits.
+ * Return the result as td::RefInt256
+ *
+ * msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms
+ * ihr_fwd_fees = ceil((msg_fwd_fees * ihr_price_factor)/2^16) nanograms
+ * bits in the root cell of a message are not included in msg.bits (lump_price pays for them)
+ *
+ * @param cells The number of cells in the message.
+ * @param bits The number of bits in the message.
+ *
+ * @returns The computed forward fees for the message as td::RefInt256j.
+ */
+td::RefInt256 MsgPrices::compute_fwd_fees256(td::uint64 cells, td::uint64 bits) const {
+  return td::make_refint(lump_price) +
+         td::rshift(td::make_refint(bit_price) * bits + td::make_refint(cell_price) * cells, 16,
+                    1);  // divide by 2^16 with ceil rounding
+}
+
 /**
  * Computes the forward fees and IHR fees for a message with the given number of cells and bits.
  *
@@ -3539,6 +3563,9 @@ td::Status FetchConfigParams::fetch_config_params(
     if (compute_phase_cfg->global_version >= 4) {
       compute_phase_cfg->prev_blocks_info = std::move(prev_blocks_info);
     }
+    if (compute_phase_cfg->global_version >= 6) {
+      compute_phase_cfg->unpacked_config_tuple = config.get_unpacked_config_tuple(now);
+    }
     compute_phase_cfg->suspended_addresses = config.get_suspended_addresses(now);
     compute_phase_cfg->size_limits = size_limits;
   }

crypto/block/transaction.h

@@ -117,6 +117,7 @@ struct ComputePhaseConfig {
   td::uint16 max_vm_data_depth = 512;
   int global_version = 0;
   Ref<vm::Tuple> prev_blocks_info;
+  Ref<vm::Tuple> unpacked_config_tuple;
   std::unique_ptr<vm::Dictionary> suspended_addresses;
   SizeLimitsConfig size_limits;
   int vm_log_verbosity = 0;