diff --git a/blockchain-explorer/blockchain-explorer-http.cpp b/blockchain-explorer/blockchain-explorer-http.cpp index e2322ff76..e5203cb7e 100644 --- a/blockchain-explorer/blockchain-explorer-http.cpp +++ b/blockchain-explorer/blockchain-explorer-http.cpp @@ -122,9 +122,8 @@ HttpAnswer& HttpAnswer::operator<<(MessageCell msg) { abort("cannot unpack internal message"); return *this; } - td::RefInt256 value; - td::Ref extra; - if (!block::unpack_CurrencyCollection(info.value, value, extra)) { + block::CurrencyCollection currency_collection; + if (!currency_collection.unpack(info.value)) { abort("cannot unpack message value"); return *this; } @@ -133,7 +132,7 @@ HttpAnswer& HttpAnswer::operator<<(MessageCell msg) { << "destination" << AddressCell{info.dest} << "\n" << "lt" << info.created_lt << "\n" << "time" << info.created_at << " (" << time_to_human(info.created_at) << ")\n" - << "value" << value << "\n"; + << "value" << currency_collection.to_str()<< "\n"; break; } default: @@ -365,6 +364,7 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) { ton::LogicalTime last_trans_lt = 0; ton::Bits256 last_trans_hash; last_trans_hash.set_zero(); + block::CurrencyCollection balance = block::CurrencyCollection::zero(); try { auto state_root = vm::MerkleProof::virtualize(acc_c.q_roots[1], 1); if (state_root.is_null()) { @@ -397,6 +397,20 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) { } last_trans_hash = acc_info.last_trans_hash; last_trans_lt = acc_info.last_trans_lt; + block::gen::Account::Record_account acc; + block::gen::AccountStorage::Record storage_rec; + if (!tlb::unpack_cell(acc_c.root, acc)) { + abort("cannot unpack Account"); + return *this; + } + if (!tlb::csr_unpack(acc.storage, storage_rec)) { + abort("cannot unpack AccountStorage"); + return *this; + } + if (!balance.unpack(storage_rec.balance)) { + abort("cannot unpack account balance"); + return *this; + } } else if (acc_c.root.not_null()) { abort(PSTRING() << "account state proof shows that account state for " << acc_c.addr.workchain << ":" << acc_c.addr.addr.to_hex() << " must be empty, but it is not"); @@ -434,6 +448,7 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) { *this << "workchain" << acc_c.addr.workchain << ""; *this << "account hex" << acc_c.addr.addr.to_hex() << ""; *this << "account" << acc_c.addr.rserialize(true) << ""; + *this << "balance" << balance.to_str() << ""; if (last_trans_lt > 0) { *this << "last transaction" << "lt=" << last_trans_lt diff --git a/crypto/block/block.h b/crypto/block/block.h index 7e42c9db5..bdb071e82 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -249,6 +249,12 @@ struct ParamLimits { bool deserialize(vm::CellSlice& cs); int classify(td::uint64 value) const; bool fits(unsigned cls, td::uint64 value) const; + void multiply_by(double x) { + CHECK(x > 0.0); + for (td::uint32& y : limits_) { + y = (td::uint32)std::min(y * x, 1e9); + } + } private: std::array limits_; diff --git a/crypto/smc-envelope/GenericAccount.cpp b/crypto/smc-envelope/GenericAccount.cpp index 04249699c..4c9bd1659 100644 --- a/crypto/smc-envelope/GenericAccount.cpp +++ b/crypto/smc-envelope/GenericAccount.cpp @@ -61,7 +61,8 @@ block::StdAddress GenericAccount::get_address(ton::WorkchainId workchain_id, return block::StdAddress(workchain_id, init_state->get_hash().bits(), true /*bounce*/); } -void GenericAccount::store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms) { +void GenericAccount::store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms, + td::Ref extra_currencies) { td::BigInt256 dest_addr; dest_addr.import_bits(dest_address.addr.as_bitslice()); cb.store_zeroes(1) @@ -73,7 +74,8 @@ void GenericAccount::store_int_message(vm::CellBuilder& cb, const block::StdAddr .store_long(dest_address.workchain, 8) .store_int256(dest_addr, 256); block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - cb.store_zeroes(9 + 64 + 32); + cb.store_maybe_ref(extra_currencies); + cb.store_zeroes(8 + 64 + 32); } td::Ref GenericAccount::create_ext_message(const block::StdAddress& address, td::Ref new_state, diff --git a/crypto/smc-envelope/GenericAccount.h b/crypto/smc-envelope/GenericAccount.h index 285553c78..93a059f99 100644 --- a/crypto/smc-envelope/GenericAccount.h +++ b/crypto/smc-envelope/GenericAccount.h @@ -36,7 +36,8 @@ class GenericAccount { static block::StdAddress get_address(ton::WorkchainId workchain_id, const td::Ref& init_state) noexcept; static td::Ref create_ext_message(const block::StdAddress& address, td::Ref new_state, td::Ref body) noexcept; - static void store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms); + static void store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms, + td::Ref extra_currencies); static td::Result get_public_key(const SmartContract& sc); static td::Result get_seqno(const SmartContract& sc); diff --git a/crypto/smc-envelope/WalletInterface.cpp b/crypto/smc-envelope/WalletInterface.cpp index e02759b23..eecec4354 100644 --- a/crypto/smc-envelope/WalletInterface.cpp +++ b/crypto/smc-envelope/WalletInterface.cpp @@ -48,7 +48,7 @@ td::Result> WalletInterface::get_init_message(const td::Ed2551 td::Ref WalletInterface::create_int_message(const Gift &gift) { vm::CellBuilder cbi; - GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms); + GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms, gift.extra_currencies); if (gift.init_state.not_null()) { cbi.store_ones(2); cbi.store_ref(gift.init_state); diff --git a/crypto/smc-envelope/WalletInterface.h b/crypto/smc-envelope/WalletInterface.h index c4e1f270e..e88ca0a61 100644 --- a/crypto/smc-envelope/WalletInterface.h +++ b/crypto/smc-envelope/WalletInterface.h @@ -37,6 +37,7 @@ class WalletInterface : public SmartContract { struct Gift { block::StdAddress destination; td::int64 gramms; + td::Ref extra_currencies; td::int32 send_mode{-1}; bool is_encrypted{false}; diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index c79924d0e..41f9c3396 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -1779,7 +1779,7 @@ Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, t int mode, int skip1, int skip2) const { if (dict1.is_null()) { assert(!skip2); - if ((mode & 1) && dict2.is_null()) { + if ((mode & 1) && dict2.not_null()) { throw CombineError{}; } return dict2; @@ -1854,11 +1854,11 @@ Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, t key_buffer[-1] = 0; // combine left subtrees auto c1 = dict_combine_with(label1.remainder->prefetch_ref(0), label2.remainder->prefetch_ref(0), key_buffer, - n - c - 1, total_key_len, combine_func); + n - c - 1, total_key_len, combine_func, mode); key_buffer[-1] = 1; // combine right subtrees auto c2 = dict_combine_with(label1.remainder->prefetch_ref(1), label2.remainder->prefetch_ref(1), key_buffer, - n - c - 1, total_key_len, combine_func); + n - c - 1, total_key_len, combine_func, mode); label1.remainder.clear(); label2.remainder.clear(); // c1 and c2 are merged left and right children of dict1 and dict2 diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index fa4cc507f..d9e6d2604 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -3730,7 +3730,7 @@ void TestNode::continue_check_validator_load3(std::unique_ptr info2, int mode, std::string file_pfx) { LOG(INFO) << "continue_check_validator_load3 for blocks " << info1->blk_id.to_str() << " and " - << info1->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx + << info2->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx << "`: comparing block creators data"; if (info1->created_total.first <= 0 || info2->created_total.first <= 0) { LOG(ERROR) << "no total created blocks statistics"; @@ -3791,13 +3791,14 @@ void TestNode::continue_check_validator_load3(std::unique_ptrvset->export_scaled_validator_weights()); for (int i = 0; i < count; i++) { int x1 = d[i].first, y1 = d[i].second; - double xe = (i < main_count ? (double)xs / main_count : 0); + bool is_masterchain_validator = i < main_count; + double xe = (is_masterchain_validator ? (double)xs / main_count : 0); double ye = shard_share[i] * (double)ys / shard_count; td::Bits256 pk = info2->vset->list[i].pubkey.as_bits256(); double p1 = create_prob(x1, .9 * xe), p2 = shard_create_prob(y1, .9 * ye, chunk_size); td::TerminalIO::out() << "val #" << i << ": pubkey " << pk.to_hex() << ", blocks created (" << x1 << "," << y1 << "), expected (" << xe << "," << ye << "), probabilities " << p1 << " and " << p2 << "\n"; - if (std::min(p1, p2) < .00001) { + if ((is_masterchain_validator ? p1 : p2) < .00001) { LOG(ERROR) << "validator #" << i << " with pubkey " << pk.to_hex() << " : serious misbehavior detected: created less than 90% of the expected amount of blocks with " "probability 99.999% : created (" @@ -3810,7 +3811,7 @@ void TestNode::continue_check_validator_load3(std::unique_ptr code:bytes data:bytes last_transaction_id:internal.transactionId block_id:ton.blockIdExt frozen_hash:bytes sync_utime:int53 = raw.FullAccountState; +raw.message source:accountAddress destination:accountAddress value:int64 extra_currencies:vector fwd_fee:int64 ihr_fee:int64 created_lt:int64 body_hash:bytes msg_data:msg.Data = raw.Message; raw.transaction address:accountAddress utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; @@ -87,7 +89,7 @@ pchan.statePayout A:int64 B:int64 = pchan.State; pchan.accountState config:pchan.config state:pchan.State description:string = AccountState; uninited.accountState frozen_hash:bytes = AccountState; -fullAccountState address:accountAddress balance:int64 last_transaction_id:internal.transactionId block_id:ton.blockIdExt sync_utime:int53 account_state:AccountState revision:int32 = FullAccountState; +fullAccountState address:accountAddress balance:int64 extra_currencies:vector last_transaction_id:internal.transactionId block_id:ton.blockIdExt sync_utime:int53 account_state:AccountState revision:int32 = FullAccountState; accountRevisionList revisions:vector = AccountRevisionList; accountList accounts:vector = AccountList; @@ -110,7 +112,7 @@ msg.dataDecrypted proof:bytes data:msg.Data = msg.DataDecrypted; msg.dataEncryptedArray elements:vector = msg.DataEncryptedArray; msg.dataDecryptedArray elements:vector = msg.DataDecryptedArray; -msg.message destination:accountAddress public_key:string amount:int64 data:msg.Data send_mode:int32 = msg.Message; +msg.message destination:accountAddress public_key:string amount:int64 extra_currencies:vector data:msg.Data send_mode:int32 = msg.Message; // // DNS diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 686bd9181..70c08459c 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tonlib/test/online.cpp b/tonlib/test/online.cpp index 52f517200..cf892c292 100644 --- a/tonlib/test/online.cpp +++ b/tonlib/test/online.cpp @@ -264,7 +264,8 @@ td::Result create_send_grams_query(Client& client, const Wallet& source data = tonlib_api::make_object(message.raw.unwrap(), message.init_state.unwrap()); } msgs.push_back(tonlib_api::make_object( - tonlib_api::make_object(destination), "", amount, std::move(data), -1)); + tonlib_api::make_object(destination), "", amount, + std::vector>{}, std::move(data), -1)); auto r_id = sync_send(client, tonlib_api::make_object( @@ -566,7 +567,7 @@ void test_multisig(Client& client, const Wallet& giver_wallet) { for (int i = 0; i < 2; i++) { // Just transfer all (some) money back in one query vm::CellBuilder icb; - ton::GenericAccount::store_int_message(icb, block::StdAddress::parse(giver_wallet.address).move_as_ok(), 1); + ton::GenericAccount::store_int_message(icb, block::StdAddress::parse(giver_wallet.address).move_as_ok(), 1, {}); icb.store_bytes("\0\0\0\0", 4); vm::CellString::store(icb, "Greatings from multisig", 35 * 8).ensure(); ton::MultisigWallet::QueryBuilder qb(wallet_id, -1 - i, icb.finalize()); diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 1e9882880..19d5f0d0a 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -177,6 +177,7 @@ static block::AccountState create_account_state(ton::tl_object_ptr extra_currencies; ton::UnixTime storage_last_paid{0}; vm::CellStorageStat storage_stat; @@ -205,6 +206,74 @@ std::string to_bytes(td::Ref cell) { return vm::std_boc_serialize(cell, vm::BagOfCells::Mode::WithCRC32C).move_as_ok().as_slice().str(); } +td::Result>> parse_extra_currencies_or_throw( + const td::Ref dict_root) { + std::vector> result; + vm::Dictionary dict{dict_root, 32}; + if (!dict.check_for_each([&](td::Ref value, td::ConstBitPtr key, int n) { + CHECK(n == 32); + int id = (int)key.get_int(n); + auto amount_ref = block::tlb::t_VarUIntegerPos_32.as_integer_skip(value.write()); + if (amount_ref.is_null() || !value->empty_ext()) { + return false; + } + td::int64 amount = amount_ref->to_long(); + if (amount == td::int64(~0ULL << 63)) { + return false; + } + result.push_back(tonlib_api::make_object(id, amount)); + return true; + })) { + return td::Status::Error("Failed to parse extra currencies dict"); + } + return result; +} + +td::Result>> parse_extra_currencies( + const td::Ref& dict_root) { + return TRY_VM(parse_extra_currencies_or_throw(dict_root)); +} + +td::Result> to_extra_currenctes_dict( + const std::vector>& extra_currencies) { + vm::Dictionary dict{32}; + for (const auto &f : extra_currencies) { + if (f->amount_ == 0) { + continue; + } + if (f->amount_ < 0) { + return td::Status::Error("Negative extra currency amount"); + } + vm::CellBuilder cb2; + block::tlb::t_VarUInteger_32.store_integer_value(cb2, *td::make_refint(f->amount_)); + if (!dict.set_builder(td::BitArray<32>(f->id_), cb2, vm::DictionaryBase::SetMode::Add)) { + return td::Status::Error("Duplicate extra currency id"); + } + } + return std::move(dict).extract_root_cell(); +} + +td::Status check_enough_extra_currencies(const td::Ref &balance, const td::Ref &amount) { + block::CurrencyCollection c1{td::zero_refint(), balance}; + block::CurrencyCollection c2{td::zero_refint(), amount}; + auto res = TRY_VM(td::Result{c1 >= c2}); + TRY_RESULT(v, std::move(res)); + if (!v) { + return TonlibError::NotEnoughFunds(); + } + return td::Status::OK(); +} + +td::Result> add_extra_currencies(const td::Ref &e1, const td::Ref &e2) { + block::CurrencyCollection c1{td::zero_refint(), e1}; + block::CurrencyCollection c2{td::zero_refint(), e2}; + TRY_RESULT_ASSIGN(c1, TRY_VM(td::Result{c1 + c2})); + if (c1.is_valid()) { + return td::Status::Error("Failed to add extra currencies"); + } + return c1.extra; +} + td::Result get_public_key(td::Slice public_key) { TRY_RESULT_PREFIX(address, block::PublicKey::parse(public_key), TonlibError::InvalidPublicKey()); return address; @@ -312,9 +381,10 @@ class AccountState { if (state.data.not_null()) { data = to_bytes(state.data); } + TRY_RESULT(extra_currencies, parse_extra_currencies(get_extra_currencies())); return tonlib_api::make_object( - get_balance(), std::move(code), std::move(data), to_transaction_id(raw().info), to_tonlib_api(raw().block_id), - raw().frozen_hash, get_sync_time()); + get_balance(), std::move(extra_currencies), std::move(code), std::move(data), to_transaction_id(raw().info), + to_tonlib_api(raw().block_id), raw().frozen_hash, get_sync_time()); } td::Result> to_wallet_v3_accountState() const { @@ -446,10 +516,11 @@ class AccountState { td::Result> to_fullAccountState() const { TRY_RESULT(account_state, to_accountState()); + TRY_RESULT(extra_currencies, parse_extra_currencies(get_extra_currencies())); return tonlib_api::make_object( tonlib_api::make_object(get_address().rserialize(true)), get_balance(), - to_transaction_id(raw().info), to_tonlib_api(raw().block_id), get_sync_time(), std::move(account_state), - get_wallet_revision()); + std::move(extra_currencies), to_transaction_id(raw().info), to_tonlib_api(raw().block_id), get_sync_time(), + std::move(account_state), get_wallet_revision()); } td::Result> to_shardAccountCell() const { @@ -549,6 +620,10 @@ class AccountState { return raw_.balance; } + td::Ref get_extra_currencies() const { + return raw_.extra_currencies; + } + const RawAccountState& raw() const { return raw_; } @@ -1341,6 +1416,7 @@ class GetRawAccountState : public td::actor::Actor { } TRY_RESULT(balance, to_balance(storage.balance)); res.balance = balance; + res.extra_currencies = storage.balance->prefetch_ref(); auto state_tag = block::gen::t_AccountState.get_tag(*storage.state); if (state_tag < 0) { return td::Status::Error("Failed to parse AccountState tag"); @@ -2007,7 +2083,9 @@ class RunEmulator : public TonlibQueryActor { account = std::move(emulation_result.move_as_ok().account); RawAccountState raw = std::move(account_state_->raw()); raw.block_id = block_id_.id; - raw.balance = account.get_balance().grams->to_long(); + block::CurrencyCollection balance = account.get_balance(); + raw.balance = balance.grams->to_long(); + raw.extra_currencies = balance.extra; raw.storage_last_paid = std::move(account.last_paid); raw.storage_stat = std::move(account.storage_stat); raw.code = std::move(account.code); @@ -3015,6 +3093,7 @@ struct ToRawTransactions { } TRY_RESULT(balance, to_balance(msg_info.value)); + TRY_RESULT(extra_currencies, parse_extra_currencies(msg_info.value->prefetch_ref())); TRY_RESULT(src, to_std_address(msg_info.src)); TRY_RESULT(dest, to_std_address(msg_info.dest)); TRY_RESULT(fwd_fee, to_balance(msg_info.fwd_fee)); @@ -3023,8 +3102,9 @@ struct ToRawTransactions { return tonlib_api::make_object( tonlib_api::make_object(src), - tonlib_api::make_object(std::move(dest)), balance, fwd_fee, ihr_fee, created_lt, - std::move(body_hash), get_data(src)); + tonlib_api::make_object(std::move(dest)), balance, + std::move(extra_currencies), fwd_fee, ihr_fee, created_lt, std::move(body_hash), + get_data(src)); } case block::gen::CommonMsgInfo::ext_in_msg_info: { block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; @@ -3034,7 +3114,8 @@ struct ToRawTransactions { TRY_RESULT(dest, to_std_address(msg_info.dest)); return tonlib_api::make_object( tonlib_api::make_object(), - tonlib_api::make_object(std::move(dest)), 0, 0, 0, 0, std::move(body_hash), + tonlib_api::make_object(std::move(dest)), 0, + std::vector>{}, 0, 0, 0, std::move(body_hash), get_data("")); } case block::gen::CommonMsgInfo::ext_out_msg_info: { @@ -3046,7 +3127,9 @@ struct ToRawTransactions { auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( tonlib_api::make_object(src), - tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), get_data(src)); + tonlib_api::make_object(), 0, + std::vector>{}, 0, 0, created_lt, std::move(body_hash), + get_data(src)); } } @@ -3502,6 +3585,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { struct Action { block::StdAddress destination; td::int64 amount; + td::Ref extra_currencies; td::int32 send_mode{-1}; bool is_encrypted{false}; @@ -3549,6 +3633,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { return TonlibError::InvalidField("amount", "can't be negative"); } res.amount = message.amount_; + TRY_RESULT_ASSIGN(res.extra_currencies, to_extra_currenctes_dict(message.extra_currencies_)); if (!message.public_key_.empty()) { TRY_RESULT(public_key, get_public_key(message.public_key_)); auto key = td::Ed25519::PublicKey(td::SecureString(public_key.key)); @@ -3959,8 +4044,10 @@ class GenericCreateSendGrams : public TonlibQueryActor { } td::int64 amount = 0; + td::Ref extra_currencies; for (auto& action : actions_) { amount += action.amount; + TRY_RESULT_ASSIGN(extra_currencies, add_extra_currencies(extra_currencies, action.extra_currencies)); } if (amount > source_->get_balance()) { @@ -3972,6 +4059,8 @@ class GenericCreateSendGrams : public TonlibQueryActor { return TonlibError::NotEnoughFunds(); } + TRY_STATUS(check_enough_extra_currencies(source_->get_extra_currencies(), extra_currencies)); + if (source_->get_wallet_type() == AccountState::RestrictedWallet) { auto r_unlocked_balance = ton::RestrictedWallet::create(source_->get_smc_state()) ->get_balance(source_->get_balance(), source_->get_sync_time()); @@ -3989,12 +4078,13 @@ class GenericCreateSendGrams : public TonlibQueryActor { auto& destination = destinations_[i]; gift.destination = destinations_[i]->get_address(); gift.gramms = action.amount; + gift.extra_currencies = action.extra_currencies; gift.send_mode = action.send_mode; // Temporary turn off this dangerous transfer - if (false && action.amount == source_->get_balance()) { - gift.gramms = -1; - } + // if (action.amount == source_->get_balance()) { + // gift.gramms = -1; + // } if (action.body.not_null()) { gift.body = action.body; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index f12e88d41..775b82e53 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -1103,9 +1103,9 @@ class TonlibCli : public td::actor::Actor { void pchan_init_2(Address addr, td::int32 pchan_id, td::int64 value, tonlib_api::object_ptr query, td::Promise promise) { std::vector> messages; - messages.push_back( - make_object(channels_[pchan_id].to_address(), "", value, - make_object(query->body_, query->init_state_), -1)); + messages.push_back(make_object( + channels_[pchan_id].to_address(), "", value, std::vector>{}, + make_object(query->body_, query->init_state_), -1)); auto action = make_object(std::move(messages), true); send_query( make_object(addr.input_key(), std::move(addr.address), 60, std::move(action), nullptr), @@ -2053,6 +2053,19 @@ class TonlibCli : public td::actor::Actor { }); } + static void print_full_account_state(const ton::tl_object_ptr& state) { + td::StringBuilder balance_str; + balance_str << "Balance: " << Grams{td::narrow_cast(state->balance_ * (state->balance_ > 0))}; + for (const auto& extra : state->extra_currencies_) { + balance_str << " + " << extra->amount_ << ".$" << extra->id_; + } + td::TerminalIO::out() << balance_str.as_cslice() << "\n"; + td::TerminalIO::out() << "Sync utime: " << state->sync_utime_ << "\n"; + td::TerminalIO::out() << "transaction.LT: " << state->last_transaction_id_->lt_ << "\n"; + td::TerminalIO::out() << "transaction.Hash: " << td::base64_encode(state->last_transaction_id_->hash_) << "\n"; + td::TerminalIO::out() << to_string(state->account_state_); + } + void get_state(td::Slice key, td::Promise promise) { TRY_RESULT_PROMISE(promise, address, to_account_address(key, false)); @@ -2061,14 +2074,7 @@ class TonlibCli : public td::actor::Actor { ton::move_tl_object_as(std::move(address.address))), promise.wrap([address_str](auto&& state) { td::TerminalIO::out() << "Address: " << address_str << "\n"; - td::TerminalIO::out() << "Balance: " - << Grams{td::narrow_cast(state->balance_ * (state->balance_ > 0))} - << "\n"; - td::TerminalIO::out() << "Sync utime: " << state->sync_utime_ << "\n"; - td::TerminalIO::out() << "transaction.LT: " << state->last_transaction_id_->lt_ << "\n"; - td::TerminalIO::out() << "transaction.Hash: " << td::base64_encode(state->last_transaction_id_->hash_) - << "\n"; - td::TerminalIO::out() << to_string(state->account_state_); + print_full_account_state(state); return td::Unit(); })); } @@ -2085,14 +2091,7 @@ class TonlibCli : public td::actor::Actor { ton::move_tl_object_as(std::move(transaction_id))), promise.wrap([address_str](auto&& state) { td::TerminalIO::out() << "Address: " << address_str << "\n"; - td::TerminalIO::out() << "Balance: " - << Grams{td::narrow_cast(state->balance_ * (state->balance_ > 0))} - << "\n"; - td::TerminalIO::out() << "Sync utime: " << state->sync_utime_ << "\n"; - td::TerminalIO::out() << "transaction.LT: " << state->last_transaction_id_->lt_ << "\n"; - td::TerminalIO::out() << "transaction.Hash: " << td::base64_encode(state->last_transaction_id_->hash_) - << "\n"; - td::TerminalIO::out() << to_string(state->account_state_); + print_full_account_state(state); return td::Unit(); })); } @@ -2220,15 +2219,29 @@ class TonlibCli : public td::actor::Actor { td::StringBuilder sb; for (tonlib_api::object_ptr& t : res->transactions_) { td::int64 balance = 0; + std::map extra_currencies; balance += t->in_msg_->value_; + for (const auto& extra : t->in_msg_->extra_currencies_) { + extra_currencies[extra->id_] += extra->amount_; + } for (auto& ot : t->out_msgs_) { balance -= ot->value_; + for (const auto& extra : ot->extra_currencies_) { + extra_currencies[extra->id_] -= extra->amount_; + } } if (balance >= 0) { sb << Grams{td::uint64(balance)}; } else { sb << "-" << Grams{td::uint64(-balance)}; } + for (const auto& [id, amount] : extra_currencies) { + if (amount > 0) { + sb << " + " << amount << ".$" << id; + } else if (amount < 0) { + sb << " - " << -amount << ".$" << id; + } + } sb << " Fee: " << Grams{td::uint64(t->fee_)}; if (t->in_msg_->source_->account_address_.empty()) { sb << " External "; @@ -2258,6 +2271,9 @@ class TonlibCli : public td::actor::Actor { sb << " To " << ot->destination_->account_address_; } sb << " " << Grams{td::uint64(ot->value_)}; + for (const auto& extra : ot->extra_currencies_) { + sb << " + " << extra->amount_ << ".$" << extra->id_; + } print_msg_data(sb, ot->msg_data_); } sb << "\n"; @@ -2322,8 +2338,9 @@ class TonlibCli : public td::actor::Actor { } else { data = make_object(message.str()); } - messages.push_back( - make_object(std::move(address.address), "", amount.nano, std::move(data), -1)); + messages.push_back(make_object( + std::move(address.address), "", amount.nano, std::vector>{}, + std::move(data), -1)); return td::Status::OK(); }; diff --git a/validator/fabric.h b/validator/fabric.h index 9a702e3ee..84de1da4c 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -89,7 +89,7 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc Ed25519_PublicKey creator, td::Ref validator_set, td::Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token = {}, unsigned mode = 0); + td::CancellationToken cancellation_token, unsigned mode, int attempt_idx = 0); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 7754430fd..2105d9bf9 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -77,6 +77,8 @@ class Collator final : public td::actor::Actor { td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; unsigned mode_ = 0; + int attempt_idx_; + bool allow_repeat_collation_ = false; ton::BlockSeqno last_block_seqno{0}; ton::BlockSeqno prev_mc_block_seqno{0}; ton::BlockSeqno new_block_seqno{0}; @@ -92,7 +94,7 @@ class Collator final : public td::actor::Actor { Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode); + td::CancellationToken cancellation_token, unsigned mode, int attempt_idx); ~Collator() override = default; bool is_busy() const { return busy_; @@ -330,6 +332,7 @@ class Collator final : public td::actor::Actor { bool insert_out_msg(Ref out_msg); bool insert_out_msg(Ref out_msg, td::ConstBitPtr msg_hash); bool register_out_msg_queue_op(bool force = false); + bool register_dispatch_queue_op(bool force = false); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); bool combine_account_transactions(); bool update_public_libraries(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 256e6c312..25f59248c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -45,11 +45,13 @@ using td::Ref; using namespace std::literals::string_literals; // Don't increase MERGE_MAX_QUEUE_LIMIT too much: merging requires cleaning the whole queue in out_msg_queue_cleanup -static const td::uint32 FORCE_SPLIT_QUEUE_SIZE = 4096; -static const td::uint32 SPLIT_MAX_QUEUE_SIZE = 100000; -static const td::uint32 MERGE_MAX_QUEUE_SIZE = 2047; -static const td::uint32 SKIP_EXTERNALS_QUEUE_SIZE = 8000; -static const int HIGH_PRIORITY_EXTERNAL = 10; // don't skip high priority externals when queue is big +static constexpr td::uint32 FORCE_SPLIT_QUEUE_SIZE = 4096; +static constexpr td::uint32 SPLIT_MAX_QUEUE_SIZE = 100000; +static constexpr td::uint32 MERGE_MAX_QUEUE_SIZE = 2047; +static constexpr td::uint32 SKIP_EXTERNALS_QUEUE_SIZE = 8000; +static constexpr int HIGH_PRIORITY_EXTERNAL = 10; // don't skip high priority externals when queue is big + +static constexpr int MAX_ATTEMPTS = 5; #define DBG(__n) dbg(__n)&& #define DSTART int __dcnt = 0; @@ -74,12 +76,15 @@ static inline bool dbg(int c) { * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the collator. * @param promise The promise to return the result. + * @param cancellation_token Token to cancel collation. + * @param mode +1 - skip storing candidate to disk. + * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. */ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, td::CancellationToken cancellation_token, - unsigned mode) + unsigned mode, int attempt_idx) : shard_(shard) , is_hardfork_(is_hardfork) , min_mc_block_id{min_masterchain_block_id} @@ -95,6 +100,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) , mode_(mode) + , attempt_idx_(attempt_idx) , perf_timer_("collate", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); @@ -111,7 +117,11 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha * The results of these queries are handled by corresponding callback functions. */ void Collator::start_up() { - LOG(WARNING) << "Collator for shard " << shard_.to_str() << " started"; + LOG(WARNING) << "Collator for shard " << shard_.to_str() << " started" + << (attempt_idx_ ? PSTRING() << " (attempt #" << attempt_idx_ << ")" : ""); + if (!check_cancelled()) { + return; + } LOG(DEBUG) << "Previous block #1 is " << prev_blocks.at(0).to_str(); if (prev_blocks.size() > 1) { LOG(DEBUG) << "Previous block #2 is " << prev_blocks.at(1).to_str(); @@ -344,8 +354,15 @@ bool Collator::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "cannot generate block candidate for " << show_shard(shard_) << " : " << error.to_string(); if (busy_) { - LOG(INFO) << "collation took " << perf_timer_.elapsed() << " s"; - main_promise(std::move(error)); + if (allow_repeat_collation_ && error.code() != ErrorCode::cancelled && attempt_idx_ + 1 < MAX_ATTEMPTS && + !is_hardfork_ && !timeout.is_in_past()) { + LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; + run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, + td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_, + attempt_idx_ + 1); + } else { + main_promise(std::move(error)); + } busy_ = false; } stop(); @@ -728,6 +745,15 @@ bool Collator::unpack_last_mc_state() { return fatal_error(limits.move_as_error()); } block_limits_ = limits.move_as_ok(); + if (attempt_idx_ == 3) { + LOG(INFO) << "Attempt #3: bytes, gas limits /= 2"; + block_limits_->bytes.multiply_by(0.5); + block_limits_->gas.multiply_by(0.5); + } else if (attempt_idx_ == 4) { + LOG(INFO) << "Attempt #4: bytes, gas limits /= 4"; + block_limits_->bytes.multiply_by(0.25); + block_limits_->gas.multiply_by(0.25); + } LOG(DEBUG) << "block limits: bytes [" << block_limits_->bytes.underload() << ", " << block_limits_->bytes.soft() << ", " << block_limits_->bytes.hard() << "]"; LOG(DEBUG) << "block limits: gas [" << block_limits_->gas.underload() << ", " << block_limits_->gas.soft() << ", " @@ -2189,6 +2215,7 @@ bool Collator::do_collate() { if (max_lt == start_lt) { ++max_lt; } + allow_repeat_collation_ = true; // NB: interchanged 1.2 and 1.1 (is this always correct?) // 1.1. re-adjust neighbors' out_msg_queues (for oneself) if (!add_trivial_neighbor()) { @@ -3715,6 +3742,10 @@ bool Collator::process_inbound_external_messages() { LOG(INFO) << "skipping processing of inbound external messages"; return true; } + if (attempt_idx_ >= 2) { + LOG(INFO) << "Attempt #" << attempt_idx_ << ": skip external messages"; + return true; + } if (out_msg_queue_size_ > SKIP_EXTERNALS_QUEUE_SIZE) { LOG(INFO) << "skipping processing of inbound external messages (except for high-priority) because out_msg_queue is " "too big (" @@ -3845,6 +3876,10 @@ bool Collator::process_dispatch_queue() { if (max_per_initiator[iter] == 0 || max_total_count[iter] == 0) { continue; } + if (iter > 0 && attempt_idx_ >= 1) { + LOG(INFO) << "Attempt #" << attempt_idx_ << ": skip process_dispatch_queue"; + break; + } vm::AugmentedDictionary cur_dispatch_queue{dispatch_queue_->get_root(), 256, block::tlb::aug_DispatchQueue}; std::map, size_t> count_per_initiator; size_t total_count = 0; @@ -3857,13 +3892,13 @@ bool Collator::process_dispatch_queue() { stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": " << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; - return true; + return register_dispatch_queue_op(true); } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing dispatch queue"; stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": timeout\n"; - return true; + return register_dispatch_queue_op(true); } StdSmcAddress src_addr; td::Ref account_dispatch_queue; @@ -3941,6 +3976,7 @@ bool Collator::process_dispatch_queue() { if (iter == 0) { have_unprocessed_account_dispatch_queue_ = false; } + register_dispatch_queue_op(true); } return true; } @@ -3964,12 +4000,7 @@ bool Collator::process_deferred_message(Ref enq_msg, StdSmcAddres return fatal_error(PSTRING() << "failed to delete message from DispatchQueue: address=" << src_addr.to_hex() << ", lt=" << lt); } - ++dispatch_queue_ops_; - if (!(dispatch_queue_ops_ & 63)) { - if (!block_limit_status_->add_proof(dispatch_queue_->get_root_cell())) { - return false; - } - } + register_dispatch_queue_op(); ++sender_generated_messages_count_[src_addr]; LogicalTime enqueued_lt = 0; @@ -4062,6 +4093,7 @@ bool Collator::process_deferred_message(Ref enq_msg, StdSmcAddres ++unprocessed_deferred_messages_[src_addr]; LOG(INFO) << "delivering deferred message from account " << src_addr.to_hex() << ", lt=" << lt << ", emitted_lt=" << emitted_lt; + block_limit_status_->add_cell(msg_env); register_new_msg(std::move(new_msg)); msg_metadata = std::move(env.metadata); return true; @@ -4241,11 +4273,7 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema } ++dispatch_dict_size; dispatch_queue_->set(src_addr, block::pack_account_dispatch_queue(dispatch_dict, dispatch_dict_size)); - ++dispatch_queue_ops_; - if (!(dispatch_queue_ops_ & 63)) { - return block_limit_status_->add_proof(dispatch_queue_->get_root_cell()); - } - return true; + return register_dispatch_queue_op(); } auto next_hop = block::interpolate_addr(src_prefix, dest_prefix, route_info.second); @@ -5130,6 +5158,23 @@ bool Collator::register_out_msg_queue_op(bool force) { } } +/** + * Registers a dispatch queue message queue operation. + * Adds the proof to the block limit status every 64 operations. + * + * @param force If true, the proof will always be added to the block limit status. + * + * @returns True if the operation was successfully registered, false otherwise. + */ +bool Collator::register_dispatch_queue_op(bool force) { + ++dispatch_queue_ops_; + if (force || !(dispatch_queue_ops_ & 63)) { + return block_limit_status_->add_proof(dispatch_queue_->get_root_cell()); + } else { + return true; + } +} + /** * Creates a new shard state and the Merkle update. * @@ -5255,9 +5300,10 @@ bool Collator::compute_out_msg_queue_info(Ref& out_msg_queue_info) { vm::CellSlice maybe_extra = cb.as_cellslice(); cb.reset(); - return register_out_msg_queue_op(true) && out_msg_queue_->append_dict_to_bool(cb) // _ out_queue:OutMsgQueue - && processed_upto_->pack(cb) // proc_info:ProcessedInfo - && cb.append_cellslice_bool(maybe_extra) // extra:(Maybe OutMsgQueueExtra) + return register_out_msg_queue_op(true) && register_dispatch_queue_op(true) && + out_msg_queue_->append_dict_to_bool(cb) // _ out_queue:OutMsgQueue + && processed_upto_->pack(cb) // proc_info:ProcessedInfo + && cb.append_cellslice_bool(maybe_extra) // extra:(Maybe OutMsgQueueExtra) && cb.finalize_to(out_msg_queue_info); } diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 8d3fa429b..16292c8cc 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -216,17 +216,18 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc Ed25519_PublicKey creator, td::Ref validator_set, td::Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode) { + td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, - min_masterchain_block_id, std::move(prev), std::move(validator_set), creator, - std::move(collator_opts), std::move(manager), timeout, std::move(promise), - std::move(cancellation_token), mode) + td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1) + << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), + shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), + creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), + std::move(cancellation_token), mode, attempt_idx) .release(); } @@ -242,7 +243,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, - std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0) + std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0, 0) .release(); } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 8ade2afa2..9210ecb3c 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1400,13 +1400,9 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { if (acc_root.not_null()) { if (mode_ & 0x40000000) { vm::MerkleProofBuilder mpb{acc_root}; - // account_none$0 = Account; - // account$1 addr:MsgAddressInt storage_stat:StorageInfo storage:AccountStorage = Account; - // account_storage$_ last_trans_lt:uint64 balance:CurrencyCollection state:AccountState = AccountStorage; - // account_active$1 _:StateInit = AccountState; - auto S = mpb.root()->load_cell(); - if (S.is_error()) { - fatal_error(S.move_as_error_prefix("Failed to load account: ")); + // This does not include code, data and libs into proof, but it includes extra currencies + if (!block::gen::t_Account.validate_ref(mpb.root())) { + fatal_error("failed to validate Account"); return; } if (!mpb.extract_proof_to(acc_root)) { diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 363446107..0a1ec14d2 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -129,7 +129,7 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, - actor_id(this), td::Timestamp::in(10.0), std::move(P)); + actor_id(this), td::Timestamp::in(10.0), std::move(P), td::CancellationToken{}, 0); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index a7f2cacf9..0c3907609 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -172,6 +172,7 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s prev_block_ids_ = std::vector{next_block_id}; cached_collated_block_ = nullptr; approved_candidates_cache_.clear(); + cancellation_token_source_.cancel(); } void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, @@ -417,6 +418,7 @@ void ValidatorGroup::destroy() { delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, td::Timestamp::in(10.0)); } + cancellation_token_source_.cancel(); stop(); } @@ -506,7 +508,8 @@ void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, t if (self_collate) { run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise)); + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), + cancellation_token_source_.get_cancellation_token(), 0); return; } if (collator_adnl_id.is_zero()) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 081716864..c278c276b 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -148,6 +148,7 @@ class ValidatorGroup : public td::actor::Actor { std::vector> promises; }; std::shared_ptr cached_collated_block_; + td::CancellationTokenSource cancellation_token_source_; void generated_block_candidate(std::shared_ptr cache, td::Result R);