From 4e2624459b4330b65433f9a0052b1c21f6b1b6a3 Mon Sep 17 00:00:00 2001 From: ton Date: Wed, 27 May 2020 22:10:46 +0400 Subject: [PATCH 1/2] integrating the existing state of TON Storage / TON Payments / CPS Fift development branches --- CMakeLists.txt | 12 +- adnl/adnl-peer.cpp | 3 +- crypto/CMakeLists.txt | 14 +- crypto/common/refcnt.cpp | 5 + crypto/common/refcnt.hpp | 2 + crypto/fift/Continuation.cpp | 398 ++++++ crypto/fift/Continuation.h | 272 +++++ crypto/fift/Dictionary.cpp | 145 ++- crypto/fift/Dictionary.h | 114 +- crypto/fift/Fift.cpp | 46 +- crypto/fift/Fift.h | 4 +- crypto/fift/IntCtx.cpp | 139 ++- crypto/fift/IntCtx.h | 76 +- crypto/fift/utils.cpp | 24 +- crypto/fift/utils.h | 3 +- crypto/fift/words.cpp | 687 ++++++----- crypto/fift/words.h | 13 - crypto/func/test/a12.fc | 4 + crypto/func/test/a12_1.fc | 3 + crypto/func/test/a12_2.fc | 3 + crypto/func/test/a12_3.fc | 3 + crypto/func/test/a12_4.fc | 3 + crypto/func/test/a12_5.fc | 3 + crypto/func/test/a12_6.fc | 3 + crypto/func/test/a12_7.fc | 4 + crypto/smc-envelope/GenericAccount.cpp | 22 +- crypto/smc-envelope/GenericAccount.h | 7 +- crypto/smc-envelope/HighloadWallet.cpp | 117 +- crypto/smc-envelope/HighloadWallet.h | 45 +- crypto/smc-envelope/HighloadWalletV2.cpp | 91 +- crypto/smc-envelope/HighloadWalletV2.h | 49 +- crypto/smc-envelope/SmartContract.cpp | 19 +- crypto/smc-envelope/SmartContract.h | 1 + crypto/smc-envelope/SmartContractCode.cpp | 32 - crypto/smc-envelope/SmartContractCode.h | 13 +- crypto/smc-envelope/WalletInterface.cpp | 77 ++ crypto/smc-envelope/WalletInterface.h | 108 +- crypto/smc-envelope/WalletV3.cpp | 106 +- crypto/smc-envelope/WalletV3.h | 135 +-- crypto/test/test-db.cpp | 242 +--- crypto/test/test-smartcont.cpp | 451 +++---- crypto/vm/cells/CellSlice.cpp | 4 +- crypto/vm/cells/CellString.cpp | 26 + crypto/vm/cells/CellString.h | 14 +- crypto/vm/db/StaticBagOfCellsDb.cpp | 39 +- crypto/vm/db/StaticBagOfCellsDb.h | 6 +- lite-client/lite-client.cpp | 2 +- rldp2/Ack.cpp | 29 + rldp2/Ack.h | 18 + rldp2/Bbr.cpp | 62 + rldp2/Bbr.h | 29 + rldp2/BdwStats.cpp | 50 + rldp2/BdwStats.h | 36 + rldp2/CMakeLists.txt | 58 + rldp2/FecHelper.cpp | 24 + rldp2/FecHelper.h | 17 + rldp2/InboundTransfer.cpp | 58 + rldp2/InboundTransfer.h | 37 + rldp2/LossSender.cpp | 135 +++ rldp2/LossSender.h | 31 + rldp2/LossStats.cpp | 23 + rldp2/LossStats.h | 19 + rldp2/OutboundTransfer.cpp | 40 + rldp2/OutboundTransfer.h | 40 + rldp2/Pacer.cpp | 45 + rldp2/Pacer.h | 39 + rldp2/RldpConnection.cpp | 369 ++++++ rldp2/RldpConnection.h | 119 ++ rldp2/RldpReceiver.cpp | 34 + rldp2/RldpReceiver.h | 36 + rldp2/RldpSender.cpp | 95 ++ rldp2/RldpSender.h | 81 ++ rldp2/RttStats.cpp | 51 + rldp2/RttStats.h | 23 + rldp2/SenderPackets.cpp | 125 ++ rldp2/SenderPackets.h | 89 ++ rldp2/rldp-in.hpp | 113 ++ rldp2/rldp.cpp | 246 ++++ rldp2/rldp.h | 45 + rldp2/rldp.hpp | 48 + storage/Bitset.h | 69 ++ storage/CMakeLists.txt | 55 + storage/LoadSpeed.cpp | 33 + storage/LoadSpeed.h | 25 + storage/MerkleTree.cpp | 342 ++++++ storage/MerkleTree.h | 77 ++ storage/NodeActor.cpp | 379 ++++++ storage/NodeActor.h | 127 ++ storage/PartsHelper.h | 259 ++++ storage/PeerActor.cpp | 400 ++++++ storage/PeerActor.h | 112 ++ storage/PeerState.cpp | 16 + storage/PeerState.h | 55 + storage/SharedState.h | 48 + storage/Torrent.cpp | 425 +++++++ storage/Torrent.h | 150 +++ storage/TorrentCreator.cpp | 173 +++ storage/TorrentCreator.h | 48 + storage/TorrentHeader.cpp | 50 + storage/TorrentHeader.h | 49 + storage/TorrentHeader.hpp | 62 + storage/TorrentInfo.cpp | 47 + storage/TorrentInfo.h | 38 + storage/TorrentMeta.cpp | 123 ++ storage/TorrentMeta.h | 37 + storage/storage-cli.cpp | 852 +++++++++++++ storage/test/storage.cpp | 1341 +++++++++++++++++++++ tdactor/td/actor/common.h | 32 +- tdactor/td/actor/core/ActorExecutor.cpp | 3 +- tdactor/td/actor/core/ActorExecutor.h | 6 + tdactor/td/actor/core/IoWorker.cpp | 10 +- tdactor/td/actor/core/IoWorker.h | 2 +- tdactor/td/actor/core/Scheduler.cpp | 11 +- tdactor/td/actor/core/Scheduler.h | 4 +- tddb/CMakeLists.txt | 25 +- tddb/td/db/utils/BlobView.cpp | 339 ++++++ tddb/td/db/utils/BlobView.h | 67 + tddb/td/db/utils/CyclicBuffer.cpp | 2 +- tdnet/td/net/UdpServer.cpp | 2 + tdutils/td/utils/Heap.h | 4 + tdutils/td/utils/PathView.h | 3 + tdutils/td/utils/Random.cpp | 3 + tdutils/td/utils/Random.h | 16 + tdutils/td/utils/Span.h | 61 +- tdutils/td/utils/Status.h | 16 + tdutils/td/utils/ThreadSafeCounter.h | 12 + tdutils/td/utils/Time.cpp | 22 + tdutils/td/utils/Time.h | 17 +- tdutils/td/utils/TimedStat.h | 24 + tdutils/td/utils/VectorQueue.h | 6 + tdutils/td/utils/bits.h | 36 + tdutils/td/utils/format.h | 5 + tdutils/td/utils/optional.h | 2 + tdutils/td/utils/tests.cpp | 9 +- tdutils/td/utils/tests.h | 2 + tdutils/test/heap.cpp | 4 +- tdutils/test/misc.cpp | 33 + test/test-rldp2.cpp | 203 ++++ tl-utils/common-utils.hpp | 9 +- tl/generate/scheme/ton_api.tl | 24 + tl/generate/scheme/ton_api.tlo | Bin 61672 -> 63696 bytes tl/generate/scheme/tonlib_api.tl | 6 - tl/generate/scheme/tonlib_api.tlo | Bin 25984 -> 25416 bytes tonlib/CMakeLists.txt | 2 +- tonlib/test/offline.cpp | 4 +- tonlib/test/online.cpp | 3 - tonlib/tonlib/TonlibClient.cpp | 132 +- tonlib/tonlib/tonlib-cli.cpp | 21 +- validator/impl/block.cpp | 2 +- validator/impl/collator.cpp | 4 +- validator/impl/liteserver.cpp | 2 +- validator/impl/proof.cpp | 2 +- validator/impl/shard.cpp | 4 +- 153 files changed, 10753 insertions(+), 1688 deletions(-) create mode 100644 crypto/fift/Continuation.cpp create mode 100644 crypto/fift/Continuation.h create mode 100644 crypto/func/test/a12.fc create mode 100644 crypto/func/test/a12_1.fc create mode 100644 crypto/func/test/a12_2.fc create mode 100644 crypto/func/test/a12_3.fc create mode 100644 crypto/func/test/a12_4.fc create mode 100644 crypto/func/test/a12_5.fc create mode 100644 crypto/func/test/a12_6.fc create mode 100644 crypto/func/test/a12_7.fc create mode 100644 crypto/smc-envelope/WalletInterface.cpp create mode 100644 rldp2/Ack.cpp create mode 100644 rldp2/Ack.h create mode 100644 rldp2/Bbr.cpp create mode 100644 rldp2/Bbr.h create mode 100644 rldp2/BdwStats.cpp create mode 100644 rldp2/BdwStats.h create mode 100644 rldp2/CMakeLists.txt create mode 100644 rldp2/FecHelper.cpp create mode 100644 rldp2/FecHelper.h create mode 100644 rldp2/InboundTransfer.cpp create mode 100644 rldp2/InboundTransfer.h create mode 100644 rldp2/LossSender.cpp create mode 100644 rldp2/LossSender.h create mode 100644 rldp2/LossStats.cpp create mode 100644 rldp2/LossStats.h create mode 100644 rldp2/OutboundTransfer.cpp create mode 100644 rldp2/OutboundTransfer.h create mode 100644 rldp2/Pacer.cpp create mode 100644 rldp2/Pacer.h create mode 100644 rldp2/RldpConnection.cpp create mode 100644 rldp2/RldpConnection.h create mode 100644 rldp2/RldpReceiver.cpp create mode 100644 rldp2/RldpReceiver.h create mode 100644 rldp2/RldpSender.cpp create mode 100644 rldp2/RldpSender.h create mode 100644 rldp2/RttStats.cpp create mode 100644 rldp2/RttStats.h create mode 100644 rldp2/SenderPackets.cpp create mode 100644 rldp2/SenderPackets.h create mode 100644 rldp2/rldp-in.hpp create mode 100644 rldp2/rldp.cpp create mode 100644 rldp2/rldp.h create mode 100644 rldp2/rldp.hpp create mode 100644 storage/Bitset.h create mode 100644 storage/CMakeLists.txt create mode 100644 storage/LoadSpeed.cpp create mode 100644 storage/LoadSpeed.h create mode 100644 storage/MerkleTree.cpp create mode 100644 storage/MerkleTree.h create mode 100644 storage/NodeActor.cpp create mode 100644 storage/NodeActor.h create mode 100644 storage/PartsHelper.h create mode 100644 storage/PeerActor.cpp create mode 100644 storage/PeerActor.h create mode 100644 storage/PeerState.cpp create mode 100644 storage/PeerState.h create mode 100644 storage/SharedState.h create mode 100644 storage/Torrent.cpp create mode 100644 storage/Torrent.h create mode 100644 storage/TorrentCreator.cpp create mode 100644 storage/TorrentCreator.h create mode 100644 storage/TorrentHeader.cpp create mode 100644 storage/TorrentHeader.h create mode 100644 storage/TorrentHeader.hpp create mode 100644 storage/TorrentInfo.cpp create mode 100644 storage/TorrentInfo.h create mode 100644 storage/TorrentMeta.cpp create mode 100644 storage/TorrentMeta.h create mode 100644 storage/storage-cli.cpp create mode 100644 storage/test/storage.cpp create mode 100644 tddb/td/db/utils/BlobView.cpp create mode 100644 tddb/td/db/utils/BlobView.h create mode 100644 test/test-rldp2.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 941865f00..cc833562a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -232,7 +232,7 @@ elseif (CLANG OR GCC) if (APPLE) #use "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/export_list" for exported symbols set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S") - #set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} -fvisibility=hidden -Wl,-dead_strip,-x,-S") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL") @@ -378,12 +378,14 @@ add_subdirectory(tdfec) add_subdirectory(keyring) add_subdirectory(fec) add_subdirectory(rldp) +add_subdirectory(rldp2) add_subdirectory(dht) add_subdirectory(overlay) add_subdirectory(catchain) add_subdirectory(validator-session) add_subdirectory(validator) add_subdirectory(blockchain-explorer) +add_subdirectory(storage) add_subdirectory(validator-engine) add_subdirectory(validator-engine-console) add_subdirectory(create-hardfork) @@ -445,7 +447,10 @@ endif() #BEGIN internal if (NOT TON_ONLY_TONLIB) add_executable(test-db test/test-td-main.cpp ${TONDB_TEST_SOURCE}) -target_link_libraries(test-db PRIVATE ton_db memprof) +target_link_libraries(test-db PRIVATE ton_db memprof tdfec) + +add_executable(test-storage test/test-td-main.cpp ${STORAGE_TEST_SOURCE}) +target_link_libraries(test-storage PRIVATE storage ton_db memprof tl_api tl-utils fec rldp2) add_executable(test-rocksdb test/test-rocksdb.cpp) target_link_libraries(test-rocksdb PRIVATE memprof tddb tdutils) @@ -469,6 +474,8 @@ add_executable(test-dht test/test-dht.cpp) target_link_libraries(test-dht adnl adnltest dht tl_api) add_executable(test-rldp test/test-rldp.cpp) target_link_libraries(test-rldp adnl adnltest dht rldp tl_api) +add_executable(test-rldp2 test/test-rldp2.cpp) +target_link_libraries(test-rldp2 adnl adnltest dht rldp2 tl_api) add_executable(test-validator-session-state test/test-validator-session-state.cpp) target_link_libraries(test-validator-session-state adnl dht rldp validatorsession tl_api) @@ -536,6 +543,7 @@ if (NOT TON_ONLY_TONLIB) add_test(test-adnl test-adnl) add_test(test-dht test-dht) add_test(test-rldp test-rldp) +add_test(test-rldp2 test-rldp2) #add_test(test-validator-session-state test-validator-session-state) add_test(test-catchain test-catchain) diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 246822eea..132f795a5 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -141,7 +141,8 @@ void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) { if (!packet.priority_addr_list().empty()) { update_addr_list(packet.priority_addr_list()); } - VLOG(ADNL_NOTICE) << this << ": dropping IN message old our reinit date " << packet.reinit_date() << " date=" << d; + VLOG(ADNL_NOTICE) << this << ": dropping IN message old our reinit date " << packet.dst_reinit_date() + << " date=" << d; auto M = OutboundAdnlMessage{adnlmessage::AdnlMessageNop{}, 0}; send_message(std::move(M)); return; diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index d8efc5862..a9a30cbda 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -124,9 +124,6 @@ set(TON_CRYPTO_SOURCE vm/db/StaticBagOfCellsDb.h vm/db/StaticBagOfCellsDb.cpp - - vm/db/BlobView.h - vm/db/BlobView.cpp ) set(TON_DB_SOURCE @@ -144,6 +141,7 @@ set(FIFT_SOURCE fift/Dictionary.cpp fift/Fift.cpp fift/IntCtx.cpp + fift/Continuation.cpp fift/SourceLookup.cpp fift/utils.cpp fift/words.cpp @@ -151,6 +149,7 @@ set(FIFT_SOURCE fift/Dictionary.h fift/Fift.h fift/IntCtx.h + fift/Continuation.h fift/SourceLookup.h fift/utils.h fift/words.h @@ -216,9 +215,7 @@ set(SMC_ENVELOPE_SOURCE smc-envelope/PaymentChannel.cpp smc-envelope/SmartContract.cpp smc-envelope/SmartContractCode.cpp - smc-envelope/TestGiver.cpp - smc-envelope/TestWallet.cpp - smc-envelope/Wallet.cpp + smc-envelope/WalletInterface.cpp smc-envelope/WalletV3.cpp smc-envelope/GenericAccount.h @@ -228,9 +225,6 @@ set(SMC_ENVELOPE_SOURCE smc-envelope/MultisigWallet.h smc-envelope/SmartContract.h smc-envelope/SmartContractCode.h - smc-envelope/TestGiver.h - smc-envelope/TestWallet.h - smc-envelope/Wallet.h smc-envelope/WalletInterface.h smc-envelope/WalletV3.h ) @@ -268,7 +262,7 @@ set(FIFT_TEST_SOURCE add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE}) target_include_directories(ton_crypto PUBLIC $ $) -target_link_libraries(ton_crypto PUBLIC ${OPENSSL_CRYPTO_LIBRARY} tdutils) +target_link_libraries(ton_crypto PUBLIC ${OPENSSL_CRYPTO_LIBRARY} tdutils tddb_utils) if (NOT WIN32) target_link_libraries(ton_crypto PUBLIC dl z) endif() diff --git a/crypto/common/refcnt.cpp b/crypto/common/refcnt.cpp index a90414f87..7ef0857cc 100644 --- a/crypto/common/refcnt.cpp +++ b/crypto/common/refcnt.cpp @@ -21,6 +21,11 @@ #include "td/utils/ScopeGuard.h" namespace td { + +Ref CntObject::clone() const { + return Ref{make_copy(), Ref::acquire_t()}; +} + namespace detail { struct SafeDeleter { public: diff --git a/crypto/common/refcnt.hpp b/crypto/common/refcnt.hpp index 7625ff7a9..ef50c3b9a 100644 --- a/crypto/common/refcnt.hpp +++ b/crypto/common/refcnt.hpp @@ -83,6 +83,7 @@ class CntObject { void assert_unique() const { assert(is_unique()); } + Ref clone() const; }; typedef Ref RefAny; @@ -159,6 +160,7 @@ struct static_cast_ref {}; namespace detail { void safe_delete(const CntObject* ptr); } + template class Ref { T* ptr; diff --git a/crypto/fift/Continuation.cpp b/crypto/fift/Continuation.cpp new file mode 100644 index 000000000..7e3b5ea2d --- /dev/null +++ b/crypto/fift/Continuation.cpp @@ -0,0 +1,398 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2020 Telegram Systems LLP +*/ +#include "Continuation.h" +#include "IntCtx.h" +#include "Dictionary.h" + +namespace fift { + +// +// FiftCont +// +bool FiftCont::print_dict_name(std::ostream& os, const IntCtx& ctx) const { + std::string word_name; + if (ctx.dictionary && ctx.dictionary->lookup_def(this, &word_name)) { + if (word_name.size() && word_name.back() == ' ') { + word_name.pop_back(); + } + os << word_name; + return true; + } + return false; +} + +std::string FiftCont::get_dict_name(const IntCtx& ctx) const { + std::string word_name; + if (ctx.dictionary && ctx.dictionary->lookup_def(this, &word_name)) { + if (word_name.size() && word_name.back() == ' ') { + word_name.pop_back(); + } + return word_name; + } + return {}; +} + +bool FiftCont::print_dummy_name(std::ostream& os, const IntCtx& ctx) const { + os << ""; + return false; +} + +bool FiftCont::print_name(std::ostream& os, const IntCtx& ctx) const { + return print_dict_name(os, ctx) || print_dummy_name(os, ctx); +} + +bool FiftCont::dump(std::ostream& os, const IntCtx& ctx) const { + bool ok = print_name(os, ctx); + os << std::endl; + return ok; +} + +// +// QuitCont +// +Ref QuitCont::run_tail(IntCtx& ctx) const { + ctx.set_exit_code(exit_code); + ctx.next.clear(); + return {}; +} + +bool QuitCont::print_name(std::ostream& os, const IntCtx& ctx) const { + os << ""; + return true; +} + +// +// SeqCont +// +Ref SeqCont::run_tail(IntCtx& ctx) const { + ctx.next = seq(second, std::move(ctx.next)); + return first; +} + +Ref SeqCont::run_modify(IntCtx& ctx) { + if (ctx.next.is_null()) { + ctx.next = std::move(second); + return std::move(first); + } else { + auto res = std::move(first); + first = std::move(second); + second = std::move(ctx.next); + ctx.next = self(); + return res; + } +} + +bool SeqCont::print_name(std::ostream& os, const IntCtx& ctx) const { + if (first.not_null()) { + return first->print_name(os, ctx); + } else { + return true; + } +} + +bool SeqCont::dump(std::ostream& os, const IntCtx& ctx) const { + if (first.not_null()) { + return first->dump(os, ctx); + } else { + return true; + } +} + +// +// TimesCont +// +Ref TimesCont::run_tail(IntCtx& ctx) const { + if (count > 1) { + ctx.next = td::make_ref(body, SeqCont::seq(after, std::move(ctx.next)), count - 1); + } else { + ctx.next = SeqCont::seq(after, std::move(ctx.next)); + } + return body; +} + +Ref TimesCont::run_modify(IntCtx& ctx) { + if (ctx.next.not_null()) { + after = SeqCont::seq(std::move(after), std::move(ctx.next)); + } + if (count > 1) { + --count; + ctx.next = self(); + return body; + } else { + ctx.next = std::move(after); + return std::move(body); + } +} + +bool TimesCont::print_name(std::ostream& os, const IntCtx& ctx) const { + os << ""; + return true; +} + +bool TimesCont::dump(std::ostream& os, const IntCtx& ctx) const { + os << " "; + return body->dump(os, ctx); +} + +// +// UntilCont +// +Ref UntilCont::run_tail(IntCtx& ctx) const { + if (ctx.stack.pop_bool()) { + return after; + } else if (ctx.next.not_null()) { + ctx.next = td::make_ref(body, SeqCont::seq(after, std::move(ctx.next))); + return body; + } else { + ctx.next = self(); + return body; + } +} + +Ref UntilCont::run_modify(IntCtx& ctx) { + if (ctx.stack.pop_bool()) { + return std::move(after); + } else { + if (ctx.next.not_null()) { + after = SeqCont::seq(after, std::move(ctx.next)); + } + ctx.next = self(); + return body; + } +} + +bool UntilCont::print_name(std::ostream& os, const IntCtx& ctx) const { + os << ""; + return true; +} + +bool UntilCont::dump(std::ostream& os, const IntCtx& ctx) const { + os << " "; + return body->dump(os, ctx); +} + +// +// WhileCont +// +Ref WhileCont::run_tail(IntCtx& ctx) const { + if (!stage) { + ctx.next = td::make_ref(cond, body, SeqCont::seq(after, std::move(ctx.next)), true); + return cond; + } + if (!ctx.stack.pop_bool()) { + return after; + } else { + ctx.next = td::make_ref(cond, body, SeqCont::seq(after, std::move(ctx.next))); + return body; + } +} + +Ref WhileCont::run_modify(IntCtx& ctx) { + if (!stage) { + if (ctx.next.not_null()) { + after = SeqCont::seq(std::move(after), std::move(ctx.next)); + } + stage = true; + ctx.next = self(); + return cond; + } + if (!ctx.stack.pop_bool()) { + return std::move(after); + } else { + if (ctx.next.not_null()) { + after = SeqCont::seq(std::move(after), std::move(ctx.next)); + } + stage = false; + ctx.next = self(); + return body; + } +} + +bool WhileCont::print_name(std::ostream& os, const IntCtx& ctx) const { + os << ""; + return true; +} + +bool WhileCont::dump(std::ostream& os, const IntCtx& ctx) const { + os << " "; + return (stage ? body : cond)->dump(os, ctx); +} + +// +// LoopCont +// +Ref LoopCont::run_tail(IntCtx& ctx) const { + return Ref(clone()); +} + +Ref LoopCont::run_modify(IntCtx& ctx) { + if (ctx.next.not_null()) { + after = SeqCont::seq(std::move(after), std::move(ctx.next)); + } + switch (state) { + case 0: + if (!init(ctx)) { + return std::move(after); + } + state = 1; + // fallthrough + case 1: + if (!pre_exec(ctx)) { + state = 3; + if (finalize(ctx)) { + return std::move(after); + } else { + return {}; + } + } + state = 2; + ctx.next = self(); + return func; + case 2: + if (post_exec(ctx)) { + state = 1; + return self(); + } + state = 3; + // fallthrough + case 3: + if (finalize(ctx)) { + return std::move(after); + } else { + return {}; + } + default: + throw IntError{"invalid LoopCont state"}; + } +} + +bool LoopCont::print_name(std::ostream& os, const IntCtx& ctx) const { + os << ""; + return true; +} + +// +// GenericLitCont +// +bool GenericLitCont::print_name(std::ostream& os, const IntCtx& ctx) const { + auto list = get_literals(); + bool sp = false; + for (auto entry : list) { + if (sp) { + os << sp; + } + sp = true; + int tp = entry.type(); + if (entry.is_int() || entry.is(vm::StackEntry::t_string) || entry.is(vm::StackEntry::t_bytes)) { + entry.dump(os); + } else { + auto cont_lit = entry.as_object(); + if (cont_lit.not_null()) { + os << "{ "; + cont_lit->print_name(os, ctx); + os << " }"; + } else { + os << ""; + } + } + } + return true; +} + +// +// SmallIntLitCont +// +Ref SmallIntLitCont::run_tail(IntCtx& ctx) const { + ctx.stack.push_smallint(value_); + return {}; +} + +std::vector SmallIntLitCont::get_literals() const { + return {td::make_refint(value_)}; +} + +// +// IntLitCont +// +Ref IntLitCont::run_tail(IntCtx& ctx) const { + ctx.stack.push_int(value_); + return {}; +} + +Ref IntLitCont::run_modify(IntCtx& ctx) { + ctx.stack.push_int(std::move(value_)); + return {}; +} + +Ref IntLitCont::literal(td::RefInt256 int_value) { + if (int_value->signed_fits_bits(64)) { + return literal(int_value->to_long()); + } else { + return td::make_ref(std::move(int_value)); + } +} + +// +// LitCont +// +Ref LitCont::run_tail(IntCtx& ctx) const { + ctx.stack.push(value_); + return {}; +} + +Ref LitCont::run_modify(IntCtx& ctx) { + ctx.stack.push(std::move(value_)); + return {}; +} + +Ref LitCont::literal(vm::StackEntry value) { + if (value.is_int()) { + return literal(std::move(value).as_int()); + } else { + return td::make_ref(std::move(value)); + } +} + +// +// MultiLitCont +// +Ref MultiLitCont::run_tail(IntCtx& ctx) const { + for (auto& value : values_) { + ctx.stack.push(value); + } + return {}; +} + +Ref MultiLitCont::run_modify(IntCtx& ctx) { + for (auto& value : values_) { + ctx.stack.push(std::move(value)); + } + return {}; +} + +MultiLitCont& MultiLitCont::push_back(vm::StackEntry new_literal) { + values_.push_back(std::move(new_literal)); + return *this; +} + +vm::StackEntry MultiLitCont::at(int idx) const { + return values_.at(idx); +} + +} // namespace fift diff --git a/crypto/fift/Continuation.h b/crypto/fift/Continuation.h new file mode 100644 index 000000000..f2c44e7b0 --- /dev/null +++ b/crypto/fift/Continuation.h @@ -0,0 +1,272 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2020 Telegram Systems LLP +*/ +#pragma once +#include "common/refcnt.hpp" +#include "common/refint.h" +#include "vm/stack.hpp" + +namespace fift { +using td::Ref; +struct IntCtx; + +/* + * + * FIFT CONTINUATIONS + * + */ + +class FiftCont : public td::CntObject { + public: + FiftCont() = default; + virtual ~FiftCont() override = default; + virtual Ref run_tail(IntCtx& ctx) const = 0; + virtual Ref run_modify(IntCtx& ctx) { + return run_tail(ctx); + } + virtual Ref handle_tail(IntCtx& ctx) const { + return {}; + } + virtual Ref handle_modify(IntCtx& ctx) { + return handle_tail(ctx); + } + virtual Ref up() const { + return {}; + } + virtual bool is_list() const { + return false; + } + virtual long long list_size() const { + return -1; + } + virtual const Ref* get_list() const { + return nullptr; + } + virtual bool is_literal() const { + return false; + } + virtual int literal_count() const { + return -1; + } + virtual std::vector get_literals() const { + return {}; + } + std::string get_dict_name(const IntCtx& ctx) const; + bool print_dict_name(std::ostream& os, const IntCtx& ctx) const; + bool print_dummy_name(std::ostream& os, const IntCtx& ctx) const; + virtual bool print_name(std::ostream& os, const IntCtx& ctx) const; + virtual bool dump(std::ostream& os, const IntCtx& ctx) const; + Ref self() const { + return Ref{this}; + } +}; + +class QuitCont : public FiftCont { + int exit_code; + + public: + QuitCont(int _exit_code = -1) : exit_code(_exit_code) { + } + Ref run_tail(IntCtx& ctx) const override; + bool print_name(std::ostream& os, const IntCtx& ctx) const override; +}; + +class SeqCont : public FiftCont { + Ref first, second; + + public: + SeqCont(Ref _first, Ref _second) : first(std::move(_first)), second(std::move(_second)) { + } + static Ref seq(Ref _first, Ref _second) { + return _second.is_null() ? std::move(_first) : td::make_ref(std::move(_first), std::move(_second)); + } + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + Ref up() const override { + return second; + } + bool print_name(std::ostream& os, const IntCtx& ctx) const override; + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + +class TimesCont : public FiftCont { + Ref body, after; + int count; + + public: + TimesCont(Ref _body, Ref _after, int _count) + : body(std::move(_body)), after(std::move(_after)), count(_count) { + } + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + Ref up() const override { + return after; + } + bool print_name(std::ostream& os, const IntCtx& ctx) const override; + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + +class UntilCont : public FiftCont { + Ref body, after; + + public: + UntilCont(Ref _body, Ref _after) : body(std::move(_body)), after(std::move(_after)) { + } + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + bool print_name(std::ostream& os, const IntCtx& ctx) const override; + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + +class WhileCont : public FiftCont { + Ref cond, body, after; + bool stage; + + public: + WhileCont(Ref _cond, Ref _body, Ref _after, bool _stage = false) + : cond(std::move(_cond)), body(std::move(_body)), after(std::move(_after)), stage(_stage) { + } + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + Ref up() const override { + return after; + } + bool print_name(std::ostream& os, const IntCtx& ctx) const override; + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + +class LoopCont : public FiftCont { + Ref func, after; + int state; + + public: + LoopCont(Ref _func, Ref _after, int _state = 0) + : func(std::move(_func)), after(std::move(_after)), state(_state) { + } + LoopCont(const LoopCont&) = default; + virtual bool init(IntCtx& ctx) { + return true; + } + virtual bool pre_exec(IntCtx& ctx) { + return true; + } + virtual bool post_exec(IntCtx& ctx) { + return true; + } + virtual bool finalize(IntCtx& ctx) { + return true; + } + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + Ref up() const override { + return after; + } + bool print_name(std::ostream& os, const IntCtx& ctx) const override; +}; + +class GenericLitCont : public FiftCont { + public: + bool is_literal() const override { + return true; + } + std::vector get_literals() const override = 0; + bool print_name(std::ostream& os, const IntCtx& ctx) const override; +}; + +class SmallIntLitCont : public GenericLitCont { + long long value_; + + public: + SmallIntLitCont(long long value) : value_(value) { + } + Ref run_tail(IntCtx& ctx) const override; + std::vector get_literals() const override; + static Ref literal(long long int_value) { + return td::make_ref(int_value); + } +}; + +class IntLitCont : public GenericLitCont { + td::RefInt256 value_; + + public: + IntLitCont(td::RefInt256 value) : value_(std::move(value)) { + } + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + std::vector get_literals() const override { + return {vm::StackEntry(value_)}; + } + static Ref literal(td::RefInt256 int_value); + static Ref literal(long long int_value) { + return SmallIntLitCont::literal(int_value); + } +}; + +class LitCont : public GenericLitCont { + vm::StackEntry value_; + + public: + LitCont(const vm::StackEntry& value) : value_(value) { + } + LitCont(vm::StackEntry&& value) : value_(std::move(value)) { + } + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + std::vector get_literals() const override { + return {value_}; + } + static Ref literal(vm::StackEntry value); + static Ref literal(td::RefInt256 int_value) { + return IntLitCont::literal(std::move(int_value)); + } + static Ref literal(long long int_value) { + return SmallIntLitCont::literal(int_value); + } +}; + +class MultiLitCont : public GenericLitCont { + std::vector values_; + + public: + MultiLitCont(const std::vector& values) : values_(values) { + } + MultiLitCont(std::vector&& values) : values_(std::move(values)) { + } + MultiLitCont(std::initializer_list value_list) : values_(value_list) { + } + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + std::vector get_literals() const override { + return values_; + } + MultiLitCont& push_back(vm::StackEntry new_literal); + vm::StackEntry at(int idx) const; +}; + +class InterpretCont : public FiftCont { + public: + InterpretCont() = default; + Ref run_tail(IntCtx&) const override; // text interpreter, defined in words.cpp + bool print_name(std::ostream& os, const IntCtx& ctx) const override { + os << ""; + return true; + } +}; + +} // namespace fift diff --git a/crypto/fift/Dictionary.cpp b/crypto/fift/Dictionary.cpp index 133dfacd9..59da278f2 100644 --- a/crypto/fift/Dictionary.cpp +++ b/crypto/fift/Dictionary.cpp @@ -20,20 +20,10 @@ namespace fift { -// -// WordDef -// -void WordDef::run(IntCtx& ctx) const { - auto next = run_tail(ctx); - while (next.not_null()) { - next = next->run_tail(ctx); - } -} - // // StackWord // -Ref StackWord::run_tail(IntCtx& ctx) const { +Ref StackWord::run_tail(IntCtx& ctx) const { f(ctx.stack); return {}; } @@ -41,7 +31,7 @@ Ref StackWord::run_tail(IntCtx& ctx) const { // // CtxWord // -Ref CtxWord::run_tail(IntCtx& ctx) const { +Ref CtxWord::run_tail(IntCtx& ctx) const { f(ctx); return {}; } @@ -49,80 +39,132 @@ Ref CtxWord::run_tail(IntCtx& ctx) const { // // CtxTailWord // -Ref CtxTailWord::run_tail(IntCtx& ctx) const { +Ref CtxTailWord::run_tail(IntCtx& ctx) const { return f(ctx); } // // WordList // -WordList::WordList(std::vector>&& _list) : list(std::move(_list)) { +WordList::WordList(std::vector>&& _list) : list(std::move(_list)) { } -WordList::WordList(const std::vector>& _list) : list(_list) { +WordList::WordList(const std::vector>& _list) : list(_list) { } -WordList& WordList::push_back(Ref word_def) { +WordList& WordList::push_back(Ref word_def) { list.push_back(std::move(word_def)); return *this; } -WordList& WordList::push_back(WordDef& wd) { +WordList& WordList::push_back(FiftCont& wd) { list.emplace_back(&wd); return *this; } -Ref WordList::run_tail(IntCtx& ctx) const { +Ref WordList::run_tail(IntCtx& ctx) const { if (list.empty()) { return {}; } - auto it = list.cbegin(), it2 = list.cend() - 1; - while (it < it2) { - (*it)->run(ctx); - ++it; + if (list.size() > 1) { + ctx.next = td::make_ref(std::move(ctx.next), Ref(this), 1); } - return *it; + return list[0]; } void WordList::close() { list.shrink_to_fit(); } -WordList& WordList::append(const std::vector>& other) { +WordList& WordList::append(const std::vector>& other) { list.insert(list.end(), other.begin(), other.end()); return *this; } -// -// DictEntry -// - -DictEntry::DictEntry(Ref _def, bool _act) : def(std::move(_def)), active(_act) { +WordList& WordList::append(const Ref* begin, const Ref* end) { + list.insert(list.end(), begin, end); + return *this; } -DictEntry::DictEntry(StackWordFunc func) : def(Ref{true, std::move(func)}), active(false) { +bool WordList::dump(std::ostream& os, const IntCtx& ctx) const { + os << "{"; + for (auto entry : list) { + os << ' '; + entry->print_name(os, ctx); + } + os << " }" << std::endl; + return true; } -DictEntry::DictEntry(CtxWordFunc func, bool _act) : def(Ref{true, std::move(func)}), active(_act) { +// +// ListCont +// + +Ref ListCont::run_tail(IntCtx& ctx) const { + auto sz = list->size(); + if (pos >= sz) { + return std::move(ctx.next); + } else if (ctx.next.not_null()) { + ctx.next = td::make_ref(SeqCont::seq(next, std::move(ctx.next)), list, pos + 1); + } else if (pos + 1 == sz) { + ctx.next = next; + } else { + ctx.next = td::make_ref(next, list, pos + 1); + } + return list->at(pos); } -DictEntry::DictEntry(CtxTailWordFunc func, bool _act) : def(Ref{true, std::move(func)}), active(_act) { +Ref ListCont::run_modify(IntCtx& ctx) { + auto sz = list->size(); + if (pos >= sz) { + return std::move(ctx.next); + } + auto cur = list->at(pos++); + if (ctx.next.not_null()) { + next = SeqCont::seq(next, std::move(ctx.next)); + } + if (pos == sz) { + ctx.next = std::move(next); + } else { + ctx.next = self(); + } + return cur; } -Ref DictEntry::get_def() const& { - return def; +bool ListCont::dump(std::ostream& os, const IntCtx& ctx) const { + std::string dict_name = list->get_dict_name(ctx); + if (!dict_name.empty()) { + os << "[in " << dict_name << ":] "; + } + std::size_t sz = list->size(), i, a = (pos >= 16 ? pos - 16 : 0), b = std::min(pos + 16, sz); + if (a > 0) { + os << "... "; + } + for (i = a; i < b; i++) { + if (i == pos) { + os << "**HERE** "; + } + list->at(i)->print_name(os, ctx); + os << ' '; + } + if (b < sz) { + os << "..."; + } + os << std::endl; + return true; } -Ref DictEntry::get_def() && { - return std::move(def); +// +// DictEntry +// + +DictEntry::DictEntry(StackWordFunc func) : def(Ref{true, std::move(func)}), active(false) { } -void DictEntry::operator()(IntCtx& ctx) const { - def->run(ctx); +DictEntry::DictEntry(CtxWordFunc func, bool _act) : def(Ref{true, std::move(func)}), active(_act) { } -bool DictEntry::is_active() const { - return active; +DictEntry::DictEntry(CtxTailWordFunc func, bool _act) : def(Ref{true, std::move(func)}), active(_act) { } // @@ -141,7 +183,7 @@ void Dictionary::def_ctx_word(std::string name, CtxWordFunc func) { } void Dictionary::def_active_word(std::string name, CtxWordFunc func) { - Ref wdef = Ref{true, std::move(func)}; + Ref wdef = Ref{true, std::move(func)}; def_word(std::move(name), {std::move(wdef), true}); } @@ -166,17 +208,32 @@ void Dictionary::undef_word(td::Slice name) { words_.erase(it); } +bool Dictionary::lookup_def(const FiftCont* cont, std::string* word_ptr) const { + if (!cont) { + return false; + } + for (const auto& entry : words_) { + if (entry.second.get_def().get() == cont) { + if (word_ptr) { + *word_ptr = entry.first; + } + return true; + } + } + return false; +} + void interpret_nop(vm::Stack& stack) { } -Ref Dictionary::nop_word_def = Ref{true, interpret_nop}; +Ref Dictionary::nop_word_def = Ref{true, interpret_nop}; // // functions for wordef // -Ref pop_exec_token(vm::Stack& stack) { +Ref pop_exec_token(vm::Stack& stack) { stack.check_underflow(1); - auto wd_ref = stack.pop().as_object(); + auto wd_ref = stack.pop().as_object(); if (wd_ref.is_null()) { throw IntError{"execution token expected"}; } diff --git a/crypto/fift/Dictionary.h b/crypto/fift/Dictionary.h index 3bd79e942..7307cdbeb 100644 --- a/crypto/fift/Dictionary.h +++ b/crypto/fift/Dictionary.h @@ -22,9 +22,11 @@ #include #include "IntCtx.h" +#include "Continuation.h" namespace fift { using td::Ref; + /* * * WORD CLASSES @@ -34,66 +36,49 @@ using td::Ref; typedef std::function StackWordFunc; typedef std::function CtxWordFunc; -class WordDef : public td::CntObject { - public: - WordDef() = default; - virtual ~WordDef() override = default; - virtual Ref run_tail(IntCtx& ctx) const = 0; - void run(IntCtx& ctx) const; - virtual bool is_list() const { - return false; - } - virtual long long list_size() const { - return -1; - } - virtual const std::vector>* get_list() const { - return nullptr; - } -}; - -class StackWord : public WordDef { +class StackWord : public FiftCont { StackWordFunc f; public: StackWord(StackWordFunc _f) : f(std::move(_f)) { } ~StackWord() override = default; - Ref run_tail(IntCtx& ctx) const override; + Ref run_tail(IntCtx& ctx) const override; }; -class CtxWord : public WordDef { +class CtxWord : public FiftCont { CtxWordFunc f; public: CtxWord(CtxWordFunc _f) : f(std::move(_f)) { } ~CtxWord() override = default; - Ref run_tail(IntCtx& ctx) const override; + Ref run_tail(IntCtx& ctx) const override; }; -typedef std::function(IntCtx&)> CtxTailWordFunc; +typedef std::function(IntCtx&)> CtxTailWordFunc; -class CtxTailWord : public WordDef { +class CtxTailWord : public FiftCont { CtxTailWordFunc f; public: CtxTailWord(CtxTailWordFunc _f) : f(std::move(_f)) { } ~CtxTailWord() override = default; - Ref run_tail(IntCtx& ctx) const override; + Ref run_tail(IntCtx& ctx) const override; }; -class WordList : public WordDef { - std::vector> list; +class WordList : public FiftCont { + std::vector> list; public: ~WordList() override = default; WordList() = default; - WordList(std::vector>&& _list); - WordList(const std::vector>& _list); - WordList& push_back(Ref word_def); - WordList& push_back(WordDef& wd); - Ref run_tail(IntCtx& ctx) const override; + WordList(std::vector>&& _list); + WordList(const std::vector>& _list); + WordList& push_back(Ref word_def); + WordList& push_back(FiftCont& wd); + Ref run_tail(IntCtx& ctx) const override; void close(); bool is_list() const override { return true; @@ -101,43 +86,73 @@ class WordList : public WordDef { long long list_size() const override { return (long long)list.size(); } - const std::vector>* get_list() const override { - return &list; + std::size_t size() const { + return list.size(); + } + const Ref& at(std::size_t idx) const { + return list.at(idx); + } + const Ref* get_list() const override { + return list.data(); } - WordList& append(const std::vector>& other); + WordList& append(const std::vector>& other); + WordList& append(const Ref* begin, const Ref* end); WordList* make_copy() const override { return new WordList(list); } + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + +class ListCont : public FiftCont { + Ref next; + Ref list; + std::size_t pos; + + public: + ListCont(Ref nxt, Ref wl, std::size_t p = 0) : next(std::move(nxt)), list(std::move(wl)), pos(p) { + } + ~ListCont() override = default; + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + Ref up() const override { + return next; + } + bool dump(std::ostream& os, const IntCtx& ctx) const override; }; class DictEntry { - Ref def; + Ref def; bool active; public: DictEntry() = delete; DictEntry(const DictEntry& ref) = default; DictEntry(DictEntry&& ref) = default; - DictEntry(Ref _def, bool _act = false); + DictEntry(Ref _def, bool _act = false) : def(std::move(_def)), active(_act) { + } DictEntry(StackWordFunc func); DictEntry(CtxWordFunc func, bool _act = false); DictEntry(CtxTailWordFunc func, bool _act = false); - //DictEntry(const std::vector>& word_list); - //DictEntry(std::vector>&& word_list); + //DictEntry(const std::vector>& word_list); + //DictEntry(std::vector>&& word_list); DictEntry& operator=(const DictEntry&) = default; DictEntry& operator=(DictEntry&&) = default; - Ref get_def() const&; - Ref get_def() &&; - void operator()(IntCtx& ctx) const; - bool is_active() const; - ~DictEntry() = default; + Ref get_def() const& { + return def; + } + Ref get_def() && { + return std::move(def); + } + bool is_active() const { + return active; + } }; /* -DictEntry::DictEntry(const std::vector>& word_list) : def(Ref{true, word_list}) { +DictEntry::DictEntry(const std::vector>& word_list) : def(Ref{true, word_list}) { } -DictEntry::DictEntry(std::vector>&& word_list) : def(Ref{true, std::move(word_list)}) { +DictEntry::DictEntry(std::vector>&& word_list) : def(Ref{true, std::move(word_list)}) { } */ @@ -156,7 +171,10 @@ class Dictionary { void def_stack_word(std::string name, StackWordFunc func); void def_word(std::string name, DictEntry word); void undef_word(td::Slice name); - + bool lookup_def(const FiftCont* cont, std::string* word_ptr = nullptr) const; + bool lookup_def(Ref cont, std::string* word_ptr = nullptr) const { + return lookup_def(cont.get(), word_ptr); + } auto begin() const { return words_.begin(); } @@ -164,7 +182,7 @@ class Dictionary { return words_.end(); } - static Ref nop_word_def; + static Ref nop_word_def; private: std::map> words_; @@ -176,7 +194,7 @@ class Dictionary { * */ -Ref pop_exec_token(vm::Stack& stack); +Ref pop_exec_token(vm::Stack& stack); Ref pop_word_list(vm::Stack& stack); void push_argcount(vm::Stack& stack, int args); } // namespace fift diff --git a/crypto/fift/Fift.cpp b/crypto/fift/Fift.cpp index a2feee590..ef265657c 100644 --- a/crypto/fift/Fift.cpp +++ b/crypto/fift/Fift.cpp @@ -37,25 +37,18 @@ td::Result Fift::interpret_file(std::string fname, std::string current_dir, return td::Status::Error("cannot locate file `" + fname + "`"); } auto file = r_file.move_as_ok(); - IntCtx ctx; std::stringstream ss(file.data); - ctx.input_stream = &ss; - ctx.filename = td::PathView(file.path).file_name().str(); - ctx.currentd_dir = td::PathView(file.path).parent_dir().str(); - ctx.include_depth = is_interactive ? 0 : 1; - return do_interpret(ctx); + IntCtx ctx{ss, td::PathView(file.path).file_name().str(), td::PathView(file.path).parent_dir().str(), + (int)!is_interactive}; + return do_interpret(ctx, is_interactive); } td::Result Fift::interpret_istream(std::istream& stream, std::string current_dir, bool is_interactive) { - IntCtx ctx; - ctx.input_stream = &stream; - ctx.filename = "stdin"; - ctx.currentd_dir = current_dir; - ctx.include_depth = is_interactive ? 0 : 1; - return do_interpret(ctx); + IntCtx ctx{stream, "stdin", current_dir, (int)!is_interactive}; + return do_interpret(ctx, is_interactive); } -td::Result Fift::do_interpret(IntCtx& ctx) { +td::Result Fift::do_interpret(IntCtx& ctx, bool is_interactive) { ctx.ton_db = &config_.ton_db; ctx.source_lookup = &config_.source_lookup; ctx.dictionary = &config_.dictionary; @@ -64,13 +57,26 @@ td::Result Fift::do_interpret(IntCtx& ctx) { if (!ctx.output_stream) { return td::Status::Error("Cannot run interpreter without output_stream"); } - try { - return funny_interpret_loop(ctx); - } catch (fift::IntError ab) { - return td::Status::Error(ab.msg); - } catch (fift::Quit q) { - return q.res; + while (true) { + auto res = ctx.run(td::make_ref()); + if (res.is_error()) { + res = ctx.add_error_loc(res.move_as_error()); + if (config_.show_backtrace) { + std::ostringstream os; + ctx.print_error_backtrace(os); + LOG(ERROR) << os.str(); + } + if (is_interactive) { + LOG(ERROR) << res.move_as_error().message(); + ctx.top_ctx(); + ctx.clear_error(); + ctx.stack.clear(); + ctx.load_next_line(); + continue; + } + } + return res; } - return 0; } + } // namespace fift diff --git a/crypto/fift/Fift.h b/crypto/fift/Fift.h index a57eef0e2..ebcf2ef4d 100644 --- a/crypto/fift/Fift.h +++ b/crypto/fift/Fift.h @@ -26,7 +26,6 @@ namespace fift { struct IntCtx; -int funny_interpret_loop(IntCtx& ctx); struct Fift { public: @@ -36,6 +35,7 @@ struct Fift { fift::Dictionary dictionary; std::ostream* output_stream{&std::cout}; std::ostream* error_stream{&std::cerr}; + bool show_backtrace{true}; }; // Fift must own ton_db and dictionary, no concurrent access is allowed explicit Fift(Config config); @@ -48,6 +48,6 @@ struct Fift { private: Config config_; - td::Result do_interpret(IntCtx& ctx); + td::Result do_interpret(IntCtx& ctx, bool is_interactive = false); }; } // namespace fift diff --git a/crypto/fift/IntCtx.cpp b/crypto/fift/IntCtx.cpp index 143b528f4..9d623b069 100644 --- a/crypto/fift/IntCtx.cpp +++ b/crypto/fift/IntCtx.cpp @@ -68,33 +68,69 @@ void CharClassifier::set_char_class(int c, int cl) { } IntCtx::Savepoint::Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir, - std::istream* new_input_stream) + std::unique_ptr new_input_stream) : ctx(_ctx) , old_line_no(_ctx.line_no) , old_need_line(_ctx.need_line) , old_filename(_ctx.filename) , old_current_dir(_ctx.currentd_dir) , old_input_stream(_ctx.input_stream) + , old_input_stream_holder(std::move(_ctx.input_stream_holder)) , old_curline(_ctx.str) - , old_curpos(_ctx.input_ptr - _ctx.str.c_str()) { + , old_curpos(_ctx.input_ptr - _ctx.str.c_str()) + , old_word(_ctx.word) { ctx.line_no = 0; ctx.filename = new_filename; ctx.currentd_dir = new_current_dir; - ctx.input_stream = new_input_stream; + ctx.input_stream = new_input_stream.get(); + ctx.input_stream_holder = std::move(new_input_stream); ctx.str = ""; ctx.input_ptr = 0; ++(ctx.include_depth); } -IntCtx::Savepoint::~Savepoint() { +bool IntCtx::Savepoint::restore(IntCtx& _ctx) { + if (restored || &ctx != &_ctx) { + return false; + } ctx.line_no = old_line_no; ctx.need_line = old_need_line; ctx.filename = old_filename; ctx.currentd_dir = old_current_dir; ctx.input_stream = old_input_stream; + ctx.input_stream_holder = std::move(old_input_stream_holder); ctx.str = old_curline; ctx.input_ptr = ctx.str.c_str() + old_curpos; + ctx.word = old_word; --(ctx.include_depth); + return restored = true; +} + +bool IntCtx::enter_ctx(std::string new_filename, std::string new_current_dir, + std::unique_ptr new_input_stream) { + if (!new_input_stream) { + return false; + } + ctx_save_stack.emplace_back(*this, std::move(new_filename), std::move(new_current_dir), std::move(new_input_stream)); + return true; +} + +bool IntCtx::leave_ctx() { + if (ctx_save_stack.empty()) { + return false; + } + bool ok = ctx_save_stack.back().restore(*this); + ctx_save_stack.pop_back(); + return ok; +} + +bool IntCtx::top_ctx() { + while (!ctx_save_stack.empty()) { + if (!leave_ctx()) { + return false; + } + } + return true; } bool IntCtx::load_next_line() { @@ -194,4 +230,99 @@ void IntCtx::check_int_exec() const { throw IntError{"internal interpret mode only"}; } } + +bool IntCtx::print_error_backtrace(std::ostream& os) const { + if (exc_cont.is_null() && exc_next.is_null()) { + os << "(no backtrace)\n"; + return false; + } + if (exc_cont.not_null()) { + os << "top: "; + exc_cont->dump(os, *this); + } + return print_backtrace(os, exc_next); +} + +bool IntCtx::print_backtrace(std::ostream& os, Ref cont) const { + for (int i = 1; cont.not_null() && i <= 16; i++) { + os << "level " << i << ": "; + cont->dump(os, *this); + cont = cont->up(); + } + if (cont.not_null()) { + os << "... more levels ...\n"; + } + return true; +} + +Ref IntCtx::throw_exception(td::Status err, Ref cur) { + exc_cont = std::move(cur); + exc_next = std::move(next); + error = std::move(err); + next.clear(); + auto cont = std::move(exc_handler); + if (cont.is_null()) { + return {}; // no Fift exception handler set + } else if (cont.is_unique()) { + return cont.unique_write().handle_modify(*this); + } else { + return cont->handle_tail(*this); + } +} + +void IntCtx::clear_error() { + error = td::Status::OK(); + exit_code = 0; +} + +td::Result IntCtx::get_result() { + if (error.is_error()) { + return error.move_as_error(); + } else { + return exit_code; + } +} + +td::Status IntCtx::add_error_loc(td::Status err) const { + if (err.is_error()) { + std::ostringstream os; + if (include_depth && line_no) { + os << filename << ":" << line_no << ":\t"; + } + if (!word.empty()) { + os << word << ":"; + } + return err.move_as_error_prefix(os.str()); + } else { + return err; + } +} + +td::Result IntCtx::run(Ref cont) { + clear_error(); + while (cont.not_null()) { + try { + if (cont.is_unique()) { + cont = cont.unique_write().run_modify(*this); + } else { + cont = cont->run_tail(*this); + } + } catch (IntError& err) { + cont = throw_exception(td::Status::Error(err.msg), std::move(cont)); + } catch (vm::VmError& err) { + cont = throw_exception(err.as_status(), std::move(cont)); + } catch (vm::VmVirtError& err) { + cont = throw_exception(err.as_status(), std::move(cont)); + } catch (vm::CellBuilder::CellWriteError&) { + cont = throw_exception(td::Status::Error("Cell builder write error"), std::move(cont)); + } catch (vm::VmFatal&) { + cont = throw_exception(td::Status::Error("fatal vm error"), std::move(cont)); + } + if (cont.is_null()) { + cont = std::move(next); + } + } + return get_result(); +} + } // namespace fift diff --git a/crypto/fift/IntCtx.h b/crypto/fift/IntCtx.h index 207216748..807a5d016 100644 --- a/crypto/fift/IntCtx.h +++ b/crypto/fift/IntCtx.h @@ -22,6 +22,11 @@ #include "crypto/vm/stack.hpp" #include "crypto/common/bitstring.h" +#include "td/utils/Status.h" + +#include "Dictionary.h" +#include "Continuation.h" + #include #include #include @@ -65,13 +70,18 @@ class CharClassifier { struct IntCtx { vm::Stack stack; + Ref next, exc_handler; + Ref exc_cont, exc_next; int state{0}; int include_depth{0}; int line_no{0}; + int exit_code{0}; + td::Status error; bool need_line{true}; std::string filename; std::string currentd_dir; std::istream* input_stream{nullptr}; + std::unique_ptr input_stream_holder; std::ostream* output_stream{nullptr}; std::ostream* error_stream{nullptr}; @@ -79,18 +89,50 @@ struct IntCtx { Dictionary* dictionary{nullptr}; SourceLookup* source_lookup{nullptr}; int* now{nullptr}; + std::string word; private: std::string str; - const char* input_ptr; + const char* input_ptr = nullptr; + + class Savepoint { + IntCtx& ctx; + int old_line_no; + bool old_need_line; + bool restored{false}; + std::string old_filename; + std::string old_current_dir; + std::istream* old_input_stream; + std::unique_ptr old_input_stream_holder; + std::string old_curline; + std::ptrdiff_t old_curpos; + std::string old_word; + + public: + Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir, + std::unique_ptr new_input_stream); + bool restore(IntCtx& _ctx); + }; + + std::vector ctx_save_stack; public: IntCtx() = default; + IntCtx(std::istream& _istream, std::string _filename, std::string _curdir = "", int _depth = 0) + : include_depth(_depth) + , filename(std::move(_filename)) + , currentd_dir(std::move(_curdir)) + , input_stream(&_istream) { + } operator vm::Stack&() { return stack; } + bool enter_ctx(std::string new_filename, std::string new_current_dir, std::unique_ptr new_input_stream); + bool leave_ctx(); + bool top_ctx(); + td::Slice scan_word_to(char delim, bool err_endl = true); td::Slice scan_word(); td::Slice scan_word_ext(const CharClassifier& classifier); @@ -127,25 +169,29 @@ struct IntCtx { state = 0; stack.clear(); } - class Savepoint { - IntCtx& ctx; - int old_line_no; - bool old_need_line; - std::string old_filename; - std::string old_current_dir; - std::istream* old_input_stream; - std::string old_curline; - std::ptrdiff_t old_curpos; - - public: - Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir, std::istream* new_input_stream); - ~Savepoint(); - }; void check_compile() const; void check_execute() const; void check_not_int_exec() const; void check_int_exec() const; + + bool print_error_backtrace(std::ostream& os) const; + bool print_backtrace(std::ostream& os, Ref cont) const; + + td::Status add_error_loc(td::Status err) const; + + void set_exit_code(int err_code) { + exit_code = err_code; + } + int get_exit_code() const { + return exit_code; + } + + void clear_error(); + td::Result get_result(); + + Ref throw_exception(td::Status err, Ref cur = {}); + td::Result run(Ref cont); }; td::StringBuilder& operator<<(td::StringBuilder& os, const IntCtx& ctx); diff --git a/crypto/fift/utils.cpp b/crypto/fift/utils.cpp index fc2b27533..b6c61c6d6 100644 --- a/crypto/fift/utils.cpp +++ b/crypto/fift/utils.cpp @@ -28,6 +28,12 @@ namespace { std::string fift_dir(std::string dir) { return dir.size() > 0 ? dir : td::PathView(td::realpath(__FILE__).move_as_ok()).parent_dir().str() + "lib/"; } +std::string smartcont_dir(std::string dir) { + return dir.size() > 0 + ? dir + : td::PathView(td::PathView(td::realpath(__FILE__).move_as_ok()).parent_dir_noslash()).parent_dir().str() + + "smartcont/"; +} td::Result load_source(std::string name, std::string dir = "") { return td::read_file_str(fift_dir(dir) + name); } @@ -49,6 +55,9 @@ td::Result load_Lisp_fif(std::string dir = "") { td::Result load_GetOpt_fif(std::string dir = "") { return load_source("GetOpt.fif", dir); } +td::Result load_wallet3_code_fif(std::string dir = "") { + return td::read_file_str(smartcont_dir(dir) + "wallet-v3-code.fif"); +} class MemoryFileLoader : public fift::FileLoader { public: @@ -98,7 +107,7 @@ class MemoryFileLoader : public fift::FileLoader { td::Result create_source_lookup(std::string main, bool need_preamble = true, bool need_asm = true, bool need_ton_util = true, bool need_lisp = true, - std::string dir = "") { + bool need_w3_code = true, std::string dir = "") { auto loader = std::make_unique(); loader->add_file("/main.fif", std::move(main)); if (need_preamble) { @@ -127,6 +136,10 @@ td::Result create_source_lookup(std::string main, bool need_ TRY_RESULT(f, load_Lisp_fif(dir)); loader->add_file("/Lisp.fif", std::move(f)); } + if (need_w3_code) { + TRY_RESULT(f, load_wallet3_code_fif(dir)); + loader->add_file("/wallet-v3-code.fif", std::move(f)); + } auto res = fift::SourceLookup(std::move(loader)); res.add_include_path("/"); return std::move(res); @@ -158,7 +171,7 @@ td::Result run_fift(fift::SourceLookup source_lookup, std::o } // namespace td::Result mem_run_fift(std::string source, std::vector args, std::string fift_dir) { std::stringstream ss; - TRY_RESULT(source_lookup, create_source_lookup(source, true, true, true, true, fift_dir)); + TRY_RESULT(source_lookup, create_source_lookup(source, true, true, true, true, true, fift_dir)); TRY_RESULT_ASSIGN(source_lookup, run_fift(std::move(source_lookup), &ss, true, std::move(args))); FiftOutput res; res.source_lookup = std::move(source_lookup); @@ -174,8 +187,9 @@ td::Result mem_run_fift(SourceLookup source_lookup, std::vector create_mem_source_lookup(std::string main, std::string fift_dir, bool need_preamble, - bool need_asm, bool need_ton_util, bool need_lisp) { - return create_source_lookup(main, need_preamble, need_asm, need_ton_util, need_lisp, fift_dir); + bool need_asm, bool need_ton_util, bool need_lisp, + bool need_w3_code) { + return create_source_lookup(main, need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, fift_dir); } td::Result> compile_asm(td::Slice asm_code, std::string fift_dir, bool is_raw) { @@ -183,7 +197,7 @@ td::Result> compile_asm(td::Slice asm_code, std::string fift_d TRY_RESULT(source_lookup, create_source_lookup(PSTRING() << "\"Asm.fif\" include\n " << (is_raw ? "<{" : "") << asm_code << "\n" << (is_raw ? "}>c" : "") << " boc>B \"res\" B>file", - true, true, true, false, fift_dir)); + true, true, true, false, false, fift_dir)); TRY_RESULT(res, run_fift(std::move(source_lookup), &ss)); TRY_RESULT(boc, res.read_file("res")); return vm::std_boc_deserialize(std::move(boc.data)); diff --git a/crypto/fift/utils.h b/crypto/fift/utils.h index 11f1e4a34..dd434fe01 100644 --- a/crypto/fift/utils.h +++ b/crypto/fift/utils.h @@ -28,7 +28,8 @@ struct FiftOutput { }; td::Result create_mem_source_lookup(std::string main, std::string fift_dir = "", bool need_preamble = true, bool need_asm = true, - bool need_ton_util = true, bool need_lisp = true); + bool need_ton_util = true, bool need_lisp = true, + bool need_w3_code = true); td::Result mem_run_fift(std::string source, std::vector args = {}, std::string fift_dir = ""); td::Result mem_run_fift(SourceLookup source_lookup, std::vector args); td::Result> compile_asm(td::Slice asm_code, std::string fift_dir = "", bool is_raw = true); diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp index cd54f4a16..cc1595dcd 100644 --- a/crypto/fift/words.cpp +++ b/crypto/fift/words.cpp @@ -62,7 +62,7 @@ void show_total_cells(std::ostream& stream) { stream << "total cells = " << vm::DataCell::get_total_data_cells() << std::endl; } -void do_compile(vm::Stack& stack, Ref word_def); +void do_compile(vm::Stack& stack, Ref word_def); void do_compile_literals(vm::Stack& stack, int count); void interpret_dot(IntCtx& ctx, bool space_after) { @@ -204,18 +204,6 @@ void interpret_negate(vm::Stack& stack) { stack.push_int(-stack.pop_int()); } -void interpret_const(vm::Stack& stack, long long val) { - stack.push_smallint(val); -} - -void interpret_big_const(vm::Stack& stack, td::RefInt256 val) { - stack.push_int(std::move(val)); -} - -void interpret_literal(vm::Stack& stack, vm::StackEntry se) { - stack.push(std::move(se)); -} - void interpret_cmp(vm::Stack& stack, const char opt[3]) { auto y = stack.pop_int(); auto x = stack.pop_int(); @@ -537,7 +525,8 @@ void interpret_hold(vm::Stack& stack) { stack.check_underflow(2); char buffer[8]; unsigned len = make_utf8_char(buffer, stack.pop_smallint_range(0x10ffff, -128)); - std::string s = stack.pop_string() + std::string{buffer, len}; + std::string s = stack.pop_string(); + s.append(buffer, len); stack.push_string(std::move(s)); } @@ -1567,78 +1556,183 @@ void interpret_dict_get(vm::Stack& stack, int sgnd, int mode) { } } -void interpret_dict_map(IntCtx& ctx, bool ext, bool sgnd) { - auto func = pop_exec_token(ctx); - int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits); - vm::Dictionary dict{ctx.stack.pop_maybe_cell(), n}, dict2{n}; - for (auto entry : dict.range(false, sgnd)) { - ctx.stack.push_builder(Ref{true}); - if (ext) { - ctx.stack.push_int(dict.key_as_integer(entry.first, sgnd)); - } - ctx.stack.push_cellslice(std::move(entry.second)); - func->run(ctx); - if (ctx.stack.pop_bool()) { - if (!dict2.set_builder(entry.first, n, ctx.stack.pop_builder())) { - throw IntError{"cannot insert value into dictionary"}; - } +class DictMapCont : public LoopCont { + int n; + bool ext; + bool sgnd; + vm::Dictionary dict, dict2; + vm::DictIterator it; + + public: + DictMapCont(Ref _func, Ref _after, int _n, Ref dict_root, bool _ext, bool _sgnd) + : LoopCont(std::move(_func), std::move(_after)) + , n(_n) + , ext(_ext) + , sgnd(_sgnd) + , dict(std::move(dict_root), n) + , dict2(n) { + } + DictMapCont(const DictMapCont&) = default; + DictMapCont* make_copy() const override { + return new DictMapCont(*this); + } + bool init(IntCtx& ctx) override { + it = dict.init_iterator(false, sgnd); + return true; + } + bool pre_exec(IntCtx& ctx) override; + bool post_exec(IntCtx& ctx) override; + bool finalize(IntCtx& ctx) override; +}; + +bool DictMapCont::pre_exec(IntCtx& ctx) { + if (it.eof()) { + return false; + } + ctx.stack.push_builder(td::make_ref()); + if (ext) { + ctx.stack.push_int(dict.key_as_integer(it.cur_pos(), sgnd)); + } + ctx.stack.push_cellslice(it.cur_value()); + return true; +} + +bool DictMapCont::post_exec(IntCtx& ctx) { + if (ctx.stack.pop_bool()) { + if (!dict2.set_builder(it.cur_pos(), n, ctx.stack.pop_builder())) { + throw IntError{"cannot insert value into dictionary"}; } - }; + } + return !(++it).eof(); +} + +bool DictMapCont::finalize(IntCtx& ctx) { ctx.stack.push_maybe_cell(std::move(dict2).extract_root_cell()); + return true; } -void interpret_dict_foreach(IntCtx& ctx, bool reverse, bool sgnd) { +Ref interpret_dict_map(IntCtx& ctx, bool ext, bool sgnd) { auto func = pop_exec_token(ctx); int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits); - vm::Dictionary dict{ctx.stack.pop_maybe_cell(), n}; - for (auto entry : dict.range(reverse, sgnd)) { - ctx.stack.push_int(dict.key_as_integer(entry.first, sgnd)); - ctx.stack.push_cellslice(std::move(entry.second)); - func->run(ctx); - if (!ctx.stack.pop_bool()) { - ctx.stack.push_bool(false); - return; - } - }; - ctx.stack.push_bool(true); + return td::make_ref(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(), ext, sgnd); +} + +class DictIterCont : public LoopCont { + int n; + bool reverse; + bool sgnd; + bool ok; + bool inited{false}; + vm::Dictionary dict; + vm::DictIterator it; + + public: + DictIterCont(Ref _func, Ref _after, int _n, Ref dict_root, bool _reverse, bool _sgnd) + : LoopCont(std::move(_func), std::move(_after)) + , n(_n) + , reverse(_reverse) + , sgnd(_sgnd) + , ok(true) + , dict(std::move(dict_root), n) { + } + DictIterCont(const DictIterCont&) = default; + DictIterCont* make_copy() const override { + return new DictIterCont(*this); + } + bool do_init(); + bool init(IntCtx& ctx) override { + return do_init(); + } + bool pre_exec(IntCtx& ctx) override; + bool post_exec(IntCtx& ctx) override; + bool finalize(IntCtx& ctx) override; + template + bool lookup(const T& key, bool strict, bool backw) { + return do_init() && it.lookup(key, strict, backw); + } +}; + +bool DictIterCont::do_init() { + if (!inited) { + it = dict.init_iterator(reverse, sgnd); + inited = true; + } + return true; +} + +bool DictIterCont::pre_exec(IntCtx& ctx) { + if (it.eof()) { + return false; + } + ctx.stack.push_int(dict.key_as_integer(it.cur_pos(), sgnd)); + ctx.stack.push_cellslice(it.cur_value()); + return true; +} + +bool DictIterCont::post_exec(IntCtx& ctx) { + ok = ctx.stack.pop_bool(); + return ok && !(++it).eof(); +} + +bool DictIterCont::finalize(IntCtx& ctx) { + ctx.stack.push_bool(ok); + return true; +} + +Ref interpret_dict_foreach(IntCtx& ctx, bool reverse, bool sgnd) { + auto func = pop_exec_token(ctx); + int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits); + return td::make_ref(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(), reverse, sgnd); } // mode: +1 = reverse, +2 = signed, +4 = strict, +8 = lookup backwards, +16 = with hint -void interpret_dict_foreach_from(IntCtx& ctx, int mode) { +Ref interpret_dict_foreach_from(IntCtx& ctx, int mode) { if (mode < 0) { mode = ctx.stack.pop_smallint_range(31); } auto func = pop_exec_token(ctx); int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits); - vm::Dictionary dict{ctx.stack.pop_maybe_cell(), n}; - vm::DictIterator it{dict, mode & 3}; - unsigned char buffer[vm::Dictionary::max_key_bytes]; + auto it_cont = td::make_ref(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(), + mode & 1, mode & 2); for (int s = (mode >> 4) & 1; s >= 0; --s) { - auto key = dict.integer_key(ctx.stack.pop_int(), n, mode & 2, buffer); + unsigned char buffer[vm::Dictionary::max_key_bytes]; + auto key = vm::Dictionary::integer_key(ctx.stack.pop_int(), n, mode & 2, buffer); if (!key.is_valid()) { throw IntError{"not enough bits for a dictionary key"}; } - it.lookup(key, mode & 4, mode & 8); - } - for (; !it.eof(); ++it) { - ctx.stack.push_int(dict.key_as_integer(it.cur_pos(), mode & 2)); - ctx.stack.push_cellslice(it.cur_value()); - func->run(ctx); - if (!ctx.stack.pop_bool()) { - ctx.stack.push_bool(false); - return; - } - }; - ctx.stack.push_bool(true); + it_cont.write().lookup(key, mode & 4, mode & 8); + } + return it_cont; } -void interpret_dict_merge(IntCtx& ctx) { - auto func = pop_exec_token(ctx); - int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits); - vm::Dictionary dict2{ctx.stack.pop_maybe_cell(), n}; - vm::Dictionary dict1{ctx.stack.pop_maybe_cell(), n}; - vm::Dictionary dict3{n}; - auto it1 = dict1.begin(), it2 = dict2.begin(); +class DictMergeCont : public LoopCont { + int n; + vm::Dictionary dict1, dict2, dict3; + vm::DictIterator it1, it2; + + public: + DictMergeCont(Ref _func, Ref _after, int _n, Ref dict1_root, Ref dict2_root) + : LoopCont(std::move(_func), std::move(_after)) + , n(_n) + , dict1(std::move(dict1_root), n) + , dict2(std::move(dict2_root), n) + , dict3(n) { + } + DictMergeCont(const DictMergeCont&) = default; + DictMergeCont* make_copy() const override { + return new DictMergeCont(*this); + } + bool init(IntCtx& ctx) override { + it1 = dict1.begin(); + it2 = dict2.begin(); + return true; + } + bool pre_exec(IntCtx& ctx) override; + bool post_exec(IntCtx& ctx) override; + bool finalize(IntCtx& ctx) override; +}; + +bool DictMergeCont::pre_exec(IntCtx& ctx) { while (!it1.eof() || !it2.eof()) { int c = it1.eof() ? 1 : (it2.eof() ? -1 : it1.cur_pos().compare(it2.cur_pos(), n)); bool ok = true; @@ -1652,29 +1746,67 @@ void interpret_dict_merge(IntCtx& ctx) { ctx.stack.push_builder(Ref{true}); ctx.stack.push_cellslice(it1.cur_value()); ctx.stack.push_cellslice(it2.cur_value()); - func->run(ctx); - if (ctx.stack.pop_bool()) { - ok = dict3.set_builder(it1.cur_pos(), n, ctx.stack.pop_builder()); - } - ++it1; - ++it2; + return true; } if (!ok) { throw IntError{"cannot insert value into dictionary"}; } } + return false; +} + +bool DictMergeCont::post_exec(IntCtx& ctx) { + if (ctx.stack.pop_bool() && !dict3.set_builder(it1.cur_pos(), n, ctx.stack.pop_builder())) { + throw IntError{"cannot insert value into dictionary"}; + } + ++it1; + ++it2; + return true; +} + +bool DictMergeCont::finalize(IntCtx& ctx) { ctx.stack.push_maybe_cell(std::move(dict3).extract_root_cell()); + return true; } -void interpret_dict_diff(IntCtx& ctx) { +Ref interpret_dict_merge(IntCtx& ctx) { auto func = pop_exec_token(ctx); int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits); - vm::Dictionary dict2{ctx.stack.pop_maybe_cell(), n}; - vm::Dictionary dict1{ctx.stack.pop_maybe_cell(), n}; - auto it1 = dict1.begin(), it2 = dict2.begin(); + auto dict2_root = ctx.stack.pop_maybe_cell(); + return td::make_ref(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(), + std::move(dict2_root)); +} + +class DictDiffCont : public LoopCont { + int n; + bool ok{true}; + vm::Dictionary dict1, dict2; + vm::DictIterator it1, it2; + + public: + DictDiffCont(Ref _func, Ref _after, int _n, Ref dict1_root, Ref dict2_root) + : LoopCont(std::move(_func), std::move(_after)) + , n(_n) + , dict1(std::move(dict1_root), n) + , dict2(std::move(dict2_root), n) { + } + DictDiffCont(const DictDiffCont&) = default; + DictDiffCont* make_copy() const override { + return new DictDiffCont(*this); + } + bool init(IntCtx& ctx) override { + it1 = dict1.begin(); + it2 = dict2.begin(); + return true; + } + bool pre_exec(IntCtx& ctx) override; + bool post_exec(IntCtx& ctx) override; + bool finalize(IntCtx& ctx) override; +}; + +bool DictDiffCont::pre_exec(IntCtx& ctx) { while (!it1.eof() || !it2.eof()) { int c = it1.eof() ? 1 : (it2.eof() ? -1 : it1.cur_pos().compare(it2.cur_pos(), n)); - bool run = true; if (c < 0) { ctx.stack.push_int(dict1.key_as_integer(it1.cur_pos())); ctx.stack.push_cellslice(it1.cur_value()); @@ -1691,20 +1823,33 @@ void interpret_dict_diff(IntCtx& ctx) { ctx.stack.push_cellslice(it1.cur_value()); ctx.stack.push_cellslice(it2.cur_value()); } else { - run = false; + ++it1; + ++it2; + continue; } ++it1; ++it2; } - if (run) { - func->run(ctx); - if (!ctx.stack.pop_bool()) { - ctx.stack.push_bool(false); - return; - } - } + return true; } - ctx.stack.push_bool(true); + return false; +} + +bool DictDiffCont::post_exec(IntCtx& ctx) { + return (ok = ctx.stack.pop_bool()); +} + +bool DictDiffCont::finalize(IntCtx& ctx) { + ctx.stack.push_bool(ok); + return true; +} + +Ref interpret_dict_diff(IntCtx& ctx) { + auto func = pop_exec_token(ctx); + int n = ctx.stack.pop_smallint_range(vm::Dictionary::max_key_bits); + auto dict2_root = ctx.stack.pop_maybe_cell(); + return td::make_ref(std::move(func), std::move(ctx.next), n, ctx.stack.pop_maybe_cell(), + std::move(dict2_root)); } void interpret_pfx_dict_add(vm::Stack& stack, vm::Dictionary::SetMode mode, bool add_builder) { @@ -1791,7 +1936,7 @@ void interpret_wordlist_begin(IntCtx& ctx) { void interpret_wordlist_end_aux(vm::Stack& stack) { Ref wordlist_ref = pop_word_list(stack); wordlist_ref.write().close(); - stack.push({vm::from_object, Ref{wordlist_ref}}); + stack.push({vm::from_object, Ref{wordlist_ref}}); } void interpret_wordlist_end(IntCtx& ctx) { @@ -1846,7 +1991,7 @@ void interpret_create(IntCtx& ctx) { interpret_create_aux(ctx, 0); } -Ref create_aux_wd{Ref{true, std::bind(interpret_create_aux, std::placeholders::_1, -1)}}; +Ref create_aux_wd{Ref{true, std::bind(interpret_create_aux, std::placeholders::_1, -1)}}; // { bl word 2 ' (create) } :: : void interpret_colon(IntCtx& ctx, int mode) { @@ -2027,39 +2172,43 @@ void interpret_parse_hex_number(vm::Stack& stack) { } void interpret_quit(IntCtx& ctx) { - throw Quit{0}; + // TODO: change to correct behavior + ctx.set_exit_code(0); + ctx.next.clear(); } void interpret_bye(IntCtx& ctx) { - throw Quit{-1}; + ctx.set_exit_code(-1); + ctx.next.clear(); } -void interpret_halt(vm::Stack& stack) { - int code = stack.pop_smallint_range(255); - throw Quit{~code}; +void interpret_halt(IntCtx& ctx) { + ctx.set_exit_code(~ctx.stack.pop_smallint_range(255)); + ctx.next.clear(); } void interpret_abort(IntCtx& ctx) { throw IntError{ctx.stack.pop_string()}; } -Ref interpret_execute(IntCtx& ctx) { +Ref interpret_execute(IntCtx& ctx) { return pop_exec_token(ctx); } -Ref interpret_execute_times(IntCtx& ctx) { +Ref interpret_execute_times(IntCtx& ctx) { int count = ctx.stack.pop_smallint_range(1000000000); - auto wd_ref = pop_exec_token(ctx); - if (!count) { + auto body = pop_exec_token(ctx); + if (count <= 0) { return {}; } - while (--count > 0) { - wd_ref->run(ctx); + if (count == 1) { + return body; } - return wd_ref; + ctx.next = td::make_ref(body, std::move(ctx.next), count - 1); + return body; } -Ref interpret_if(IntCtx& ctx) { +Ref interpret_if(IntCtx& ctx) { auto true_ref = pop_exec_token(ctx); if (ctx.stack.pop_bool()) { return true_ref; @@ -2068,7 +2217,7 @@ Ref interpret_if(IntCtx& ctx) { } } -Ref interpret_ifnot(IntCtx& ctx) { +Ref interpret_ifnot(IntCtx& ctx) { auto false_ref = pop_exec_token(ctx); if (ctx.stack.pop_bool()) { return {}; @@ -2077,7 +2226,7 @@ Ref interpret_ifnot(IntCtx& ctx) { } } -Ref interpret_cond(IntCtx& ctx) { +Ref interpret_cond(IntCtx& ctx) { auto false_ref = pop_exec_token(ctx); auto true_ref = pop_exec_token(ctx); if (ctx.stack.pop_bool()) { @@ -2087,23 +2236,17 @@ Ref interpret_cond(IntCtx& ctx) { } } -void interpret_while(IntCtx& ctx) { - auto body_ref = pop_exec_token(ctx); - auto cond_ref = pop_exec_token(ctx); - while (true) { - cond_ref->run(ctx); - if (!ctx.stack.pop_bool()) { - break; - } - body_ref->run(ctx); - } +Ref interpret_while(IntCtx& ctx) { + auto body = pop_exec_token(ctx); + auto cond = pop_exec_token(ctx); + ctx.next = td::make_ref(cond, std::move(body), std::move(ctx.next), true); + return cond; } -void interpret_until(IntCtx& ctx) { - auto body_ref = pop_exec_token(ctx); - do { - body_ref->run(ctx); - } while (!ctx.stack.pop_bool()); +Ref interpret_until(IntCtx& ctx) { + auto body = pop_exec_token(ctx); + ctx.next = td::make_ref(body, std::move(ctx.next)); + return body; } void interpret_tick(IntCtx& ctx) { @@ -2133,25 +2276,39 @@ void interpret_find(IntCtx& ctx) { } } -void interpret_tick_nop(vm::Stack& stack) { - stack.push({vm::from_object, Dictionary::nop_word_def}); +void interpret_leave_source(IntCtx& ctx) { + if (!ctx.leave_ctx()) { + throw IntError{"cannot leave included file interpretation context"}; + } } -void interpret_include(IntCtx& ctx) { +Ref interpret_include(IntCtx& ctx) { auto fname = ctx.stack.pop_string(); auto r_file = ctx.source_lookup->lookup_source(fname, ctx.currentd_dir); if (r_file.is_error()) { throw IntError{"cannot locate file `" + fname + "`"}; } auto file = r_file.move_as_ok(); - std::stringstream ss(std::move(file.data)); - IntCtx::Savepoint save{ctx, td::PathView(file.path).file_name().str(), td::PathView(file.path).parent_dir().str(), - &ss}; - funny_interpret_loop(ctx); + auto ss = std::make_unique(std::move(file.data)); + if (!ctx.enter_ctx(td::PathView(file.path).file_name().str(), td::PathView(file.path).parent_dir().str(), + std::move(ss))) { + throw IntError{"cannot enter included file interpretation context"}; + } + ctx.next = SeqCont::seq(td::make_ref(interpret_leave_source), std::move(ctx.next)); + return td::make_ref(); } -void interpret_skip_source(vm::Stack& stack) { - throw SkipToEof(); +td::Ref exit_interpret{true}; + +Ref interpret_skip_source(IntCtx& ctx) { + auto cont = exit_interpret->get().as_object(); + ctx.next.clear(); + /* + if (cont.is_null()) { + throw IntError{"no interpreter exit point set"}; + } + */ + return cont; } void interpret_words(IntCtx& ctx) { @@ -2161,6 +2318,14 @@ void interpret_words(IntCtx& ctx) { *ctx.output_stream << std::endl; } +void interpret_print_backtrace(IntCtx& ctx) { + ctx.print_backtrace(*ctx.output_stream, ctx.next); +} + +void interpret_print_continuation(IntCtx& ctx) { + ctx.print_backtrace(*ctx.output_stream, pop_exec_token(ctx)); +} + void interpret_pack_std_smc_addr(vm::Stack& stack) { block::StdAddress a; stack.check_underflow(3); @@ -2450,17 +2615,17 @@ void interpret_get_fixed_cmdline_arg(vm::Stack& stack, int n) { } // n -- executes $n -void interpret_get_cmdline_arg(IntCtx& ctx) { +Ref interpret_get_cmdline_arg(IntCtx& ctx) { int n = ctx.stack.pop_smallint_range(999999); if (n) { interpret_get_fixed_cmdline_arg(ctx.stack, n); - return; + return {}; } auto entry = ctx.dictionary->lookup("$0 "); if (!entry) { throw IntError{"-?"}; } else { - (*entry)(ctx); + return entry->get_def(); } } @@ -2493,16 +2658,16 @@ void interpret_getenv_exists(vm::Stack& stack) { } // x1 .. xn n 'w --> -void interpret_execute_internal(IntCtx& ctx) { - Ref word_def = pop_exec_token(ctx); +Ref interpret_execute_internal(IntCtx& ctx) { + Ref word_def = pop_exec_token(ctx); int count = ctx.stack.pop_smallint_range(255); ctx.stack.check_underflow(count); - word_def->run(ctx); + return word_def; } // wl x1 .. xn n 'w --> wl' void interpret_compile_internal(vm::Stack& stack) { - Ref word_def = pop_exec_token(stack); + Ref word_def = pop_exec_token(stack); int count = stack.pop_smallint_range(255); do_compile_literals(stack, count); if (word_def != Dictionary::nop_word_def) { @@ -2510,12 +2675,14 @@ void interpret_compile_internal(vm::Stack& stack) { } } -void do_compile(vm::Stack& stack, Ref word_def) { +void do_compile(vm::Stack& stack, Ref word_def) { Ref wl_ref = pop_word_list(stack); if (word_def != Dictionary::nop_word_def) { - if ((td::uint64)word_def->list_size() <= 1) { + auto list_size = word_def->list_size(); + if ((td::uint64)list_size <= 1) { // inline short definitions - wl_ref.write().append(*(word_def->get_list())); + auto list = word_def->get_list(); + wl_ref.write().append(list, list + list_size); } else { wl_ref.write().push_back(word_def); } @@ -2524,19 +2691,7 @@ void do_compile(vm::Stack& stack, Ref word_def) { } void compile_one_literal(WordList& wlist, vm::StackEntry val) { - using namespace std::placeholders; - if (val.type() == vm::StackEntry::t_int) { - auto x = std::move(val).as_int(); - if (!x->signed_fits_bits(257)) { - throw IntError{"invalid numeric literal"}; - } else if (x->signed_fits_bits(td::BigIntInfo::word_shift)) { - wlist.push_back(Ref{true, std::bind(interpret_const, _1, x->to_long())}); - } else { - wlist.push_back(Ref{true, std::bind(interpret_big_const, _1, std::move(x))}); - } - } else { - wlist.push_back(Ref{true, std::bind(interpret_literal, _1, std::move(val))}); - } + wlist.push_back(LitCont::literal(std::move(val))); } void do_compile_literals(vm::Stack& stack, int count) { @@ -2548,8 +2703,16 @@ void do_compile_literals(vm::Stack& stack, int count) { if (wl_ref.is_null()) { throw IntError{"list of words expected"}; } - for (int i = count - 1; i >= 0; i--) { - compile_one_literal(wl_ref.write(), std::move(stack[i])); + if (count >= 2) { + std::vector literals; + for (int i = count - 1; i >= 0; i--) { + literals.push_back(std::move(stack[i])); + } + wl_ref.write().push_back(td::make_ref(std::move(literals))); + } else { + for (int i = count - 1; i >= 0; i--) { + compile_one_literal(wl_ref.write(), std::move(stack[i])); + } } stack.pop_many(count + 1); stack.push({vm::from_object, wl_ref}); @@ -2654,13 +2817,13 @@ void init_words_common(Dictionary& d) { d.def_stack_word("or ", interpret_or); d.def_stack_word("xor ", interpret_xor); // integer constants - d.def_stack_word("false ", std::bind(interpret_const, _1, 0)); - d.def_stack_word("true ", std::bind(interpret_const, _1, -1)); - d.def_stack_word("0 ", std::bind(interpret_const, _1, 0)); - d.def_stack_word("1 ", std::bind(interpret_const, _1, 1)); - d.def_stack_word("2 ", std::bind(interpret_const, _1, 2)); - d.def_stack_word("-1 ", std::bind(interpret_const, _1, -1)); - d.def_stack_word("bl ", std::bind(interpret_const, _1, 32)); + d.def_word("false ", IntLitCont::literal(0)); + d.def_word("true ", IntLitCont::literal(-1)); + d.def_word("0 ", IntLitCont::literal(0)); + d.def_word("1 ", IntLitCont::literal(1)); + d.def_word("2 ", IntLitCont::literal(2)); + d.def_word("-1 ", IntLitCont::literal(-1)); + d.def_word("bl ", IntLitCont::literal(32)); // integer comparison d.def_stack_word("cmp ", std::bind(interpret_cmp, _1, "\xff\x00\x01")); d.def_stack_word("= ", std::bind(interpret_cmp, _1, "\x00\xff\x00")); @@ -2835,16 +2998,16 @@ void init_words_common(Dictionary& d) { d.def_stack_word("pfxdict!+ ", std::bind(interpret_pfx_dict_add, _1, vm::Dictionary::SetMode::Add, false)); d.def_stack_word("pfxdict! ", std::bind(interpret_pfx_dict_add, _1, vm::Dictionary::SetMode::Set, false)); d.def_stack_word("pfxdict@ ", interpret_pfx_dict_get); - d.def_ctx_word("dictmap ", std::bind(interpret_dict_map, _1, false, false)); - d.def_ctx_word("dictmapext ", std::bind(interpret_dict_map, _1, true, false)); - d.def_ctx_word("idictmapext ", std::bind(interpret_dict_map, _1, true, true)); - d.def_ctx_word("dictforeach ", std::bind(interpret_dict_foreach, _1, false, false)); - d.def_ctx_word("idictforeach ", std::bind(interpret_dict_foreach, _1, false, true)); - d.def_ctx_word("dictforeachrev ", std::bind(interpret_dict_foreach, _1, true, false)); - d.def_ctx_word("idictforeachrev ", std::bind(interpret_dict_foreach, _1, true, true)); - d.def_ctx_word("dictforeachfromx ", std::bind(interpret_dict_foreach_from, _1, -1)); - d.def_ctx_word("dictmerge ", interpret_dict_merge); - d.def_ctx_word("dictdiff ", interpret_dict_diff); + d.def_ctx_tail_word("dictmap ", std::bind(interpret_dict_map, _1, false, false)); + d.def_ctx_tail_word("dictmapext ", std::bind(interpret_dict_map, _1, true, false)); + d.def_ctx_tail_word("idictmapext ", std::bind(interpret_dict_map, _1, true, true)); + d.def_ctx_tail_word("dictforeach ", std::bind(interpret_dict_foreach, _1, false, false)); + d.def_ctx_tail_word("idictforeach ", std::bind(interpret_dict_foreach, _1, false, true)); + d.def_ctx_tail_word("dictforeachrev ", std::bind(interpret_dict_foreach, _1, true, false)); + d.def_ctx_tail_word("idictforeachrev ", std::bind(interpret_dict_foreach, _1, true, true)); + d.def_ctx_tail_word("dictforeachfromx ", std::bind(interpret_dict_foreach_from, _1, -1)); + d.def_ctx_tail_word("dictmerge ", interpret_dict_merge); + d.def_ctx_tail_word("dictdiff ", interpret_dict_diff); // slice/bitstring constants d.def_active_word("x{", interpret_bitstring_hex_literal); d.def_active_word("b{", interpret_bitstring_binary_literal); @@ -2880,8 +3043,8 @@ void init_words_common(Dictionary& d) { d.def_ctx_tail_word("if ", interpret_if); d.def_ctx_tail_word("ifnot ", interpret_ifnot); d.def_ctx_tail_word("cond ", interpret_cond); - d.def_ctx_word("while ", interpret_while); - d.def_ctx_word("until ", interpret_until); + d.def_ctx_tail_word("while ", interpret_while); + d.def_ctx_tail_word("until ", interpret_until); // compiler control d.def_active_word("[ ", interpret_internal_interpret_begin); d.def_active_word("] ", interpret_internal_interpret_end); @@ -2890,9 +3053,9 @@ void init_words_common(Dictionary& d) { d.def_stack_word("({) ", interpret_wordlist_begin_aux); d.def_stack_word("(}) ", interpret_wordlist_end_aux); d.def_stack_word("(compile) ", interpret_compile_internal); - d.def_ctx_word("(execute) ", interpret_execute_internal); + d.def_ctx_tail_word("(execute) ", interpret_execute_internal); d.def_active_word("' ", interpret_tick); - d.def_stack_word("'nop ", interpret_tick_nop); + d.def_word("'nop ", LitCont::literal({vm::from_object, Dictionary::nop_word_def})); // dictionary manipulation d.def_ctx_word("find ", interpret_find); d.def_ctx_word("create ", interpret_create); @@ -2904,20 +3067,23 @@ void init_words_common(Dictionary& d) { d.def_ctx_word("(forget) ", interpret_forget_aux); d.def_ctx_word("forget ", interpret_forget); d.def_ctx_word("words ", interpret_words); + d.def_ctx_word(".bt ", interpret_print_backtrace); + d.def_ctx_word("cont. ", interpret_print_continuation); // input parse d.def_ctx_word("word ", interpret_word); d.def_ctx_word("(word) ", interpret_word_ext); d.def_ctx_word("skipspc ", interpret_skipspc); - d.def_ctx_word("include ", interpret_include); - d.def_stack_word("skip-to-eof ", interpret_skip_source); + d.def_ctx_tail_word("include ", interpret_include); + d.def_ctx_tail_word("skip-to-eof ", interpret_skip_source); + d.def_word("'exit-interpret ", LitCont::literal(exit_interpret)); d.def_ctx_word("abort ", interpret_abort); d.def_ctx_word("quit ", interpret_quit); d.def_ctx_word("bye ", interpret_bye); - d.def_stack_word("halt ", interpret_halt); + d.def_ctx_word("halt ", interpret_halt); // cmdline args - d.def_stack_word("$* ", std::bind(interpret_literal, _1, vm::StackEntry{cmdline_args})); + d.def_word("$* ", LitCont::literal(cmdline_args)); d.def_stack_word("$# ", interpret_get_cmdline_arg_count); - d.def_ctx_word("$() ", interpret_get_cmdline_arg); + d.def_ctx_tail_word("$() ", interpret_get_cmdline_arg); } void init_words_ton(Dictionary& d) { @@ -2934,7 +3100,7 @@ void init_words_vm(Dictionary& d, bool enable_debug) { using namespace std::placeholders; vm::init_op_cp0(enable_debug); // vm run - d.def_stack_word("vmlibs ", std::bind(interpret_literal, _1, vm::StackEntry{vm_libraries})); + d.def_word("vmlibs ", LitCont::literal(vm_libraries)); // d.def_ctx_word("runvmcode ", std::bind(interpret_run_vm, _1, 0x40)); // d.def_ctx_word("runvm ", std::bind(interpret_run_vm, _1, 0x45)); d.def_ctx_word("runvmx ", std::bind(interpret_run_vm, _1, -1)); @@ -2947,7 +3113,7 @@ void init_words_vm(Dictionary& d, bool enable_debug) { void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* const argv[]) { using namespace std::placeholders; LOG(DEBUG) << "import_cmdlist_args(" << arg0 << "," << n << ")"; - d.def_stack_word("$0 ", std::bind(interpret_literal, _1, vm::StackEntry{arg0})); + d.def_word("$0 ", LitCont::literal(arg0)); vm::StackEntry list; for (int i = n - 1; i >= 0; i--) { list = vm::StackEntry::cons(vm::StackEntry{argv[i]}, list); @@ -2978,104 +3144,81 @@ td::RefInt256 numeric_value(std::string s) { return num; } -int funny_interpret_loop(IntCtx& ctx) { - while (ctx.load_next_line()) { - if (ctx.is_sb()) { - continue; - } - std::ostringstream errs; - bool ok = true; - while (ok) { +Ref interpret_compile_execute(IntCtx& ctx) { + if (ctx.state > 0) { + interpret_compile_internal(ctx.stack); + return {}; + } else { + return interpret_execute_internal(ctx); + } +} + +Ref InterpretCont::run_tail(IntCtx& ctx) const { + if (!ctx.get_input() && !ctx.load_next_line()) { + return {}; + } + while (true) { + if (!ctx.is_sb()) { ctx.skipspc(); - const char* ptr = ctx.get_input(); - if (!*ptr) { + if (*ctx.get_input()) { break; } - std::string Word; - Word.reserve(128); - auto entry = ctx.dictionary->lookup(""); - std::string entry_word; - const char* ptr_end = ptr; - while (*ptr && *ptr != ' ' && *ptr != '\t') { - Word += *ptr++; - auto cur = ctx.dictionary->lookup(Word); - if (cur) { - entry = cur; - entry_word = Word; - ptr_end = ptr; - } - } - auto cur = ctx.dictionary->lookup(Word + " "); - if (cur || !entry) { - entry = std::move(cur); - ctx.set_input(ptr); - ctx.skipspc(); - } else { - Word = entry_word; - ctx.set_input(ptr_end); - } - try { - if (entry) { - if (entry->is_active()) { - (*entry)(ctx); - } else { - ctx.stack.push_smallint(0); - ctx.stack.push({vm::from_object, entry->get_def()}); - } - } else { - auto res = numeric_value_ext(Word); - ctx.stack.push(std::move(res.first)); - if (res.second.not_null()) { - ctx.stack.push(std::move(res.second)); - push_argcount(ctx, 2); - } else { - push_argcount(ctx, 1); - } - } - if (ctx.state > 0) { - interpret_compile_internal(ctx.stack); - } else { - interpret_execute_internal(ctx); - } - } catch (IntError& ab) { - errs << ctx << Word << ": " << ab.msg; - ok = false; - } catch (vm::VmError& ab) { - errs << ctx << Word << ": " << ab.get_msg(); - ok = false; - } catch (vm::CellBuilder::CellWriteError) { - errs << ctx << Word << ": Cell builder write error"; - ok = false; - } catch (vm::VmFatal) { - errs << ctx << Word << ": fatal vm error"; - ok = false; - } catch (Quit& q) { - if (ctx.include_depth) { - throw; - } - if (!q.res) { - ok = false; - } else { - return q.res; - } - } catch (SkipToEof) { - return 0; - } - }; - if (!ok) { - auto err_msg = errs.str(); - if (!err_msg.empty()) { - LOG(ERROR) << err_msg; - } - ctx.clear(); - if (ctx.include_depth) { - throw IntError{"error interpreting included file `" + ctx.filename + "` : " + err_msg}; - } - } else if (!ctx.state && !ctx.include_depth) { + } + if (!ctx.state && !ctx.include_depth) { *ctx.output_stream << " ok" << std::endl; } + if (!ctx.load_next_line()) { + return {}; + } + } + const char* ptr = ctx.get_input(); + std::string Word; + Word.reserve(128); + auto entry = ctx.dictionary->lookup(""); + std::string entry_word; + const char* ptr_end = ptr; + while (*ptr && *ptr != ' ' && *ptr != '\t') { + Word += *ptr++; + auto cur = ctx.dictionary->lookup(Word); + if (cur) { + entry = cur; + entry_word = Word; + ptr_end = ptr; + } + } + auto cur = ctx.dictionary->lookup(Word + " "); + if (cur || !entry) { + entry = std::move(cur); + ctx.set_input(ptr); + ctx.skipspc(); + } else { + Word = entry_word; + ctx.set_input(ptr_end); + } + ctx.word = Word; + static Ref compile_exec_ref = td::make_ref(interpret_compile_execute); + if (!entry) { + // numbers + auto res = numeric_value_ext(Word); + ctx.stack.push(std::move(res.first)); + if (res.second.not_null()) { + ctx.stack.push(std::move(res.second)); + push_argcount(ctx, 2); + } else { + push_argcount(ctx, 1); + } + } else if (!entry->is_active()) { + // ordinary word + ctx.stack.push_smallint(0); + ctx.stack.push({vm::from_object, entry->get_def()}); + } else { + // active word + ctx.next = SeqCont::seq(compile_exec_ref, SeqCont::seq(self(), std::move(ctx.next))); + return entry->get_def(); } - return 0; + exit_interpret->set({vm::from_object, ctx.next}); + ctx.next = SeqCont::seq(self(), std::move(ctx.next)); + return compile_exec_ref; } } // namespace fift diff --git a/crypto/fift/words.h b/crypto/fift/words.h index de5b43893..65466b70c 100644 --- a/crypto/fift/words.h +++ b/crypto/fift/words.h @@ -21,23 +21,10 @@ namespace fift { -// thrown by 'quit', 'bye' and 'halt' for exiting to top level -struct Quit { - int res; - Quit() : res(0) { - } - Quit(int _res) : res(_res) { - } -}; - -struct SkipToEof {}; - void init_words_common(Dictionary& dictionary); void init_words_vm(Dictionary& dictionary, bool debug_enabled = false); void init_words_ton(Dictionary& dictionary); void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* const argv[]); -int funny_interpret_loop(IntCtx& ctx); - } // namespace fift diff --git a/crypto/func/test/a12.fc b/crypto/func/test/a12.fc new file mode 100644 index 000000000..4f668ed3a --- /dev/null +++ b/crypto/func/test/a12.fc @@ -0,0 +1,4 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o) { + return (a, i, o, j, e, f, g, h, k, l, m, d, b, c, n); + ;; optimal code is 6-byte: s11 s12 XCHG2 2 5 BLKSWAP s13 s13 s11 XCHG3 +} diff --git a/crypto/func/test/a12_1.fc b/crypto/func/test/a12_1.fc new file mode 100644 index 000000000..45c97ee5f --- /dev/null +++ b/crypto/func/test/a12_1.fc @@ -0,0 +1,3 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j) { + return (f, j, a, b, i, g, d, e, h, c); +} diff --git a/crypto/func/test/a12_2.fc b/crypto/func/test/a12_2.fc new file mode 100644 index 000000000..8ee93d273 --- /dev/null +++ b/crypto/func/test/a12_2.fc @@ -0,0 +1,3 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j) { + return (f, d, j, a, i, g, b, e, h, c); +} diff --git a/crypto/func/test/a12_3.fc b/crypto/func/test/a12_3.fc new file mode 100644 index 000000000..faa5c9c20 --- /dev/null +++ b/crypto/func/test/a12_3.fc @@ -0,0 +1,3 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) { + return (e, h, d, a, c, g, f, b, i, j, k); ;; optimal code is 4 ops, 8 bytes +} diff --git a/crypto/func/test/a12_4.fc b/crypto/func/test/a12_4.fc new file mode 100644 index 000000000..bf0d797eb --- /dev/null +++ b/crypto/func/test/a12_4.fc @@ -0,0 +1,3 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) { + return (b, a, c, e, g, d, f, k, i, j, h); ;; optimal code is 4 ops, 8 bytes +} diff --git a/crypto/func/test/a12_5.fc b/crypto/func/test/a12_5.fc new file mode 100644 index 000000000..4c7235801 --- /dev/null +++ b/crypto/func/test/a12_5.fc @@ -0,0 +1,3 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) { + return (c, g, d, k, a, f, e, h, i, j, b); ;; optimal code is 6 ops, 6 bytes +} diff --git a/crypto/func/test/a12_6.fc b/crypto/func/test/a12_6.fc new file mode 100644 index 000000000..49982541d --- /dev/null +++ b/crypto/func/test/a12_6.fc @@ -0,0 +1,3 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) { + return (h, e, d, j, k, f, i, a, b, c, g); ;; optimal code is 3 ops, 6 bytes +} diff --git a/crypto/func/test/a12_7.fc b/crypto/func/test/a12_7.fc new file mode 100644 index 000000000..99c42b0c4 --- /dev/null +++ b/crypto/func/test/a12_7.fc @@ -0,0 +1,4 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o, int p) { + return (m, f, l, j, b, e, a, d, p, k, c, n, g, i, h, o); ;; optimal code is 6 ops, 11 bytes + ;; s12 s0 s14 XCHG3 s0 s5 XCHG 6 8 BLKSWAP s3 s3 s9 XCHG3 s10 s14 s15 XCHG3 s13 s9 s9 XCHG3 +} diff --git a/crypto/smc-envelope/GenericAccount.cpp b/crypto/smc-envelope/GenericAccount.cpp index 91b2edf76..d4a7317ff 100644 --- a/crypto/smc-envelope/GenericAccount.cpp +++ b/crypto/smc-envelope/GenericAccount.cpp @@ -46,7 +46,8 @@ bool unpack_grams(td::Ref cs, td::uint64& amount) { } } // namespace smc -td::Ref GenericAccount::get_init_state(td::Ref code, td::Ref data) noexcept { +td::Ref GenericAccount::get_init_state(const td::Ref& code, + const td::Ref& data) noexcept { return vm::CellBuilder() .store_zeroes(2) .store_ones(2) @@ -136,4 +137,23 @@ td::Result GenericAccount::get_public_key(const SmartCon }; return TRY_VM(do_get_public_key()); } + +td::Result GenericAccount::get_seqno(const SmartContract& sc) { + return TRY_VM([&]() -> td::Result { + auto answer = sc.run_get_method("seqno"); + if (!answer.success) { + return td::Status::Error("seqno get method failed"); + } + return static_cast(answer.stack.write().pop_long_range(std::numeric_limits::max())); + }()); +} +td::Result GenericAccount::get_wallet_id(const SmartContract& sc) { + return TRY_VM([&]() -> td::Result { + auto answer = sc.run_get_method("wallet_id"); + if (!answer.success) { + return td::Status::Error("seqno get method failed"); + } + return static_cast(answer.stack.write().pop_long_range(std::numeric_limits::max())); + }()); +} } // namespace ton diff --git a/crypto/smc-envelope/GenericAccount.h b/crypto/smc-envelope/GenericAccount.h index 6100b6dd4..285553c78 100644 --- a/crypto/smc-envelope/GenericAccount.h +++ b/crypto/smc-envelope/GenericAccount.h @@ -29,12 +29,17 @@ bool unpack_grams(td::Ref cs, td::uint64& amount); } // namespace smc class GenericAccount { public: - static td::Ref get_init_state(td::Ref code, td::Ref data) noexcept; + static td::Ref get_init_state(const td::Ref& code, const td::Ref& data) noexcept; + static td::Ref get_init_state(const SmartContract::State& state) noexcept { + return get_init_state(state.code, state.data); + } static block::StdAddress get_address(ton::WorkchainId workchain_id, const td::Ref& init_state) noexcept; static td::Ref create_ext_message(const block::StdAddress& address, td::Ref new_state, td::Ref body) noexcept; static void store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms); static td::Result get_public_key(const SmartContract& sc); + static td::Result get_seqno(const SmartContract& sc); + static td::Result get_wallet_id(const SmartContract& sc); }; } // namespace ton diff --git a/crypto/smc-envelope/HighloadWallet.cpp b/crypto/smc-envelope/HighloadWallet.cpp index 7996d095f..69843fcff 100644 --- a/crypto/smc-envelope/HighloadWallet.cpp +++ b/crypto/smc-envelope/HighloadWallet.cpp @@ -27,48 +27,12 @@ #include namespace ton { -td::optional HighloadWallet::guess_revision(const vm::Cell::Hash& code_hash) { - for (td::int32 i = 1; i <= 2; i++) { - if (get_init_code(i)->get_hash() == code_hash) { - return i; - } - } - return {}; -} -td::optional HighloadWallet::guess_revision(const block::StdAddress& address, - const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) { - for (td::int32 i = 1; i <= 2; i++) { - if (GenericAccount::get_address(address.workchain, get_init_state(public_key, wallet_id, i)) == address) { - return i; - } - } - return {}; -} -td::Ref HighloadWallet::get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, - td::int32 revision) noexcept { - auto code = get_init_code(revision); - auto data = get_init_data(public_key, wallet_id); - return GenericAccount::get_init_state(std::move(code), std::move(data)); -} -td::Ref HighloadWallet::get_init_message(const td::Ed25519::PrivateKey& private_key, - td::uint32 wallet_id) noexcept { - td::uint32 seqno = 0; - td::uint32 valid_until = std::numeric_limits::max(); - auto append_message = [&](auto&& cb) -> vm::CellBuilder& { - cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32); - CHECK(cb.store_maybe_ref({})); - return cb; - }; - auto signature = private_key.sign(append_message(vm::CellBuilder()).finalize()->get_hash().as_slice()).move_as_ok(); - - return append_message(vm::CellBuilder().store_bytes(signature)).finalize(); -} - -td::Ref HighloadWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, - td::uint32 seqno, td::uint32 valid_until, - td::Span gifts) noexcept { - CHECK(gifts.size() <= max_gifts_size); +td::Result> HighloadWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, + td::uint32 valid_until, td::Span gifts) const { + TRY_RESULT(wallet_id, get_wallet_id()); + TRY_RESULT(seqno, get_seqno()); + CHECK(gifts.size() <= get_max_gifts_size()); vm::Dictionary messages(16); for (size_t i = 0; i < gifts.size(); i++) { auto& gift = gifts[i]; @@ -91,63 +55,36 @@ td::Ref HighloadWallet::make_a_gift_message(const td::Ed25519::Private return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); } -td::Ref HighloadWallet::get_init_code(td::int32 revision) noexcept { - return SmartContractCode::get_code(SmartContractCode::HighloadWalletV1, revision); -} - -vm::CellHash HighloadWallet::get_init_code_hash() noexcept { - return get_init_code(0)->get_hash(); -} - -td::Ref HighloadWallet::get_init_data(const td::Ed25519::PublicKey& public_key, - td::uint32 wallet_id) noexcept { +td::Ref HighloadWallet::get_init_data(const InitData& init_data) noexcept { return vm::CellBuilder() - .store_long(0, 32) - .store_long(wallet_id, 32) - .store_bytes(public_key.as_octet_string()) + .store_long(init_data.seqno, 32) + .store_long(init_data.wallet_id, 32) + .store_bytes(init_data.public_key) .finalize(); } -td::Result HighloadWallet::get_seqno() const { - return TRY_VM(get_seqno_or_throw()); -} - -td::Result HighloadWallet::get_seqno_or_throw() const { - if (state_.data.is_null()) { - return 0; - } - //FIXME use get method - return static_cast(vm::load_cell_slice(state_.data).fetch_ulong(32)); -} - td::Result HighloadWallet::get_wallet_id() const { - return TRY_VM(get_wallet_id_or_throw()); -} - -td::Result HighloadWallet::get_wallet_id_or_throw() const { - if (state_.data.is_null()) { - return 0; - } - //FIXME use get method - auto cs = vm::load_cell_slice(state_.data); - cs.skip_first(32); - return static_cast(cs.fetch_ulong(32)); + return TRY_VM([&]() -> td::Result { + if (state_.data.is_null()) { + return 0; + } + auto cs = vm::load_cell_slice(state_.data); + cs.skip_first(32); + return static_cast(cs.fetch_ulong(32)); + }()); } td::Result HighloadWallet::get_public_key() const { - return TRY_VM(get_public_key_or_throw()); -} - -td::Result HighloadWallet::get_public_key_or_throw() const { - if (state_.data.is_null()) { - return td::Status::Error("data is null"); - } - //FIXME use get method - auto cs = vm::load_cell_slice(state_.data); - cs.skip_first(64); - td::SecureString res(td::Ed25519::PublicKey::LENGTH); - cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast(res.size())); - return td::Ed25519::PublicKey(std::move(res)); + return TRY_VM([&]() -> td::Result { + if (state_.data.is_null()) { + return td::Status::Error("data is null"); + } + auto cs = vm::load_cell_slice(state_.data); + cs.skip_first(64); + td::SecureString res(td::Ed25519::PublicKey::LENGTH); + cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast(res.size())); + return td::Ed25519::PublicKey(std::move(res)); + }()); } } // namespace ton diff --git a/crypto/smc-envelope/HighloadWallet.h b/crypto/smc-envelope/HighloadWallet.h index ce37af49d..db36caaee 100644 --- a/crypto/smc-envelope/HighloadWallet.h +++ b/crypto/smc-envelope/HighloadWallet.h @@ -26,41 +26,24 @@ #include "vm/cells/CellString.h" namespace ton { -class HighloadWallet : public ton::SmartContract, public WalletInterface { - public: - explicit HighloadWallet(State state) : ton::SmartContract(std::move(state)) { - } +struct HighloadWalletTraits { + using InitData = WalletInterface::DefaultInitData; + static constexpr unsigned max_message_size = vm::CellString::max_bytes; static constexpr unsigned max_gifts_size = 254; - static td::Ref get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, - td::int32 revision) noexcept; - static td::Ref get_init_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id) noexcept; - static td::Ref make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, - td::uint32 seqno, td::uint32 valid_until, td::Span gifts) noexcept; - - static td::Ref get_init_code(td::int32 revision) noexcept; - static vm::CellHash get_init_code_hash() noexcept; - static td::Ref get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) noexcept; - static td::optional guess_revision(const vm::Cell::Hash& code_hash); - static td::optional guess_revision(const block::StdAddress& address, - const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id); - td::Result get_seqno() const; - td::Result get_wallet_id() const; + static constexpr auto code_type = SmartContractCode::HighloadWalletV1; +}; - td::Result> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, - td::Span gifts) const override { - TRY_RESULT(seqno, get_seqno()); - TRY_RESULT(wallet_id, get_wallet_id()); - return make_a_gift_message(private_key, wallet_id, seqno, valid_until, gifts); - } - size_t get_max_gifts_size() const override { - return max_gifts_size; +class HighloadWallet : public WalletBase { + public: + explicit HighloadWallet(State state) : WalletBase(std::move(state)) { } - td::Result get_public_key() const override; + td::Result> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, + td::Span gifts) const override; + static td::Ref get_init_data(const InitData& init_data) noexcept; - private: - td::Result get_seqno_or_throw() const; - td::Result get_wallet_id_or_throw() const; - td::Result get_public_key_or_throw() const; + // can't use get methods for compatibility with old revisions + td::Result get_wallet_id() const override; + td::Result get_public_key() const override; }; } // namespace ton diff --git a/crypto/smc-envelope/HighloadWalletV2.cpp b/crypto/smc-envelope/HighloadWalletV2.cpp index bae249b41..e0454233e 100644 --- a/crypto/smc-envelope/HighloadWalletV2.cpp +++ b/crypto/smc-envelope/HighloadWalletV2.cpp @@ -27,33 +27,10 @@ #include namespace ton { -td::optional HighloadWalletV2::guess_revision(const vm::Cell::Hash& code_hash) { - for (td::int32 i = 1; i <= 2; i++) { - if (get_init_code(i)->get_hash() == code_hash) { - return i; - } - } - return {}; -} -td::optional HighloadWalletV2::guess_revision(const block::StdAddress& address, - const td::Ed25519::PublicKey& public_key, - td::uint32 wallet_id) { - for (td::int32 i = 1; i <= 2; i++) { - if (GenericAccount::get_address(address.workchain, get_init_state(public_key, wallet_id, i)) == address) { - return i; - } - } - return {}; -} -td::Ref HighloadWalletV2::get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, - td::int32 revision) noexcept { - auto code = get_init_code(revision); - auto data = get_init_data(public_key, wallet_id); - return GenericAccount::get_init_state(std::move(code), std::move(data)); -} -td::Ref HighloadWalletV2::get_init_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, - td::uint32 valid_until) noexcept { +td::Result> HighloadWalletV2::get_init_message(const td::Ed25519::PrivateKey& private_key, + td::uint32 valid_until) const noexcept { + TRY_RESULT(wallet_id, get_wallet_id()); td::uint32 id = -1; auto append_message = [&](auto&& cb) -> vm::CellBuilder& { cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(id, 32); @@ -65,10 +42,11 @@ td::Ref HighloadWalletV2::get_init_message(const td::Ed25519::PrivateK return append_message(vm::CellBuilder().store_bytes(signature)).finalize(); } -td::Ref HighloadWalletV2::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, - td::uint32 wallet_id, td::uint32 valid_until, - td::Span gifts) noexcept { - CHECK(gifts.size() <= max_gifts_size); +td::Result> HighloadWalletV2::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, + td::uint32 valid_until, + td::Span gifts) const { + TRY_RESULT(wallet_id, get_wallet_id()); + CHECK(gifts.size() <= get_max_gifts_size()); vm::Dictionary messages(16); for (size_t i = 0; i < gifts.size(); i++) { auto& gift = gifts[i]; @@ -96,49 +74,34 @@ td::Ref HighloadWalletV2::make_a_gift_message(const td::Ed25519::Priva return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); } -td::Ref HighloadWalletV2::get_init_code(td::int32 revision) noexcept { - return SmartContractCode::get_code(SmartContractCode::HighloadWalletV2, revision); -} - -vm::CellHash HighloadWalletV2::get_init_code_hash() noexcept { - return get_init_code(0)->get_hash(); -} - -td::Ref HighloadWalletV2::get_init_data(const td::Ed25519::PublicKey& public_key, - td::uint32 wallet_id) noexcept { +td::Ref HighloadWalletV2::get_init_data(const InitData& init_data) noexcept { vm::CellBuilder cb; - cb.store_long(wallet_id, 32).store_long(0, 64).store_bytes(public_key.as_octet_string()); + cb.store_long(init_data.wallet_id, 32).store_long(init_data.seqno, 64).store_bytes(init_data.public_key); CHECK(cb.store_maybe_ref({})); return cb.finalize(); } td::Result HighloadWalletV2::get_wallet_id() const { - return TRY_VM(get_wallet_id_or_throw()); -} - -td::Result HighloadWalletV2::get_wallet_id_or_throw() const { - if (state_.data.is_null()) { - return 0; - } - //FIXME use get method - auto cs = vm::load_cell_slice(state_.data); - return static_cast(cs.fetch_ulong(32)); + return TRY_VM([&]() -> td::Result { + if (state_.data.is_null()) { + return 0; + } + auto cs = vm::load_cell_slice(state_.data); + return static_cast(cs.fetch_ulong(32)); + }()); } td::Result HighloadWalletV2::get_public_key() const { - return TRY_VM(get_public_key_or_throw()); -} - -td::Result HighloadWalletV2::get_public_key_or_throw() const { - if (state_.data.is_null()) { - return td::Status::Error("data is null"); - } - //FIXME use get method - auto cs = vm::load_cell_slice(state_.data); - cs.skip_first(96); - td::SecureString res(td::Ed25519::PublicKey::LENGTH); - cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast(res.size())); - return td::Ed25519::PublicKey(std::move(res)); + return TRY_VM([&]() -> td::Result { + if (state_.data.is_null()) { + return td::Status::Error("data is null"); + } + auto cs = vm::load_cell_slice(state_.data); + cs.skip_first(96); + td::SecureString res(td::Ed25519::PublicKey::LENGTH); + cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast(res.size())); + return td::Ed25519::PublicKey(std::move(res)); + }()); } } // namespace ton diff --git a/crypto/smc-envelope/HighloadWalletV2.h b/crypto/smc-envelope/HighloadWalletV2.h index 975cf644c..6b5e6dba2 100644 --- a/crypto/smc-envelope/HighloadWalletV2.h +++ b/crypto/smc-envelope/HighloadWalletV2.h @@ -26,41 +26,26 @@ #include "vm/cells/CellString.h" namespace ton { -class HighloadWalletV2 : public ton::SmartContract, public WalletInterface { - public: - explicit HighloadWalletV2(State state) : ton::SmartContract(std::move(state)) { - } +struct HighloadWalletV2Traits { + using InitData = WalletInterface::DefaultInitData; + static constexpr unsigned max_message_size = vm::CellString::max_bytes; static constexpr unsigned max_gifts_size = 254; - static td::Ref get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, - td::int32 revision) noexcept; - static td::Ref get_init_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, - td::uint32 valid_until) noexcept; - - static td::Ref make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, - td::uint32 valid_until, td::Span gifts) noexcept; - - static td::Ref get_init_code(td::int32 revision) noexcept; - static vm::CellHash get_init_code_hash() noexcept; - static td::Ref get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) noexcept; - static td::optional guess_revision(const vm::Cell::Hash& code_hash); - static td::optional guess_revision(const block::StdAddress& address, - const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id); - - td::Result get_wallet_id() const; - - td::Result> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, - td::Span gifts) const override { - TRY_RESULT(wallet_id, get_wallet_id()); - return make_a_gift_message(private_key, wallet_id, valid_until, gifts); - } - size_t get_max_gifts_size() const override { - return max_gifts_size; + static constexpr auto code_type = SmartContractCode::HighloadWalletV2; +}; +class HighloadWalletV2 : public WalletBase { + public: + explicit HighloadWalletV2(State state) : WalletBase(std::move(state)) { } + td::Result> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, + td::Span gifts) const override; + static td::Ref get_init_data(const InitData& init_data) noexcept; + td::Result> get_init_message(const td::Ed25519::PrivateKey& private_key, + td::uint32 valid_until = std::numeric_limits::max()) const + noexcept; + + // can't use get methods for compatibility with old revisions + td::Result get_wallet_id() const override; td::Result get_public_key() const override; - - private: - td::Result get_wallet_id_or_throw() const; - td::Result get_public_key_or_throw() const; }; } // namespace ton diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 79a2a8239..f3485be9b 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -30,6 +30,14 @@ #include "td/utils/crypto.h" namespace ton { +int SmartContract::Answer::output_actions_count(td::Ref list) { + int i = -1; + do { + ++i; + list = load_cell_slice(std::move(list)).prefetch_ref(); + } while (list.not_null()); + return i; +} namespace { td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref body) { @@ -66,15 +74,6 @@ td::Ref prepare_vm_c7(td::uint32 now, td::uint64 balance) { return vm::make_tuple_ref(std::move(tuple)); } -static int output_actions_count(td::Ref list) { - int i = -1; - do { - ++i; - list = load_cell_slice(std::move(list)).prefetch_ref(); - } while (list.not_null()); - return i; -} - SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, vm::GasLimits gas, bool ignore_chksig) { auto gas_credit = gas.gas_credit; @@ -133,7 +132,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref actions; td::int32 code; td::int64 gas_used; + static int output_actions_count(td::Ref list); }; struct Args { diff --git a/crypto/smc-envelope/SmartContractCode.cpp b/crypto/smc-envelope/SmartContractCode.cpp index 5ae940a1f..d10c4b5c8 100644 --- a/crypto/smc-envelope/SmartContractCode.cpp +++ b/crypto/smc-envelope/SmartContractCode.cpp @@ -38,8 +38,6 @@ const auto& get_map() { map[name] = vm::std_boc_deserialize(td::base64_decode(code_str).move_as_ok()).move_as_ok(); }; #include "smartcont/auto/multisig-code.cpp" -#include "smartcont/auto/simple-wallet-ext-code.cpp" -#include "smartcont/auto/simple-wallet-code.cpp" #include "smartcont/auto/wallet-code.cpp" #include "smartcont/auto/highload-wallet-code.cpp" #include "smartcont/auto/highload-wallet-v2-code.cpp" @@ -65,18 +63,6 @@ const auto& get_map() { "QE0VNggED0Dm+hMfJgUXO68qIH+QFUEIf5EPKjAvQE0fgAf44WIYAQ9HhvpSCYAtMH1DAB+wCRMuIBs+" "ZbgyWhyEA0gED0Q4rmMcgSyx8Tyz/L//QAye1UCAAE0DACASAGBwAXvZznaiaGmvmOuF/8AEG+X5dqJoaY+Y6Z/p/" "5j6AmipEEAgegc30JjJLb/JXdHxQANCCAQPSWb6UyURCUMFMDud4gkzM2AZIyMOKz"); - with_tvm_code("simple-wallet-r1", - "te6ccgEEAQEAAAAAUwAAov8AIN0gggFMl7qXMO1E0NcLH+Ck8mCBAgDXGCDXCx/tRNDTH9P/" - "0VESuvKhIvkBVBBE+RDyovgAAdMfMSDXSpbTB9QC+wDe0aTIyx/L/8ntVA=="); - with_tvm_code("simple-wallet-r2", - "te6ccgEBAQEAXwAAuv8AIN0gggFMl7ohggEznLqxnHGw7UTQ0x/XC//jBOCk8mCBAgDXGCDXCx/tRNDTH9P/" - "0VESuvKhIvkBVBBE+RDyovgAAdMfMSDXSpbTB9QC+wDe0aTIyx/L/8ntVA=="); - with_tvm_code("wallet-r1", - "te6ccgEBAQEAVwAAqv8AIN0gggFMl7qXMO1E0NcLH+Ck8mCDCNcYINMf0x8B+CO78mPtRNDTH9P/0VExuvKhA/" - "kBVBBC+RDyovgAApMg10qW0wfUAvsA6NGkyMsfy//J7VQ="); - with_tvm_code("wallet-r2", - "te6ccgEBAQEAYwAAwv8AIN0gggFMl7ohggEznLqxnHGw7UTQ0x/XC//jBOCk8mCDCNcYINMf0x8B+CO78mPtRNDTH9P/" - "0VExuvKhA/kBVBBC+RDyovgAApMg10qW0wfUAvsA6NGkyMsfy//J7VQ="); with_tvm_code("wallet3-r1", "te6ccgEBAQEAYgAAwP8AIN0gggFMl7qXMO1E0NcLH+Ck8mCDCNcYINMf0x/TH/gjE7vyY+1E0NMf0x/T/" "9FRMrryoVFEuvKiBPkBVBBV+RDyo/gAkyDXSpbTB9QC+wDo0QGkyMsfyx/L/8ntVA=="); @@ -123,22 +109,10 @@ td::Result> SmartContractCode::load(td::Slice name) { td::Span SmartContractCode::get_revisions(Type type) { switch (type) { - case Type::WalletV1: { - static int res[] = {1, 2}; - return res; - } - case Type::WalletV2: { - static int res[] = {1, 2}; - return res; - } case Type::WalletV3: { static int res[] = {1, 2}; return res; } - case Type::WalletV1Ext: { - static int res[] = {-1}; - return res; - } case Type::HighloadWalletV1: { static int res[] = {-1, 1, 2}; return res; @@ -191,14 +165,8 @@ td::Ref SmartContractCode::get_code(Type type, int ext_revision) { auto revision = validate_revision(type, ext_revision).move_as_ok(); auto basename = [](Type type) -> td::Slice { switch (type) { - case Type::WalletV1: - return "simple-wallet"; - case Type::WalletV2: - return "wallet"; case Type::WalletV3: return "wallet3"; - case Type::WalletV1Ext: - return "simple-wallet-ext"; case Type::HighloadWalletV1: return "highload-wallet"; case Type::HighloadWalletV2: diff --git a/crypto/smc-envelope/SmartContractCode.h b/crypto/smc-envelope/SmartContractCode.h index e9ca1c543..85be35318 100644 --- a/crypto/smc-envelope/SmartContractCode.h +++ b/crypto/smc-envelope/SmartContractCode.h @@ -26,18 +26,7 @@ class SmartContractCode { public: static td::Result> load(td::Slice name); - enum Type { - WalletV1 = 1, - WalletV1Ext, - WalletV2, - WalletV3, - HighloadWalletV1, - HighloadWalletV2, - ManualDns, - Multisig, - PaymentChannel, - RestrictedWallet - }; + enum Type { WalletV3 = 4, HighloadWalletV1, HighloadWalletV2, ManualDns, Multisig, PaymentChannel, RestrictedWallet }; static td::Span get_revisions(Type type); static td::Result validate_revision(Type type, int revision); static td::Ref get_code(Type type, int revision = 0); diff --git a/crypto/smc-envelope/WalletInterface.cpp b/crypto/smc-envelope/WalletInterface.cpp new file mode 100644 index 000000000..4c2feca91 --- /dev/null +++ b/crypto/smc-envelope/WalletInterface.cpp @@ -0,0 +1,77 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "WalletInterface.h" + +namespace ton { +td::Result WalletInterface::get_balance(td::uint64 account_balance, td::uint32 now) const { + return TRY_VM([&]() -> td::Result { + Answer answer = this->run_get_method(Args().set_method_id("balance").set_balance(account_balance).set_now(now)); + if (!answer.success) { + return td::Status::Error("balance get method failed"); + } + return static_cast(answer.stack.write().pop_long()); + }()); +} + +td::Result WalletInterface::get_public_key() const { + return GenericAccount::get_public_key(*this); +}; + +td::Result WalletInterface::get_seqno() const { + return GenericAccount::get_seqno(*this); +} + +td::Result WalletInterface::get_wallet_id() const { + return GenericAccount::get_wallet_id(*this); +} + +td::Result> WalletInterface::get_init_message(const td::Ed25519::PrivateKey &private_key, + td::uint32 valid_until) const { + return make_a_gift_message(private_key, valid_until, {}); +} + +td::Ref WalletInterface::create_int_message(const Gift &gift) { + vm::CellBuilder cbi; + GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms); + if (gift.init_state.not_null()) { + cbi.store_ones(2); + cbi.store_ref(gift.init_state); + } else { + cbi.store_zeroes(1); + } + cbi.store_zeroes(1); + store_gift_message(cbi, gift); + return cbi.finalize(); +} +void WalletInterface::store_gift_message(vm::CellBuilder &cb, const Gift &gift) { + if (gift.body.not_null()) { + auto body = vm::load_cell_slice(gift.body); + //TODO: handle error + CHECK(cb.append_cellslice_bool(body)); + return; + } + + if (gift.is_encrypted) { + cb.store_long(1, 32); + } else { + cb.store_long(0, 32); + } + vm::CellString::store(cb, gift.message, 35 * 8).ensure(); +} +} // namespace ton diff --git a/crypto/smc-envelope/WalletInterface.h b/crypto/smc-envelope/WalletInterface.h index d4113a6f6..296cca0e7 100644 --- a/crypto/smc-envelope/WalletInterface.h +++ b/crypto/smc-envelope/WalletInterface.h @@ -21,13 +21,17 @@ #include "td/utils/common.h" #include "Ed25519.h" #include "block/block.h" +#include "block/block-parse.h" #include "vm/cells/CellString.h" #include "SmartContract.h" +#include "SmartContractCode.h" #include "GenericAccount.h" +#include + namespace ton { -class WalletInterface { +class WalletInterface : public SmartContract { public: struct Gift { block::StdAddress destination; @@ -39,49 +43,91 @@ class WalletInterface { td::Ref body; td::Ref init_state; }; + struct DefaultInitData { + td::SecureString public_key; + td::uint32 wallet_id{0}; + td::uint32 seqno{0}; + DefaultInitData() = default; + DefaultInitData(td::Slice key, td::uint32 wallet_id) : public_key(key), wallet_id(wallet_id) { + } + }; + + WalletInterface(State state) : SmartContract(std::move(state)) { + } virtual ~WalletInterface() { } virtual size_t get_max_gifts_size() const = 0; + virtual size_t get_max_message_size() const = 0; virtual td::Result> make_a_gift_message(const td::Ed25519::PrivateKey &private_key, td::uint32 valid_until, td::Span gifts) const = 0; - virtual td::Result get_public_key() const { - return td::Status::Error("Unsupported"); + + virtual td::Result get_seqno() const; + virtual td::Result get_wallet_id() const; + virtual td::Result get_balance(td::uint64 account_balance, td::uint32 now) const; + virtual td::Result get_public_key() const; + + td::Result> get_init_message(const td::Ed25519::PrivateKey &private_key, + td::uint32 valid_until = std::numeric_limits::max()) const; + + static td::Ref create_int_message(const Gift &gift); + static void store_gift_message(vm::CellBuilder &cb, const Gift &gift); +}; + +template +class WalletBase : public WalletInterface { + public: + using Traits = TraitsT; + using InitData = typename Traits::InitData; + + explicit WalletBase(State state) : WalletInterface(std::move(state)) { } - td::Result> get_init_message( - const td::Ed25519::PrivateKey &private_key, - td::uint32 valid_until = std::numeric_limits::max()) const { - return make_a_gift_message(private_key, valid_until, {}); + size_t get_max_gifts_size() const override { + return Traits::max_gifts_size; } - static td::Ref create_int_message(const Gift &gift) { - vm::CellBuilder cbi; - GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms); - if (gift.init_state.not_null()) { - cbi.store_ones(2); - cbi.store_ref(gift.init_state); - } else { - cbi.store_zeroes(1); - } - cbi.store_zeroes(1); - store_gift_message(cbi, gift); - return cbi.finalize(); + size_t get_max_message_size() const override { + return Traits::max_message_size; } - static void store_gift_message(vm::CellBuilder &cb, const Gift &gift) { - if (gift.body.not_null()) { - auto body = vm::load_cell_slice(gift.body); - //TODO: handle error - CHECK(cb.append_cellslice_bool(body)); - return; - } - if (gift.is_encrypted) { - cb.store_long(1, 32); - } else { - cb.store_long(0, 32); + static td::Ref create(State state) { + return td::Ref(true, std::move(state)); + } + static td::Ref get_init_code(int revision) { + return SmartContractCode::get_code(get_code_type(), revision); + }; + static State get_init_state(int revision, const InitData &init_data) { + return {get_init_code(revision), WalletT::get_init_data(init_data)}; + } + static SmartContractCode::Type get_code_type() { + return Traits::code_type; + } + static td::optional guess_revision(const vm::Cell::Hash &code_hash) { + for (auto revision : ton::SmartContractCode::get_revisions(get_code_type())) { + auto code = get_init_code(revision); + if (code->get_hash() == code_hash) { + return revision; + } } - vm::CellString::store(cb, gift.message, 35 * 8).ensure(); + return {}; + } + static td::Span get_revisions() { + return ton::SmartContractCode::get_revisions(get_code_type()); + } + static td::optional guess_revision(block::StdAddress &address, const InitData &init_data) { + for (auto revision : get_revisions()) { + if (WalletT(get_init_state(revision, init_data)).get_address(address.workchain) == address) { + return revision; + } + } + return {}; + } + static td::Ref create(const InitData &init_data, int revision) { + return td::Ref(true, State{get_init_code(revision), WalletT::get_init_data(init_data)}); + } + CntObject *make_copy() const override { + return new WalletT(get_state()); } }; diff --git a/crypto/smc-envelope/WalletV3.cpp b/crypto/smc-envelope/WalletV3.cpp index 5bd7cfcd0..f027f8bc8 100644 --- a/crypto/smc-envelope/WalletV3.cpp +++ b/crypto/smc-envelope/WalletV3.cpp @@ -27,36 +27,11 @@ #include namespace ton { -td::Ref WalletV3::get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, - td::int32 revision) noexcept { - auto code = get_init_code(revision); - auto data = get_init_data(public_key, wallet_id); - return GenericAccount::get_init_state(std::move(code), std::move(data)); -} - -td::optional WalletV3::guess_revision(const vm::Cell::Hash& code_hash) { - for (td::int32 i = 1; i <= 2; i++) { - if (get_init_code(i)->get_hash() == code_hash) { - return i; - } - } - return {}; -} -td::optional WalletV3::guess_revision(const block::StdAddress& address, - const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id) { - for (td::int32 i = 1; i <= 2; i++) { - if (GenericAccount::get_address(address.workchain, get_init_state(public_key, wallet_id, i)) == address) { - return i; - } - } - return {}; -} - -td::Ref WalletV3::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, - td::uint32 seqno, td::uint32 valid_until, - td::Span gifts) noexcept { - CHECK(gifts.size() <= max_gifts_size); - +td::Result> WalletV3::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, + td::uint32 valid_until, td::Span gifts) const { + CHECK(gifts.size() <= get_max_gifts_size()); + TRY_RESULT(seqno, get_seqno()); + TRY_RESULT(wallet_id, get_wallet_id()); vm::CellBuilder cb; cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32); @@ -73,63 +48,36 @@ td::Ref WalletV3::make_a_gift_message(const td::Ed25519::PrivateKey& p return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); } -td::Ref WalletV3::get_init_code(td::int32 revision) noexcept { - return SmartContractCode::get_code(ton::SmartContractCode::WalletV3, revision); -} - -vm::CellHash WalletV3::get_init_code_hash() noexcept { - return get_init_code()->get_hash(); -} - -td::Ref WalletV3::get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, - td::uint32 seqno) noexcept { +td::Ref WalletV3::get_init_data(const InitData& init_data) noexcept { return vm::CellBuilder() - .store_long(seqno, 32) - .store_long(wallet_id, 32) - .store_bytes(public_key.as_octet_string()) + .store_long(init_data.seqno, 32) + .store_long(init_data.wallet_id, 32) + .store_bytes(init_data.public_key) .finalize(); } -td::Result WalletV3::get_seqno() const { - return TRY_VM(get_seqno_or_throw()); -} - -td::Result WalletV3::get_seqno_or_throw() const { - if (state_.data.is_null()) { - return 0; - } - //FIXME use get method - return static_cast(vm::load_cell_slice(state_.data).fetch_ulong(32)); -} - td::Result WalletV3::get_wallet_id() const { - return TRY_VM(get_wallet_id_or_throw()); -} - -td::Result WalletV3::get_wallet_id_or_throw() const { - if (state_.data.is_null()) { - return 0; - } - //FIXME use get method - auto cs = vm::load_cell_slice(state_.data); - cs.skip_first(32); - return static_cast(cs.fetch_ulong(32)); + return TRY_VM([&]() -> td::Result { + if (state_.data.is_null()) { + return 0; + } + auto cs = vm::load_cell_slice(state_.data); + cs.skip_first(32); + return static_cast(cs.fetch_ulong(32)); + }()); } td::Result WalletV3::get_public_key() const { - return TRY_VM(get_public_key_or_throw()); -} - -td::Result WalletV3::get_public_key_or_throw() const { - if (state_.data.is_null()) { - return td::Status::Error("data is null"); - } - //FIXME use get method - auto cs = vm::load_cell_slice(state_.data); - cs.skip_first(64); - td::SecureString res(td::Ed25519::PublicKey::LENGTH); - cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast(res.size())); - return td::Ed25519::PublicKey(std::move(res)); + return TRY_VM([&]() -> td::Result { + if (state_.data.is_null()) { + return td::Status::Error("data is null"); + } + auto cs = vm::load_cell_slice(state_.data); + cs.skip_first(64); + td::SecureString res(td::Ed25519::PublicKey::LENGTH); + cs.fetch_bytes(res.as_mutable_slice().ubegin(), td::narrow_cast(res.size())); + return td::Ed25519::PublicKey(std::move(res)); + }()); } } // namespace ton diff --git a/crypto/smc-envelope/WalletV3.h b/crypto/smc-envelope/WalletV3.h index b3e332dda..926e7ffbd 100644 --- a/crypto/smc-envelope/WalletV3.h +++ b/crypto/smc-envelope/WalletV3.h @@ -26,135 +26,30 @@ #include "vm/cells/CellString.h" namespace ton { -class WalletV3 : public ton::SmartContract, public WalletInterface { - public: - explicit WalletV3(State state) : ton::SmartContract(std::move(state)) { - } - explicit WalletV3(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, td::uint32 seqno = 0) - : WalletV3(State{get_init_code(), get_init_data(public_key, wallet_id, seqno)}) { - } - static constexpr unsigned max_message_size = vm::CellString::max_bytes; - static constexpr unsigned max_gifts_size = 4; - - static td::optional guess_revision(const vm::Cell::Hash& code_hash); - static td::optional guess_revision(const block::StdAddress& address, - const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id); - static td::Ref get_init_state(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, - td::int32 revision = 0) noexcept; - static td::Ref make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 wallet_id, - td::uint32 seqno, td::uint32 valid_until, td::Span gifts) noexcept; - static td::Ref get_init_code(td::int32 revision = 0) noexcept; - static vm::CellHash get_init_code_hash() noexcept; - static td::Ref get_init_data(const td::Ed25519::PublicKey& public_key, td::uint32 wallet_id, - td::uint32 seqno = 0) noexcept; +struct WalletV3Traits { + using InitData = WalletInterface::DefaultInitData; - td::Result get_seqno() const; - td::Result get_wallet_id() const; + static constexpr unsigned max_message_size = vm::CellString::max_bytes; + static constexpr unsigned max_gifts_size = 4; + static constexpr auto code_type = SmartContractCode::WalletV3; +}; - using WalletInterface::get_init_message; - td::Result> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, - td::Span gifts) const override { - TRY_RESULT(seqno, get_seqno()); - TRY_RESULT(wallet_id, get_wallet_id()); - return make_a_gift_message(private_key, wallet_id, seqno, valid_until, gifts); - } - size_t get_max_gifts_size() const override { - return max_gifts_size; +class WalletV3 : public WalletBase { + public: + explicit WalletV3(State state) : WalletBase(std::move(state)) { } - td::Result get_public_key() const override; + td::Result> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, + td::Span gifts) const override; + static td::Ref get_init_data(const InitData& init_data) noexcept; - private: - td::Result get_seqno_or_throw() const; - td::Result get_wallet_id_or_throw() const; - td::Result get_public_key_or_throw() const; + // can't use get methods for compatibility with old revisions + td::Result get_wallet_id() const override; + td::Result get_public_key() const override; }; } // namespace ton -#include "smc-envelope/SmartContractCode.h" -#include "smc-envelope/GenericAccount.h" -#include "block/block-parse.h" -#include namespace ton { -template -class WalletBase : public SmartContract, public WalletInterface { - public: - using Traits = TraitsT; - using InitData = typename Traits::InitData; - - explicit WalletBase(State state) : SmartContract(std::move(state)) { - } - static td::Ref create(State state) { - return td::Ref(true, std::move(state)); - } - static td::Ref get_init_code(int revision) { - return SmartContractCode::get_code(get_code_type(), revision); - }; - size_t get_max_gifts_size() const override { - return Traits::max_gifts_size; - } - static SmartContractCode::Type get_code_type() { - return Traits::code_type; - } - static td::optional guess_revision(const vm::Cell::Hash& code_hash) { - for (auto i : ton::SmartContractCode::get_revisions(get_code_type())) { - auto code = SmartContractCode::get_code(get_code_type(), i); - if (code->get_hash() == code_hash) { - return i; - } - } - return {}; - } - - static td::Ref create(const InitData& init_data, int revision) { - return td::Ref(true, State{get_init_code(revision), WalletT::get_init_data(init_data)}); - } - - td::Result get_seqno() const { - return TRY_VM([&]() -> td::Result { - Answer answer = this->run_get_method("seqno"); - if (!answer.success) { - return td::Status::Error("seqno get method failed"); - } - return static_cast(answer.stack.write().pop_long_range(std::numeric_limits::max())); - }()); - } - td::Result get_wallet_id() const { - return TRY_VM([&]() -> td::Result { - Answer answer = this->run_get_method("wallet_id"); - if (!answer.success) { - return td::Status::Error("seqno get method failed"); - } - return static_cast(answer.stack.write().pop_long_range(std::numeric_limits::max())); - }()); - } - - td::Result get_balance(td::uint64 account_balance, td::uint32 now) const { - return TRY_VM([&]() -> td::Result { - Answer answer = this->run_get_method(Args().set_method_id("balance").set_balance(account_balance).set_now(now)); - if (!answer.success) { - return td::Status::Error("balance get method failed"); - } - return static_cast(answer.stack.write().pop_long()); - }()); - } - - td::Result get_public_key() const override { - return TRY_VM([&]() -> td::Result { - Answer answer = this->run_get_method("get_public_key"); - if (!answer.success) { - return td::Status::Error("get_public_key get method failed"); - } - auto key_int = answer.stack.write().pop_int(); - LOG(ERROR) << key_int->bit_size(false); - td::SecureString bytes(32); - if (!key_int->export_bytes(bytes.as_mutable_slice().ubegin(), bytes.size(), false)) { - return td::Status::Error("not a public key"); - } - return td::Ed25519::PublicKey(std::move(bytes)); - }()); - }; -}; struct RestrictedWalletTraits { struct InitData { diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 361c99476..413d774fa 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -20,9 +20,9 @@ #include "vm/cellslice.h" #include "vm/cells.h" #include "common/AtomicRef.h" +#include "vm/cells/CellString.h" #include "vm/cells/MerkleProof.h" #include "vm/cells/MerkleUpdate.h" -#include "vm/db/BlobView.h" #include "vm/db/CellStorage.h" #include "vm/db/CellHashTable.h" #include "vm/db/TonDb.h" @@ -33,6 +33,7 @@ #include "td/utils/crypto.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" +#include "td/utils/Span.h" #include "td/utils/Status.h" #include "td/utils/Timer.h" #include "td/utils/filesystem.h" @@ -44,8 +45,12 @@ #include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" +#include "td/db/utils/BlobView.h" #include "td/db/RocksDb.h" #include "td/db/MemoryKeyValue.h" +#include "td/db/utils/CyclicBuffer.h" + +#include "td/fec/fec.h" #include #include @@ -433,17 +438,10 @@ class RandomBagOfCells { }; }; -template -void random_shuffle(td::MutableSpan v, td::Random::Xorshift128plus &rnd) { - for (std::size_t i = 1; i < v.size(); i++) { - auto pos = static_cast(rnd() % (i + 1)); - std::swap(v[i], v[pos]); - } -} Ref gen_random_cell(int size, td::Random::Xorshift128plus &rnd, bool with_prunned_branches = true, std::vector> cells = {}) { if (!cells.empty()) { - random_shuffle(td::MutableSpan>(cells), rnd); + td::random_shuffle(as_mutable_span(cells), rnd); cells.resize(cells.size() % rnd()); } return RandomBagOfCells(size, rnd, with_prunned_branches, std::move(cells)).get_root(); @@ -451,7 +449,7 @@ Ref gen_random_cell(int size, td::Random::Xorshift128plus &rnd, bool with_ std::vector> gen_random_cells(int roots, int size, td::Random::Xorshift128plus &rnd, bool with_prunned_branches = true, std::vector> cells = {}) { if (!cells.empty()) { - random_shuffle(td::MutableSpan>(cells), rnd); + td::random_shuffle(as_mutable_span(cells), rnd); cells.resize(cells.size() % rnd()); } return RandomBagOfCells(size, rnd, with_prunned_branches, std::move(cells)).get_random_roots(roots, rnd); @@ -788,7 +786,7 @@ TEST(TonDb, DynamicBoc) { old_root_serialization = serialize_boc(cell); // Check that DynamicBagOfCells properly loads cells - cell = vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(td::BufferSlice(old_root_serialization))) + cell = vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(td::BufferSlice(old_root_serialization))) .move_as_ok() ->get_root_cell(0) .move_as_ok(); @@ -1599,11 +1597,11 @@ class BenchBocDeserializer : public td::Benchmark { auto blob = [&] { switch (config_.blob_type) { case BenchBocDeserializerConfig::File: - return vm::FileBlobView::create("serialization").move_as_ok(); + return td::FileBlobView::create("serialization").move_as_ok(); case BenchBocDeserializerConfig::Memory: - return vm::BufferSliceBlobView::create(serialization_.clone()); + return td::BufferSliceBlobView::create(serialization_.clone()); case BenchBocDeserializerConfig::FileMemoryMap: - return vm::FileMemoryMappingBlobView::create("serialization").move_as_ok(); + return td::FileMemoryMappingBlobView::create("serialization").move_as_ok(); default: UNREACHABLE(); } @@ -2083,222 +2081,6 @@ TEST(Ref, AtomicRef) { LOG(ERROR) << String::total_strings.sum(); } -class FileMerkleTree { - public: - FileMerkleTree(size_t chunks_count, td::Ref root = {}) { - log_n_ = 0; - while ((size_t(1) << log_n_) < chunks_count) { - log_n_++; - } - n_ = size_t(1) << log_n_; - mark_.resize(n_ * 2); - proof_.resize(n_ * 2); - - CHECK(n_ == chunks_count); // TODO: support other chunks_count - //auto x = vm::CellBuilder().finalize(); - root_ = std::move(root); - } - - struct Chunk { - td::size_t index{0}; - td::Slice hash; - }; - - void remove_chunk(td::size_t index) { - CHECK(index < n_); - index += n_; - while (proof_[index].not_null()) { - proof_[index] = {}; - index /= 2; - } - } - - bool has_chunk(td::size_t index) const { - CHECK(index < n_); - index += n_; - return proof_[index].not_null(); - } - - void add_chunk(td::size_t index, td::Slice hash) { - CHECK(hash.size() == 32); - CHECK(index < n_); - index += n_; - auto cell = vm::CellBuilder().store_bytes(hash).finalize(); - CHECK(proof_[index].is_null()); - proof_[index] = std::move(cell); - for (index /= 2; index != 0; index /= 2) { - CHECK(proof_[index].is_null()); - auto &left = proof_[index * 2]; - auto &right = proof_[index * 2 + 1]; - if (left.not_null() && right.not_null()) { - proof_[index] = vm::CellBuilder().store_ref(left).store_ref(right).finalize(); - } else { - mark_[index] = mark_id_; - } - } - } - - td::Status validate_proof(td::Ref new_root) { - // TODO: check structure - return td::Status::OK(); - } - - td::Status add_proof(td::Ref new_root) { - TRY_STATUS(validate_proof(new_root)); - auto combined = vm::MerkleProof::combine_fast_raw(root_, new_root); - if (combined.is_null()) { - return td::Status::Error("Can't combine proofs"); - } - root_ = std::move(combined); - return td::Status::OK(); - } - - td::Status try_add_chunks(td::Span chunks) { - for (auto chunk : chunks) { - if (has_chunk(chunk.index)) { - return td::Status::Error("Already has chunk"); - } - } - mark_id_++; - for (auto chunk : chunks) { - add_chunk(chunk.index, chunk.hash); - } - auto r_new_root = merge(root_, 1); - if (r_new_root.is_error()) { - for (auto chunk : chunks) { - remove_chunk(chunk.index); - } - return r_new_root.move_as_error(); - } - root_ = r_new_root.move_as_ok(); - return td::Status::OK(); - } - - td::Result> merge(td::Ref root, size_t index) { - const auto &down = proof_[index]; - if (down.not_null()) { - if (down->get_hash() != root->get_hash(0)) { - return td::Status::Error("Hash mismatch"); - } - return down; - } - - if (mark_[index] != mark_id_) { - return root; - } - - vm::CellSlice cs(vm::NoVm(), root); - if (cs.is_special()) { - return td::Status::Error("Proof is not enough to validate chunks"); - } - - CHECK(cs.size_refs() == 2); - vm::CellBuilder cb; - cb.store_bits(cs.fetch_bits(cs.size())); - TRY_RESULT(left, merge(cs.fetch_ref(), index * 2)); - TRY_RESULT(right, merge(cs.fetch_ref(), index * 2 + 1)); - cb.store_ref(std::move(left)).store_ref(std::move(right)); - return cb.finalize(); - } - - void init_proof() { - CHECK(proof_[1].not_null()); - root_ = proof_[1]; - } - - td::Result> gen_proof(size_t l, size_t r) { - auto usage_tree = std::make_shared(); - auto usage_cell = vm::UsageCell::create(root_, usage_tree->root_ptr()); - TRY_STATUS(do_gen_proof(std::move(usage_cell), 0, n_ - 1, l, r)); - auto res = vm::MerkleProof::generate_raw(root_, usage_tree.get()); - CHECK(res.not_null()); - return res; - } - - private: - td::size_t n_; // n = 2^log_n - td::size_t log_n_; - td::size_t mark_id_{0}; - std::vector mark_; // n_ * 2 - std::vector> proof_; // n_ * 2 - td::Ref root_; - - td::Status do_gen_proof(td::Ref node, size_t il, size_t ir, size_t l, size_t r) { - if (ir < l || il > r) { - return td::Status::OK(); - } - if (l <= il && ir <= r) { - return td::Status::OK(); - } - vm::CellSlice cs(vm::NoVm(), std::move(node)); - if (cs.is_special()) { - return td::Status::Error("Can't generate a proof"); - } - CHECK(cs.size_refs() == 2); - auto ic = (il + ir) / 2; - TRY_STATUS(do_gen_proof(cs.fetch_ref(), il, ic, l, r)); - TRY_STATUS(do_gen_proof(cs.fetch_ref(), ic + 1, ir, l, r)); - return td::Status::OK(); - } -}; - -TEST(FileMerkleTree, Manual) { - // create big random file - size_t chunk_size = 768; - // for simplicity numer of chunks in a file is a power of two - size_t chunks_count = 1 << 16; - size_t file_size = chunk_size * chunks_count; - td::Timer timer; - LOG(INFO) << "Generate random string"; - const auto file = td::rand_string('a', 'z', td::narrow_cast(file_size)); - LOG(INFO) << timer; - - timer = {}; - LOG(INFO) << "Calculate all hashes"; - std::vector hashes(chunks_count); - for (size_t i = 0; i < chunks_count; i++) { - td::sha256(td::Slice(file).substr(i * chunk_size, chunk_size), hashes[i].as_slice()); - } - LOG(INFO) << timer; - - timer = {}; - LOG(INFO) << "Init merkle tree"; - FileMerkleTree tree(chunks_count); - for (size_t i = 0; i < chunks_count; i++) { - tree.add_chunk(i, hashes[i].as_slice()); - } - tree.init_proof(); - LOG(INFO) << timer; - - auto root_proof = tree.gen_proof(0, chunks_count - 1).move_as_ok(); - - // first download each chunk one by one - - for (size_t stride : {1 << 6, 1}) { - timer = {}; - LOG(INFO) << "Gen all proofs, stride = " << stride; - for (size_t i = 0; i < chunks_count; i += stride) { - tree.gen_proof(i, i + stride - 1).move_as_ok(); - } - LOG(INFO) << timer; - timer = {}; - LOG(INFO) << "Proof size: " << vm::std_boc_serialize(tree.gen_proof(0, stride - 1).move_as_ok()).ok().size(); - LOG(INFO) << "Download file, stride = " << stride; - { - FileMerkleTree new_tree(chunks_count, root_proof); - for (size_t i = 0; i < chunks_count; i += stride) { - new_tree.add_proof(tree.gen_proof(i, i + stride - 1).move_as_ok()).ensure(); - std::vector chunks; - for (size_t j = 0; j < stride; j++) { - chunks.push_back({i + j, hashes[i + j].as_slice()}); - } - new_tree.try_add_chunks(chunks).ensure(); - } - } - LOG(INFO) << timer; - } -} - //TEST(Tmp, Boc) { //LOG(ERROR) << "A"; //auto data = td::read_file("boc"); diff --git a/crypto/test/test-smartcont.cpp b/crypto/test/test-smartcont.cpp index fb82493e5..370a36e85 100644 --- a/crypto/test/test-smartcont.cpp +++ b/crypto/test/test-smartcont.cpp @@ -34,9 +34,6 @@ #include "smc-envelope/MultisigWallet.h" #include "smc-envelope/SmartContract.h" #include "smc-envelope/SmartContractCode.h" -#include "smc-envelope/TestGiver.h" -#include "smc-envelope/TestWallet.h" -#include "smc-envelope/Wallet.h" #include "smc-envelope/WalletV3.h" #include "smc-envelope/HighloadWallet.h" #include "smc-envelope/HighloadWalletV2.h" @@ -66,62 +63,6 @@ std::string load_source(std::string name) { return td::read_file_str(current_dir() + "../../crypto/" + name).move_as_ok(); } -td::Ref get_test_wallet_source() { - std::string code = R"ABCD( -SETCP0 DUP IFNOTRET // return if recv_internal -DUP 85143 INT EQUAL OVER 78748 INT EQUAL OR IFJMP:<{ // "seqno" and "get_public_key" get-methods - 1 INT AND c4 PUSHCTR CTOS 32 LDU 256 PLDU CONDSEL // cnt or pubk -}> -INC 32 THROWIF // fail unless recv_external -512 INT LDSLICEX DUP 32 PLDU // sign cs cnt -c4 PUSHCTR CTOS 32 LDU 256 LDU ENDS // sign cs cnt cnt' pubk -s1 s2 XCPU // sign cs cnt pubk cnt' cnt -EQUAL 33 THROWIFNOT // ( seqno mismatch? ) -s2 PUSH HASHSU // sign cs cnt pubk hash -s0 s4 s4 XC2PU // pubk cs cnt hash sign pubk -CHKSIGNU // pubk cs cnt ? -34 THROWIFNOT // signature mismatch -ACCEPT -SWAP 32 LDU NIP -DUP SREFS IF:<{ - // 3 INT 35 LSHIFT# 3 INT RAWRESERVE // reserve all but 103 Grams from the balance - 8 LDU LDREF // pubk cnt mode msg cs - s0 s2 XCHG SENDRAWMSG // pubk cnt cs ; ( message sent ) -}> -ENDS -INC NEWC 32 STU 256 STU ENDC c4 POPCTR -)ABCD"; - return fift::compile_asm(code).move_as_ok(); -} - -td::Ref get_wallet_source() { - std::string code = R"ABCD( -SETCP0 DUP IFNOTRET // return if recv_internal - DUP 85143 INT EQUAL OVER 78748 INT EQUAL OR IFJMP:<{ // "seqno" and "get_public_key" get-methods - 1 INT AND c4 PUSHCTR CTOS 32 LDU 256 PLDU CONDSEL // cnt or pubk - }> - INC 32 THROWIF // fail unless recv_external - 9 PUSHPOW2 LDSLICEX DUP 32 LDU 32 LDU // signature in_msg msg_seqno valid_until cs - SWAP NOW LEQ 35 THROWIF // signature in_msg msg_seqno cs - c4 PUSH CTOS 32 LDU 256 LDU ENDS // signature in_msg msg_seqno cs stored_seqno public_key - s3 s1 XCPU // signature in_msg public_key cs stored_seqno msg_seqno stored_seqno - EQUAL 33 THROWIFNOT // signature in_msg public_key cs stored_seqno - s0 s3 XCHG HASHSU // signature stored_seqno public_key cs hash - s0 s4 s2 XC2PU CHKSIGNU 34 THROWIFNOT // cs stored_seqno public_key - ACCEPT - s0 s2 XCHG // public_key stored_seqno cs - WHILE:<{ - DUP SREFS // public_key stored_seqno cs _40 - }>DO<{ // public_key stored_seqno cs - // 3 INT 35 LSHIFT# 3 INT RAWRESERVE // reserve all but 103 Grams from the balance - 8 LDU LDREF s0 s2 XCHG // public_key stored_seqno cs _45 mode - SENDRAWMSG // public_key stored_seqno cs - }> - ENDS INC // public_key seqno' - NEWC 32 STU 256 STU ENDC c4 POP -)ABCD"; - return fift::compile_asm(code).move_as_ok(); -} td::Ref get_wallet_v3_source() { std::string code = R"ABCD( SETCP0 DUP IFNOTRET // return if recv_internal @@ -150,140 +91,37 @@ SETCP0 DUP IFNOTRET // return if recv_internal return fift::compile_asm(code).move_as_ok(); } -TEST(Tonlib, TestWallet) { - LOG(ERROR) << td::base64_encode(std_boc_serialize(get_test_wallet_source()).move_as_ok()); - CHECK(get_test_wallet_source()->get_hash() == ton::TestWallet::get_init_code()->get_hash()); - auto fift_output = fift::mem_run_fift(load_source("smartcont/new-wallet.fif"), {"aba", "0"}).move_as_ok(); - - auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data; - auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet-query.boc").move_as_ok().data; - auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet.addr").move_as_ok().data; - - td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}}; - auto pub_key = priv_key.get_public_key().move_as_ok(); - auto init_state = ton::TestWallet::get_init_state(pub_key); - auto init_message = ton::TestWallet::get_init_message_new(priv_key); - auto address = ton::GenericAccount::get_address(0, init_state); - - CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); - - td::Ref res = ton::GenericAccount::create_ext_message(address, init_state, init_message); - - LOG(ERROR) << "-------"; - vm::load_cell_slice(res).print_rec(std::cerr); - LOG(ERROR) << "-------"; - vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr); - CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash()); - - fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/wallet.fif")).ensure(); - auto dest = block::StdAddress::parse("Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX").move_as_ok(); - fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup), - {"aba", "new-wallet", "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "123", - "321", "-C", "TEST"}) - .move_as_ok(); - auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; - ton::TestWallet::Gift gift; - gift.destination = dest; - gift.message = "TEST"; - gift.gramms = 321000000000ll; - ton::TestWallet wallet(priv_key.get_public_key().move_as_ok(), 123); - ASSERT_EQ(123u, wallet.get_seqno().ok()); - CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet.get_public_key().ok().as_octet_string()); - auto gift_message = ton::GenericAccount::create_ext_message( - address, {}, wallet.make_a_gift_message(priv_key, 0, {gift}).move_as_ok()); - LOG(ERROR) << "-------"; - vm::load_cell_slice(gift_message).print_rec(std::cerr); - LOG(ERROR) << "-------"; - vm::load_cell_slice(vm::std_boc_deserialize(wallet_query).move_as_ok()).print_rec(std::cerr); - CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash()); -} - -td::Ref get_wallet_source_fc() { - return fift::compile_asm(load_source("smartcont/wallet-code.fif"), "", false).move_as_ok(); -} - -TEST(Tonlib, Wallet) { - LOG(ERROR) << td::base64_encode(std_boc_serialize(get_wallet_source()).move_as_ok()); - CHECK(get_wallet_source()->get_hash() == ton::Wallet::get_init_code()->get_hash()); - - auto fift_output = fift::mem_run_fift(load_source("smartcont/new-wallet-v2.fif"), {"aba", "0"}).move_as_ok(); - - auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data; - auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet-query.boc").move_as_ok().data; - auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet.addr").move_as_ok().data; - - td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}}; - auto pub_key = priv_key.get_public_key().move_as_ok(); - auto init_state = ton::Wallet::get_init_state(pub_key); - auto init_message = ton::Wallet::get_init_message_new(priv_key); - auto address = ton::GenericAccount::get_address(0, init_state); - - CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); - - td::Ref res = ton::GenericAccount::create_ext_message(address, init_state, init_message); - - LOG(ERROR) << "-------"; - vm::load_cell_slice(res).print_rec(std::cerr); - LOG(ERROR) << "-------"; - vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr); - CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash()); - - fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/wallet-v2.fif")).ensure(); - class ZeroOsTime : public fift::OsTime { - public: - td::uint32 now() override { - return 0; - } - }; - fift_output.source_lookup.set_os_time(std::make_unique()); - auto dest = block::StdAddress::parse("Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX").move_as_ok(); - fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup), - {"aba", "new-wallet", "-C", "TESTv2", - "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "123", "321"}) - .move_as_ok(); - auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; - ton::TestWallet::Gift gift; - gift.destination = dest; - gift.message = "TESTv2"; - gift.gramms = 321000000000ll; - ton::Wallet wallet(priv_key.get_public_key().move_as_ok(), 123); - ASSERT_EQ(123u, wallet.get_seqno().ok()); - CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet.get_public_key().ok().as_octet_string()); - auto gift_message = ton::GenericAccount::create_ext_message( - address, {}, wallet.make_a_gift_message(priv_key, 60, {gift}).move_as_ok()); - LOG(ERROR) << "-------"; - vm::load_cell_slice(gift_message).print_rec(std::cerr); - LOG(ERROR) << "-------"; - vm::load_cell_slice(vm::std_boc_deserialize(wallet_query).move_as_ok()).print_rec(std::cerr); - CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash()); -} - TEST(Tonlib, WalletV3) { LOG(ERROR) << td::base64_encode(std_boc_serialize(get_wallet_v3_source()).move_as_ok()); - CHECK(get_wallet_v3_source()->get_hash() == ton::WalletV3::get_init_code()->get_hash()); + CHECK(get_wallet_v3_source()->get_hash() == ton::WalletV3::get_init_code(2)->get_hash()); auto fift_output = fift::mem_run_fift(load_source("smartcont/new-wallet-v3.fif"), {"aba", "0", "239"}).move_as_ok(); - auto new_wallet_pk = fift_output.source_lookup.read_file("new-wallet.pk").move_as_ok().data; auto new_wallet_query = fift_output.source_lookup.read_file("new-wallet-query.boc").move_as_ok().data; auto new_wallet_addr = fift_output.source_lookup.read_file("new-wallet.addr").move_as_ok().data; td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}}; auto pub_key = priv_key.get_public_key().move_as_ok(); - auto init_state = ton::WalletV3::get_init_state(pub_key, 239); - auto init_message = - ton::WalletV3(priv_key.get_public_key().move_as_ok(), 239).get_init_message(priv_key).move_as_ok(); - auto address = ton::GenericAccount::get_address(0, init_state); + ton::WalletV3::InitData init_data; + init_data.public_key = pub_key.as_octet_string(); + init_data.wallet_id = 239; + auto wallet = ton::WalletV3::create(init_data, 2); + ASSERT_EQ(239u, wallet->get_wallet_id().ok()); + ASSERT_EQ(0u, wallet->get_seqno().ok()); + auto address = wallet->get_address(); CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); - td::Ref res = ton::GenericAccount::create_ext_message(address, init_state, init_message); - + auto init_message = wallet->get_init_message(priv_key).move_as_ok(); + td::Ref ext_init_message = ton::GenericAccount::create_ext_message( + address, ton::GenericAccount::get_init_state(wallet->get_state()), init_message); LOG(ERROR) << "-------"; - vm::load_cell_slice(res).print_rec(std::cerr); + vm::load_cell_slice(ext_init_message).print_rec(std::cerr); LOG(ERROR) << "-------"; vm::load_cell_slice(vm::std_boc_deserialize(new_wallet_query).move_as_ok()).print_rec(std::cerr); - CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == res->get_hash()); + CHECK(vm::std_boc_deserialize(new_wallet_query).move_as_ok()->get_hash() == ext_init_message->get_hash()); + + CHECK(wallet.write().send_external_message(init_message).success); fift_output.source_lookup.write_file("/main.fif", load_source("smartcont/wallet-v3.fif")).ensure(); class ZeroOsTime : public fift::OsTime { @@ -296,7 +134,7 @@ TEST(Tonlib, WalletV3) { auto dest = block::StdAddress::parse("Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX").move_as_ok(); fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup), {"aba", "new-wallet", "-C", "TESTv3", - "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "239", "123", "321"}) + "Ef9Tj6fMJP+OqhAdhKXxq36DL+HYSzCc3+9O6UNzqsgPfYFX", "239", "1", "321"}) .move_as_ok(); auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; @@ -305,15 +143,14 @@ TEST(Tonlib, WalletV3) { gift.message = "TESTv3"; gift.gramms = 321000000000ll; - ton::WalletV3 wallet(priv_key.get_public_key().move_as_ok(), 239, 123); - ASSERT_EQ(239u, wallet.get_wallet_id().ok()); - ASSERT_EQ(123u, wallet.get_seqno().ok()); - CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet.get_public_key().ok().as_octet_string()); + ASSERT_EQ(239u, wallet->get_wallet_id().ok()); + ASSERT_EQ(1u, wallet->get_seqno().ok()); + CHECK(priv_key.get_public_key().ok().as_octet_string() == wallet->get_public_key().ok().as_octet_string()); CHECK(priv_key.get_public_key().ok().as_octet_string() == - ton::GenericAccount::get_public_key(wallet).ok().as_octet_string()); + ton::GenericAccount::get_public_key(*wallet).ok().as_octet_string()); auto gift_message = ton::GenericAccount::create_ext_message( - address, {}, wallet.make_a_gift_message(priv_key, 60, {gift}).move_as_ok()); + address, {}, wallet->make_a_gift_message(priv_key, 60, {gift}).move_as_ok()); LOG(ERROR) << "-------"; vm::load_cell_slice(gift_message).print_rec(std::cerr); LOG(ERROR) << "-------"; @@ -334,20 +171,21 @@ TEST(Tonlib, HighloadWallet) { td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}}; auto pub_key = priv_key.get_public_key().move_as_ok(); - auto init_state = ton::HighloadWallet::get_init_state(pub_key, 239, -1); - auto init_message = ton::HighloadWallet::get_init_message(priv_key, 239); - auto address = ton::GenericAccount::get_address(0, init_state); + ton::HighloadWallet::InitData init_data(pub_key.as_octet_string(), 239); - ton::HighloadWallet wallet( - {ton::HighloadWallet::get_init_code(-1), ton::HighloadWallet::get_init_data(pub_key, 239)}); - ASSERT_EQ(239u, wallet.get_wallet_id().ok()); - ASSERT_EQ(0u, wallet.get_seqno().ok()); - CHECK(pub_key.as_octet_string() == wallet.get_public_key().ok().as_octet_string()); - CHECK(pub_key.as_octet_string() == ton::GenericAccount::get_public_key(wallet).ok().as_octet_string()); + auto wallet = ton::HighloadWallet::create(init_data, -1); + auto address = wallet->get_address(); + CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); + ASSERT_EQ(239u, wallet->get_wallet_id().ok()); + ASSERT_EQ(0u, wallet->get_seqno().ok()); + CHECK(pub_key.as_octet_string() == wallet->get_public_key().ok().as_octet_string()); + CHECK(pub_key.as_octet_string() == ton::GenericAccount::get_public_key(*wallet).ok().as_octet_string()); CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); - td::Ref res = ton::GenericAccount::create_ext_message(address, init_state, init_message); + auto init_message = wallet->get_init_message(priv_key).move_as_ok(); + td::Ref res = ton::GenericAccount::create_ext_message( + address, ton::GenericAccount::get_init_state(wallet->get_state()), init_message); LOG(ERROR) << "---smc-envelope----"; vm::load_cell_slice(res).print_rec(std::cerr); @@ -382,12 +220,14 @@ TEST(Tonlib, HighloadWallet) { return 0; } }; + init_data.seqno = 123; + wallet = ton::HighloadWallet::create(init_data, -1); fift_output.source_lookup.set_os_time(std::make_unique()); fift_output = fift::mem_run_fift(std::move(fift_output.source_lookup), {"aba", "new-wallet", "239", "123", "order"}) .move_as_ok(); auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; auto gift_message = ton::GenericAccount::create_ext_message( - address, {}, ton::HighloadWallet::make_a_gift_message(priv_key, 239, 123, 60, gifts)); + address, {}, wallet->make_a_gift_message(priv_key, 60, gifts).move_as_ok()); LOG(ERROR) << "---smc-envelope----"; vm::load_cell_slice(gift_message).print_rec(std::cerr); LOG(ERROR) << "---fift scripts----"; @@ -416,19 +256,21 @@ TEST(Tonlib, HighloadWalletV2) { td::Ed25519::PrivateKey priv_key{td::SecureString{new_wallet_pk}}; auto pub_key = priv_key.get_public_key().move_as_ok(); - auto init_state = ton::HighloadWalletV2::get_init_state(pub_key, 239, -1); - auto init_message = ton::HighloadWalletV2::get_init_message(priv_key, 239, 65535); - auto address = ton::GenericAccount::get_address(0, init_state); + ton::HighloadWalletV2::InitData init_data(pub_key.as_octet_string(), 239); + + auto wallet = ton::HighloadWalletV2::create(init_data, -1); + auto address = wallet->get_address(); - ton::HighloadWalletV2 wallet( - {ton::HighloadWalletV2::get_init_code(-1), ton::HighloadWalletV2::get_init_data(pub_key, 239)}); - ASSERT_EQ(239u, wallet.get_wallet_id().ok()); - CHECK(pub_key.as_octet_string() == wallet.get_public_key().ok().as_octet_string()); - CHECK(pub_key.as_octet_string() == ton::GenericAccount::get_public_key(wallet).ok().as_octet_string()); + ASSERT_EQ(239u, wallet->get_wallet_id().ok()); + wallet->get_seqno().ensure_error(); + CHECK(pub_key.as_octet_string() == wallet->get_public_key().ok().as_octet_string()); + CHECK(pub_key.as_octet_string() == ton::GenericAccount::get_public_key(*wallet).ok().as_octet_string()); CHECK(address.addr.as_slice() == td::Slice(new_wallet_addr).substr(0, 32)); - td::Ref res = ton::GenericAccount::create_ext_message(address, init_state, init_message); + auto init_message = wallet->get_init_message(priv_key, 65535).move_as_ok(); + td::Ref res = ton::GenericAccount::create_ext_message( + address, ton::GenericAccount::get_init_state(wallet->get_state()), init_message); LOG(ERROR) << "---smc-envelope----"; vm::load_cell_slice(res).print_rec(std::cerr); @@ -462,7 +304,7 @@ TEST(Tonlib, HighloadWalletV2) { fift::mem_run_fift(std::move(fift_output.source_lookup), {"aba", "new-wallet", "239", "order"}).move_as_ok(); auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; auto gift_message = ton::GenericAccount::create_ext_message( - address, {}, ton::HighloadWalletV2::make_a_gift_message(priv_key, 239, 60, gifts)); + address, {}, wallet->make_a_gift_message(priv_key, 60, gifts).move_as_ok()); LOG(ERROR) << "---smc-envelope----"; vm::load_cell_slice(gift_message).print_rec(std::cerr); LOG(ERROR) << "---fift scripts----"; @@ -470,28 +312,6 @@ TEST(Tonlib, HighloadWalletV2) { CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == gift_message->get_hash()); } -TEST(Tonlib, TestGiver) { - auto address = - block::StdAddress::parse("-1:60c04141c6a7b96d68615e7a91d265ad0f3a9a922e9ae9c901d4fa83f5d3c0d0").move_as_ok(); - LOG(ERROR) << address.bounceable; - auto fift_output = fift::mem_run_fift(load_source("smartcont/testgiver.fif"), - {"aba", address.rserialize(), "0", "6.666", "wallet-query"}) - .move_as_ok(); - LOG(ERROR) << fift_output.output; - - auto wallet_query = fift_output.source_lookup.read_file("wallet-query.boc").move_as_ok().data; - - ton::TestGiver::Gift gift; - gift.gramms = 1000000000ll * 6666 / 1000; - gift.message = "GIFT"; - gift.destination = address; - td::Ed25519::PrivateKey key{td::SecureString()}; - auto res = ton::GenericAccount::create_ext_message(ton::TestGiver::address(), {}, - ton::TestGiver().make_a_gift_message(key, 0, {gift}).move_as_ok()); - vm::CellSlice(vm::NoVm(), res).print_rec(std::cerr); - CHECK(vm::std_boc_deserialize(wallet_query).move_as_ok()->get_hash() == res->get_hash()); -} - TEST(Tonlib, RestrictedWallet) { //auto source_lookup = fift::create_mem_source_lookup(load_source("smartcont/new-restricted-wallet2.fif")).move_as_ok(); //source_lookup @@ -585,74 +405,130 @@ TEST(Tonlib, RestrictedWallet3) { CHECK(wallet->get_seqno().move_as_ok() == 2); } -class SimpleWallet : public ton::SmartContract { +template +void check_wallet_seqno(td::Ref wallet, td::uint32 seqno) { + ASSERT_EQ(seqno, wallet->get_seqno().ok()); +} +void check_wallet_seqno(td::Ref wallet, td::uint32 seqno) { +} +void check_wallet_seqno(td::Ref wallet, td::uint32 seqno) { +} +template +void check_wallet_state(td::Ref wallet, td::uint32 seqno, td::uint32 wallet_id, td::Slice public_key) { + ASSERT_EQ(wallet_id, wallet->get_wallet_id().ok()); + ASSERT_EQ(public_key, wallet->get_public_key().ok().as_octet_string().as_slice()); + check_wallet_seqno(wallet, seqno); +} + +struct CreatedWallet { + td::optional priv_key; + block::StdAddress address; + td::Ref wallet; +}; + +template +class InitWallet { public: - SimpleWallet(State state) : SmartContract(std::move(state)) { + CreatedWallet operator()(int revision) const { + ton::WalletInterface::DefaultInitData init_data; + auto priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto pub_key = priv_key.get_public_key().move_as_ok(); + + init_data.seqno = 0; + init_data.wallet_id = 123; + init_data.public_key = pub_key.as_octet_string(); + + auto wallet = T::create(init_data, revision); + auto address = wallet->get_address(); + check_wallet_state(wallet, 0, 123, init_data.public_key); + CHECK(wallet.write().send_external_message(wallet->get_init_message(priv_key).move_as_ok()).success); + + CreatedWallet res; + res.wallet = std::move(wallet); + res.address = std::move(address); + res.priv_key = std::move(priv_key); + return res; } +}; - const State& get_state() const { - return state_; - } - SimpleWallet* make_copy() const override { - return new SimpleWallet{state_}; - } +template <> +CreatedWallet InitWallet::operator()(int revision) const { + auto init_priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto init_pub_key = init_priv_key.get_public_key().move_as_ok(); + auto priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto pub_key = priv_key.get_public_key().move_as_ok(); - static td::Ref create_empty() { - return td::Ref(true, - State{ton::SmartContractCode::get_code(ton::SmartContractCode::WalletV1Ext), {}}); - } - static td::Ref create(td::Ref data) { - return td::Ref( - true, State{ton::SmartContractCode::get_code(ton::SmartContractCode::WalletV1Ext), std::move(data)}); - } - static td::Ref create_fast(td::Ref data) { - return td::Ref( - true, State{ton::SmartContractCode::get_code(ton::SmartContractCode::WalletV1), std::move(data)}); - } + ton::RestrictedWallet::InitData init_data; + init_data.init_key = init_pub_key.as_octet_string(); + init_data.main_key = pub_key.as_octet_string(); + init_data.wallet_id = 123; + auto wallet = ton::RestrictedWallet::create(init_data, 1); + check_wallet_state(wallet, 0, 123, init_data.init_key); - td::int32 seqno() const { - auto res = run_get_method("seqno"); - return res.stack.write().pop_smallint_range(1000000000); - } + auto address = wallet->get_address(); - td::Ref create_init_state(td::Slice public_key) const { - td::RefInt256 pk{true}; - pk.write().import_bytes(public_key.ubegin(), public_key.size(), false); - auto res = run_get_method("create_init_state", {pk}); - return res.stack.write().pop_cell(); - } + td::uint64 x = 100 * 1000000000ull; + ton::RestrictedWallet::Config config; + config.start_at = 1; + config.limits = {{-32768, x}, {92, x * 3 / 4}, {183, x * 1 / 2}, {366, x * 1 / 4}, {548, 0}}; + CHECK(wallet.write().send_external_message(wallet->get_init_message(init_priv_key, 10, config).move_as_ok()).success); + CHECK(wallet->get_seqno().move_as_ok() == 1); - td::Ref prepare_send_message(td::Ref msg, td::int8 mode = 3) const { - auto res = run_get_method("prepare_send_message", {td::make_refint(mode), msg}); - return res.stack.write().pop_cell(); - } + CreatedWallet res; + res.wallet = std::move(wallet); + res.address = std::move(address); + res.priv_key = std::move(priv_key); + return res; +} - static td::Ref sign_message(vm::Ref body, const td::Ed25519::PrivateKey& pk) { - auto signature = pk.sign(body->get_hash().as_slice()).move_as_ok(); - return vm::CellBuilder().store_bytes(signature.as_slice()).append_cellslice(vm::load_cell_slice(body)).finalize(); +template +void do_test_wallet(int revision) { + auto res = InitWallet()(revision); + auto priv_key = res.priv_key.unwrap(); + auto address = std::move(res.address); + auto iwallet = std::move(res.wallet); + auto public_key = priv_key.get_public_key().move_as_ok().as_octet_string(); + ; + check_wallet_state(iwallet, 1, 123, public_key); + + // lets send a lot of messages + std::vector gifts; + for (size_t i = 0; i < iwallet->get_max_gifts_size(); i++) { + ton::WalletInterface::Gift gift; + gift.gramms = 1; + gift.destination = address; + gift.message = std::string(iwallet->get_max_message_size(), 'z'); + gifts.push_back(gift); } -}; -TEST(Smartcon, Simple) { - auto private_key = td::Ed25519::generate_private_key().move_as_ok(); - auto public_key = private_key.get_public_key().move_as_ok().as_octet_string(); + td::uint32 valid_until = 10000; + auto send_gifts = iwallet->make_a_gift_message(priv_key, valid_until, gifts).move_as_ok(); - auto w_lib = SimpleWallet::create_empty(); - auto init_data = w_lib->create_init_state(public_key); - - auto w = SimpleWallet::create(init_data); - LOG(ERROR) << w->code_size(); - auto fw = SimpleWallet::create_fast(init_data); - LOG(ERROR) << fw->code_size(); - LOG(ERROR) << w->seqno(); + { + auto cwallet = iwallet; + CHECK(!cwallet.write() + .send_external_message(send_gifts, ton::SmartContract::Args().set_now(valid_until + 1)) + .success); + } + //TODO: make wallet work (or not) with now == valid_until + auto ans = iwallet.write().send_external_message(send_gifts, ton::SmartContract::Args().set_now(valid_until - 1)); + CHECK(ans.success); + CHECK((int)gifts.size() <= ans.output_actions_count(ans.actions)); + check_wallet_state(iwallet, 2, 123, public_key); +} - for (int i = 0; i < 20; i++) { - auto msg = w->sign_message(w->prepare_send_message(vm::CellBuilder().finalize()), private_key); - w.write().send_external_message(msg); - fw.write().send_external_message(msg); +template +void do_test_wallet() { + for (auto revision : T::get_revisions()) { + do_test_wallet(revision); } - ASSERT_EQ(20, w->seqno()); - CHECK(w->get_state().data->get_hash() == fw->get_state().data->get_hash()); +} + +TEST(Tonlib, Wallet) { + do_test_wallet(); + do_test_wallet(); + do_test_wallet(); + do_test_wallet(); } namespace std { // ouch @@ -1157,7 +1033,7 @@ class CheckedDns { } } void update(const Action& action) { - return update(td::Span(&action, 1)); + return update(td::span_one(action)); } std::vector resolve(td::Slice name, td::int16 category) { @@ -1337,16 +1213,13 @@ TEST(Smartcont, DnsManual) { { CheckedDns::Action e[4] = {CheckedDns::Action{"", 0, ""}, CheckedDns::Action{"a.b.c", 1, "hello"}, CheckedDns::Action{"a.b.c", 2, "world"}, CheckedDns::Action{"x.y.z", 3, "abc"}}; - dns.update(td::Span(e, 4)); + dns.update(td::span(e, 4)); } dns.resolve("a.b.c", 1); dns.resolve("a.b.c", 2); dns.resolve("x.y.z", 3); - { - CheckedDns::Action e[1] = {CheckedDns::Action{"x.y.z", 0, ""}}; - dns.update(td::Span(e, 1)); - } + dns.update(td::span_one(CheckedDns::Action{"x.y.z", 0, ""})); dns.resolve("a.b.c", 1); dns.resolve("a.b.c", 2); @@ -1355,7 +1228,7 @@ TEST(Smartcont, DnsManual) { { CheckedDns::Action e[3] = {CheckedDns::Action{"x.y.z", 0, ""}, CheckedDns::Action{"x.y.z", 1, "xxx"}, CheckedDns::Action{"x.y.z", 2, "yyy"}}; - dns.update(td::Span(e, 3)); + dns.update(td::span(e, 3)); } dns.resolve("a.b.c", 1); dns.resolve("a.b.c", 2); diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 916fc0238..90b78558b 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -48,7 +48,7 @@ Cell::LoadedCell load_cell_nothrow(const Ref& ref) { auto res = ref->load_cell(); if (res.is_ok()) { auto ld = res.move_as_ok(); - CHECK(ld.virt.get_virtualization() == 0 || ld.data_cell->special_type() != Cell::SpecialType::PrunnedBranch); + //CHECK(ld.virt.get_virtualization() == 0 || ld.data_cell->special_type() != Cell::SpecialType::PrunnedBranch); return ld; } return {}; @@ -58,7 +58,7 @@ Cell::LoadedCell load_cell_nothrow(const Ref& ref, int mode) { auto res = ref->load_cell(); if (res.is_ok()) { auto ld = res.move_as_ok(); - CHECK(ld.virt.get_virtualization() == 0 || ld.data_cell->special_type() != Cell::SpecialType::PrunnedBranch); + //CHECK(ld.virt.get_virtualization() == 0 || ld.data_cell->special_type() != Cell::SpecialType::PrunnedBranch); if ((mode >> (ld.data_cell->is_special() ? 1 : 0)) & 1) { return ld; } diff --git a/crypto/vm/cells/CellString.cpp b/crypto/vm/cells/CellString.cpp index 9889dc8e2..b4738f88c 100644 --- a/crypto/vm/cells/CellString.cpp +++ b/crypto/vm/cells/CellString.cpp @@ -79,6 +79,19 @@ td::Result CellString::load(CellSlice &cs, unsigned int top_bits) { CHECK(to.offs == (int)size); return res; } +td::Result> CellString::create(td::Slice slice, unsigned int top_bits) { + vm::CellBuilder cb; + TRY_STATUS(store(cb, slice, top_bits)); + return cb.finalize(); +} +bool CellString::fetch_to(CellSlice &cs, std::string &res, unsigned int top_bits) { + auto r_str = load(cs, top_bits); + if (r_str.is_error()) { + return false; + } + res = r_str.move_as_ok(); + return true; +} td::Status CellText::store(CellBuilder &cb, td::Slice slice, unsigned int top_bits) { td::uint32 size = td::narrow_cast(slice.size() * 8); @@ -154,4 +167,17 @@ td::Result CellText::load(CellSlice &cs) { CHECK(to.offs == (int)size); return res; } +td::Result> CellText::create(td::Slice slice, unsigned int top_bits) { + vm::CellBuilder cb; + TRY_STATUS(store(cb, slice, top_bits)); + return cb.finalize(); +} +bool CellText::fetch_to(CellSlice &cs, std::string &res) { + auto r_str = load(cs); + if (r_str.is_error()) { + return false; + } + res = r_str.move_as_ok(); + return true; +} } // namespace vm diff --git a/crypto/vm/cells/CellString.h b/crypto/vm/cells/CellString.h index ab93203f8..78b63f359 100644 --- a/crypto/vm/cells/CellString.h +++ b/crypto/vm/cells/CellString.h @@ -31,11 +31,8 @@ class CellString { static td::Status store(CellBuilder &cb, td::Slice slice, unsigned int top_bits = Cell::max_bits); static td::Status store(CellBuilder &cb, td::BitSlice slice, unsigned int top_bits = Cell::max_bits); static td::Result load(CellSlice &cs, unsigned int top_bits = Cell::max_bits); - static td::Result> create(td::Slice slice, unsigned int top_bits = Cell::max_bits) { - vm::CellBuilder cb; - TRY_STATUS(store(cb, slice, top_bits)); - return cb.finalize(); - } + static td::Result> create(td::Slice slice, unsigned int top_bits = Cell::max_bits); + static bool fetch_to(CellSlice &cs, std::string &res, unsigned int top_bits = Cell::max_bits); private: template @@ -50,11 +47,8 @@ class CellText { static td::Status store(CellBuilder &cb, td::Slice slice, unsigned int top_bits = Cell::max_bits); static td::Status store(CellBuilder &cb, td::BitSlice slice, unsigned int top_bits = Cell::max_bits); static td::Result load(CellSlice &cs); - static td::Result> create(td::Slice slice, unsigned int top_bits = Cell::max_bits) { - vm::CellBuilder cb; - TRY_STATUS(store(cb, slice, top_bits)); - return cb.finalize(); - } + static td::Result> create(td::Slice slice, unsigned int top_bits = Cell::max_bits); + static bool fetch_to(CellSlice &cs, std::string &res); private: template diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 882915758..9c00a98ce 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -187,9 +187,9 @@ class StaticBagOfCellsDbBaselineImpl : public StaticBagOfCellsDb { } }; -td::Result> StaticBagOfCellsDbBaseline::create(std::unique_ptr data) { - std::string buf(data->size(), '\0'); - TRY_RESULT(slice, data->view(buf, 0)); +td::Result> StaticBagOfCellsDbBaseline::create(td::BlobView data) { + std::string buf(data.size(), '\0'); + TRY_RESULT(slice, data.view(buf, 0)); return create(slice); } @@ -211,7 +211,7 @@ td::Result> StaticBagOfCellsDbBaseline::crea // class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { public: - explicit StaticBagOfCellsDbLazyImpl(std::unique_ptr data, StaticBagOfCellsDbLazy::Options options) + explicit StaticBagOfCellsDbLazyImpl(td::BlobView data, StaticBagOfCellsDbLazy::Options options) : data_(std::move(data)), options_(std::move(options)) { get_thread_safe_counter().add(1); } @@ -240,7 +240,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { private: std::atomic should_cache_cells_{true}; - std::unique_ptr data_; + td::BlobView data_; StaticBagOfCellsDbLazy::Options options_; bool has_info_{false}; BagOfCells::Info info_; @@ -313,8 +313,8 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { char arr[8]; td::RwMutex::ReadLock guard; if (info_.has_index) { - TRY_RESULT(new_offset_view, data_->view(td::MutableSlice(arr, info_.offset_byte_size), - info_.index_offset + idx * info_.offset_byte_size)); + TRY_RESULT(new_offset_view, data_.view(td::MutableSlice(arr, info_.offset_byte_size), + info_.index_offset + idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { guard = index_data_rw_mutex_.lock_read().move_as_ok(); @@ -331,8 +331,8 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { return 0; } char arr[8]; - TRY_RESULT(idx_view, data_->view(td::MutableSlice(arr, info_.ref_byte_size), - info_.roots_offset + root_i * info_.ref_byte_size)); + TRY_RESULT(idx_view, data_.view(td::MutableSlice(arr, info_.ref_byte_size), + info_.roots_offset + root_i * info_.ref_byte_size)); CHECK(idx_view.size() == (size_t)info_.ref_byte_size); return info_.read_ref(idx_view.ubegin()); } @@ -369,17 +369,17 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { return td::Status::OK(); } std::string header(1000, '\0'); - TRY_RESULT(header_view, data_->view(td::MutableSlice(header).truncate(data_->size()), 0)) + TRY_RESULT(header_view, data_.view(td::MutableSlice(header).truncate(data_.size()), 0)) auto parse_res = info_.parse_serialized_header(header_view); if (parse_res <= 0) { return td::Status::Error("bag-of-cell error: failed to read header"); } - if (info_.total_size < data_->size()) { + if (info_.total_size < data_.size()) { return td::Status::Error("bag-of-cell error: not enough data"); } if (options_.check_crc32c && info_.has_crc32c) { std::string buf(td::narrow_cast(info_.total_size), '\0'); - TRY_RESULT(data, data_->view(td::MutableSlice(buf), 0)); + TRY_RESULT(data, data_.view(td::MutableSlice(buf), 0)); unsigned crc_computed = td::crc32c(td::Slice{data.ubegin(), data.uend() - 4}); unsigned crc_stored = td::as(data.uend() - 4); if (crc_computed != crc_stored) { @@ -407,8 +407,8 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { auto buf_slice = td::MutableSlice(buf.data(), buf.size()); for (; index_i_ <= idx; index_i_++) { auto offset = td::narrow_cast(info_.data_offset + index_offset_); - CHECK(data_->size() >= offset); - TRY_RESULT(cell, data_->view(buf_slice.copy().truncate(data_->size() - offset), offset)); + CHECK(data_.size() >= offset); + TRY_RESULT(cell, data_.view(buf_slice.copy().truncate(data_.size() - offset), offset)); CellSerializationInfo cell_info; TRY_STATUS(cell_info.init(cell, info_.ref_byte_size)); index_offset_ += cell_info.end_offset; @@ -455,7 +455,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { TRY_RESULT(cell_location, get_cell_location(idx)); auto buf = alloc(cell_location.end - cell_location.begin); - TRY_RESULT(cell_slice, data_->view(buf.as_slice(), cell_location.begin)); + TRY_RESULT(cell_slice, data_.view(buf.as_slice(), cell_location.begin)); TRY_RESULT(res, deserialize_any_cell(idx, cell_slice, cell_location.should_cache)); return std::move(res); } @@ -470,7 +470,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { TRY_RESULT(cell_location, get_cell_location(idx)); auto buf = alloc(cell_location.end - cell_location.begin); - TRY_RESULT(cell_slice, data_->view(buf.as_slice(), cell_location.begin)); + TRY_RESULT(cell_slice, data_.view(buf.as_slice(), cell_location.begin)); TRY_RESULT(res, deserialize_data_cell(idx, cell_slice, cell_location.should_cache)); return std::move(res); } @@ -528,18 +528,17 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } }; -td::Result> StaticBagOfCellsDbLazy::create(std::unique_ptr data, - Options options) { +td::Result> StaticBagOfCellsDbLazy::create(td::BlobView data, Options options) { return std::make_shared(std::move(data), std::move(options)); } td::Result> StaticBagOfCellsDbLazy::create(td::BufferSlice data, Options options) { - return std::make_shared(vm::BufferSliceBlobView::create(std::move(data)), + return std::make_shared(td::BufferSliceBlobView::create(std::move(data)), std::move(options)); } td::Result> StaticBagOfCellsDbLazy::create(std::string data, Options options) { - return create(BufferSliceBlobView::create(td::BufferSlice(data)), std::move(options)); + return create(td::BufferSliceBlobView::create(td::BufferSlice(data)), std::move(options)); } } // namespace vm diff --git a/crypto/vm/db/StaticBagOfCellsDb.h b/crypto/vm/db/StaticBagOfCellsDb.h index 124c868ae..03c45ee88 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.h +++ b/crypto/vm/db/StaticBagOfCellsDb.h @@ -19,7 +19,7 @@ #pragma once #include "vm/cells.h" -#include "vm/db/BlobView.h" +#include "td/db/utils/BlobView.h" #include "td/utils/Status.h" @@ -41,7 +41,7 @@ class StaticBagOfCellsDb : public std::enable_shared_from_this> create(std::unique_ptr data); + static td::Result> create(td::BlobView data); static td::Result> create(td::Slice data); }; @@ -52,7 +52,7 @@ class StaticBagOfCellsDbLazy { } bool check_crc32c{false}; }; - static td::Result> create(std::unique_ptr data, Options options = {}); + static td::Result> create(td::BlobView data, Options options = {}); static td::Result> create(td::BufferSlice data, Options options = {}); static td::Result> create(std::string data, Options options = {}); }; diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index fa866b847..8b9c7c1af 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -1129,7 +1129,7 @@ td::Result, std::shared_ptr>> la td::BufferSlice data) { vm::StaticBagOfCellsDbLazy::Options options; options.check_crc32c = true; - TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(std::move(data)), options)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(std::move(data)), options)); TRY_RESULT(rc, boc->get_root_count()); if (rc != 1) { return td::Status::Error(-668, "bag-of-cells is not standard (exactly one root cell expected)"); diff --git a/rldp2/Ack.cpp b/rldp2/Ack.cpp new file mode 100644 index 000000000..2d71f9562 --- /dev/null +++ b/rldp2/Ack.cpp @@ -0,0 +1,29 @@ +#include "Ack.h" + +namespace ton { +namespace rldp2 { + +bool Ack::on_got_packet(td::uint32 seqno) { + if (seqno > max_seqno) { + td::uint32 diff = seqno - max_seqno; + if (diff >= 32) { + received_mask = 0; + } else { + received_mask <<= diff; + } + max_seqno = seqno; + } + td::uint32 offset = max_seqno - seqno; + if (offset < 32) { + td::uint32 mask = 1 << offset; + if ((received_mask & mask) == 0) { + received_count++; + received_mask |= mask; + return true; + } + } + + return false; +} +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/Ack.h b/rldp2/Ack.h new file mode 100644 index 000000000..092ea938a --- /dev/null +++ b/rldp2/Ack.h @@ -0,0 +1,18 @@ +#pragma once + +#include "td/utils/int_types.h" + +namespace ton { +namespace rldp2 { +// Helper for receiver +// Also this information is sent to the sender as an acknowlegement. +struct Ack { + td::uint32 max_seqno{0}; + td::uint32 received_mask{0}; + td::uint32 received_count{0}; + + // returns true if we know that packet is new and hasn't been received yet + bool on_got_packet(td::uint32 seqno); +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/Bbr.cpp b/rldp2/Bbr.cpp new file mode 100644 index 000000000..93d06f827 --- /dev/null +++ b/rldp2/Bbr.cpp @@ -0,0 +1,62 @@ +#include "Bbr.h" + +#include "BdwStats.h" +#include "RttStats.h" + +#include "td/utils/Random.h" + +namespace ton { +namespace rldp2 { +void Bbr::step(const RttStats &rtt_stats, const BdwStats &bdw_stats, td::uint64 in_flight, td::Timestamp now) { + rtt_min_ = rtt_stats.windowed_min_rtt; + bdw_max_ = bdw_stats.windowed_max_bdw; + if (bdw_max_ > bdw_peak_ * 1.25) { + bdw_peak_ = bdw_max_; + bdw_peak_at_round = rtt_stats.rtt_round; + //LOG(ERROR) << "NEW PEAK " << bdw_peak_ * 768; + } + + if (state_ == State::Start && bdw_peak_at_round + 3 < rtt_stats.rtt_round) { + //LOG(ERROR) << "START -> DRAIN"; + state_ = State::Drain; + } + + if (state_ == State::Drain && (double)in_flight < bdw_max_ * rtt_min_) { + //LOG(ERROR) << "DRAIN -> BPROBE BDW"; + state_ = State::ProbeBdw; + probe_bdw_cycle_ = td::Random::fast(1, 5); + probe_bdw_cycle_at_ = now; + } + + if (state_ == State::ProbeBdw && td::Timestamp::in(rtt_stats.windowed_min_rtt, probe_bdw_cycle_at_).is_in_past(now)) { + probe_bdw_cycle_at_ = now; + probe_bdw_cycle_ = (probe_bdw_cycle_ + 1) % 6; + //LOG(ERROR) << "NEW PROBE BDW CYCLE"; + } + + //TODO: ProbeRtt state. Don't want to implenent now without proper tests +} + +double Bbr::get_rate() const { + if (state_ == State::Start) { + return bdw_max_ * 2.8; + } + if (state_ == State::Drain) { + return bdw_max_ / 2.8; + } + if (state_ == State::ProbeBdw) { + constexpr double probe_bdw_gain[6] = {0.75, 1, 1, 1, 1, 1.25}; + return probe_bdw_gain[probe_bdw_cycle_] * bdw_max_; + } + UNREACHABLE(); +} + +td::uint32 Bbr::get_window_size() const { + if (state_ == State::Start || state_ == State::Drain) { + return td::max(td::uint32(bdw_max_ * rtt_min_ * 2.8 + 1), 10u); + } + return td::max(td::uint32(bdw_max_ * rtt_min_ * 2 + 1), 10u); +} + +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/Bbr.h b/rldp2/Bbr.h new file mode 100644 index 000000000..29d4f8577 --- /dev/null +++ b/rldp2/Bbr.h @@ -0,0 +1,29 @@ +#pragma once +#include "td/utils/int_types.h" +#include "td/utils/Time.h" + +namespace ton { +namespace rldp2 { +struct RttStats; +struct BdwStats; + +struct Bbr { + public: + void step(const RttStats &rtt_stats, const BdwStats &bdw_stats, td::uint64 in_flight, td::Timestamp now); + double get_rate() const; + + td::uint32 get_window_size() const; + + private: + double bdw_peak_{-1}; + td::uint32 bdw_peak_at_round{0}; + td::uint32 probe_bdw_cycle_{0}; + td::Timestamp probe_bdw_cycle_at_; + double rtt_min_{0}; + double bdw_max_{0}; + enum class State { Start, Drain, ProbeRtt, ProbeBdw } state_ = State::Start; +}; + +} // namespace rldp2 +} // namespace ton + diff --git a/rldp2/BdwStats.cpp b/rldp2/BdwStats.cpp new file mode 100644 index 000000000..d0bc75714 --- /dev/null +++ b/rldp2/BdwStats.cpp @@ -0,0 +1,50 @@ +#include "BdwStats.h" + +namespace ton { +namespace rldp2 { + +BdwStats::PacketInfo BdwStats::on_packet_send(td::Timestamp first_sent_at) const { + PacketInfo packet; + packet.delivered_now = delivered_now; + packet.first_sent_at = first_sent_at; + packet.delivered_count = delivered_count; + packet.is_paused = static_cast(paused_at_); + return packet; +} + +void BdwStats::on_packet_ack(const PacketInfo &info, td::Timestamp sent_at, td::Timestamp now) { + if (paused_at_.is_in_past(info.delivered_now)) { + paused_at_ = {}; + } + auto sent_passed = sent_at.at() - info.first_sent_at.at(); + auto ack_passed = now.at() - info.delivered_now.at(); + auto passed = td::max(sent_passed, ack_passed); + if (passed < 0.01) { + LOG(ERROR) << "Invalid passed " << passed; + } + auto delivered = delivered_count - info.delivered_count; + on_rate_sample((double)delivered / passed, now, info.is_paused); +} + +void BdwStats::on_update(td::Timestamp now, td::uint64 delivered_count_diff) { + this->delivered_now = now; + this->delivered_count += delivered_count_diff; +} + +void BdwStats::on_pause(td::Timestamp now) { + paused_at_ = now; +} + +void BdwStats::on_rate_sample(double rate, td::Timestamp now, bool is_paused) { + // ignore decrease of rate if is_paused == true + if (is_paused && rate < windowed_max_bdw) { + return; + } + windowed_max_bdw_stat.add_event(rate, now.at()); + auto windowed_max_bdw_sample = windowed_max_bdw_stat.get_stat(now.at()).get_stat(); + if (windowed_max_bdw_sample) { + windowed_max_bdw = windowed_max_bdw_sample.value(); + } +} +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/BdwStats.h b/rldp2/BdwStats.h new file mode 100644 index 000000000..a953169bf --- /dev/null +++ b/rldp2/BdwStats.h @@ -0,0 +1,36 @@ +#pragma once + +#include "td/utils/Time.h" +#include "td/utils/TimedStat.h" + +namespace ton { +namespace rldp2 { +struct BdwStats { + struct State {}; + + struct PacketInfo { + td::Timestamp first_sent_at; + + td::Timestamp delivered_now; + td::uint64 delivered_count{0}; + bool is_paused{false}; + }; + + PacketInfo on_packet_send(td::Timestamp first_sent_at) const; + void on_packet_ack(const PacketInfo &info, td::Timestamp sent_at, td::Timestamp now); + + void on_update(td::Timestamp now, td::uint64 delivered_count_diff); + + void on_pause(td::Timestamp now); + double windowed_max_bdw{0}; + + private: + td::Timestamp delivered_now; + td::uint64 delivered_count{0}; + td::TimedStat> windowed_max_bdw_stat{5, 0}; + td::Timestamp paused_at_; + + void on_rate_sample(double rate, td::Timestamp now, bool is_paused); +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt new file mode 100644 index 000000000..f93f6e1e0 --- /dev/null +++ b/rldp2/CMakeLists.txt @@ -0,0 +1,58 @@ +cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) + +if (NOT OPENSSL_FOUND) + find_package(OpenSSL REQUIRED) +endif() + +if (NOT GSL_FOUND) + find_package(GSL) +endif() + +set(RLDP_SOURCE + Ack.cpp + Bbr.cpp + BdwStats.cpp + FecHelper.cpp + InboundTransfer.cpp + LossSender.cpp + LossStats.cpp + OutboundTransfer.cpp + Pacer.cpp + rldp.cpp + RldpReceiver.cpp + RldpSender.cpp + RldpConnection.cpp + RttStats.cpp + SenderPackets.cpp + + Ack.h + Bbr.h + BdwStats.h + FecHelper.h + InboundTransfer.h + LossSender.h + LossStats.h + OutboundTransfer.h + Pacer.h + rldp.h + rldp.hpp + RldpReceiver.h + RldpSender.h + RldpConnection.h + RttStats.h + SenderPackets.h +) + +add_library(rldp2 STATIC ${RLDP_SOURCE}) + +target_include_directories(rldp PUBLIC + $ + $/.. + ${OPENSSL_INCLUDE_DIR} +) +if (GSL_FOUND) + target_link_libraries(rldp2 PRIVATE GSL::gsl) + target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1) +endif() +target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api) + diff --git a/rldp2/FecHelper.cpp b/rldp2/FecHelper.cpp new file mode 100644 index 000000000..0f53b7ccd --- /dev/null +++ b/rldp2/FecHelper.cpp @@ -0,0 +1,24 @@ +#include "FecHelper.h" + +#include "td/utils/check.h" + +namespace ton { +namespace rldp2 { +td::uint32 FecHelper::get_fec_symbols_count() const { + constexpr td::uint32 x = 5; + constexpr td::uint32 y = 5; + // smallest (symbols_count + x + y * i) > received_symbols_count + if (symbols_count + x > received_symbols_count) { + return symbols_count + x; + } + td::uint32 i = (received_symbols_count - (symbols_count + x)) / y + 1; + return symbols_count + x + i * y; +} + +td::uint32 FecHelper::get_left_fec_symbols_count() const { + auto fec_symbols_count = get_fec_symbols_count(); + CHECK(fec_symbols_count > received_symbols_count); + return fec_symbols_count - received_symbols_count; +} +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/FecHelper.h b/rldp2/FecHelper.h new file mode 100644 index 000000000..b3e656555 --- /dev/null +++ b/rldp2/FecHelper.h @@ -0,0 +1,17 @@ +#pragma once + +#include "td/utils/int_types.h" + +namespace ton { +namespace rldp2 { + +struct FecHelper { + td::uint32 symbols_count{0}; + td::uint32 received_symbols_count{0}; + + td::uint32 get_fec_symbols_count() const; + td::uint32 get_left_fec_symbols_count() const; +}; + +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/InboundTransfer.cpp b/rldp2/InboundTransfer.cpp new file mode 100644 index 000000000..c9ef9feaf --- /dev/null +++ b/rldp2/InboundTransfer.cpp @@ -0,0 +1,58 @@ +#include "InboundTransfer.h" + +#include "common/errorcode.h" + +namespace ton { +namespace rldp2 { +size_t InboundTransfer::total_size() const { + return data_.size(); +} + +std::map &InboundTransfer::parts() { + return parts_; +} + +bool InboundTransfer::is_part_completed(td::uint32 part_i) { + return parts_.count(part_i) == 0 && part_i < next_part_; +} + +td::Result InboundTransfer::get_part(td::uint32 part_i, const ton::fec::FecType &fec_type) { + auto it = parts_.find(part_i); + if (it != parts_.end()) { + return &it->second; + } + //TODO: pass offset off and process even newer parts. + //LOG_CHECK(next_part_ >= part_i) << next_part_ << " >= " << part_i; + if (next_part_ == part_i && parts_.size() < 20) { + auto offset = offset_; + offset_ += fec_type.size(); + if (offset_ > total_size()) { + return td::Status::Error(ErrorCode::protoviolation, + PSTRING() << "too big part: offset=" << offset_ << " total_size=" << total_size() + << " total_size=" << fec_type.size() << " part=" << part_i); + } + + auto decoder = fec_type.create_decoder().move_as_ok(); + auto it = parts_.emplace(part_i, Part{std::move(decoder), RldpReceiver(RldpSender::Config()), offset}); + next_part_++; + return &it.first->second; + } + return nullptr; +} + +void InboundTransfer::finish_part(td::uint32 part_i, td::Slice data) { + auto it = parts_.find(part_i); + CHECK(it != parts_.end()); + data_.as_slice().substr(it->second.offset).copy_from(data); + parts_.erase(it); +} + +td::optional> InboundTransfer::try_finish() { + if (parts_.empty() && offset_ == data_.size()) { + return std::move(data_); + } + return {}; +} + +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/InboundTransfer.h b/rldp2/InboundTransfer.h new file mode 100644 index 000000000..86cfbd172 --- /dev/null +++ b/rldp2/InboundTransfer.h @@ -0,0 +1,37 @@ +#pragma once + +#include "td/utils/optional.h" + +#include "fec/fec.h" + +#include "RldpReceiver.h" + +#include + +namespace ton { +namespace rldp2 { +struct InboundTransfer { + struct Part { + std::unique_ptr decoder; + RldpReceiver receiver; + size_t offset; + }; + + explicit InboundTransfer(size_t total_size) : data_(total_size) { + } + + size_t total_size() const; + std::map &parts(); + bool is_part_completed(td::uint32 part_i); + td::Result get_part(td::uint32 part_i, const ton::fec::FecType &fec_type); + void finish_part(td::uint32 part_i, td::Slice data); + td::optional> try_finish(); + + private: + std::map parts_; + td::uint32 next_part_{0}; + size_t offset_{0}; + td::BufferSlice data_; +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/LossSender.cpp b/rldp2/LossSender.cpp new file mode 100644 index 000000000..46da0533f --- /dev/null +++ b/rldp2/LossSender.cpp @@ -0,0 +1,135 @@ +#include "LossSender.h" + +#include "td/utils/logging.h" + +#if TON_HAVE_GSL +#include +#endif + +#include + +namespace ton { +namespace rldp2 { +namespace { +// works for 1e-x, where x in {1..10} +double ndtri_fast(double p) { + if (p < 2e-10) { + return 6.361340902404; + } + if (p < 2e-9) { + return 5.997807015008; + } + if (p < 2e-8) { + return 5.612001244175; + } + if (p < 2e-7) { + return 5.199337582193; + } + if (p < 2e-6) { + return 4.753424308823; + } + if (p < 2e-5) { + return 4.264890793923; + } + if (p < 2e-4) { + return 3.719016485456; + } + if (p < 2e-3) { + return 3.090232306168; + } + if (p < 2e-2) { + return 2.326347874041; + } + return 1.281551565545; +} +} // namespace + +LossSender::LossSender(double loss, double p) : loss_(loss), p_(p) { + v_.resize(2); + v_[0] = 1; + res_.push_back(0); + S_ = ndtri_fast(p_); + sigma_ = p * (1 - p); + //LOG(ERROR) << S_ << " " << ndtri(1 - p_); + //CHECK(fabs(S_ - ndtri(1 - p_)) < 1e-6); +} + +int LossSender::send_n(int n) { + if (n < 50) { + return send_n_exact(n); + } + return send_n_approx_nbd(n); +} + +int LossSender::send_n_exact(int n) { + while ((int)res_.size() <= n) { + step(); + } + return res_[n]; +} + +int LossSender::send_n_approx_norm(int n) { + double a = (1 - loss_) * (1 - loss_); + double b = loss_ * (loss_ - 1) * (2 * n + S_ * S_); + double c = loss_ * loss_ * n * n + S_ * S_ * n * loss_ * (loss_ - 1); + double x = ((-b + sqrt(b * b - 4 * a * c)) / (2 * a)); + return (int)(x + n + 1); +} + +int LossSender::send_n_approx_nbd(int n) { +#if TON_HAVE_GSL + auto mean = n * loss_ / (1 - loss_); + auto var = sqrt(mean / (1 - loss_)); + auto min_k = static_cast(mean + var); + auto max_k = min_k + static_cast(var + 1) * 15; + while (min_k + 1 < max_k) { + int k = (min_k + max_k) / 2; + if (gsl_cdf_negative_binomial_P(k, 1 - loss_, n) > 1 - p_) { + max_k = k; + } else { + min_k = k; + } + } + return max_k + n; +#endif + return send_n_approx_norm(n); +} + +int LossSender::send_n_approx_pd(int n) { +#if TON_HAVE_GSL + for (int k = 0;; k++) { + if (gsl_cdf_poisson_P(k, (n + k) * loss_) > 1 - p_) { + return k + n; + } + } +#endif + return send_n_approx_norm(n); +} +bool LossSender::has_good_approx() { +#if TON_HAVE_GSL + return true; +#else + return false; +#endif +} + +void LossSender::step() { + n_++; + v_.push_back(0); + v_[n_] = v_[n_ - 1]; + for (int j = n_; j >= 0; j--) { + v_[j + 1] += v_[j] * loss_; + v_[j] *= (1 - loss_); + } + + while (res_i_ < n_ && v_[res_i_] < 1 - p_) { + res_i_++; + } + auto left_ = n_ - res_i_; + if ((int)res_.size() == left_) { + res_.push_back(n_); + } +} + +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/LossSender.h b/rldp2/LossSender.h new file mode 100644 index 000000000..2a27a8d00 --- /dev/null +++ b/rldp2/LossSender.h @@ -0,0 +1,31 @@ +#pragma once +#include + +namespace ton { +namespace rldp2 { +struct LossSender { + LossSender(double loss, double p); + int send_n(int n); + + int send_n_exact(int n); + int send_n_approx_norm(int n); + int send_n_approx_nbd(int n); + int send_n_approx_pd(int n); + + bool has_good_approx(); + + private: + double loss_; + double p_; + double S_; + double sigma_; + int n_{0}; + std::vector v_; + std::vector res_; + int res_i_{0}; + + void step(); +}; + +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/LossStats.cpp b/rldp2/LossStats.cpp new file mode 100644 index 000000000..9eca2e9db --- /dev/null +++ b/rldp2/LossStats.cpp @@ -0,0 +1,23 @@ +#include "LossStats.h" +#include "td/utils/misc.h" + +#include + +namespace ton { +namespace rldp2 { +void LossStats::on_update(td::uint32 ack, td::uint32 lost) { + ack_ += ack; + lost_ += lost; + + if (ack_ + lost_ > 1000) { + auto new_loss = td::clamp((double)lost_ / (ack_ + lost_), 0.001, 0.2); + if (fabs(new_loss - loss) > 5e-3) { + prob = LossSender(new_loss, 1e-9); + } + loss = new_loss; + ack_ = 0; + lost_ = 0; + } +} +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/LossStats.h b/rldp2/LossStats.h new file mode 100644 index 000000000..19e5427c9 --- /dev/null +++ b/rldp2/LossStats.h @@ -0,0 +1,19 @@ +#pragma once + +#include "LossSender.h" + +#include "td/utils/int_types.h" + +namespace ton { +namespace rldp2 { +struct LossStats { + void on_update(td::uint32 ack, td::uint32 lost); + double loss = 0.1; + LossSender prob{0.1, 1e-9}; + + private: + td::uint32 ack_{0}; + td::uint32 lost_{0}; +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/OutboundTransfer.cpp b/rldp2/OutboundTransfer.cpp new file mode 100644 index 000000000..1449cc097 --- /dev/null +++ b/rldp2/OutboundTransfer.cpp @@ -0,0 +1,40 @@ +#include "OutboundTransfer.h" + +namespace ton { +namespace rldp2 { +size_t OutboundTransfer::total_size() const { + return data_.size(); +} +std::map &OutboundTransfer::parts(const RldpSender::Config &config) { + while (parts_.size() < 20) { + auto offset = next_part_ * part_size(); + if (offset >= data_.size()) { + break; + } + td::BufferSlice D = data_.from_slice(data_.as_slice().substr(offset).truncate(part_size())); + ton::fec::FecType fec_type = td::fec::RaptorQEncoder::Parameters{D.size(), symbol_size(), 0}; + auto encoder = fec_type.create_encoder(std::move(D)).move_as_ok(); + auto symbols_count = fec_type.symbols_count(); + parts_.emplace(next_part_, Part{std::move(encoder), RldpSender(config, symbols_count), std::move(fec_type)}); + next_part_++; + } + return parts_; +} + +void OutboundTransfer::drop_part(td::uint32 part_i) { + parts_.erase(part_i); +} + +OutboundTransfer::Part *OutboundTransfer::get_part(td::uint32 part_i) { + auto it = parts_.find(part_i); + if (it == parts_.end()) { + return nullptr; + } + return &it->second; +} + +bool OutboundTransfer::is_done() const { + return next_part_ * part_size() >= data_.size() && parts_.empty(); +} +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/OutboundTransfer.h b/rldp2/OutboundTransfer.h new file mode 100644 index 000000000..249659bd6 --- /dev/null +++ b/rldp2/OutboundTransfer.h @@ -0,0 +1,40 @@ +#pragma once + +#include "RldpSender.h" +#include "fec/fec.h" + +#include + +namespace ton { +namespace rldp2 { +struct OutboundTransfer { + public: + struct Part { + std::unique_ptr encoder; + RldpSender sender; + ton::fec::FecType fec_type; + }; + + OutboundTransfer(td::BufferSlice data) : data_(std::move(data)) { + } + + size_t total_size() const; + std::map &parts(const RldpSender::Config &config); + void drop_part(td::uint32 part_i); + Part *get_part(td::uint32 part_i); + bool is_done() const; + + private: + td::BufferSlice data_; + std::map parts_; + td::uint32 next_part_{0}; + + static size_t part_size() { + return 2000000; + } + static size_t symbol_size() { + return 768; + } +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/Pacer.cpp b/rldp2/Pacer.cpp new file mode 100644 index 000000000..933695ee3 --- /dev/null +++ b/rldp2/Pacer.cpp @@ -0,0 +1,45 @@ +#include "Pacer.h" +namespace ton { +namespace rldp2 { +Pacer::Pacer(Options options) + : speed_(options.initial_speed) + , capacity_(options.initial_capacity) + , max_capacity_(options.max_capacity) + , time_granularity_(options.time_granularity) { +} + +td::Timestamp Pacer::wakeup_at() const { + return wakeup_at_; +} + +void Pacer::set_speed(double speed) { + if (speed < 1) { + speed = 1; + } + speed_ = speed; +} + +td::optional Pacer::send(double size, td::Timestamp now) { + update_capacity(now); + + if (size < capacity_) { + capacity_ -= size; + return {}; + } + + size -= capacity_; + capacity_ = 0; + wakeup_at_ = td::Timestamp::in(size / speed_, now); + capacity_at_ = wakeup_at_; + return wakeup_at_; +} + +void Pacer::update_capacity(td::Timestamp now) { + if (capacity_at_ && capacity_at_.is_in_past(now)) { + capacity_ += (now.at() - capacity_at_.at()) * speed_; + capacity_ = td::min(capacity_, td::max(max_capacity_, speed_ * time_granularity_)); + } + capacity_at_ = now; +} +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/Pacer.h b/rldp2/Pacer.h new file mode 100644 index 000000000..560858b46 --- /dev/null +++ b/rldp2/Pacer.h @@ -0,0 +1,39 @@ +#pragma once +#include "td/utils/optional.h" +#include "td/utils/Time.h" + +namespace ton { +namespace rldp2 { +// NB: Should be careful with max_capacity < time_granularity * speed +// We may send packet of any size. +// After that we will be put to sleep till wakeup_at(). +// When we are awake we send packet of any size again. +// Logic is - we don't have to wait to send a packet - it is poinless. +// But we have to wait for some time after packet is sent +class Pacer { + public: + struct Options { + Options() { + } + double initial_capacity{20}; + double initial_speed{10}; + double max_capacity{40}; + double time_granularity{0.001}; + }; + Pacer(Options options = {}); + td::Timestamp wakeup_at() const; + void set_speed(double speed); + td::optional send(double size, td::Timestamp now); + + private: + double speed_; + double capacity_; + double max_capacity_; + double time_granularity_; + td::Timestamp capacity_at_; + td::Timestamp wakeup_at_; + + void update_capacity(td::Timestamp now); +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/RldpConnection.cpp b/rldp2/RldpConnection.cpp new file mode 100644 index 000000000..12034e6ec --- /dev/null +++ b/rldp2/RldpConnection.cpp @@ -0,0 +1,369 @@ +#include "RldpConnection.h" + +#include "td/utils/overloaded.h" +#include "td/utils/Random.h" +#include "td/utils/tl_helpers.h" + +#include "tl-utils/tl-utils.hpp" +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" + +#include "common/errorcode.h" + +#include "td/actor//actor.h" + +namespace ton { +namespace rldp2 { +void RldpConnection::add_limit(td::Timestamp timeout, Limit limit) { + CHECK(timeout); + auto p = limits_set_.insert(limit); + LOG_CHECK(p.second) << limit.transfer_id.to_hex(); + limits_heap_.insert(timeout.at(), const_cast(&*p.first)); +} + +td::Timestamp RldpConnection::next_limit_expires_at() { + if (limits_heap_.empty()) { + return td::Timestamp::never(); + } + return td::Timestamp::at(limits_heap_.top_key()); +} + +void RldpConnection::drop_limits(TransferId id) { + Limit limit; + limit.transfer_id = id; + auto it = limits_set_.find(limit); + if (it == limits_set_.end()) { + return; + } + limits_heap_.erase(const_cast(static_cast(&*it))); + limits_set_.erase(it); +} + +void RldpConnection::on_inbound_completed(TransferId transfer_id, td::Timestamp now) { + inbound_transfers_.erase(transfer_id); + completed_set_.insert(transfer_id); + completed_queue_.push(CompletedId{transfer_id, now.in(20)}); + while (completed_queue_.size() > 128 && completed_queue_.front().timeout.is_in_past(now)) { + completed_set_.erase(completed_queue_.pop().transfer_id); + } +} + +td::Timestamp RldpConnection::loop_limits(td::Timestamp now) { + while (!limits_heap_.empty() && td::Timestamp::at(limits_heap_.top_key()).is_in_past(now)) { + auto *limit = static_cast(limits_heap_.pop()); + auto error = td::Status::Error(ErrorCode::timeout, "timeout"); + if (limit->is_inbound) { + on_inbound_completed(limit->transfer_id, now); + to_receive_.emplace_back(limit->transfer_id, std::move(error)); + } else { + auto it = outbound_transfers_.find(limit->transfer_id); + if (it != outbound_transfers_.end()) { + for (auto &part : it->second.parts(RldpSender::Config{})) { + in_flight_count_ -= part.second.sender.get_inflight_symbols_count(); + } + outbound_transfers_.erase(it); + to_on_sent_.emplace_back(limit->transfer_id, std::move(error)); + } else { + LOG(ERROR) << "Timeout on unknown transfer " << limit->transfer_id.to_hex(); + } + } + limits_set_.erase(*limit); + } + + return next_limit_expires_at(); +} + +void RldpConnection::set_receive_limits(TransferId transfer_id, td::Timestamp timeout, td::uint64 max_size) { + CHECK(timeout); + Limit limit; + limit.transfer_id = transfer_id; + limit.max_size = max_size; + limit.is_inbound = true; + add_limit(timeout, limit); +} + +RldpConnection::RldpConnection() { + bdw_stats_.on_update(td::Timestamp::now(), 0); + + rtt_stats_.windowed_min_rtt = 0.5; + bdw_stats_.windowed_max_bdw = 10; +} + +void RldpConnection::send(TransferId transfer_id, td::BufferSlice data, td::Timestamp timeout) { + if (transfer_id.is_zero()) { + td::Random::secure_bytes(transfer_id.as_slice()); + } else { + if (outbound_transfers_.find(transfer_id) != outbound_transfers_.end()) { + LOG(WARNING) << "Skip resend of " << transfer_id.to_hex(); + return; + } + } + + if (timeout) { + Limit limit; + limit.transfer_id = transfer_id; + limit.max_size = 0; + limit.is_inbound = false; + add_limit(timeout, limit); + } + outbound_transfers_.emplace(transfer_id, OutboundTransfer{std::move(data)}); +} + +void RldpConnection::receive_raw(td::BufferSlice packet) { + auto F = ton::fetch_tl_object(std::move(packet), true); + if (F.is_error()) { + return; + } + downcast_call(*F.move_as_ok(), [&](auto &obj) { this->receive_raw_obj(obj); }); +} + +void RldpConnection::loop_bbr(td::Timestamp now) { + bbr_.step(rtt_stats_, bdw_stats_, in_flight_count_, td::Timestamp::now()); + //LOG(ERROR) << td::format::as_time(rtt_stats_.windowed_min_rtt) << " " + //<< td::format::as_size((td::int64)bdw_stats_.windowed_max_bdw * 768) << " " << rtt_stats_.rtt_round; + double speed = bbr_.get_rate(); + td::uint32 congestion_window = bbr_.get_window_size(); + + static td::Timestamp next; + //FIXME: remove this UNSAFE debug output + if (next.is_in_past(now)) { + next = td::Timestamp::in(1, now); + if (td::actor::core::ActorExecuteContext::get()->actor().get_actor_info_ptr()->get_name() == "Alice") { + LOG(ERROR) << "speed=" << td::format::as_size((td::int64)speed * 768) << " " + << "cgw=" << td::format::as_size((td::int64)congestion_window * 768) << " " + << "loss=" << loss_stats_.loss * 100 << "%"; + } + } + + pacer_.set_speed(speed); + congestion_window_ = congestion_window; +} + +td::Timestamp RldpConnection::run(ConnectionCallback &callback) { + auto now = td::Timestamp::now(); + loop_bbr(now); + + td::Timestamp alarm_timestamp; + td::VectorQueue *> queue; + for (auto &outbound : outbound_transfers_) { + queue.push(&outbound); + } + while (!queue.empty()) { + auto outbound = queue.pop(); + auto o_timeout = step(outbound->first, outbound->second, now); + if (o_timeout) { + alarm_timestamp.relax(o_timeout.unwrap()); + } else { + queue.push(outbound); + } + } + + if (in_flight_count_ > congestion_window_) { + bdw_stats_.on_pause(now); + } + + for (auto &inbound : inbound_transfers_) { + alarm_timestamp.relax(run(inbound.first, inbound.second)); + } + + alarm_timestamp.relax(loop_limits(td::Timestamp::now())); + + for (auto &data : to_receive_) { + callback.receive(data.first, std::move(data.second)); + } + for (auto &raw : to_send_raw_) { + callback.send_raw(std::move(raw)); + } + to_send_raw_.clear(); + to_receive_.clear(); + for (auto &res : to_on_sent_) { + callback.on_sent(res.first, std::move(res.second)); + } + to_on_sent_.clear(); + + return alarm_timestamp; +} + +td::Timestamp RldpConnection::run(const TransferId &transfer_id, InboundTransfer &inbound) { + td::Timestamp wakeup_at; + bool has_actions = true; + while (has_actions) { + has_actions = false; + for (auto &it : inbound.parts()) { + auto &inbound = it.second; + inbound.receiver.next_action(td::Timestamp::now()) + .visit(td::overloaded([&](const RldpReceiver::ActionWait &wait) { wakeup_at.relax(wait.wait_till); }, + [&](const RldpReceiver::ActionSendAck &send) { + send_packet(ton::create_serialize_tl_object( + transfer_id, it.first, send.ack.max_seqno, send.ack.received_mask, + send.ack.received_count)); + inbound.receiver.on_ack_sent(td::Timestamp::now()); + has_actions = true; + })); + } + } + return wakeup_at; +} + +td::optional RldpConnection::step(const TransferId &transfer_id, OutboundTransfer &outbound, + td::Timestamp now) { + bool only_probe = in_flight_count_ > congestion_window_; + + td::Timestamp wakeup_at; + if (!pacer_.wakeup_at().is_in_past(now)) { + wakeup_at = pacer_.wakeup_at(); + only_probe = true; + } + + for (auto &it : outbound.parts(RldpSender::Config{})) { + auto &part = it.second; + + Guard guard(in_flight_count_, part.sender); + auto action = part.sender.next_action(now, only_probe); + + bool was_send = false; + action.visit(td::overloaded( + [&](const RldpSender::ActionSend &send) { + auto seqno = send.seqno - 1; + if (part.encoder->get_info().ready_symbol_count <= seqno) { + part.encoder->prepare_more_symbols(); + } + auto symbol = part.encoder->gen_symbol(seqno).data; + send_packet(ton::create_serialize_tl_object( + transfer_id, part.fec_type.tl(), it.first, outbound.total_size(), seqno, std::move(symbol))); + if (!send.is_probe) { + pacer_.send(1, now); + } + part.sender.on_send(send.seqno, now, send.is_probe, rtt_stats_, bdw_stats_); + if (send.is_probe) { + //LOG(ERROR) << "PROBE " << it.first << " " << send.seqno; + } + //LOG(ERROR) << "SEND"; + was_send = true; + }, + [&](const RldpSender::ActionWait &wait) { + //LOG(ERROR) << "WAIT"; + wakeup_at.relax(wait.wait_till); + })); + if (was_send) { + return {}; + } + } + + return wakeup_at; +} + +void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_messagePart &part) { + if (completed_set_.count(part.transfer_id_) > 0) { + send_packet(ton::create_serialize_tl_object(part.transfer_id_, part.part_)); + return; + } + + auto r_total_size = td::narrow_cast_safe(part.total_size_); + if (r_total_size.is_error()) { + return; + } + auto r_fec_type = ton::fec::FecType::create(std::move(part.fec_type_)); + if (r_fec_type.is_error()) { + return; + } + + auto total_size = r_total_size.move_as_ok(); + + auto transfer_id = part.transfer_id_; + + // check total_size limits + td::uint64 max_size = default_mtu(); + Limit key; + key.transfer_id = transfer_id; + auto limit_it = limits_set_.find(key); + bool has_limit = limit_it != limits_set_.end(); + if (has_limit && limit_it->max_size != 0) { + max_size = limit_it->max_size; + } + if (total_size > max_size) { + LOG(INFO) << "Drop too big rldp query " << part.total_size_ << " > " << max_size; + return; + } + + auto it = inbound_transfers_.find(transfer_id); + if (it == inbound_transfers_.end()) { + if (!has_limit) { + // set timeout even for small inbound queries + // TODO: other party stil may ddos us with small transfers + set_receive_limits(transfer_id, td::Timestamp::in(10), max_size); + } + it = inbound_transfers_.emplace(transfer_id, InboundTransfer{total_size}).first; + } + + auto &inbound = it->second; + auto o_res = [&]() -> td::optional> { + TRY_RESULT(in_part, inbound.get_part(part.part_, r_fec_type.move_as_ok())); + if (!in_part) { + if (inbound.is_part_completed(part.part_)) { + send_packet(ton::create_serialize_tl_object(transfer_id, part.part_)); + } + return {}; + } + if (in_part->receiver.on_received(part.seqno_, td::Timestamp::now())) { + TRY_STATUS_PREFIX(in_part->decoder->add_symbol({static_cast(part.seqno_), std::move(part.data_)}), + td::Status::Error(ErrorCode::protoviolation, "invalid symbol")); + if (in_part->decoder->may_try_decode()) { + auto r_data = in_part->decoder->try_decode(false); + if (r_data.is_ok()) { + inbound.finish_part(part.part_, r_data.move_as_ok().data); + } + } + } + return inbound.try_finish(); + }(); + + if (o_res) { + drop_limits(transfer_id); + on_inbound_completed(transfer_id, td::Timestamp::now()); + to_receive_.emplace_back(transfer_id, o_res.unwrap()); + } +} + +void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_complete &complete) { + auto transfer_id = complete.transfer_id_; + auto it = outbound_transfers_.find(transfer_id); + if (it == outbound_transfers_.end()) { + return; + } + + auto *part = it->second.get_part(complete.part_); + if (part) { + in_flight_count_ -= part->sender.get_inflight_symbols_count(); + it->second.drop_part(complete.part_); + } + + if (it->second.is_done()) { + drop_limits(it->first); + to_on_sent_.emplace_back(it->first, td::Unit()); + outbound_transfers_.erase(it); + } +} + +void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_confirm &confirm) { + auto transfer_id = confirm.transfer_id_; + auto it = outbound_transfers_.find(transfer_id); + if (it == outbound_transfers_.end()) { + return; + } + auto *part = it->second.get_part(confirm.part_); + if (!part) { + return; + } + Guard guard(in_flight_count_, part->sender); + Ack ack; + ack.max_seqno = confirm.max_seqno_; + ack.received_count = confirm.received_count_; + ack.received_mask = confirm.received_mask_; + auto update = part->sender.on_ack(ack, 0, td::Timestamp::now(), rtt_stats_, bdw_stats_, loss_stats_); + // update.new_received event + // update.o_loss_at event +} + +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/RldpConnection.h b/rldp2/RldpConnection.h new file mode 100644 index 000000000..e0adaeb49 --- /dev/null +++ b/rldp2/RldpConnection.h @@ -0,0 +1,119 @@ +#pragma once + +#include "Bbr.h" +#include "InboundTransfer.h" +#include "LossStats.h" +#include "OutboundTransfer.h" +#include "Pacer.h" +#include "RttStats.h" + +#include "common/bitstring.h" + +#include "td/utils/buffer.h" +#include "td/utils/Heap.h" +#include "td/utils/VectorQueue.h" + +#include + +namespace ton { +namespace rldp2 { +using TransferId = td::Bits256; +class ConnectionCallback { + public: + virtual ~ConnectionCallback() { + } + virtual void send_raw(td::BufferSlice small_datagram) = 0; + virtual void receive(TransferId transfer_id, td::Result r_data) = 0; + virtual void on_sent(TransferId transfer_id, td::Result state) = 0; +}; + +class RldpConnection { + public: + RldpConnection(); + RldpConnection(RldpConnection &&other) = delete; + RldpConnection &operator=(RldpConnection &&other) = delete; + void send(TransferId tranfer_id, td::BufferSlice data, td::Timestamp timeout = td::Timestamp::never()); + void set_receive_limits(TransferId transfer_id, td::Timestamp timeout, td::uint64 max_size); + + void receive_raw(td::BufferSlice packet); + + td::Timestamp run(ConnectionCallback &callback); + + void set_default_mtu(td::uint64 mtu) { + default_mtu_ = mtu; + } + td::uint64 default_mtu() const { + return default_mtu_; + } + + private: + td::uint64 default_mtu_ = 7680; + + std::map outbound_transfers_; + td::uint32 in_flight_count_{0}; + std::map inbound_transfers_; + + struct Limit : public td::HeapNode { + TransferId transfer_id; + td::uint64 max_size; + bool is_inbound; + bool operator<(const Limit &other) const { + return transfer_id < other.transfer_id; + } + }; + td::KHeap limits_heap_; + std::set limits_set_; + + struct CompletedId { + TransferId transfer_id; + td::Timestamp timeout; + }; + td::VectorQueue completed_queue_; + std::set completed_set_; + + void add_limit(td::Timestamp timeout, Limit limit); + td::Timestamp next_limit_expires_at(); + void drop_limits(TransferId id); + void on_inbound_completed(TransferId transfer_id, td::Timestamp now); + td::Timestamp loop_limits(td::Timestamp now); + + void loop_bbr(td::Timestamp now); + + RttStats rtt_stats_; + BdwStats bdw_stats_; + LossStats loss_stats_; + Bbr bbr_; + Pacer pacer_; + td::uint32 congestion_window_{0}; + + std::vector to_send_raw_; + std::vector>> to_receive_; + std::vector>> to_on_sent_; + + void send_packet(td::BufferSlice packet) { + to_send_raw_.push_back(std::move(packet)); + }; + + td::Timestamp run(const TransferId &transfer_id, InboundTransfer &inbound); + struct Guard { + td::uint32 &in_flight_count; + const RldpSender &sender; + td::uint32 before_in_flight{sender.get_inflight_symbols_count()}; + + Guard(td::uint32 &in_flight_count, const RldpSender &sender) : in_flight_count(in_flight_count), sender(sender){}; + ~Guard() { + in_flight_count -= before_in_flight; + in_flight_count += sender.get_inflight_symbols_count(); + } + }; + + td::optional step(const TransferId &transfer_id, OutboundTransfer &outbound, td::Timestamp now); + + void receive_raw_obj(ton::ton_api::rldp2_messagePart &part); + + void receive_raw_obj(ton::ton_api::rldp2_complete &part); + + void receive_raw_obj(ton::ton_api::rldp2_confirm &part); +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/RldpReceiver.cpp b/rldp2/RldpReceiver.cpp new file mode 100644 index 000000000..ed97c5340 --- /dev/null +++ b/rldp2/RldpReceiver.cpp @@ -0,0 +1,34 @@ +#include "RldpReceiver.h" + +namespace ton { +namespace rldp2 { +td::Variant RldpReceiver::next_action(td::Timestamp now) { + if (send_ack_at_ && (send_ack_at_.is_in_past(now))) { + return ActionSendAck{ack}; + } + return ActionWait{send_ack_at_}; +} + +void RldpReceiver::on_ack_sent(td::Timestamp now) { + if (cnt_ != 0) { + //LOG(ERROR) << "RESEND ACK " << cnt_; + } + cnt_++; + if (cnt_ > 7) { + send_ack_at_ = {}; + } else { + send_ack_at_.relax(td::Timestamp::at(now.at() + config_.ack_delay * (1 << cnt_))); + } +} + +bool RldpReceiver::on_received(td::uint32 seqno, td::Timestamp now) { + if (!ack.on_got_packet(seqno)) { + return false; + } + cnt_ = 0; + send_ack_at_.relax(td::Timestamp::at(now.at() + config_.ack_delay)); + return true; +} + +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/RldpReceiver.h b/rldp2/RldpReceiver.h new file mode 100644 index 000000000..b41a55f88 --- /dev/null +++ b/rldp2/RldpReceiver.h @@ -0,0 +1,36 @@ +#pragma once + +#include "Ack.h" +#include "RldpSender.h" + +namespace ton { +namespace rldp2 { +class RldpReceiver { + public: + RldpReceiver() = default; + RldpReceiver(RldpSender::Config config) : config_(config) { + } + + struct ActionSendAck { + Ack ack; + }; + + struct ActionWait { + td::Timestamp wait_till; + }; + + td::Variant next_action(td::Timestamp now); + + bool on_received(td::uint32 seqno, td::Timestamp now); + + void on_ack_sent(td::Timestamp now); + + private: + Ack ack; + td::Timestamp send_ack_at_; + td::uint32 cnt_{0}; + + RldpSender::Config config_; +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/RldpSender.cpp b/rldp2/RldpSender.cpp new file mode 100644 index 000000000..6964689af --- /dev/null +++ b/rldp2/RldpSender.cpp @@ -0,0 +1,95 @@ +#include "RldpSender.h" + +#include "RttStats.h" +#include "LossStats.h" +#include "BdwStats.h" + +#include "td/utils/misc.h" + +namespace ton { +namespace rldp2 { +td::Variant RldpSender::next_action(td::Timestamp now, + bool only_probe) { + if (!only_probe && extra_symbols_ > get_inflight_symbols_count()) { + //LOG(ERROR) << fec_helper_.symbols_count << " " << fec_helper_.get_extra_symbols_count(); + return ActionSend{packets_.next_seqno(), false}; + } + + return next_probe(now); +} + +td::Variant RldpSender::next_probe(td::Timestamp now) { + if (probe_timeout_.is_in_past(now)) { + return ActionSend{packets_.next_seqno(), true}; + } + return ActionWait{probe_timeout_}; +} + +SenderPackets::Update RldpSender::on_ack(const Ack &ack, double ack_delay, td::Timestamp now, RttStats &rtt_stats, + BdwStats &bdw_stats, LossStats &loss_stats) { + //LOG(ERROR) << "ON ACK " << ack.max_seqno << " " << ack.received_mask << " " << ack.received_count; + auto update = packets_.on_ack(ack); + if (!update.was_max_updated) { + CHECK(!update.new_received); + return update; + } + + // update rtt + ack_delay = td::clamp(ack_delay, 0.0, config_.max_ack_delay); + auto rtt_sample = now.at() - packets_.max_packet().sent_at.at(); + rtt_stats.on_rtt_sample(rtt_sample, ack_delay, now); + + bdw_stats.on_update(now, update.new_received); + bdw_stats.on_packet_ack(packets_.max_packet().bdw_packet_info, packets_.max_packet().sent_at, now); + + // drop ready packets + SenderPackets::Limits limits; + limits.sent_at = td::Timestamp::at(now.at() - get_loss_delay(rtt_stats)); + limits.seqno = sub_or_zero(packets_.max_packet().seqno, get_loss_seqno_delay()); + update.drop_update = packets_.drop_packets(limits); + + loss_stats.on_update(update.drop_update.new_ack, update.drop_update.new_lost); + + fec_helper_.received_symbols_count = packets_.received_count(); + extra_symbols_ = loss_stats.prob.send_n(fec_helper_.get_left_fec_symbols_count()); + return update; +} + +void RldpSender::on_send(td::uint32 seqno, td::Timestamp now, bool is_probe, const RttStats &rtt_stats, + const BdwStats &bdw_stats) { + SenderPackets::Packet packet; + packet.is_in_flight = true; + packet.sent_at = now; + packet.seqno = seqno; + packet.size = 0; + packet.bdw_packet_info = bdw_stats.on_packet_send(packets_.first_sent_at(now)); + packets_.send(packet); + + probe_timeout_ = td::Timestamp::at(now.at() + get_probe_delay(rtt_stats)); + + if (is_probe) { + //LOG(ERROR) << get_probe_delay(rtt_stats) << " " << rtt_stats.last_rtt << " " << packets_.in_flight_count() << " " + //<< packets_.received_count(); + probe_k_ = std::min(probe_k_ * 2, 10u); + } else { + probe_k_ = 1; + } +} + +double RldpSender::get_loss_delay(const RttStats &rtt_stats) { + auto rtt = std::max(rtt_stats.last_rtt, rtt_stats.smoothed_rtt); + if (rtt < 0) { + rtt = config_.initial_rtt; + } + return rtt * 8 / 7; +} + +double RldpSender::get_probe_delay(const RttStats &rtt_stats) { + if (rtt_stats.last_rtt < 0) { + return config_.initial_rtt * 2; + } else { + return (rtt_stats.smoothed_rtt + rtt_stats.rtt_var * 4 + config_.max_ack_delay) * probe_k_; + } +} +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/RldpSender.h b/rldp2/RldpSender.h new file mode 100644 index 000000000..a5f582b7e --- /dev/null +++ b/rldp2/RldpSender.h @@ -0,0 +1,81 @@ +#pragma once + +#include "td/utils/Time.h" +#include "td/utils/Variant.h" + +#include "FecHelper.h" +#include "SenderPackets.h" + +namespace ton { +namespace rldp2 { +struct Ack; +struct BdwStats; +struct RttStats; +struct LossStats; + +inline td::uint32 sub_or_zero(td::uint32 a, td::uint32 b) { + if (a < b) { + return 0; + } + return a - b; +} + +class RldpSender { + public: + struct Config { + static constexpr double DEFAULT_MAX_ACK_DELAY = 0.01; + static constexpr td::uint32 DEFAULT_PACKET_TRESHOLD = 3; + static constexpr double DEFAULT_INITIAL_RTT = 0.5; + + double max_ack_delay{DEFAULT_MAX_ACK_DELAY}; + double ack_delay{DEFAULT_MAX_ACK_DELAY}; + td::uint32 packet_treshold{DEFAULT_PACKET_TRESHOLD}; + double initial_rtt{DEFAULT_INITIAL_RTT}; + }; + + RldpSender() = default; + RldpSender(Config config, td::uint32 symbols_count) : config_(config) { + fec_helper_.symbols_count = symbols_count; + extra_symbols_ = fec_helper_.get_left_fec_symbols_count(); + } + + struct ActionWait { + td::Timestamp wait_till; + }; + + struct ActionSend { + td::uint32 seqno; + bool is_probe; + }; + + td::Variant next_action(td::Timestamp now, bool only_probe = false); + td::Variant next_probe(td::Timestamp now); + + td::uint32 get_inflight_symbols_count() const { + return packets_.in_flight_count(); + } + + SenderPackets::Update on_ack(const Ack &ack, double ack_delay, td::Timestamp now, RttStats &rtt_stats, + BdwStats &bdw_stats, LossStats &loss_stats); + + void on_send(td::uint32 seqno, td::Timestamp now, bool is_probe, const RttStats &rtt_stats, + const BdwStats &bdw_state); + + private: + Config config_; + SenderPackets packets_; + FecHelper fec_helper_; + td::Timestamp probe_timeout_; + td::uint32 probe_k_{1}; + td::uint32 extra_symbols_{0}; + + double get_loss_delay(const RttStats &rtt_stats); + + double get_probe_delay(const RttStats &rtt_stats); + + td::uint32 get_loss_seqno_delay() { + return config_.packet_treshold; + } +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/RttStats.cpp b/rldp2/RttStats.cpp new file mode 100644 index 000000000..4e998e2cd --- /dev/null +++ b/rldp2/RttStats.cpp @@ -0,0 +1,51 @@ +#include "RttStats.h" +#include + +namespace ton { +namespace rldp2 { +void RttStats::on_rtt_sample(double rtt_sample, double ack_delay, td::Timestamp now) { + if (rtt_sample < 0.001 || rtt_sample > 10) { + LOG(WARNING) << "Suspicious rtt sample " << rtt_sample; + return; + } + if (ack_delay < -1e-9 || ack_delay > 10) { + LOG(WARNING) << "Suspicious ack_delay " << ack_delay; + return; + } + rtt_sample = td::max(0.01, rtt_sample); + + last_rtt = rtt_sample; + + windowed_min_rtt_stat.add_event(rtt_sample, now.at()); + auto windowed_min_rtt_sample = windowed_min_rtt_stat.get_stat(now.at()).get_stat(); + if (windowed_min_rtt_sample) { + windowed_min_rtt = windowed_min_rtt_sample.value(); + } + + if (smoothed_rtt < 0) { + // ignore ack_delay just because + min_rtt = last_rtt; + smoothed_rtt = last_rtt; + rtt_var = last_rtt / 2; + } else { + if (rtt_sample < min_rtt) { + min_rtt = rtt_sample; + } + + double adjusted_rtt = rtt_sample; + if (adjusted_rtt - ack_delay > min_rtt) { + adjusted_rtt -= ack_delay; + } + + smoothed_rtt += (adjusted_rtt - smoothed_rtt) / 8; + double var = fabs(smoothed_rtt - adjusted_rtt); + rtt_var += (var - rtt_var) / 4; + } + + if (td::Timestamp::in(smoothed_rtt, rtt_round_at).is_in_past(now)) { + rtt_round_at = now; + rtt_round++; + } +} +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/RttStats.h b/rldp2/RttStats.h new file mode 100644 index 000000000..fca1385c8 --- /dev/null +++ b/rldp2/RttStats.h @@ -0,0 +1,23 @@ +#pragma once + +#include "td/utils/Time.h" +#include "td/utils/TimedStat.h" + +namespace ton { +namespace rldp2 { +struct RttStats { + void on_rtt_sample(double rtt_sample, double ack_delay, td::Timestamp now); + + double min_rtt = -1; + double windowed_min_rtt = -1; + double last_rtt = -1; + double smoothed_rtt = -1; + double rtt_var = -1; + td::uint32 rtt_round{0}; + + private: + td::Timestamp rtt_round_at; + td::TimedStat> windowed_min_rtt_stat{5, 0}; +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/SenderPackets.cpp b/rldp2/SenderPackets.cpp new file mode 100644 index 000000000..d0969fef4 --- /dev/null +++ b/rldp2/SenderPackets.cpp @@ -0,0 +1,125 @@ +#include "SenderPackets.h" + +#include "td/utils/bits.h" + +namespace ton { +namespace rldp2 { +td::uint32 SenderPackets::next_seqno() const { + return last_seqno_ + 1; +} + +SenderPackets::DropUpdate SenderPackets::drop_packets(const Limits &limits) { + while (!packets.empty()) { + auto &packet = packets.front(); + if (!limits.should_drop(packet)) { + break; + } + mark_ack_or_lost(packet); + packets.pop(); + } + DropUpdate update; + update.new_ack = total_ack_ - last_total_ack_; + update.new_lost = total_lost_ - last_total_lost_; + last_total_ack_ = total_ack_; + last_total_lost_ = total_lost_; + update.o_loss_at = std::move(last_loss_); + return update; +} + +SenderPackets::Update SenderPackets::on_ack(Ack ack) { + ack.max_seqno = td::min(ack.max_seqno, last_seqno_); + ack.received_count = td::min(ack.received_count, ack.max_seqno); + + // TODO: seqno of rldp and seqno of a packet must be completly separate seqnos + Update update; + if (received_count_ < ack.received_count) { + update.new_received = ack.received_count - received_count_; + left_ack_ += update.new_received; + left_ack_ = td::min(left_ack_, in_flight_count_); + received_count_ = ack.received_count; + } + + if (max_packet_.seqno > ack.max_seqno) { + return update; + } + + auto packet = get_packet(ack.max_seqno); + if (!packet) { + return update; + } + + if (max_packet_.seqno < ack.max_seqno) { + update.was_max_updated = true; + max_packet_ = *packet; + } + + for (td::uint32 i : td::BitsRange(ack.received_mask)) { + if (ack.max_seqno < i) { + break; + } + auto seqno = ack.max_seqno - i; + auto packet = get_packet(seqno); + if (!packet) { + break; + } + mark_ack(*packet); + } + + return update; +} +void SenderPackets::mark_ack_or_lost(Packet &packet) { + if (left_ack_) { + mark_ack(packet); + } else { + mark_lost(packet); + } +} + +void SenderPackets::mark_lost(Packet &packet) { + if (!packet.is_in_flight) { + return; + } + total_lost_++; + in_flight_count_--; + packet.is_in_flight = false; + last_loss_ = packet.sent_at; +} + +void SenderPackets::mark_ack(Packet &packet) { + if (!packet.is_in_flight) { + return; + } + if (left_ack_ > 0) { + left_ack_--; + } + total_ack_++; + in_flight_count_--; + packet.is_in_flight = false; +} + +SenderPackets::Packet *SenderPackets::get_packet(td::uint32 seqno) { + if (packets.empty()) { + return nullptr; + } + auto front_seqno = packets.front().seqno; + if (front_seqno > seqno) { + return nullptr; + } + td::uint32 index = seqno - front_seqno; + if (index >= packets.size()) { + return nullptr; + } + auto packet = packets.data() + index; + CHECK(packet->seqno == seqno); + return packet; +} + +void SenderPackets::send(Packet packet) { + CHECK(next_seqno() == packet.seqno); + packets.push(packet); + last_seqno_++; + in_flight_count_ += packet.is_in_flight; +} + +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/SenderPackets.h b/rldp2/SenderPackets.h new file mode 100644 index 000000000..6282e1ce0 --- /dev/null +++ b/rldp2/SenderPackets.h @@ -0,0 +1,89 @@ +#pragma once + +#include "td/utils/VectorQueue.h" +#include "Ack.h" +#include "BdwStats.h" + +namespace ton { +namespace rldp2 { +class SenderPackets { + public: + struct Packet { + bool is_in_flight{false}; + td::Timestamp sent_at; + td::uint32 seqno{0}; + td::uint32 size{0}; + + BdwStats::PacketInfo bdw_packet_info; + }; + + struct Limits { + td::Timestamp sent_at; + td::uint32 seqno{0}; + bool should_drop(const Packet &packet) const { + return !packet.is_in_flight || packet.sent_at < sent_at || packet.seqno < seqno; + } + }; + + struct DropUpdate { + td::uint32 new_ack{0}; // ~= new_received + td::uint32 new_lost{0}; + td::optional o_loss_at; + }; + + struct Update { + bool was_max_updated{false}; + td::uint32 new_received{0}; + + DropUpdate drop_update; + }; + + td::VectorQueue packets; + + void send(Packet packet); + + td::uint32 next_seqno() const; + DropUpdate drop_packets(const Limits &limits); + + Update on_ack(Ack ack); + + td::uint32 in_flight_count() const { + return in_flight_count_; + } + td::uint32 received_count() const { + return received_count_; + } + const Packet &max_packet() const { + return max_packet_; + } + td::Timestamp first_sent_at(td::Timestamp now) const { + if (!packets.empty()) { + now.relax(packets.front().sent_at); + } + return now; + } + + private: + td::uint32 in_flight_count_{0}; // sum(packet.is_in_flight for packet in packets) + td::uint32 received_count_{0}; + td::uint32 last_seqno_{0}; + Packet max_packet_; + + td::uint32 total_ack_{0}; + td::uint32 total_lost_{0}; + td::uint32 last_total_ack_{0}; + td::uint32 last_total_lost_{0}; + + td::optional last_loss_; + td::uint32 left_ack_{0}; + + void mark_ack_or_lost(Packet &packet); + + void mark_lost(Packet &packet); + + void mark_ack(Packet &packet); + + Packet *get_packet(td::uint32 seqno); +}; +} // namespace rldp2 +} // namespace ton diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp new file mode 100644 index 000000000..afb6b395a --- /dev/null +++ b/rldp2/rldp-in.hpp @@ -0,0 +1,113 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once + +#include "rldp.hpp" + +#include "tl-utils/tl-utils.hpp" +#include "adnl/adnl-query.h" +#include "adnl/adnl-peer-table.h" + +#include "td/utils/List.h" + +#include +#include + +namespace ton { + +namespace rldp2 { + +class RldpLru : public td::ListNode { + public: + TransferId transfer_id() { + return transfer_id_; + } + + RldpLru(TransferId transfer_id) : transfer_id_(transfer_id) { + } + RldpLru() { + } + + static RldpLru *from_list_node(td::ListNode *node) { + return static_cast(node); + } + + private: + TransferId transfer_id_; +}; + +class RldpConnectionActor; +class RldpIn : public RldpImpl { + public: + static constexpr td::uint64 mtu() { + return (1ull << 37); + } + static constexpr td::uint32 lru_size() { + return 128; + } + void on_sent(TransferId transfer_id, td::Result state); + + void send_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override; + void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + td::BufferSlice data) override; + + void send_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, + td::Promise promise, td::Timestamp timeout, td::BufferSlice data) override { + send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu()); + } + void send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, + td::Promise promise, td::Timestamp timeout, td::BufferSlice data, + td::uint64 max_answer_size) override; + void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); + + void receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data); + + void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_message &message); + void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_query &message); + void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_answer &message); + void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + td::Result data); + + void add_id(adnl::AdnlNodeIdShort local_id) override; + + RldpIn(td::actor::ActorId adnl) : adnl_(adnl) { + } + + private: + std::unique_ptr make_adnl_callback(); + + td::actor::ActorId adnl_; + + std::map, td::actor::ActorOwn> + connections_; + + std::map> queries_; + + std::set local_ids_; + + td::actor::ActorId create_connection(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst); +}; + +} // namespace rldp2 + +} // namespace ton diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp new file mode 100644 index 000000000..06f47448b --- /dev/null +++ b/rldp2/rldp.cpp @@ -0,0 +1,246 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include "rldp-in.hpp" +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" +#include "td/utils/Random.h" +#include "fec/fec.h" +#include "RldpConnection.h" + +namespace ton { + +namespace rldp2 { + +class RldpConnectionActor : public td::actor::Actor, private ConnectionCallback { + public: + RldpConnectionActor(td::actor::ActorId rldp, adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, + td::actor::ActorId adnl) + : rldp_(std::move(rldp)), src_(src), dst_(dst), adnl_(std::move(adnl)){}; + + void send(TransferId transfer_id, td::BufferSlice query, td::Timestamp timeout = td::Timestamp::never()) { + connection_.send(transfer_id, std::move(query), timeout); + yield(); + } + void set_receive_limits(TransferId transfer_id, td::Timestamp timeout, td::uint64 max_size) { + connection_.set_receive_limits(transfer_id, timeout, max_size); + } + void receive_raw(td::BufferSlice data) { + connection_.receive_raw(std::move(data)); + yield(); + } + + private: + td::actor::ActorId rldp_; + adnl::AdnlNodeIdShort src_; + adnl::AdnlNodeIdShort dst_; + td::actor::ActorId adnl_; + RldpConnection connection_; + + void loop() override { + alarm_timestamp() = connection_.run(*this); + } + + void send_raw(td::BufferSlice data) override { + send_closure(adnl_, &adnl::Adnl::send_message, src_, dst_, std::move(data)); + } + void receive(TransferId transfer_id, td::Result data) override { + send_closure(rldp_, &RldpIn::receive_message, dst_, src_, transfer_id, std::move(data)); + } + void on_sent(TransferId transfer_id, td::Result state) override { + send_closure(rldp_, &RldpIn::on_sent, transfer_id, std::move(state)); + } +}; + +namespace { +TransferId get_random_transfer_id() { + TransferId transfer_id; + td::Random::secure_bytes(transfer_id.as_slice()); + return transfer_id; +} +TransferId get_responce_transfer_id(TransferId transfer_id) { + return transfer_id ^ TransferId::ones(); +} +} // namespace + +void RldpIn::send_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) { + return send_message_ex(src, dst, td::Timestamp::in(10.0), std::move(data)); +} + +void RldpIn::send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + td::BufferSlice data) { + td::Bits256 id; + td::Random::secure_bytes(id.as_slice()); + + auto B = serialize_tl_object(create_tl_object(id, std::move(data)), true); + + auto transfer_id = get_random_transfer_id(); + send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); +} + +void RldpIn::send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, + td::Promise promise, td::Timestamp timeout, td::BufferSlice data, + td::uint64 max_answer_size) { + auto query_id = adnl::AdnlQuery::random_query_id(); + + auto date = static_cast(timeout.at_unix()) + 1; + auto B = serialize_tl_object(create_tl_object(query_id, max_answer_size, date, std::move(data)), + true); + + auto connection = create_connection(src, dst); + auto transfer_id = get_random_transfer_id(); + auto response_transfer_id = get_responce_transfer_id(transfer_id); + send_closure(connection, &RldpConnectionActor::set_receive_limits, response_transfer_id, timeout, max_answer_size); + send_closure(connection, &RldpConnectionActor::send, transfer_id, std::move(B), timeout); + + queries_.emplace(response_transfer_id, std::move(promise)); +} + +void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data) { + auto B = serialize_tl_object(create_tl_object(query_id, std::move(data)), true); + + send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); +} + +void RldpIn::receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data) { + send_closure(create_connection(local_id, source), &RldpConnectionActor::receive_raw, std::move(data)); +} + +td::actor::ActorId RldpIn::create_connection(adnl::AdnlNodeIdShort src, + adnl::AdnlNodeIdShort dst) { + auto it = connections_.find(std::make_pair(src, dst)); + if (it != connections_.end()) { + return it->second.get(); + } + auto connection = td::actor::create_actor("RldpConnection", actor_id(this), src, dst, adnl_); + auto res = connection.get(); + connections_[std::make_pair(src, dst)] = std::move(connection); + return res; +} + +void RldpIn::receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + td::Result r_data) { + if (r_data.is_error()) { + auto it = queries_.find(transfer_id); + if (it != queries_.end()) { + it->second.set_error(r_data.move_as_error()); + queries_.erase(it); + } else { + VLOG(RLDP_INFO) << "received error to unknown transfer_id " << transfer_id << " " << r_data.error(); + } + return; + } + + auto data = r_data.move_as_ok(); + //LOG(ERROR) << "RECEIVE MESSAGE " << data.size(); + auto F = fetch_tl_object(std::move(data), true); + if (F.is_error()) { + VLOG(RLDP_INFO) << "failed to parse rldp packet [" << source << "->" << local_id << "]: " << F.move_as_error(); + return; + } + + ton_api::downcast_call(*F.move_as_ok().get(), + [&](auto &obj) { this->process_message(source, local_id, transfer_id, obj); }); +} + +void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_message &message) { + td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver, source, local_id, std::move(message.data_)); +} + +void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_query &message) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source, local_id, + timeout = td::Timestamp::at_unix(message.timeout_), query_id = message.query_id_, + max_answer_size = static_cast(message.max_answer_size_), + transfer_id](td::Result R) { + if (R.is_ok()) { + auto data = R.move_as_ok(); + if (data.size() > max_answer_size) { + VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; + } else { + td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones(), std::move(data)); + } + } else { + VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); + } + }); + VLOG(RLDP_DEBUG) << "delivering rldp query"; + td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver_query, source, local_id, std::move(message.data_), + std::move(P)); +} + +void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_answer &message) { + auto it = queries_.find(transfer_id); + if (it != queries_.end()) { + it->second.set_value(std::move(message.data_)); + queries_.erase(it); + } else { + VLOG(RLDP_INFO) << "received answer to unknown query " << message.query_id_; + } +} + +void RldpIn::on_sent(TransferId transfer_id, td::Result state) { + //TODO: completed transfer +} + +void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) { + if (local_ids_.count(local_id) == 1) { + return; + } + + std::vector X{adnl::Adnl::int_to_bytestring(ton_api::rldp2_messagePart::ID), + adnl::Adnl::int_to_bytestring(ton_api::rldp2_confirm::ID), + adnl::Adnl::int_to_bytestring(ton_api::rldp2_complete::ID)}; + for (auto &x : X) { + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, x, make_adnl_callback()); + } + + local_ids_.insert(local_id); +} + +std::unique_ptr RldpIn::make_adnl_callback() { + class Callback : public adnl::Adnl::Callback { + private: + td::actor::ActorId id_; + + public: + Callback(td::actor::ActorId id) : id_(id) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + td::actor::send_closure(id_, &RldpIn::receive_message_part, src, dst, std::move(data)); + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + promise.set_error(td::Status::Error(ErrorCode::notready, "rldp does not support queries")); + } + }; + + return std::make_unique(actor_id(this)); +} + +td::actor::ActorOwn Rldp::create(td::actor::ActorId adnl) { + return td::actor::create_actor("rldp", td::actor::actor_dynamic_cast(adnl)); +} + +} // namespace rldp2 + +} // namespace ton diff --git a/rldp2/rldp.h b/rldp2/rldp.h new file mode 100644 index 000000000..f4b269854 --- /dev/null +++ b/rldp2/rldp.h @@ -0,0 +1,45 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include "adnl/adnl.h" + +namespace ton { + +namespace rldp2 { + +class Rldp : public adnl::AdnlSenderInterface { + public: + virtual ~Rldp() = default; + + static constexpr td::uint64 default_mtu() { + return adnl::Adnl::get_mtu(); + } + + virtual void add_id(adnl::AdnlNodeIdShort local_id) = 0; + + virtual void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + td::BufferSlice data) = 0; + + static td::actor::ActorOwn create(td::actor::ActorId adnl); +}; + +} // namespace rldp2 + +} // namespace ton diff --git a/rldp2/rldp.hpp b/rldp2/rldp.hpp new file mode 100644 index 000000000..824217db7 --- /dev/null +++ b/rldp2/rldp.hpp @@ -0,0 +1,48 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once + +#include "rldp.h" + +#include "tl-utils/tl-utils.hpp" +#include "adnl/adnl-query.h" + +#include + +namespace ton { + +namespace rldp2 { + +constexpr int VERBOSITY_NAME(RLDP_WARNING) = verbosity_WARNING; +constexpr int VERBOSITY_NAME(RLDP_NOTICE) = verbosity_INFO; +constexpr int VERBOSITY_NAME(RLDP_INFO) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(RLDP_DEBUG) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(RLDP_EXTRA_DEBUG) = verbosity_DEBUG + 1; + +using TransferId = td::Bits256; + +class RldpImpl : public Rldp { + public: + //virtual void transfer_completed(TransferId transfer_id) = 0; + //virtual void in_transfer_completed(TransferId transfer_id) = 0; +}; + +} // namespace rldp2 + +} // namespace ton diff --git a/storage/Bitset.h b/storage/Bitset.h new file mode 100644 index 000000000..1ed569e9e --- /dev/null +++ b/storage/Bitset.h @@ -0,0 +1,69 @@ +#pragma once + +#include "td/utils/Slice.h" +#include "td/utils/logging.h" + +namespace td { +struct Bitset { + public: + td::Slice as_slice() const { + return td::Slice(bits_).substr(0, (bits_size_ + 7) / 8); + } + bool get(size_t offset) const { + auto i = offset / 8; + if (i >= bits_.size()) { + return false; + } + auto bit_i = offset % 8; + return (bits_[i] & (1 << bit_i)) != 0; + } + + void reserve(size_t offset) { + auto i = offset / 8; + if (i >= bits_.size()) { + bits_.resize(i + 1); + } + } + + bool set_one(size_t offset) { + auto i = offset / 8; + auto bit_i = offset % 8; + bits_size_ = std::max(bits_size_, offset + 1); + if (i >= bits_.size()) { + bits_.resize(std::max(i + 1, bits_.size() * 2)); + } + auto mask = 1 << bit_i; + if ((bits_[i] & mask) == 0) { + bits_[i] |= mask; + count_++; + return true; + } + return false; + } + + size_t ones_count() const { + return count_; + } + + void set_raw(std::string bits) { + bits_ = std::move(bits); + bits_size_ = 0; + count_ = 0; + for (size_t n = size(), i = 0; i < n; i++) { + if (get(i)) { + count_++; + bits_size_ = i + 1; + } + } + } + + size_t size() const { + return bits_.size() * 8; + } + + private: + std::string bits_; + size_t bits_size_{0}; + size_t count_{0}; +}; +} // namespace td diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt new file mode 100644 index 000000000..fca967659 --- /dev/null +++ b/storage/CMakeLists.txt @@ -0,0 +1,55 @@ +cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) + +if (NOT OPENSSL_FOUND) + find_package(OpenSSL REQUIRED) +endif() + + +set(STORAGE_SOURCE + LoadSpeed.cpp + MerkleTree.cpp + NodeActor.cpp + PeerActor.cpp + PeerState.cpp + Torrent.cpp + TorrentCreator.cpp + TorrentHeader.cpp + TorrentInfo.cpp + TorrentMeta.cpp + + Bitset.h + LoadSpeed.h + MerkleTree.h + NodeActor.h + PartsHelper.h + PeerActor.h + PeerState.h + SharedState.h + Torrent.h + TorrentCreator.h + TorrentHeader.h + TorrentInfo.h + TorrentMeta.h +) + +set(STORAGE_CLI_SOURCE + storage-cli.cpp +) + +add_library(storage ${STORAGE_SOURCE}) +target_link_libraries(storage tdutils tdactor tddb ton_crypto tl_api ${JEMALLOC_LIBRARIES}) +target_include_directories(storage PUBLIC + $ +) + +add_executable(storage-cli ${STORAGE_CLI_SOURCE}) +target_link_libraries(storage-cli storage overlay tdutils tdactor adnl tl_api dht + rldp rldp2 catchain validatorsession full-node validator ton_validator validator + fift-lib memprof terminal ${JEMALLOC_LIBRARIES}) + +set(STORAGE_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/storage.cpp + PARENT_SCOPE +) +# Do not install it yet +#install(TARGETS storage-cli RUNTIME DESTINATION bin) diff --git a/storage/LoadSpeed.cpp b/storage/LoadSpeed.cpp new file mode 100644 index 000000000..274eb5f91 --- /dev/null +++ b/storage/LoadSpeed.cpp @@ -0,0 +1,33 @@ +#include "LoadSpeed.h" + +#include "td/utils/format.h" + +namespace ton { +void LoadSpeed::add(td::size_t size, td::Timestamp now) { + total_size_ += size; + events_.push(Event{size, now}); + update(now); +} +double LoadSpeed::speed(td::Timestamp now) const { + update(now); + return total_size_ / duration(); +} + +td::StringBuilder &operator<<(td::StringBuilder &sb, const LoadSpeed &speed) { + return sb << td::format::as_size(static_cast(speed.speed())) << "/s"; +} + +void LoadSpeed::update(td::Timestamp now) const { + while (duration() > 60) { + total_size_ -= events_.front().size; + events_.pop(); + } +} +double LoadSpeed::duration() const { + double res = 5; + if (events_.size() > 1) { + res = std::max(res, events_.back().at.at() - events_.front().at.at()); + } + return res; +} +} // namespace ton diff --git a/storage/LoadSpeed.h b/storage/LoadSpeed.h new file mode 100644 index 000000000..a56626e5b --- /dev/null +++ b/storage/LoadSpeed.h @@ -0,0 +1,25 @@ +#pragma once + +#include "td/utils/StringBuilder.h" +#include "td/utils/Time.h" +#include "td/utils/VectorQueue.h" + +namespace ton { +class LoadSpeed { + public: + void add(td::size_t size, td::Timestamp now); + double speed(td::Timestamp now = td::Timestamp::now()) const; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const LoadSpeed &speed); + + private: + struct Event { + td::size_t size; + td::Timestamp at; + }; + mutable td::VectorQueue events_; + mutable td::size_t total_size_{0}; + + double duration() const; + void update(td::Timestamp now) const; +}; +} // namespace ton diff --git a/storage/MerkleTree.cpp b/storage/MerkleTree.cpp new file mode 100644 index 000000000..8a90a4dc1 --- /dev/null +++ b/storage/MerkleTree.cpp @@ -0,0 +1,342 @@ +#include "MerkleTree.h" + +#include "common/bitstring.h" +#include "td/utils/UInt.h" + +#include "vm/cells/CellSlice.h" +#include "vm/cells/MerkleProof.h" +#include "vm/cellslice.h" +#include "vm/excno.hpp" + +namespace ton { +static td::Ref unpack_proof(td::Ref root) { + vm::CellSlice cs(vm::NoVm(), root); + CHECK(cs.special_type() == vm::Cell::SpecialType::MerkleProof); + return cs.fetch_ref(); +} + +td::uint32 MerkleTree::get_depth() const { + return log_n_; +} +td::Ref MerkleTree::get_root(size_t depth_limit) const { + if (depth_limit > log_n_ || root_proof_.is_null()) { + return root_proof_; + } + + auto usage_tree = std::make_shared(); + auto root_raw = vm::MerkleProof::virtualize(root_proof_, 1); + auto usage_cell = vm::UsageCell::create(root_raw, usage_tree->root_ptr()); + do_gen_proof(std::move(usage_cell), unpack_proof(root_proof_), depth_limit); + auto res = vm::MerkleProof::generate(root_raw, usage_tree.get()); + CHECK(res.not_null()); + return res; +} + +void MerkleTree::do_gen_proof(td::Ref node, td::Ref node_raw, size_t depth_limit) const { + if (depth_limit == 0) { + return; + } + // check if it is possible to load node without breaking virtualization + vm::CellSlice cs_raw(vm::NoVm(), std::move(node_raw)); + if (cs_raw.is_special()) { + return; + } + vm::CellSlice cs(vm::NoVm(), std::move(node)); + while (cs.have_refs()) { + do_gen_proof(cs.fetch_ref(), cs_raw.fetch_ref(), depth_limit - 1); + } +} + +td::Bits256 MerkleTree::get_root_hash() const { + CHECK(root_hash_); + return root_hash_.value(); +} + +MerkleTree::MerkleTree(size_t chunks_count, td::Bits256 root_hash) { + init_begin(chunks_count); + root_hash_ = root_hash; + init_finish(); +} + +MerkleTree::MerkleTree(size_t chunks_count, td::Ref root_proof) { + init_begin(chunks_count); + root_hash_ = unpack_proof(root_proof)->get_hash(0).as_array(); + root_proof_ = std::move(root_proof); + init_finish(); +} + +MerkleTree::MerkleTree(td::Span chunks) { + init_begin(chunks.size()); + + for (size_t i = 0; i < chunks.size(); i++) { + CHECK(chunks[i].index == i); + init_add_chunk(i, chunks[i].hash.as_slice()); + } + + init_finish(); +} + +void MerkleTree::init_begin(size_t chunks_count) { + log_n_ = 0; + while ((size_t(1) << log_n_) < chunks_count) { + log_n_++; + } + n_ = size_t(1) << log_n_; + total_blocks_ = chunks_count; + mark_.resize(n_ * 2); + proof_.resize(n_ * 2); + + td::UInt256 null{}; + auto cell = vm::CellBuilder().store_bytes(null.as_slice()).finalize(); + for (auto i = chunks_count; i < n_; i++) { + proof_[i + n_] = cell; + } +} + +void MerkleTree::init_add_chunk(size_t index, td::Slice hash) { + CHECK(index < total_blocks_); + CHECK(proof_[index + n_].is_null()); + proof_[index + n_] = vm::CellBuilder().store_bytes(hash).finalize(); +} + +void MerkleTree::init_finish() { + for (size_t i = n_ - 1; i >= 1; i--) { + auto j = i * 2; + if (proof_[j].is_null()) { + continue; + } + if (i + 1 < n_ && proof_[i + 1].not_null() && proof_[j]->get_hash() == proof_[j + 2]->get_hash() && + proof_[j + 1]->get_hash() == proof_[j + 3]->get_hash()) { + // minor optimization for same chunks + proof_[i] = proof_[i + 1]; + } else { + proof_[i] = vm::CellBuilder().store_ref(proof_[j]).store_ref(proof_[j + 1]).finalize(); + } + } + if (proof_[1].not_null()) { + init_proof(); + } + CHECK(root_hash_); +} + +void MerkleTree::remove_chunk(td::size_t index) { + CHECK(index < n_); + index += n_; + while (proof_[index].not_null()) { + proof_[index] = {}; + index /= 2; + } +} + +bool MerkleTree::has_chunk(td::size_t index) const { + CHECK(index < n_); + index += n_; + return proof_[index].not_null(); +} + +void MerkleTree::add_chunk(td::size_t index, td::Slice hash) { + CHECK(hash.size() == 32); + CHECK(index < n_); + index += n_; + auto cell = vm::CellBuilder().store_bytes(hash).finalize(); + CHECK(proof_[index].is_null()); + proof_[index] = std::move(cell); + mark_[index] = mark_id_; + for (index /= 2; index != 0; index /= 2) { + CHECK(proof_[index].is_null()); + auto &left = proof_[index * 2]; + auto &right = proof_[index * 2 + 1]; + if (left.not_null() && right.not_null()) { + proof_[index] = vm::CellBuilder().store_ref(left).store_ref(right).finalize(); + mark_[index] = mark_id_; + } + } +} + +static td::Status do_validate(td::Ref ref, size_t depth) { + vm::CellSlice cs(vm::NoVm(), std::move(ref)); + if (cs.is_special()) { + if (cs.special_type() != vm::Cell::SpecialType::PrunnedBranch) { + return td::Status::Error("Unexpected special cell"); + } + return td::Status::OK(); + } + if (depth == 0) { + if (cs.size() != 256) { + return td::Status::Error("List in proof must have 256 bits"); + } + if (cs.size_refs() != 0) { + return td::Status::Error("List in proof must have zero refs"); + } + } else { + if (cs.size() != 0) { + return td::Status::Error("Node in proof must have zero bits"); + } + if (cs.size_refs() != 2) { + return td::Status::Error("Node in proof must have two refs"); + } + TRY_STATUS(do_validate(cs.fetch_ref(), depth - 1)); + TRY_STATUS(do_validate(cs.fetch_ref(), depth - 1)); + } + return td::Status::OK(); +} + +td::Status MerkleTree::validate_proof(td::Ref new_root) { + // 1. depth <= log_n + // 2. each non special node has two refs and nothing else + // 3. each list contains only hash + // 4. all special nodes are merkle proofs + vm::CellSlice cs(vm::NoVm(), new_root); + if (cs.special_type() != vm::Cell::SpecialType::MerkleProof) { + return td::Status::Error("Proof must be a mekle proof cell"); + } + auto root = cs.fetch_ref(); + if (root_hash_ && root->get_hash(0).as_slice() != root_hash_.value().as_slice()) { + return td::Status::Error("Proof has invalid root hash"); + } + return do_validate(std::move(root), log_n_); +} + +td::Status MerkleTree::add_proof(td::Ref new_root) { + CHECK(root_proof_.not_null() || root_hash_); + TRY_STATUS(validate_proof(new_root)); + if (root_proof_.not_null()) { + auto combined = vm::MerkleProof::combine_fast(root_proof_, std::move(new_root)); + if (combined.is_null()) { + return td::Status::Error("Can't combine proofs"); + } + root_proof_ = std::move(combined); + } else { + root_proof_ = std::move(new_root); + } + return td::Status::OK(); +} + +td::Status MerkleTree::validate_existing_chunk(const Chunk &chunk) { + vm::CellSlice cs(vm::NoVm(), proof_[chunk.index + n_]); + CHECK(cs.size() == chunk.hash.size()); + if (cs.as_bitslice().compare(chunk.hash.cbits()) != 0) { + return td::Status::Error("Hash mismatch"); + } + return td::Status::OK(); +} + +td::Status MerkleTree::try_add_chunks(td::Span chunks) { + td::Bitset bitmask; + add_chunks(chunks, bitmask); + for (size_t i = 0; i < chunks.size(); i++) { + if (!bitmask.get(i)) { + return td::Status::Error(PSLICE() << "Invalid chunk #" << chunks[i].index); + } + } + return td::Status::OK(); +} + +void MerkleTree::add_chunks(td::Span chunks, td::Bitset &bitmask) { + if (root_proof_.is_null()) { + return; + } + + mark_id_++; + bitmask.reserve(chunks.size()); + for (size_t i = 0; i < chunks.size(); i++) { + const auto &chunk = chunks[i]; + if (has_chunk(chunk.index)) { + if (validate_existing_chunk(chunk).is_ok()) { + bitmask.set_one(i); + } + continue; + } + add_chunk(chunk.index, chunk.hash.as_slice()); + } + + root_proof_ = vm::CellBuilder::create_merkle_proof(merge(unpack_proof(root_proof_), 1)); + + for (size_t i = 0; i < chunks.size(); i++) { + const auto &chunk = chunks[i]; + if (has_chunk(chunk.index) && mark_[chunk.index + n_] == mark_id_) { + bitmask.set_one(i); + } + } +} + +td::Ref MerkleTree::merge(td::Ref root, size_t index) { + const auto &down = proof_[index]; + if (down.not_null()) { + if (down->get_hash() != root->get_hash(0)) { + proof_[index] = {}; + } else { + return down; + } + } + + if (mark_[index] != mark_id_ || index >= n_) { + return root; + } + + vm::CellSlice cs(vm::NoVm(), root); + if (cs.is_special()) { + cleanup_add(index); + return root; + } + + CHECK(cs.size_refs() == 2); + vm::CellBuilder cb; + cb.store_bits(cs.fetch_bits(cs.size())); + auto left = merge(cs.fetch_ref(), index * 2); + auto right = merge(cs.fetch_ref(), index * 2 + 1); + cb.store_ref(std::move(left)).store_ref(std::move(right)); + return cb.finalize(); +} + +void MerkleTree::cleanup_add(size_t index) { + if (mark_[index] != mark_id_) { + return; + } + proof_[index] = {}; + if (index >= n_) { + return; + } + cleanup_add(index * 2); + cleanup_add(index * 2 + 1); +} + +void MerkleTree::init_proof() { + CHECK(proof_[1].not_null()); + td::Bits256 new_root_hash = proof_[1]->get_hash(0).as_array(); + CHECK(!root_hash_ || root_hash_.value() == new_root_hash); + root_hash_ = new_root_hash; + root_proof_ = vm::CellBuilder::create_merkle_proof(proof_[1]); +} + +td::Result> MerkleTree::gen_proof(size_t l, size_t r) { + if (root_proof_.is_null()) { + return td::Status::Error("got no proofs yet"); + } + auto usage_tree = std::make_shared(); + auto root_raw = vm::MerkleProof::virtualize(root_proof_, 1); + auto usage_cell = vm::UsageCell::create(root_raw, usage_tree->root_ptr()); + TRY_STATUS(TRY_VM(do_gen_proof(std::move(usage_cell), 0, n_ - 1, l, r))); + auto res = vm::MerkleProof::generate(root_raw, usage_tree.get()); + CHECK(res.not_null()); + return res; +} + +td::Status MerkleTree::do_gen_proof(td::Ref node, size_t il, size_t ir, size_t l, size_t r) const { + if (ir < l || il > r) { + return td::Status::OK(); + } + if (l <= il && ir <= r) { + return td::Status::OK(); + } + vm::CellSlice cs(vm::NoVm(), std::move(node)); + if (cs.is_special()) { + return td::Status::Error("Can't generate a proof"); + } + CHECK(cs.size_refs() == 2); + auto ic = (il + ir) / 2; + TRY_STATUS(do_gen_proof(cs.fetch_ref(), il, ic, l, r)); + TRY_STATUS(do_gen_proof(cs.fetch_ref(), ic + 1, ir, l, r)); + return td::Status::OK(); +} +} // namespace ton diff --git a/storage/MerkleTree.h b/storage/MerkleTree.h new file mode 100644 index 000000000..9234a94d1 --- /dev/null +++ b/storage/MerkleTree.h @@ -0,0 +1,77 @@ +#pragma once + +#include "td/utils/optional.h" +#include "td/utils/Slice.h" +#include "vm/cells.h" + +#include "Bitset.h" + +namespace ton { +// merkle_node$_ {n:#} left:^(ton::MerkleTree n) right:^(ton::MerkleTree n) = ton::MerkleTree (n + 1); +// merkle_leaf$_ hash:bits256 = ton::MerkleTree 0; + +class MerkleTree { + public: + td::uint32 get_depth() const; + td::Ref get_root(size_t depth_limit = std::numeric_limits::max()) const; + td::Bits256 get_root_hash() const; + + MerkleTree(size_t chunks_count, td::Bits256 root_hash); + MerkleTree(size_t chunks_count, td::Ref root_proof); + + struct Chunk { + td::size_t index{0}; + td::Bits256 hash; + }; + + explicit MerkleTree(td::Span chunks); + + MerkleTree() = default; + void init_begin(size_t chunks_count); + void init_add_chunk(td::size_t index, td::Slice hash); + void init_finish(); + + // merge external proof with an existing proof + td::Status add_proof(td::Ref new_root); + // generate proof for all chunks from l to r inclusive + td::Result> gen_proof(size_t l, size_t r); + + // Trying to add and validate list of chunks simultaniously + td::Status try_add_chunks(td::Span chunks); + + // Returns bitmask of successfully added chunks + // Intended to be used during validation of a torrent. + // We got arbitrary chunks read from disk, and we got an arbirary proof. + // Now we can say about some chunks that they are correct. This ia a general way + // to do this. + // + // NB: already added chunks are simply validated. One should be careful + // not to process them twice + void add_chunks(td::Span chunks, td::Bitset &bitmask); + + private: + td::uint64 total_blocks_; + td::size_t n_; // n = 2^log_n + td::uint32 log_n_; + td::size_t mark_id_{0}; + std::vector mark_; // n_ * 2 + std::vector> proof_; // n_ * 2 + + td::optional root_hash_; + td::Ref root_proof_; + + td::Status validate_proof(td::Ref new_root); + bool has_chunk(td::size_t index) const; + void remove_chunk(td::size_t index); + + void add_chunk(td::size_t index, td::Slice hash); + void init_proof(); + + td::Ref merge(td::Ref root, size_t index); + void cleanup_add(size_t index); + td::Status do_gen_proof(td::Ref node, size_t il, size_t ir, size_t l, size_t r) const; + void do_gen_proof(td::Ref node, td::Ref node_raw, size_t depth_limit) const; + td::Status validate_existing_chunk(const Chunk &chunk); +}; + +} // namespace ton diff --git a/storage/NodeActor.cpp b/storage/NodeActor.cpp new file mode 100644 index 000000000..63e49989d --- /dev/null +++ b/storage/NodeActor.cpp @@ -0,0 +1,379 @@ +#include "NodeActor.h" + +#include "vm/boc.h" + +#include "td/utils/Enumerator.h" +#include "td/utils/tests.h" + +namespace ton { +NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, bool should_download) + : self_id_(self_id) + , torrent_(std::move(torrent)) + , callback_(std::move(callback)) + , should_download_(should_download) + , parts_helper_(torrent.get_info().pieces_count()) { +} + +void NodeActor::start_peer(PeerId peer_id, td::Promise> promise) { + peers_[peer_id]; + loop(); + auto it = peers_.find(peer_id); + if (it == peers_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error("Won't start peer now")); + return; + } + promise.set_value(it->second.actor.get()); +} + +void NodeActor::on_signal_from_peer(PeerId peer_id) { + loop_peer(peer_id, peers_[peer_id]); +} + +void NodeActor::start_up() { + callback_->register_self(actor_id(this)); + auto pieces_count = torrent_.get_info().pieces_count(); + parts_.parts.resize(pieces_count); + + auto header = torrent_.get_header_parts_range(); + for (td::uint32 i = static_cast(header.begin); i < header.end; i++) { + parts_helper_.set_part_priority(i, 255); + } + for (td::uint32 i = 0; i < pieces_count; i++) { + if (torrent_.is_piece_ready(i)) { + parts_helper_.on_self_part_ready(i); + parts_.parts[i].ready = true; + } + } + loop(); +} + +void NodeActor::loop_will_upload() { + if (peers_.empty()) { + return; + } + + if (!will_upload_at_.is_in_past()) { + alarm_timestamp().relax(will_upload_at_); + return; + } + + will_upload_at_ = td::Timestamp::in(5); + alarm_timestamp().relax(will_upload_at_); + std::vector> peers; + for (auto &it : peers_) { + auto state = it.second.state.lock(); + bool needed = false; + if (state->peer_state_) { + needed = state->peer_state_.value().want_download; + } + peers.emplace_back(!needed, !state->node_state_.want_download, -state->download.speed(), it.first); + } + std::sort(peers.begin(), peers.end()); + + if (peers.size() > 5) { + std::swap(peers[4], peers[td::Random::fast(5, (int)peers.size() - 1)]); + peers.resize(5); + } + + std::set peers_set; + for (auto id : peers) { + peers_set.insert(std::get(id)); + } + + for (auto &it : peers_) { + auto will_upload = peers_set.count(it.first) > 0; + auto state = it.second.state.lock(); + if (state->node_state_.will_upload != will_upload) { + state->node_state_.will_upload = will_upload; + state->notify_peer(); + } + } +} + +void NodeActor::loop() { + loop_get_peers(); + loop_start_stop_peers(); + loop_queries(); + loop_will_upload(); + + if (!ready_parts_.empty()) { + for (auto &it : peers_) { + auto state = it.second.state.lock(); + state->node_ready_parts_.insert(state->node_ready_parts_.end(), ready_parts_.begin(), ready_parts_.end()); + state->notify_peer(); + } + ready_parts_.clear(); + } + + if (torrent_.is_completed() && !is_completed_) { + is_completed_ = true; + callback_->on_completed(); + } +} + +std::string NodeActor::get_stats_str() { + td::StringBuilder sb; + sb << "Node " << self_id_ << " " << torrent_.get_ready_parts_count() << "\t" << download_; + sb << "\toutq " << parts_.total_queries; + sb << "\n"; + for (auto &it : peers_) { + auto state = it.second.state.lock(); + sb << "\tPeer " << it.first; + sb << "\t" << parts_helper_.get_ready_parts(it.second.peer_token).ones_count(); + sb << "\t" << state->download; + if (state->peer_state_) { + auto &peer_state = state->peer_state_.value(); + sb << "\t up:" << peer_state.will_upload; + sb << "\tdown:" << peer_state.want_download; + sb << "\tcnt:" << parts_helper_.get_want_download_count(it.second.peer_token); + } + sb << "\toutq:" << state->node_queries_.size(); + sb << "\tinq:" << state->peer_queries_.size(); + auto &node_state = state->node_state_; + sb << "\tNup:" << node_state.will_upload; + sb << "\tNdown:" << node_state.want_download; + sb << "\n"; + } + + auto o_n = torrent_.get_files_count(); + if (o_n) { + // by default all parts priority == 1 + auto n = o_n.unwrap(); + file_priority_.resize(n, 1); + for (size_t i = 0; i < n; i++) { + auto size = torrent_.get_file_size(i); + auto ready_size = torrent_.get_file_ready_size(i); + sb << "#" << i << " " << torrent_.get_file_name(i) << "\t" << 100 * ready_size / size << "%% " + << td::format::as_size(ready_size) << "/" << td::format::as_size(size) << "\t priority=" << file_priority_[i] + << "\n"; + } + } + return sb.as_cslice().str(); +} + +void NodeActor::set_file_priority(size_t i, td::uint8 priority) { + auto o_files_count = torrent_.get_files_count(); + if (!o_files_count) { + return; + } + auto files_count = o_files_count.unwrap(); + if (file_priority_.size() != files_count) { + // by default all parts priority == 1 + file_priority_.resize(files_count, 1); + } + + if (i >= files_count) { + for (td::uint32 part_i = 0; part_i < torrent_.get_info().pieces_count(); part_i++) { + parts_helper_.set_part_priority(part_i, priority); + } + for (auto &p : file_priority_) { + p = priority; + } + return; + } + if (file_priority_[i] == priority) { + return; + } + file_priority_[i] = priority; + auto range = torrent_.get_file_parts_range(i); + td::uint32 begin = static_cast(range.begin); + td::uint32 end = static_cast(range.end); + for (td::uint32 i = begin; i < end; i++) { + if (i == begin || i + 1 == end) { + auto chunks = torrent_.chunks_by_piece(i); + td::uint8 max_priority = 0; + for (auto chunk_id : chunks) { + if (chunk_id == 0) { + max_priority = 255; + } else { + max_priority = td::max(max_priority, file_priority_[chunk_id - 1]); + } + } + parts_helper_.set_part_priority(i, max_priority); + } else { + parts_helper_.set_part_priority(i, priority); + } + } + yield(); +} + +void NodeActor::set_should_download(bool should_download) { + should_download_ = should_download; + yield(); +} + +void NodeActor::tear_down() { + callback_->on_closed(std::move(torrent_)); +} + +void NodeActor::loop_start_stop_peers() { + for (auto &it : peers_) { + auto &peer = it.second; + auto peer_id = it.first; + + if (peer.notifier.empty()) { + peer.notifier = td::actor::create_actor("Notifier", actor_id(this), peer_id); + } + + if (peer.actor.empty()) { + LOG(ERROR) << "Init Peer " << self_id_ << " -> " << peer_id; + auto state = peer.state.lock(); + state->node = peer.notifier.get(); + for (td::uint32 i = 0; i < parts_.parts.size(); i++) { + if (parts_.parts[i].ready) { + state->node_ready_parts_.push_back(i); + } + } + peer.peer_token = parts_helper_.register_peer(peer_id); + peer.actor = callback_->create_peer(self_id_, peer_id, peer.state); + } + } +} + +void NodeActor::loop_queries() { + if (!should_download_) { + return; + } + for (auto &it : peers_) { + auto peer_token = it.second.peer_token; + auto state = it.second.state.lock(); + if (!state->peer_state_) { + parts_helper_.set_peer_limit(peer_token, 0); + continue; + } + if (!state->peer_state_.value().will_upload) { + parts_helper_.set_peer_limit(peer_token, 0); + continue; + } + parts_helper_.set_peer_limit(peer_token, + td::narrow_cast(MAX_PEER_TOTAL_QUERIES - state->node_queries_.size())); + } + + auto parts = parts_helper_.get_rarest_parts(MAX_TOTAL_QUERIES); + for (auto &part : parts) { + auto it = peers_.find(part.peer_id); + CHECK(it != peers_.end()); + auto state = it->second.state.lock(); + CHECK(state->peer_state_); + CHECK(state->peer_state_.value().will_upload); + CHECK(state->node_queries_.size() < MAX_PEER_TOTAL_QUERIES); + auto part_id = part.part_id; + state->node_queries_[static_cast(part_id)]; + parts_helper_.lock_part(part_id); + parts_.total_queries++; + parts_.parts[part_id].query_to_peer = part.peer_id; + state->notify_peer(); + } +} + +void NodeActor::loop_get_peers() { + if (has_get_peers_) { + return; + } + if (next_get_peers_at_.is_in_past()) { + callback_->get_peers(promise_send_closure(td::actor::actor_id(this), &NodeActor::got_peers)); + has_get_peers_ = true; + return; + } + alarm_timestamp().relax(next_get_peers_at_); +} + +void NodeActor::got_peers(td::Result> r_peers) { + if (r_peers.is_error()) { + next_get_peers_at_ = td::Timestamp::in(GET_PEER_RETRY_TIMEOUT); + } else { + auto peers = r_peers.move_as_ok(); + for (auto &peer : peers) { + if (peer == self_id_) { + continue; + } + peers_[peer]; + } + next_get_peers_at_ = td::Timestamp::in(GET_PEER_EACH); + } + has_get_peers_ = false; + loop(); +} + +void NodeActor::loop_peer(const PeerId &peer_id, Peer &peer) { + auto state = peer.state.lock(); + CHECK(!state->peer.empty()); + + for (auto part_id : state->peer_ready_parts_) { + parts_helper_.on_peer_part_ready(peer.peer_token, part_id); + } + state->peer_ready_parts_.clear(); + + // Answer queries from peer + bool should_notify_peer = false; + + auto want_download = parts_helper_.get_want_download_count(peer.peer_token) > 0; + if (state->node_state_.want_download != want_download) { + state->node_state_.want_download = want_download; + should_notify_peer = true; + } + + for (auto it = state->peer_queries_.begin(); it != state->peer_queries_.end();) { + if (it->second) { + it++; + } else { + should_notify_peer = true; + it->second = [&]() -> td::Result { + if (!state->node_state_.will_upload) { + return td::Status::Error("Won't upload"); + } + TRY_RESULT(proof, torrent_.get_piece_proof(it->first)); + TRY_RESULT(data, torrent_.get_piece_data(it->first)); + PeerState::Part res; + TRY_RESULT(proof_serialized, vm::std_boc_serialize(std::move(proof))); + res.proof = std::move(proof_serialized); + res.data = td::BufferSlice(std::move(data)); + return std::move(res); + }(); + } + } + + // Handle results from peer + for (auto it = state->node_queries_.begin(); it != state->node_queries_.end();) { + if (it->second) { + auto part_id = it->first; + auto r_unit = it->second.unwrap().move_fmap([&](PeerState::Part part) -> td::Result { + TRY_RESULT(proof, vm::std_boc_deserialize(part.proof)); + TRY_STATUS(torrent_.add_piece(part_id, part.data.as_slice(), std::move(proof))); + download_.add(part.data.size(), td::Timestamp::now()); + return td::Unit(); + }); + + parts_.parts[part_id].query_to_peer = {}; + parts_.total_queries--; + it = state->node_queries_.erase(it); + parts_helper_.unlock_part(part_id); + + if (r_unit.is_ok()) { + on_part_ready(part_id); + } else { + //LOG(ERROR) << "Failed " << part_id; + } + } else { + it++; + } + } + + if (should_notify_peer) { + state->notify_peer(); + } + + yield(); +} + +void NodeActor::on_part_ready(PartId part_id) { + parts_helper_.on_self_part_ready(part_id); + CHECK(!parts_.parts[part_id].ready); + parts_.parts[part_id].ready = true; + for (auto &peer : peers_) { + // TODO: notify only peer want_download_count == 0 + peer.second.state.unsafe()->notify_node(); + } + ready_parts_.push_back(part_id); +} +} // namespace ton diff --git a/storage/NodeActor.h b/storage/NodeActor.h new file mode 100644 index 000000000..3b4f57e55 --- /dev/null +++ b/storage/NodeActor.h @@ -0,0 +1,127 @@ +#pragma once + +#include "LoadSpeed.h" +#include "PartsHelper.h" +#include "PeerActor.h" +#include "Torrent.h" + +#include "td/utils/Random.h" + +#include + +namespace ton { +class NodeActor : public td::actor::Actor { + public: + class Callback { + public: + virtual ~Callback() { + } + virtual td::actor::ActorOwn create_peer(PeerId self_id, PeerId peer_id, + td::SharedState state) = 0; + virtual void get_peers(td::Promise> peers) = 0; + virtual void register_self(td::actor::ActorId self) = 0; + + //TODO: proper callbacks + virtual void on_completed() = 0; + virtual void on_closed(ton::Torrent torrent) = 0; + }; + + NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, bool should_download = true); + void start_peer(PeerId peer_id, td::Promise> promise); + + ton::Torrent *with_torrent() { + return &torrent_; + } + std::string get_stats_str(); + + void set_file_priority(size_t i, td::uint8 priority); + void set_should_download(bool should_download); + + private: + PeerId self_id_; + ton::Torrent torrent_; + std::vector file_priority_; + td::unique_ptr callback_; + bool should_download_{false}; + + class Notifier : public td::actor::Actor { + public: + Notifier(td::actor::ActorId node, PeerId peer_id) : node_(std::move(node)), peer_id_(peer_id) { + } + + void wake_up() override { + send_closure(node_, &NodeActor::on_signal_from_peer, peer_id_); + } + + private: + td::actor::ActorId node_; + PeerId peer_id_; + }; + + struct Peer { + td::actor::ActorOwn actor; + td::actor::ActorOwn notifier; + td::SharedState state; + PartsHelper::PeerToken peer_token; + }; + + std::map peers_; + + struct QueryId { + PeerId peer; + PartId part; + + auto key() const { + return std::tie(peer, part); + } + bool operator<(const QueryId &other) const { + return key() < other.key(); + } + }; + + struct PartsSet { + struct Info { + td::optional query_to_peer; + bool ready{false}; + }; + size_t total_queries{0}; + std::vector parts; + }; + + PartsSet parts_; + PartsHelper parts_helper_; + std::vector ready_parts_; + LoadSpeed download_; + + td::Timestamp next_get_peers_at_; + bool has_get_peers_{false}; + static constexpr double GET_PEER_RETRY_TIMEOUT = 5; + static constexpr double GET_PEER_EACH = 5; + + bool is_completed_{false}; + + td::Timestamp will_upload_at_; + + void on_signal_from_peer(PeerId peer_id); + + void start_up() override; + + void loop() override; + + void tear_down() override; + + void loop_start_stop_peers(); + + static constexpr size_t MAX_TOTAL_QUERIES = 20; + static constexpr size_t MAX_PEER_TOTAL_QUERIES = 5; + void loop_queries(); + bool try_send_query(); + bool try_send_part(PartId part_id); + void loop_get_peers(); + void got_peers(td::Result> r_peers); + void loop_peer(const PeerId &peer_id, Peer &peer); + void on_part_ready(PartId part_id); + + void loop_will_upload(); +}; +} // namespace ton diff --git a/storage/PartsHelper.h b/storage/PartsHelper.h new file mode 100644 index 000000000..0f76c352f --- /dev/null +++ b/storage/PartsHelper.h @@ -0,0 +1,259 @@ +#include "PeerState.h" +#include "Bitset.h" + +#include "td/utils/Random.h" +#include "td/utils/Status.h" + +namespace ton { +struct PartsHelper { + public: + PartsHelper(size_t parts_count) : parts_(parts_count), peers_(64) { + peers_[0].is_valid = true; + } + using PartId = size_t; + using PeerToken = size_t; + + PeerToken register_self() { + return self_token_; + } + PeerToken register_peer(PeerId peer_id) { + PeerToken new_peer_token{free_peer_tokens_.empty() ? next_peer_token_ : free_peer_tokens_.back()}; + auto it = peer_id_to_token_.emplace(peer_id, new_peer_token); + if (it.second) { + if (free_peer_tokens_.empty()) { + next_peer_token_++; + peers_.resize(next_peer_token_); + } else { + free_peer_tokens_.pop_back(); + } + auto peer = get_peer(new_peer_token, true); + peer->is_valid = true; + peer->peer_id = peer_id; + peer->want_download_count = 0; + } + return it.first->second; + } + void forget_peer(PeerToken peer_token) { + CHECK(peer_token != self_token_); + free_peer_tokens_.push_back(peer_token); + auto *peer = get_peer(peer_token); + peer_id_to_token_.erase(get_peer(peer_token)->peer_id); + *peer = Peer{}; + peer->rnd = td::Random::fast_uint32(); + } + + void set_peer_limit(PeerToken peer, td::uint32 limit) { + get_peer(peer)->limit = limit; + } + + void on_peer_part_ready(PeerToken peer_token, PartId part_id) { + auto peer = get_peer(peer_token); + if (!peer->ready_parts.set_one(part_id)) { + return; + } + auto part = get_part(part_id); + if (part->is_ready) { + return; + } + peer->want_download_count++; + if (!part->rnd) { + part->rnd = td::Random::fast_uint32(); + } + change_key(part_id, part->rnd, part->peers_count, part->peers_count + 1, part->priority, part->priority); + part->peers_count++; + } + + void lock_part(PartId part_id) { + auto *part = get_part(part_id); + CHECK(!part->is_locked); + part->is_locked = true; + } + void unlock_part(PartId part_id) { + auto *part = get_part(part_id); + CHECK(part->is_locked); + part->is_locked = false; + } + + void set_part_priority(PartId part_id, td::uint8 priority) { + auto *part = get_part(part_id); + if (part->is_ready) { + return; + } + change_key(part_id, part->rnd, part->peers_count, part->peers_count, part->priority, priority); + part->priority = priority; + } + + td::uint8 get_part_priority(PartId part_id) { + auto *part = get_part(part_id); + return part->priority; + } + + void on_self_part_ready(PartId part_id) { + auto peer = get_peer(self_token_); + if (!peer->ready_parts.set_one(part_id)) { + return; + } + auto part = get_part(part_id); + CHECK(!part->is_ready); + part->is_ready = true; + for (auto &peer : peers_) { + if (peer.ready_parts.get(part_id)) { + peer.want_download_count--; + } + } + change_key(part_id, part->rnd, part->peers_count, 0, part->priority, part->priority); + } + + struct RarePart { + PartId part_id; + PeerId peer_id; + }; + std::vector get_rarest_parts(size_t max_count) { + struct It { + std::set::iterator begin, end; + PeerId peer_id; + td::uint32 limit{0}; + bool empty() const { + return begin == end || limit == 0; + } + auto key() const { + return std::tie(*begin, peer_id); + } + bool operator<(const It &other) const { + return key() < other.key(); + } + }; + std::set its; + + for (auto &peer : peers_) { + if (!peer.is_valid || peer.limit == 0) { + continue; + } + It it; + it.begin = peer.rarest_parts.begin(); + it.end = peer.rarest_parts.end(); + it.peer_id = peer.peer_id; + it.limit = peer.limit; + if (it.empty()) { + continue; + } + its.insert(it); + } + + std::vector res; + while (res.size() < max_count && !its.empty()) { + auto it = *its.begin(); + its.erase(its.begin()); + auto part_id = it.begin->part_id; + if ((res.empty() || res.back().part_id != part_id) && !get_part(part_id)->is_locked) { + res.push_back({part_id, it.peer_id}); + CHECK(get_peer(register_peer(it.peer_id))->ready_parts.get(part_id)); + it.limit--; + } + it.begin++; + if (it.empty()) { + continue; + } + its.insert(it); + } + return res; + } + + td::uint32 get_want_download_count(PeerToken peer_token) { + return get_peer(peer_token, false)->want_download_count; + } + const td::Bitset &get_ready_parts(PeerToken peer_token) { + return get_peer(peer_token, false)->ready_parts; + } + + private: + PeerToken self_token_{0}; + size_t parts_count_; + struct Part { + bool is_locked{false}; + bool is_ready{false}; + td::uint8 priority{1}; + td::uint32 rnd{0}; + td::uint32 peers_count{0}; // invalid for ready parts + }; + struct Peer { + PeerId peer_id{0}; + bool is_valid{false}; + td::uint32 rnd{0}; + td::uint32 limit{0}; + td::Bitset ready_parts; + // sum_i (peer.ready_parts[i] && !node.ready_parts[i]) + td::uint32 want_download_count{0}; + + // peers_count - count of peers which has this part + // key_count = !is_ready * peers_count; + struct Key { + td::uint8 priority{0}; + td::uint32 count{0}; + PartId part_id{0}; + td::uint32 rnd{0}; + auto key() const { + return std::make_tuple(255 - priority, count, rnd, part_id); + } + bool operator<(const Key &other) const { + return key() < other.key(); + } + }; + std::set rarest_parts; // TODO: use vector instead of set + }; + + std::vector parts_; + std::vector peers_; + td::uint32 next_peer_token_{1}; + std::map peer_id_to_token_; + std::vector free_peer_tokens_; + + Part *get_part(PartId part_id) { + CHECK(part_id < parts_.size()); + return &parts_[part_id]; + } + Peer *get_peer(PeerToken peer_token, bool can_be_uninited = false) { + CHECK(peer_token < peers_.size()); + auto res = &peers_[peer_token]; + CHECK(res->is_valid || can_be_uninited); + return res; + } + + void change_key(PartId part_id, td::uint32 rnd, td::uint32 from_count, td::uint32 to_count, td::uint8 from_priority, + td::uint8 to_priority) { + if (from_count == 0 && to_count == 0) { + return; + } + if (from_count == to_count && from_priority == to_priority) { + return; + } + for (auto &peer : peers_) { + if (!peer.is_valid) { + continue; + } + if (peer.peer_id == 0) { + continue; + } + if (!peer.ready_parts.get(part_id)) { + continue; + } + //TODO: xor is not a perfect solution as it keeps a lot order between part_ids + Peer::Key key; + key.part_id = part_id; + key.rnd = rnd ^ peer.rnd; + + if (from_count != 0 && from_priority != 0) { + key.count = from_count; + key.priority = from_priority; + peer.rarest_parts.erase(key); + } + + if (to_count != 0 && to_priority != 0) { + key.count = to_count; + key.priority = to_priority; + peer.rarest_parts.insert(key); + } + } + } +}; +} // namespace ton diff --git a/storage/PeerActor.cpp b/storage/PeerActor.cpp new file mode 100644 index 000000000..4b23ba49b --- /dev/null +++ b/storage/PeerActor.cpp @@ -0,0 +1,400 @@ +#include "PeerActor.h" +#include "auto/tl/ton_api.hpp" + +#include "tl-utils/tl-utils.hpp" + +#include "td/utils/overloaded.h" +#include "td/utils/Random.h" + +namespace ton { + +PeerState::State from_ton_api(const ton::ton_api::storage_state &state) { + PeerState::State res; + res.want_download = state.want_download_; + res.will_upload = state.will_upload_; + return res; +} + +ton::ton_api::object_ptr to_ton_api(const PeerState::State &state) { + return ton::ton_api::make_object(state.will_upload, state.want_download); +} + +PeerActor::PeerActor(td::unique_ptr callback, td::SharedState state) + : callback_(std::move(callback)), state_(std::move(state)) { + CHECK(callback_); +} + +template +td::uint64 PeerActor::create_and_send_query(ArgsT &&... args) { + return send_query(ton::create_serialize_tl_object(std::forward(args)...)); +} + +td::uint64 PeerActor::send_query(td::BufferSlice query) { + auto query_id = next_query_id_++; + //LOG(ERROR) << "send_query " << to_string(ton::fetch_tl_object(std::move(query), true).ok()); + callback_->send_query(query_id, std::move(query)); + return query_id; +} + +void PeerActor::schedule_loop() { + yield(); +} + +void PeerActor::notify_node() { + need_notify_node_ = true; +} + +void PeerActor::execute_query(td::BufferSlice query, td::Promise promise) { + TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(std::move(query), true)); + //LOG(ERROR) << "execute_query " << to_string(f); + ton::ton_api::downcast_call( + *f, td::overloaded( + [&](ton::ton_api::storage_ping &ping) { + execute_ping(static_cast(ping.session_id_), std::move(promise)); + }, + [&](ton::ton_api::storage_addUpdate &add_update) { execute_add_update(add_update, std::move(promise)); }, + [&](ton::ton_api::storage_getPiece &get_piece) { execute_get_piece(get_piece, std::move(promise)); }, + [&](auto &other) { promise.set_error(td::Status::Error("Unknown function")); })); + schedule_loop(); +} + +void PeerActor::on_ping_result(td::Result r_answer) { + ping_query_id_ = {}; + if (r_answer.is_ok()) { + on_pong(); + } +} + +void PeerActor::on_pong() { + wait_pong_till_ = td::Timestamp::in(4); + state_.lock()->peer_online_ = true; + notify_node(); +} + +void PeerActor::on_update_result(td::Result r_answer) { + update_query_id_ = {}; + if (r_answer.is_ok()) { + peer_is_inited_ = true; + have_pieces_list_.clear(); + } +} + +void PeerActor::on_get_piece_result(PartId piece_id, td::Result r_answer) { + auto state = state_.lock(); + auto it = state->node_queries_.find(piece_id); + if (it == state->node_queries_.end()) { + LOG(ERROR) << "???"; + return; + } + //TODO: handle errors ??? + it->second = [&]() -> td::Result { + TRY_RESULT(slice, std::move(r_answer)); + TRY_RESULT(piece, ton::fetch_result(slice.as_slice())); + PeerState::Part res; + res.data = std::move(piece->data_); + res.proof = std::move(piece->proof_); + return std::move(res); + }(); + notify_node(); +} + +void PeerActor::on_update_state_result(td::Result r_answer) { + if (r_answer.is_error()) { + update_state_query_.query_id = {}; + } +} + +void PeerActor::on_query_result(td::uint64 query_id, td::Result r_answer) { + if (r_answer.is_ok()) { + on_pong(); + state_.lock()->download.add(r_answer.ok().size(), td::Timestamp::now()); + } + if (ping_query_id_ && ping_query_id_.value() == query_id) { + on_ping_result(std::move(r_answer)); + } else if (update_query_id_ && update_query_id_.value() == query_id) { + on_update_result(std::move(r_answer)); + } else if (update_state_query_.query_id && update_state_query_.query_id.value() == query_id) { + on_update_state_result(std::move(r_answer)); + } else { + for (auto &query_it : node_get_piece_) { + if (query_it.second.query_id && query_it.second.query_id.value() == query_id) { + on_get_piece_result(query_it.first, std::move(r_answer)); + query_it.second.query_id = {}; + } + } + } + + schedule_loop(); +} + +void PeerActor::start_up() { + callback_->register_self(actor_id(this)); + + node_session_id_ = td::Random::secure_uint64(); + + auto state = state_.lock(); + state->peer = actor_id(this); + + notify_node(); + schedule_loop(); +} + +void PeerActor::loop() { + loop_ping(); + loop_pong(); + + loop_update_init(); + loop_update_state(); + loop_update_pieces(); + + loop_node_get_piece(); + loop_peer_get_piece(); + + loop_notify_node(); +} + +void PeerActor::loop_pong() { + if (wait_pong_till_ && wait_pong_till_.is_in_past()) { + wait_pong_till_ = {}; + LOG(INFO) << "Disconnected"; + state_.lock()->peer_online_ = false; + notify_node(); + } + alarm_timestamp().relax(wait_pong_till_); +} + +void PeerActor::loop_ping() { + if (ping_query_id_) { + return; + } + if (!next_ping_at_.is_in_past()) { + alarm_timestamp().relax(next_ping_at_); + return; + } + + next_ping_at_ = td::Timestamp::in(2); + alarm_timestamp().relax(next_ping_at_); + ping_query_id_ = create_and_send_query(node_session_id_); +} + +td::BufferSlice PeerActor::create_update_query(ton::tl_object_ptr update) { + auto session_id = static_cast(peer_session_id_.value()); + auto seqno = static_cast(++node_seqno_); + return ton::create_serialize_tl_object(session_id, seqno, std::move(update)); +} + +void PeerActor::loop_update_init() { + if (!peer_session_id_) { + return; + } + if (update_query_id_) { + return; + } + if (peer_is_inited_) { + return; + } + + update_have_pieces(); + have_pieces_list_.clear(); + + auto state = state_.lock(); + auto query = create_update_query(ton::create_tl_object( + td::BufferSlice(have_pieces_.as_slice()), to_ton_api(state->node_state_))); + + // take care about update_state_query initial state + update_state_query_.state = state->node_state_; + update_state_query_.query_id = 0; + + update_query_id_ = send_query(std::move(query)); +} + +void PeerActor::loop_update_state() { + if (!peer_is_inited_) { + return; + } + + auto state = state_.lock(); + if (!(update_state_query_.state == state->node_state_)) { + update_state_query_.state = state->node_state_; + update_state_query_.query_id = {}; + } + + if (update_state_query_.query_id) { + return; + } + + auto query = create_update_query( + ton::create_tl_object(to_ton_api(update_state_query_.state))); + update_state_query_.query_id = send_query(std::move(query)); +} + +void PeerActor::update_have_pieces() { + auto state = state_.lock(); + have_pieces_list_.insert(have_pieces_list_.end(), state->node_ready_parts_.begin(), state->node_ready_parts_.end()); + for (auto piece_id : state->node_ready_parts_) { + have_pieces_.set_one(piece_id); + } + state->node_ready_parts_.clear(); +} + +void PeerActor::loop_update_pieces() { + if (update_query_id_) { + return; + } + + if (!peer_is_inited_) { + return; + } + + update_have_pieces(); + + if (!have_pieces_list_.empty()) { + auto query = create_update_query(ton::create_tl_object( + td::transform(have_pieces_list_, [](auto x) { return static_cast(x); }))); + update_query_id_ = send_query(std::move(query)); + } +} + +void PeerActor::loop_node_get_piece() { + auto state = state_.lock(); + + for (auto it = node_get_piece_.begin(); it != node_get_piece_.end();) { + auto other_it = state->node_queries_.find(it->first); + if (other_it == state->node_queries_.end() || other_it->second) { + it = node_get_piece_.erase(it); + } else { + it++; + } + } + + for (auto &query_it : state->node_queries_) { + if (query_it.second) { + continue; + } + node_get_piece_.emplace(query_it.first, NodePieceQuery{}); + } + + for (auto &query_it : node_get_piece_) { + if (query_it.second.query_id) { + continue; + } + + query_it.second.query_id = + create_and_send_query(static_cast(query_it.first)); + } +} + +void PeerActor::loop_peer_get_piece() { + auto state = state_.lock(); + + // process answers + for (auto &it : state->peer_queries_) { + if (!it.second) { + continue; + } + auto promise_it = peer_get_piece_.find(it.first); + if (promise_it == peer_get_piece_.end()) { + continue; + } + promise_it->second.promise.set_result(it.second.unwrap().move_map([](PeerState::Part part) { + return ton::create_serialize_tl_object(std::move(part.proof), std::move(part.data)); + })); + peer_get_piece_.erase(promise_it); + } + + // erase unneeded queries + for (auto it = state->peer_queries_.begin(); it != state->peer_queries_.end();) { + if (peer_get_piece_.count(it->first) == 0) { + it = state->peer_queries_.erase(it); + notify_node(); + } else { + it++; + } + } + + // create queries + for (auto &query_it : peer_get_piece_) { + auto res = state->peer_queries_.emplace(query_it.first, td::optional>()); + if (res.second) { + notify_node(); + } + } +} + +void PeerActor::loop_notify_node() { + if (!need_notify_node_) { + return; + } + need_notify_node_ = false; + state_.lock()->notify_node(); +} + +void PeerActor::execute_ping(td::uint64 session_id, td::Promise promise) { + if (!peer_session_id_ || peer_session_id_.value() != session_id) { + peer_session_id_ = session_id; + peer_is_inited_ = false; + + update_query_id_ = {}; + update_state_query_.query_id = {}; + } + + promise.set_value(ton::create_serialize_tl_object()); +} + +void PeerActor::execute_add_update(ton::ton_api::storage_addUpdate &add_update, td::Promise promise) { + auto session_id = static_cast(add_update.session_id_); + if (session_id != node_session_id_) { + promise.set_error(td::Status::Error(404, "INVALID_SESSION")); + return; + } + + promise.set_value(ton::create_serialize_tl_object()); + + auto state = state_.lock(); + auto add_piece = [&](PartId id) { + if (!peer_have_pieces_.get(id)) { + peer_have_pieces_.set_one(id); + state->peer_ready_parts_.push_back(id); + notify_node(); + } + }; + + auto seqno = static_cast(add_update.seqno_); + auto update_peer_state = [&](PeerState::State peer_state) { + if (peer_seqno_ >= seqno) { + return; + } + if (state->peer_state_ && state->peer_state_.value() == peer_state) { + return; + } + peer_seqno_ = seqno; + state->peer_state_ = peer_state; + notify_node(); + }; + + //LOG(ERROR) << "Got " << to_string(add_update); + downcast_call(*add_update.update_, + td::overloaded( + [&](ton::ton_api::storage_updateHavePieces &have_pieces) { + for (auto id : have_pieces.piece_id_) { + add_piece(id); + } + }, + [&](ton::ton_api::storage_updateState &state) { update_peer_state(from_ton_api(*state.state_)); }, + [&](ton::ton_api::storage_updateInit &init) { + update_peer_state(from_ton_api(*init.state_)); + td::Bitset new_bitset; + new_bitset.set_raw(init.have_pieces_.as_slice().str()); + for (auto size = new_bitset.size(), i = size_t(0); i < size; i++) { + if (new_bitset.get(i)) { + add_piece(static_cast(i)); + } + } + })); +} + +void PeerActor::execute_get_piece(ton::ton_api::storage_getPiece &get_piece, td::Promise promise) { + PartId piece_id = get_piece.piece_id_; + peer_get_piece_[piece_id] = {std::move(promise)}; +} +} // namespace ton diff --git a/storage/PeerActor.h b/storage/PeerActor.h new file mode 100644 index 000000000..397d76a81 --- /dev/null +++ b/storage/PeerActor.h @@ -0,0 +1,112 @@ +#pragma once + +#include "Bitset.h" +#include "PeerState.h" +#include "SharedState.h" + +#include "td/utils/optional.h" + +#include "auto/tl/ton_api.h" + +namespace ton { +class PeerActor : public td::actor::Actor { + public: + class Callback { + public: + virtual ~Callback() { + } + virtual void register_self(td::actor::ActorId self) = 0; + virtual void send_query(td::uint64 query_id, td::BufferSlice query) = 0; + }; + + PeerActor(td::unique_ptr callback, td::SharedState state); + + void execute_query(td::BufferSlice query, td::Promise promise); + void on_query_result(td::uint64 query_id, td::Result r_answer); + + private: + td::unique_ptr callback_; + td::SharedState state_; + bool need_notify_node_{false}; + + td::uint64 next_query_id_{0}; + + // ping + td::Timestamp next_ping_at_; + td::optional ping_query_id_; + td::Timestamp wait_pong_till_; + + // startSession + td::uint64 node_session_id_; + td::Bitset peer_have_pieces_; + + // update + td::optional peer_session_id_; + td::optional update_query_id_; + bool peer_is_inited_{false}; + td::uint32 node_seqno_{0}; + td::Bitset have_pieces_; + std::vector have_pieces_list_; + td::uint32 peer_seqno_{0}; + + // update state + struct UpdateState { + td::optional query_id; + PeerState::State state; + }; + UpdateState update_state_query_; + + // getPiece + struct NodePieceQuery { + td::optional query_id; + }; + std::map node_get_piece_; + + struct PeerPieceQuery { + td::Promise promise; + }; + std::map peer_get_piece_; + + void start_up() override; + + void loop() override; + + void loop_notify_node(); + + void loop_pong(); + void execute_ping(td::uint64 session_id, td::Promise promise); + void on_ping_result(td::Result r_answer); + void on_pong(); + + void loop_ping(); + + void loop_update_init(); + void loop_update_pieces(); + void update_have_pieces(); + + void loop_update_state(); + + td::BufferSlice create_update_query(ton::tl_object_ptr update); + + void loop_node_get_piece(); + + void loop_peer_get_piece(); + + void execute_add_update(ton::ton_api::storage_addUpdate &add_update, td::Promise promise); + + void execute_get_piece(ton::ton_api::storage_getPiece &get_piece, td::Promise promise); + + void on_update_result(td::Result r_answer); + + void on_get_piece_result(PartId piece_id, td::Result r_answer); + + void on_update_state_result(td::Result r_answer); + + template + td::uint64 create_and_send_query(ArgsT &&... args); + td::uint64 send_query(td::BufferSlice query); + + void schedule_loop(); + void notify_node(); +}; +} // namespace ton diff --git a/storage/PeerState.cpp b/storage/PeerState.cpp new file mode 100644 index 000000000..3c1657306 --- /dev/null +++ b/storage/PeerState.cpp @@ -0,0 +1,16 @@ +#include "PeerState.h" +namespace ton { +void PeerState::notify_node() { + if (node.empty()) { + return; + } + td::actor::send_signals_later(node, td::actor::ActorSignals::wakeup()); +} + +void PeerState::notify_peer() { + if (peer.empty()) { + return; + } + td::actor::send_signals_later(peer, td::actor::ActorSignals::wakeup()); +} +} // namespace ton diff --git a/storage/PeerState.h b/storage/PeerState.h new file mode 100644 index 000000000..0e948d7e5 --- /dev/null +++ b/storage/PeerState.h @@ -0,0 +1,55 @@ +#pragma once +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/optional.h" + +#include "td/actor/actor.h" + +#include + +#include "LoadSpeed.h" + +namespace ton { +using PeerId = td::uint64; +using PartId = td::uint32; + +struct PeerState { + struct State { + bool will_upload{false}; + bool want_download{false}; + auto key() const { + return std::tie(will_upload, want_download); + } + bool operator==(const State &other) const { + return key() == other.key(); + } + }; + State node_state_; + td::optional peer_state_; + bool peer_online_{false}; + + struct Part { + td::BufferSlice proof; + td::BufferSlice data; + }; + std::map>> node_queries_; + std::map>> peer_queries_; + + // Peer -> Node + // update are added to this vector, so reader will be able to process all changes + std::vector peer_ready_parts_; + + // Node -> Peer + // writer writes all new parts to this vector. This state will be eventually synchornized with a peer + std::vector node_ready_parts_; + + td::actor::ActorId<> node; + td::actor::ActorId<> peer; + + LoadSpeed upload; + LoadSpeed download; + + void notify_node(); + void notify_peer(); +}; +} // namespace ton diff --git a/storage/SharedState.h b/storage/SharedState.h new file mode 100644 index 000000000..0d5297522 --- /dev/null +++ b/storage/SharedState.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include "td/utils/MovableValue.h" +#include + +namespace td { +template +class SharedState { + public: + friend class Guard; + class Guard { + public: + Guard(Guard &&) = default; + Guard(SharedState *self) : self(self) { + CHECK(!self->data_->is_locked.exchange(true)); + } + ~Guard() { + if (self.get()) { + CHECK(self.get()->data_->is_locked.exchange(false)); + } + } + T *get() { + return &self.get()->data_->data; + } + T *operator->() { + return get(); + } + + private: + td::MovableValue *> self; + }; + + auto lock() { + return Guard{this}; + } + auto unsafe() { + return &data_->data; + } + + private: + struct Data { + std::atomic is_locked{}; + T data; + }; + std::shared_ptr data_{std::make_shared()}; +}; +} // namespace td diff --git a/storage/Torrent.cpp b/storage/Torrent.cpp new file mode 100644 index 000000000..3c1464f79 --- /dev/null +++ b/storage/Torrent.cpp @@ -0,0 +1,425 @@ +#include "Torrent.h" + +#include "td/utils/Status.h" +#include "td/utils/crypto.h" +#include "td/utils/port/Stat.h" +#include "td/utils/tl_helpers.h" + +namespace ton { + +Torrent::Torrent(TorrentMeta meta) + : info_(meta.info) + , merkle_tree_(info_.pieces_count(), info_.root_hash) + , piece_is_ready_(info_.pieces_count(), false) { + not_ready_piece_count_ = piece_is_ready_.size(); + header_pieces_count_ = (info_.header_size + info_.piece_size - 1) / info_.piece_size; + not_ready_pending_piece_count_ = header_pieces_count_; + + if (meta.header) { + set_header(meta.header.unwrap()); + } else { + header_str_ = td::BufferSlice(info_.header_size); + } + if (meta.root_proof.not_null()) { + merkle_tree_.add_proof(meta.root_proof); + } +} + +td::Result Torrent::open(Options options, TorrentMeta meta) { + Torrent res(std::move(meta)); + if (!options.in_memory) { + if (options.root_dir.empty()) { + options.root_dir = "."; + } + res.set_root_dir(options.root_dir); + } + if (options.validate) { + res.validate(); + } + return std::move(res); +} + +td::Result Torrent::open(Options options, td::Slice meta_str) { + TRY_RESULT(meta, TorrentMeta::deserialize(meta_str)); + return open(std::move(options), std::move(meta)); +} + +const Torrent::Info &Torrent::get_info() const { + return info_; +} + +struct IterateInfo { + td::uint64 piece_offset; + td::uint64 chunk_offset; + td::uint64 size; +}; + +template +td::Status Torrent::iterate_piece(Info::PieceInfo piece, F &&f) { + auto chunk_it = std::lower_bound(chunks_.begin(), chunks_.end(), piece.offset, [](auto &chunk, auto &piece_offset) { + return chunk.offset + chunk.size <= piece_offset; + }); + + td::uint64 size = 0; + for (; chunk_it != chunks_.end(); chunk_it++) { + if (chunk_it->offset >= piece.offset + piece.size) { + break; + } + if (chunk_it->size == 0) { + continue; + } + auto l = td::max(chunk_it->offset, piece.offset); + auto r = td::min(chunk_it->offset + chunk_it->size, piece.offset + piece.size); + CHECK(l < r); + + IterateInfo info; + info.piece_offset = l - piece.offset; + info.chunk_offset = l - chunk_it->offset; + info.size = r - l; + size += info.size; + TRY_STATUS(f(chunk_it, info)); + } + LOG_CHECK(size == piece.size) << size << " vs " << piece.size; + return td::Status::OK(); +} + +bool Torrent::is_piece_ready(td::uint64 piece_i) const { + CHECK(piece_i < info_.pieces_count()); + return piece_is_ready_[piece_i]; +} + +td::optional Torrent::get_files_count() const { + if (header_) { + return header_.value().files_count; + } + return {}; +} +td::CSlice Torrent::get_file_name(size_t i) const { + return chunks_.at(i + 1).name; +} +td::uint64 Torrent::get_file_size(size_t i) const { + return chunks_.at(i + 1).size; +} +td::uint64 Torrent::get_file_ready_size(size_t i) const { + return chunks_.at(i + 1).ready_size; +} + +Torrent::PartsRange Torrent::get_file_parts_range(size_t i) { + auto begin = chunks_.at(i + 1).offset; + auto end = begin + chunks_.at(i + 1).size; + + PartsRange res; + res.begin = begin / info_.piece_size; + res.end = (end + info_.piece_size - 1) / info_.piece_size; + return res; +} + +Torrent::PartsRange Torrent::get_header_parts_range() { + PartsRange res; + res.begin = 0; + res.end = header_pieces_count_; + return res; +} + +TD_WARN_UNUSED_RESULT td::Status Torrent::ChunkState::get_piece(td::MutableSlice dest, td::uint64 offset, + Cache *cache) { + if (dest.empty()) { + return td::Status::OK(); + } + + if (cache != nullptr) { + auto global_offset = offset + this->offset; + if (cache->offset > global_offset || cache->offset + cache->size < global_offset + dest.size()) { + auto load_size = td::min(size - offset, (td::uint64)cache->slice.size()); + cache->size = 0; + TRY_STATUS(get_piece(cache->slice.as_slice().truncate(load_size), offset)); + cache->offset = global_offset; + cache->size = load_size; + } + + dest.copy_from(cache->slice.as_slice().substr(global_offset - cache->offset, dest.size())); + CHECK(cache->slice.size() >= dest.size()); + return td::Status::OK(); + } + + TRY_RESULT(size, data.view_copy(dest, offset)); + if (size != dest.size()) { + return td::Status::Error("Failed to read the whole chunk"); + } + return td::Status::OK(); +} + +std::string Torrent::get_stats_str() const { + td::StringBuilder sb; + auto o_n = get_files_count(); + if (!o_n) { + return "NO HEADER YET\n"; + } + for (size_t i = 0, n = o_n.unwrap(); i < n; i++) { + auto size = get_file_size(i); + auto ready_size = get_file_ready_size(i); + sb << get_file_name(i) << "\t" << 100 * ready_size / size << "%% " << td::format::as_size(ready_size) << "/" + << td::format::as_size(size) << "\n"; + } + return sb.as_cslice().str(); +} + +void Torrent::validate() { + CHECK(header_); + + std::fill(piece_is_ready_.begin(), piece_is_ready_.end(), false); + not_ready_piece_count_ = info_.pieces_count(); + + for (auto &chunk : chunks_) { + chunk.ready_size = 0; + if (root_dir_) { + if (td::stat(get_chunk_path(chunk.name)).is_error()) { + continue; + } + } + init_chunk_data(chunk); + } + + std::vector hashes; + std::vector chunks; + + auto flush = [&] { + td::Bitset bitmask; + merkle_tree_.add_chunks(chunks, bitmask); + for (size_t i = 0; i < chunks.size(); i++) { + if (!bitmask.get(i)) { + continue; + } + + auto piece_i = chunks[i].index; + auto piece = info_.get_piece_info(piece_i); + iterate_piece(piece, [&](auto it, auto info) { + it->ready_size += info.size; + return td::Status::OK(); + }); + piece_is_ready_[piece_i] = true; + ready_parts_count_++; + + CHECK(not_ready_piece_count_); + not_ready_piece_count_--; + } + + hashes.clear(); + chunks.clear(); + }; + + td::BufferSlice buf(info_.piece_size); + ChunkState::Cache cache; + cache.slice = td::BufferSlice(td::max(8u << 20, info_.piece_size)); + for (size_t piece_i = 0; piece_i < info_.pieces_count(); piece_i++) { + auto piece = info_.get_piece_info(piece_i); + td::Sha256State sha256; + sha256.init(); + bool skipped = false; + auto is_ok = iterate_piece(piece, [&](auto it, auto info) { + if (!it->data) { + skipped = true; + return td::Status::Error("No such file"); + } + if (!it->has_piece(info.chunk_offset, info.size)) { + return td::Status::Error("Don't have piece"); + } + auto dest = buf.as_slice().truncate(info.size); + TRY_STATUS(it->get_piece(dest, info.chunk_offset, &cache)); + sha256.feed(dest); + //LOG(ERROR) << dest; + return td::Status::OK(); + }); + if (is_ok.is_error()) { + LOG_IF(ERROR, !skipped) << "Failed: " << is_ok; + LOG(ERROR) << "Failed: " << is_ok; + continue; + } + MerkleTree::Chunk chunk; + chunk.index = piece_i; + sha256.extract(chunk.hash.as_slice()); + + chunks.push_back(chunk); + } + flush(); +} + +td::Result Torrent::get_piece_data(td::uint64 piece_i) { + CHECK(piece_i < info_.pieces_count()); + if (!piece_is_ready_[piece_i]) { + return td::Status::Error("Piece is not ready"); + } + auto it = pending_pieces_.find(piece_i); + if (it != pending_pieces_.end()) { + return it->second; + } + auto piece = info_.get_piece_info(piece_i); + + std::string res(piece.size, '\0'); + iterate_piece(piece, [&](auto it, auto info) { + return it->get_piece(td::MutableSlice(res).substr(info.piece_offset, info.size), info.chunk_offset); + }); + return res; +} + +td::Result> Torrent::get_piece_proof(td::uint64 piece_i) { + CHECK(piece_i < info_.pieces_count()); + return merkle_tree_.gen_proof(piece_i, piece_i); +} + +td::Status Torrent::add_piece(td::uint64 piece_i, td::Slice data, td::Ref proof) { + TRY_STATUS(merkle_tree_.add_proof(proof)); + //LOG(ERROR) << "Add piece #" << piece_i; + CHECK(piece_i < info_.pieces_count()); + if (piece_is_ready_[piece_i]) { + return td::Status::OK(); + } + piece_is_ready_[piece_i] = true; + ready_parts_count_++; + ton::MerkleTree::Chunk chunk; + chunk.index = piece_i; + td::sha256(data, chunk.hash.as_slice()); + TRY_STATUS(merkle_tree_.try_add_chunks({chunk})); + + if (chunks_.empty()) { + return add_header_piece(piece_i, data); + } + + return add_validated_piece(piece_i, data); +} + +td::Status Torrent::add_header_piece(td::uint64 piece_i, td::Slice data) { + pending_pieces_[piece_i] = data.str(); + + if (piece_i < header_pieces_count_) { + //LOG(ERROR) << "Add header piece #" << piece_i; + auto piece = info_.get_piece_info(piece_i); + auto dest = header_str_.as_slice().substr(piece.offset); + data.truncate(dest.size()); + dest.copy_from(data); + not_ready_pending_piece_count_--; + if (not_ready_pending_piece_count_ == 0) { + //LOG(ERROR) << "Got full header"; + TorrentHeader header; + TRY_STATUS(td::unserialize(header, header_str_.as_slice())); // TODO: it is a fatal error + set_header(header); + for (auto &it : pending_pieces_) { + TRY_STATUS(add_validated_piece(it.first, it.second)); + } + pending_pieces_.clear(); + } + } else { + LOG(ERROR) << "PENDING"; + } + + return td::Status::OK(); +} + +std::string Torrent::get_chunk_path(td::Slice name) { + return PSTRING() << root_dir_.value() << TD_DIR_SLASH << header_.value().dir_name << TD_DIR_SLASH << name; +} + +td::Status Torrent::init_chunk_data(ChunkState &chunk) { + if (chunk.data) { + return td::Status::OK(); + } + if (root_dir_) { + TRY_RESULT(data, td::FileNoCacheBlobView::create(get_chunk_path(chunk.name), chunk.size, true)); + chunk.data = std::move(data); + } else { + chunk.data = td::BufferSliceBlobView::create(td::BufferSlice(chunk.size)); + } + return td::Status::OK(); +} + +td::Status Torrent::add_validated_piece(td::uint64 piece_i, td::Slice data) { + CHECK(!chunks_.empty()); + + auto piece = info_.get_piece_info(piece_i); + + TRY_STATUS(iterate_piece(piece, [&](auto it, auto info) { + TRY_STATUS(init_chunk_data(*it)); + return it->add_piece(data.substr(info.piece_offset, info.size), info.chunk_offset); + })); + piece_is_ready_[piece_i] = true; + not_ready_piece_count_--; + + return td::Status::OK(); +} + +bool Torrent::is_completed() const { + return not_ready_piece_count_ == 0; +} + +td::Result Torrent::read_file(td::Slice name) { + for (auto &chunk : chunks_) { + if (chunk.name == name) { + td::BufferSlice res(chunk.size); + TRY_STATUS(chunk.get_piece(res.as_slice(), 0)); + return std::move(res); + } + } + return td::Status::Error("Unknown name"); +} + +Torrent::GetMetaOptions::GetMetaOptions() = default; +std::string Torrent::get_meta_str(const GetMetaOptions &options) const { + return get_meta(options).serialize(); +} + +TorrentMeta Torrent::get_meta(const GetMetaOptions &options) const { + TorrentMeta torrent_file; + if (options.with_header) { + torrent_file.header = header_; + } + torrent_file.info = info_; + torrent_file.info.init_cell(); + if (options.with_proof) { + torrent_file.root_proof = merkle_tree_.get_root(options.proof_depth_limit); + } + return torrent_file; +} + +Torrent::Torrent(Info info, td::optional header, ton::MerkleTree tree, std::vector chunks) + : info_(info) + , header_(std::move(header)) + , merkle_tree_(std::move(tree)) + , piece_is_ready_(info_.pieces_count(), true) + , ready_parts_count_{info_.pieces_count()} + , chunks_(std::move(chunks)) { +} + +void Torrent::set_header(const TorrentHeader &header) { + header_ = header; + auto add_chunk = [&](td::Slice name, td::uint64 offset, td::uint64 size) { + ChunkState chunk; + chunk.name = name.str(); + chunk.ready_size = 0; + chunk.size = size; + chunk.offset = offset; + chunks_.push_back(std::move(chunk)); + }; + add_chunk("", 0, header.serialization_size()); + chunks_.back().data = td::BufferSliceBlobView::create(header.serialize()); + for (size_t i = 0; i < header.files_count; i++) { + auto l = header.get_data_begin(i); + auto r = header.get_data_end(i); + add_chunk(header.get_name(i), l, r - l); + } +} + +size_t Torrent::get_ready_parts_count() const { + return ready_parts_count_; +} + +std::vector Torrent::chunks_by_piece(td::uint64 piece_id) { + std::vector res; + auto piece = info_.get_piece_info(piece_id); + auto is_ok = iterate_piece(piece, [&](auto it, auto info) { + res.push_back(it - chunks_.begin()); + return td::Status::OK(); + }); + return res; +} + +} // namespace ton diff --git a/storage/Torrent.h b/storage/Torrent.h new file mode 100644 index 000000000..baa1d7a5c --- /dev/null +++ b/storage/Torrent.h @@ -0,0 +1,150 @@ +#pragma once +#include "MerkleTree.h" +#include "TorrentMeta.h" + +#include "td/utils/buffer.h" +#include "td/db/utils/BlobView.h" + +#include + +namespace ton { +class Torrent { + public: + class Creator; + friend class Creator; + using Info = TorrentInfo; + + struct Options { + std::string root_dir; + bool in_memory{false}; + bool validate{false}; + }; + + // creation + static td::Result open(Options options, TorrentMeta meta); + static td::Result open(Options options, td::Slice meta_str); + void validate(); + + std::string get_stats_str() const; + + const Info &get_info() const; + + // get piece and proof + td::Result get_piece_data(td::uint64 piece_i); + td::Result> get_piece_proof(td::uint64 piece_i); + + // add piece (with an optional proof) + td::Status add_piece(td::uint64 piece_i, td::Slice data, td::Ref proof); + //TODO: add multiple chunks? Merkle tree supports much more general interface + + bool is_completed() const; + + // Checks that file is ready and returns its content. + // Intened mostly for in-memory usage and for tests + td::Result read_file(td::Slice name); + + struct GetMetaOptions { + GetMetaOptions(); + size_t proof_depth_limit{std::numeric_limits::max()}; + bool with_header{true}; + bool with_proof{true}; + + GetMetaOptions &without_header() { + with_header = false; + return *this; + } + GetMetaOptions &without_proof() { + with_proof = false; + return *this; + } + GetMetaOptions &with_proof_depth_limit(size_t limit) { + proof_depth_limit = limit; + return *this; + } + }; + std::string get_meta_str(const GetMetaOptions &options = {}) const; + TorrentMeta get_meta(const GetMetaOptions &options = {}) const; + + // Some api for inspection of a current state + bool is_piece_ready(td::uint64 piece_i) const; + + td::optional get_files_count() const; + td::CSlice get_file_name(size_t i) const; + td::uint64 get_file_size(size_t i) const; + td::uint64 get_file_ready_size(size_t i) const; + + struct PartsRange { + td::uint64 begin{0}; + td::uint64 end{0}; + }; + PartsRange get_file_parts_range(size_t i); + PartsRange get_header_parts_range(); + + size_t get_ready_parts_count() const; + + std::vector chunks_by_piece(td::uint64 piece_id); + + private: + Info info_; + td::optional root_dir_; + + // While header is not completly available all pieces are stored in memory + td::BufferSlice header_str_; + td::optional header_; + size_t not_ready_pending_piece_count_{0}; + size_t header_pieces_count_{0}; + std::map pending_pieces_; + + ton::MerkleTree merkle_tree_; + + std::vector piece_is_ready_; + size_t not_ready_piece_count_{0}; + size_t ready_parts_count_{0}; + + struct ChunkState { + std::string name; + td::uint64 offset{0}; + td::uint64 size{0}; + td::uint64 ready_size{0}; + td::BlobView data; + + struct Cache { + td::uint64 offset{0}; + td::uint64 size{0}; + td::BufferSlice slice; + }; + + bool is_ready() const { + return ready_size == size; + } + + TD_WARN_UNUSED_RESULT td::Status add_piece(td::Slice piece, td::uint64 offset) { + TRY_RESULT(written, data.write(piece, offset)); + CHECK(written == piece.size()); + ready_size += written; + return td::Status::OK(); + } + bool has_piece(td::uint64 offset, td::uint64 size) { + return data.size() >= offset + size; + } + TD_WARN_UNUSED_RESULT td::Status get_piece(td::MutableSlice dest, td::uint64 offset, Cache *cache = nullptr); + }; + std::vector chunks_; + + explicit Torrent(Info info, td::optional header, ton::MerkleTree tree, std::vector chunk); + explicit Torrent(TorrentMeta meta); + void set_root_dir(std::string root_dir) { + root_dir_ = std::move(root_dir); + } + + std::string get_chunk_path(td::Slice name); + td::Status init_chunk_data(ChunkState &chunk); + template + td::Status iterate_piece(Info::PieceInfo piece, F &&f); + + td::Status add_header_piece(td::uint64 piece_i, td::Slice data); + td::Status add_validated_piece(td::uint64 piece_i, td::Slice data); + void set_header(const TorrentHeader &header); +}; + +} // namespace ton diff --git a/storage/TorrentCreator.cpp b/storage/TorrentCreator.cpp new file mode 100644 index 000000000..640a2b853 --- /dev/null +++ b/storage/TorrentCreator.cpp @@ -0,0 +1,173 @@ +#include "TorrentCreator.h" + +#include "td/db/utils/CyclicBuffer.h" + +#include "td/utils/crypto.h" +#include "td/utils/PathView.h" +#include "td/utils/port/path.h" +#include "td/utils/tl_helpers.h" + +namespace ton { +td::Result Torrent::Creator::create_from_path(Options options, td::CSlice raw_path) { + TRY_RESULT(path, td::realpath(raw_path)); + TRY_RESULT(stat, td::stat(path)); + if (stat.is_dir_) { + if (!path.empty() && path.back() != TD_DIR_SLASH) { + path += TD_DIR_SLASH; + } + if (!options.dir_name) { + options.dir_name = td::PathView::dir_and_file(path).str(); + } + Torrent::Creator creator(options); + td::Status status; + auto walk_status = td::WalkPath::run(path, [&](td::CSlice name, td::WalkPath::Type type) { + if (type == td::WalkPath::Type::NotDir) { + status = creator.add_file(td::PathView::relative(name, path), name); + if (status.is_error()) { + return td::WalkPath::Action::Abort; + } + } + return td::WalkPath::Action::Continue; + }); + TRY_STATUS(std::move(status)); + TRY_STATUS(std::move(walk_status)); + return creator.finalize(); + } else { + Torrent::Creator creator(options); + TRY_STATUS(creator.add_file(td::PathView(path).file_name(), path)); + return creator.finalize(); + } +} + +td::Result Torrent::Creator::create_from_blobs(Options options, td::Span blobs) { + Torrent::Creator creator(options); + for (auto &blob : blobs) { + TRY_STATUS(creator.add_blob(blob.name, blob.data)); + } + return creator.finalize(); +} + +td::Status Torrent::Creator::add_blob(td::Slice name, td::Slice blob) { + return add_blob(name, td::BufferSliceBlobView::create(td::BufferSlice(blob))); +} + +td::Status Torrent::Creator::add_blob(td::Slice name, td::BlobView blob) { + File file; + file.name = name.str(); + file.data = std::move(blob); + files_.push_back(std::move(file)); + return td::Status::OK(); +} + +TD_WARN_UNUSED_RESULT td::Status Torrent::Creator::add_file(td::Slice name, td::CSlice path) { + LOG(INFO) << "Add file " << name << " " << path; + TRY_RESULT(data, td::FileNoCacheBlobView::create(path)); + return add_blob(name, std::move(data)); +} + +td::Result Torrent::Creator::finalize() { + TorrentHeader header; + TRY_RESULT(files_count, td::narrow_cast_safe(files_.size())); + header.files_count = files_count; + header.data_index.resize(files_count); + header.name_index.resize(files_count); + td::uint64 data_offset = 0; + for (size_t i = 0; i < files_.size(); i++) { + header.names += files_[i].name; + header.name_index[i] = header.names.size(); + data_offset += files_[i].data.size(); + header.data_index[i] = data_offset; + } + header.tot_names_size = header.names.size(); + if (options_.dir_name) { + header.dir_name = options_.dir_name.value(); + } + + // Now we should stream all data to calculate sha256 of all pieces + + std::string buffer; + td::CyclicBuffer::Options cb_options; + cb_options.chunk_size = td::max(options_.piece_size * 16, (1 << 20) / options_.piece_size * options_.piece_size); + cb_options.count = 2; + + auto reader_writer = td::CyclicBuffer::create(cb_options); + auto reader = std::move(reader_writer.first); + auto writer = std::move(reader_writer.second); + + auto header_size = header.serialization_size(); + auto file_size = header_size + data_offset; + auto chunks_count = (file_size + options_.piece_size - 1) / options_.piece_size; + ton::MerkleTree tree; + tree.init_begin(chunks_count); + std::vector chunks; + size_t chunk_i = 0; + auto flush_reader = [&](bool force) { + while (true) { + auto slice = reader.prepare_read(); + slice.truncate(options_.piece_size); + if (slice.empty() || (slice.size() != options_.piece_size && !force)) { + break; + } + td::UInt256 hash; + sha256(slice, hash.as_slice()); + CHECK(chunk_i < chunks_count); + tree.init_add_chunk(chunk_i, hash.as_slice()); + chunk_i++; + reader.confirm_read(slice.size()); + } + }; + td::uint64 offset = 0; + auto add_blob = [&](auto &&data, td::Slice name) { + td::uint64 data_offset = 0; + while (data_offset < data.size()) { + auto dest = writer.prepare_write(); + CHECK(dest.size() != 0); + dest.truncate(data.size() - data_offset); + TRY_RESULT(got_size, data.view_copy(dest, data_offset)); + CHECK(got_size != 0); + data_offset += got_size; + writer.confirm_write(got_size); + flush_reader(false); + } + + Torrent::ChunkState chunk; + chunk.name = name.str(); + chunk.offset = offset; + chunk.size = data.size(); + chunk.ready_size = chunk.size; + chunk.data = std::move(data); + + offset += chunk.size; + chunks.push_back(std::move(chunk)); + return td::Status::OK(); + }; + + Torrent::Info info; + auto header_str = td::serialize(header); + CHECK(header_size == header_str.size()); + info.header_size = header_str.size(); + td::sha256(header_str, info.header_hash.as_slice()); + + add_blob(td::BufferSliceBlobView::create(td::BufferSlice(header_str)), "").ensure(); + for (auto &file : files_) { + add_blob(std::move(file.data), file.name).ensure(); + } + flush_reader(true); + tree.init_finish(); + CHECK(chunk_i == chunks_count); + CHECK(offset == file_size); + + info.header_size = header.serialization_size(); + info.piece_size = options_.piece_size; + info.description = options_.description; + info.file_size = file_size; + info.depth = tree.get_depth(); + info.root_hash = tree.get_root_hash(); + + info.init_cell(); + + Torrent torrent(info, std::move(header), std::move(tree), std::move(chunks)); + + return std::move(torrent); +} +} // namespace ton diff --git a/storage/TorrentCreator.h b/storage/TorrentCreator.h new file mode 100644 index 000000000..8e80a0d05 --- /dev/null +++ b/storage/TorrentCreator.h @@ -0,0 +1,48 @@ +#pragma once +#include "Torrent.h" + +#include "td/utils/optional.h" +#include "td/utils/SharedSlice.h" +#include "td/db/utils/BlobView.h" + +namespace ton { +class Torrent::Creator { + public: + struct Options { + td::uint32 piece_size{128 * 768}; + + // override default dir_name + // should't be used in a usual workflow + td::optional dir_name; + + std::string description; + }; + + // If path is a file create a torrent with one file in it. + // If path is a directory, create a torrent with whole directory. + static td::Result create_from_path(Options options, td::CSlice path); + struct Blob { + td::Slice name; + td::Slice data; + }; + static td::Result create_from_blobs(Options options, td::Span blobs); + + // This api is mostly for internal usage. One should prefer static methods + explicit Creator(Options options) : options_(options) { + } + TD_WARN_UNUSED_RESULT td::Status add_blob(td::Slice name, td::Slice blob); + TD_WARN_UNUSED_RESULT td::Status add_blob(td::Slice name, td::BlobView blob); + TD_WARN_UNUSED_RESULT td::Status add_file(td::Slice name, td::CSlice path); + td::Result finalize(); + + private: + td::Status init(); + + Options options_; + struct File { + std::string name; + td::BlobView data; + }; + std::vector files_; +}; +} // namespace ton diff --git a/storage/TorrentHeader.cpp b/storage/TorrentHeader.cpp new file mode 100644 index 000000000..b8bb58295 --- /dev/null +++ b/storage/TorrentHeader.cpp @@ -0,0 +1,50 @@ +#include "TorrentHeader.hpp" + +#include "td/utils/tl_helpers.h" + +namespace ton { +td::CSlice TorrentHeader::get_dir_name() const { + return dir_name; +} + +td::uint32 TorrentHeader::get_files_count() const { + return files_count; +} + +td::uint64 TorrentHeader::get_data_begin(td::uint64 file_i) const { + return get_data_offset(file_i); +} +td::uint64 TorrentHeader::get_data_end(td::uint64 file_i) const { + return get_data_offset(file_i + 1); +} + +td::uint64 TorrentHeader::serialization_size() const { + return td::tl_calc_length(*this); +} +td::uint64 TorrentHeader::get_data_offset(td::uint64 offset_i) const { + td::uint64 res = serialization_size(); + if (offset_i > 0) { + CHECK(offset_i <= files_count); + res += data_index[offset_i - 1]; + } + return res; +} +td::BufferSlice TorrentHeader::serialize() const { + return td::BufferSlice(td::serialize(*this)); +} +td::uint64 TorrentHeader::get_data_size(td::uint64 file_i) const { + auto res = data_index[file_i]; + if (file_i > 0) { + res -= data_index[file_i - 1]; + } + return res; +} + +td::Slice TorrentHeader::get_name(td::uint64 file_i) const { + CHECK(file_i < files_count); + auto from = file_i == 0 ? 0 : name_index[file_i - 1]; + auto till = name_index[file_i]; + return td::Slice(names).substr(from, till - from); +} + +} // namespace ton diff --git a/storage/TorrentHeader.h b/storage/TorrentHeader.h new file mode 100644 index 000000000..a2fde56b2 --- /dev/null +++ b/storage/TorrentHeader.h @@ -0,0 +1,49 @@ +#pragma once + +#include "td/utils/Slice.h" +#include "td/utils/buffer.h" + +namespace ton { +// fec_info_none#c82a1964 = FecInfo; +// +// torrent_header#9128aab7 +// files_count:uint32 +// tot_name_size:uint64 +// tot_data_size:uint64 +// fec:FecInfo +// dir_name_size:uint32 +// dir_name:(dir_name_size * [uint8]) +// name_index:(files_count * [uint64]) +// data_index:(files_count * [uint64]) +// names:(file_names_size * [uint8]) +// data:(tot_data_size * [uint8]) +// = TorrentHeader; + +struct TorrentHeader { + td::uint32 files_count{0}; + td::uint64 tot_names_size{0}; + td::uint64 tot_data_size{0}; + //fec_none + std::string dir_name; + std::vector name_index; + std::vector data_index; + std::string names; + + td::uint64 get_data_begin(td::uint64 file_i) const; + td::uint64 get_data_end(td::uint64 file_i) const; + td::uint64 get_data_offset(td::uint64 offset_i) const; + td::uint64 get_data_size(td::uint64 file_i) const; + td::Slice get_name(td::uint64 file_i) const; + td::CSlice get_dir_name() const; + td::uint32 get_files_count() const; + + td::uint64 serialization_size() const; + td::BufferSlice serialize() const; + + static constexpr td::uint32 type = 0x9128aab7; + template + void store(StorerT &storer) const; + template + void parse(ParserT &parser); +}; +} // namespace ton diff --git a/storage/TorrentHeader.hpp b/storage/TorrentHeader.hpp new file mode 100644 index 000000000..c2fd2c48d --- /dev/null +++ b/storage/TorrentHeader.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include "td/utils/tl_helpers.h" + +#include "TorrentHeader.h" +namespace ton { +template +void TorrentHeader::store(StorerT &storer) const { + using td::store; + store(type, storer); + store(files_count, storer); + store(tot_names_size, storer); + store(tot_data_size, storer); + td::uint32 fec_type = 0xc82a1964; + store(fec_type, storer); + td::uint32 dir_name_size = td::narrow_cast(dir_name.size()); + store(dir_name_size, storer); + storer.store_slice(dir_name); + CHECK(name_index.size() == files_count); + CHECK(data_index.size() == files_count); + for (auto x : name_index) { + store(x, storer); + } + for (auto x : data_index) { + store(x, storer); + } + CHECK(tot_names_size == names.size()); + storer.store_slice(names); +} + +template +void TorrentHeader::parse(ParserT &parser) { + using td::parse; + td::uint32 got_type; + parse(got_type, parser); + if (got_type != type) { + parser.set_error("Unknown fec type"); + return; + } + parse(files_count, parser); + parse(tot_names_size, parser); + parse(tot_data_size, parser); + td::uint32 fec_type; + parse(fec_type, parser); + td::uint32 dir_name_size; + parse(dir_name_size, parser); + dir_name = parser.template fetch_string_raw(dir_name_size); + if (fec_type != 0xc82a1964) { + parser.set_error("Unknown fec type"); + return; + } + name_index.resize(files_count); + for (auto &x : name_index) { + parse(x, parser); + } + data_index.resize(files_count); + for (auto &x : data_index) { + parse(x, parser); + } + names = parser.template fetch_string_raw(tot_names_size); +} +} // namespace ton diff --git a/storage/TorrentInfo.cpp b/storage/TorrentInfo.cpp new file mode 100644 index 000000000..bd6b4cc60 --- /dev/null +++ b/storage/TorrentInfo.cpp @@ -0,0 +1,47 @@ +#include "TorrentInfo.h" + +#include "vm/cells/CellString.h" +#include "vm/cellslice.h" + +#include "td/utils/misc.h" + +namespace ton { +bool TorrentInfo::pack(vm::CellBuilder &cb) const { + return cb.store_long_bool(depth, 32) && cb.store_long_bool(piece_size, 32) && cb.store_long_bool(file_size, 64) && + cb.store_bits_bool(root_hash) && cb.store_long_bool(header_size, 64) && cb.store_bits_bool(header_hash) && + vm::CellText::store(cb, description).is_ok(); +} + +bool TorrentInfo::unpack(vm::CellSlice &cs) { + return cs.fetch_uint_to(32, depth) && cs.fetch_uint_to(32, piece_size) && cs.fetch_uint_to(64, file_size) && + cs.fetch_bits_to(root_hash) && cs.fetch_uint_to(64, header_size) && cs.fetch_bits_to(header_hash) && + vm::CellText::fetch_to(cs, description); +} + +vm::Cell::Hash TorrentInfo::get_hash() const { + return as_cell()->get_hash(); +} + +void TorrentInfo::init_cell() { + vm::CellBuilder cb; + CHECK(pack(cb)); + cell_ = cb.finalize(); +} + +td::Ref TorrentInfo::as_cell() const { + CHECK(cell_.not_null()); + return cell_; +} + +td::uint64 TorrentInfo::pieces_count() const { + return (file_size + piece_size - 1) / piece_size; +} + +TorrentInfo::PieceInfo TorrentInfo::get_piece_info(td::uint64 piece_i) const { + PieceInfo info; + info.offset = piece_size * piece_i; + CHECK(info.offset < file_size); + info.size = td::min(static_cast(piece_size), file_size - info.offset); + return info; +} +} // namespace ton diff --git a/storage/TorrentInfo.h b/storage/TorrentInfo.h new file mode 100644 index 000000000..ff95c3b56 --- /dev/null +++ b/storage/TorrentInfo.h @@ -0,0 +1,38 @@ +#pragma once + +#include "td/utils/Slice.h" +#include "td/utils/UInt.h" + +#include "vm/cells.h" + +namespace ton { +// torrent_info depth:# piece_size:uint32 file_size:uint64 root_hash:(## 256) header_size:uint64 header_hash:(## 256) description:Text = TorrentInfo; +struct TorrentInfo { + td::uint32 depth{0}; + td::uint32 piece_size{768 * 128}; + td::uint64 file_size{0}; + td::Bits256 root_hash; + td::uint64 header_size; + td::Bits256 header_hash; + std::string description; + + bool pack(vm::CellBuilder &cb) const; + bool unpack(vm::CellSlice &cs); + + void init_cell(); + vm::Cell::Hash get_hash() const; + td::Ref as_cell() const; + + struct PieceInfo { + td::uint64 offset; + td::uint64 size; + }; + + td::uint64 pieces_count() const; + PieceInfo get_piece_info(td::uint64 piece_i) const; + + private: + td::Ref cell_; +}; + +} // namespace ton diff --git a/storage/TorrentMeta.cpp b/storage/TorrentMeta.cpp new file mode 100644 index 000000000..4b6745f16 --- /dev/null +++ b/storage/TorrentMeta.cpp @@ -0,0 +1,123 @@ +#include "TorrentMeta.h" + +#include "TorrentHeader.hpp" + +#include "td/utils/crypto.h" +#include "td/utils/tl_helpers.h" +#include "td/utils/UInt.h" + +#include "vm/boc.h" +#include "vm/cells/MerkleProof.h" +#include "vm/cellslice.h" + +namespace ton { + +td::Result TorrentMeta::deserialize(td::Slice data) { + TorrentMeta res; + TRY_STATUS(td::unserialize(res, data)); + if (res.header) { + td::Bits256 header_hash; + td::sha256(td::serialize(res.header.value()), header_hash.as_slice()); + if (header_hash != res.info.header_hash) { + return td::Status::Error("Header hash mismatch"); + } + } else { + LOG(ERROR) << "NO HEADER"; + } + if (res.root_proof.not_null()) { + auto root = vm::MerkleProof::virtualize(res.root_proof, 1); + if (root.is_null()) { + return td::Status::Error("Root proof is not a merkle proof"); + } + if (root->get_hash().as_slice() != res.info.root_hash.as_slice()) { + return td::Status::Error("Root proof hash mismatch"); + } + } + res.info.init_cell(); + return res; +} + +std::string TorrentMeta::serialize() const { + return td::serialize(*this); +} + +template +void TorrentMeta::store(StorerT &storer) const { + using td::store; + td::uint32 flags = 0; + + if (root_proof.not_null()) { + flags |= 1; + } + + if (header) { + flags |= 2; + } + store(flags, storer); + + const auto info_boc = vm::std_boc_serialize(info.as_cell()).move_as_ok(); + td::uint32 info_boc_size = td::narrow_cast(info_boc.size()); + + td::BufferSlice root_proof_boc; + td::uint32 root_proof_boc_size{0}; + if ((flags & 1) != 0) { + root_proof_boc = vm::std_boc_serialize(root_proof).move_as_ok(); + root_proof_boc_size = td::narrow_cast(root_proof_boc.size()); + } + + store(info_boc_size, storer); + if ((flags & 1) != 0) { + store(root_proof_boc_size, storer); + } + storer.store_slice(info_boc.as_slice()); + if ((flags & 1) != 0) { + storer.store_slice(root_proof_boc.as_slice()); + } + + if ((flags & 2) != 0) { + store(header.value(), storer); + } +} + +template +void TorrentMeta::parse(ParserT &parser) { + using td::parse; + td::uint32 flags; + parse(flags, parser); + + td::uint32 info_boc_size; + td::uint32 root_proof_boc_size; + parse(info_boc_size, parser); + if ((flags & 1) != 0) { + parse(root_proof_boc_size, parser); + } + auto info_boc_str = parser.template fetch_string_raw(info_boc_size); + auto r_info_cell = vm::std_boc_deserialize(info_boc_str); + if (r_info_cell.is_error()) { + parser.set_error(r_info_cell.error().to_string()); + return; + } + + if ((flags & 1) != 0) { + auto root_proof_str = parser.template fetch_string_raw(root_proof_boc_size); + auto r_root_proof = vm::std_boc_deserialize(root_proof_str); + if (r_root_proof.is_error()) { + parser.set_error(r_root_proof.error().to_string()); + return; + } + root_proof = r_root_proof.move_as_ok(); + } + + auto cs = vm::load_cell_slice(r_info_cell.move_as_ok()); + if (!info.unpack(cs)) { + parser.set_error("Failed to parse TorrentInfo"); + return; + } + + if ((flags & 2) != 0) { + TorrentHeader new_header; + parse(new_header, parser); + header = std::move(new_header); + } +} +} // namespace ton diff --git a/storage/TorrentMeta.h b/storage/TorrentMeta.h new file mode 100644 index 000000000..8b7c33773 --- /dev/null +++ b/storage/TorrentMeta.h @@ -0,0 +1,37 @@ +#pragma once + +#include "TorrentHeader.h" +#include "TorrentInfo.h" + +#include "td/utils/optional.h" + +namespace ton { +// +// torrent_file#6a7181e0 flags:(## 32) info_boc_size:uint32 +// root_proof_boc_size:flags.0?uint32 +// info_boc:(info_boc_size * [uint8]) +// root_proof_boc:flags.0?(root_proof_boc_size * [uint8]) +// header:flags.1?TorrentHeader = TorrentMeta; +// +struct TorrentMeta { + TorrentMeta() = default; + explicit TorrentMeta(TorrentInfo info, td::Ref root_proof = {}, td::optional header = {}) + : info(std::move(info)), root_proof(std::move(root_proof)), header(std::move(header)) { + } + + TorrentInfo info; + td::Ref root_proof; + td::optional header; + + static td::Result deserialize(td::Slice data); + + std::string serialize() const; + + static constexpr td::uint64 type = 0x6a7181e0; + template + void store(StorerT &storer) const; + + template + void parse(ParserT &parser); +}; +} // namespace ton diff --git a/storage/storage-cli.cpp b/storage/storage-cli.cpp new file mode 100644 index 000000000..65fb40879 --- /dev/null +++ b/storage/storage-cli.cpp @@ -0,0 +1,852 @@ +#include "adnl/adnl.h" +#include "common/bigint.hpp" +#include "common/bitstring.h" +#include "dht/dht.h" +#include "keys/encryptor.h" +#include "overlay/overlay.h" +#include "rldp/rldp.h" +#include "rldp2/rldp.h" + +#include "td/utils/JsonBuilder.h" +#include "td/utils/port/signals.h" +#include "td/utils/Parser.h" +#include "td/utils/overloaded.h" +#include "td/utils/OptionsParser.h" +#include "td/utils/PathView.h" +#include "td/utils/Random.h" +#include "td/utils/misc.h" +#include "td/utils/filesystem.h" +#include "td/utils/port/path.h" + +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "terminal/terminal.h" + +#include "Torrent.h" +#include "TorrentCreator.h" +#include "NodeActor.h" + +#include "auto/tl/ton_api_json.h" + +#include +#include +#include +#include + +namespace ton_rldp = ton::rldp2; + +struct StorageCliOptions { + std::string config; + bool enable_readline{true}; + std::string db_root{"dht-db/"}; + + td::IPAddress addr; + + td::optional cmd; +}; + +using AdnlCategory = td::int32; + +class PeerManager : public td::actor::Actor { + public: + PeerManager(ton::adnl::AdnlNodeIdShort adnl_id, ton::overlay::OverlayIdFull overlay_id, + td::actor::ActorId overlays, td::actor::ActorId adnl, + td::actor::ActorId rldp) + : adnl_id_(std::move(adnl_id)) + , overlay_id_(std::move(overlay_id)) + , overlays_(std::move(overlays)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { + CHECK(register_adnl_id(adnl_id_) == 1); + } + void start_up() override { + // TODO: forbid broadcasts? + auto rules = ton::overlay::OverlayPrivacyRules{ton::overlay::Overlays::max_fec_broadcast_size()}; + class Callback : public ton::overlay::Overlays::Callback { + public: + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(ton::PublicKeyHash src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + } + }; + send_closure(overlays_, &ton::overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_.clone(), + std::make_unique(), rules); + } + void tear_down() override { + send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_.compute_short_id()); + } + void send_query(ton::PeerId src, ton::PeerId dst, td::BufferSlice query, td::Promise promise) { + TRY_RESULT_PROMISE(promise, src_id, peer_to_andl(src)); + TRY_RESULT_PROMISE(promise, dst_id, peer_to_andl(dst)); + query = ton::create_serialize_tl_object_suffix( + std::move(query), overlay_id_.compute_short_id().bits256_value()); + send_closure(rldp_, &ton_rldp::Rldp::send_query_ex, src_id, dst_id, "", std::move(promise), td::Timestamp::in(10), + std::move(query), 1 << 25); + } + + void execute_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) { + data = data.from_slice(data.as_slice().substr(4 + 32)); + auto src_id = register_adnl_id(src); + auto dst_id = register_adnl_id(dst); + auto it = peers_.find(std::make_pair(dst_id, src_id)); + if (it == peers_.end()) { + auto node_it = nodes_.find(dst_id); + if (node_it == nodes_.end()) { + LOG(ERROR) << "Unknown query destination"; + promise.set_error(td::Status::Error("Unknown query destination")); + return; + } + if (!node_it->second.is_alive()) { + LOG(ERROR) << "Expired query destination"; + promise.set_error(td::Status::Error("Unknown query destination")); + return; + } + send_closure(node_it->second, &ton::NodeActor::start_peer, src_id, + [promise = std::move(promise), + data = std::move(data)](td::Result> r_peer) mutable { + TRY_RESULT_PROMISE(promise, peer, std::move(r_peer)); + send_closure(peer, &ton::PeerActor::execute_query, std::move(data), std::move(promise)); + }); + return; + } + send_closure(it->second, &ton::PeerActor::execute_query, std::move(data), std::move(promise)); + } + + void register_peer(ton::PeerId src, ton::PeerId dst, td::actor::ActorId peer) { + peers_[std::make_pair(src, dst)] = std::move(peer); + register_src(src, [](td::Result res) { res.ensure(); }); + } + + void register_node(ton::PeerId src, td::actor::ActorId node) { + nodes_[src] = std::move(node); + register_src(src, [](td::Result res) { res.ensure(); }); + } + + void unregister_node(ton::PeerId src, td::actor::ActorId node) { + auto it = nodes_.find(src); + CHECK(it != nodes_.end()); + if (it->second == node) { + nodes_.erase(it); + } + unregister_src(src, [](td::Result res) { res.ensure(); }); + } + + void unregister_peer(ton::PeerId src, ton::PeerId dst, td::actor::ActorId peer) { + auto it = peers_.find(std::make_pair(src, dst)); + CHECK(it != peers_.end()); + if (it->second == peer) { + peers_.erase(it); + } + unregister_src(src, [](td::Result res) { res.ensure(); }); + } + + void unregister_src(ton::PeerId src, td::Promise promise) { + TRY_RESULT_PROMISE(promise, src_id, peer_to_andl(src)); + if (--subscribed_peers_[src] == 0) { + LOG(ERROR) << "Unsubscribe " << src_id; + subscribed_peers_.erase(src); + send_closure(adnl_, &ton::adnl::Adnl::unsubscribe, src_id, + ton::create_serialize_tl_object( + overlay_id_.compute_short_id().bits256_value()) + .as_slice() + .str()); + } + promise.set_value({}); + } + void register_src(ton::PeerId src, td::Promise promise) { + TRY_RESULT_PROMISE(promise, src_id, peer_to_andl(src)); + class Callback : public ton::adnl::Adnl::Callback { + public: + Callback(td::actor::ActorId peer_manager) : peer_manager_(std::move(peer_manager)) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + send_closure(peer_manager_, &PeerManager::execute_query, std::move(src), std::move(dst), std::move(data), + std::move(promise)); + } + + private: + td::actor::ActorId peer_manager_; + }; + + if (subscribed_peers_[src]++ == 0) { + LOG(ERROR) << "Subscribe " << src_id; + send_closure(adnl_, &ton::adnl::Adnl::subscribe, src_id, + ton::create_serialize_tl_object( + overlay_id_.compute_short_id().bits256_value()) + .as_slice() + .str(), + std::make_unique(actor_id(this))); + } + promise.set_value({}); + } + + td::Result peer_to_andl(ton::PeerId id) { + if (id <= 0 || id > adnl_ids_.size()) { + return td::Status::Error(PSLICE() << "Invalid peer id " << id); + } + return adnl_ids_[id - 1]; + } + + ton::PeerId register_adnl_id(ton::adnl::AdnlNodeIdShort id) { + auto it = adnl_to_peer_id_.emplace(id, next_peer_id_); + if (it.second) { + LOG(ERROR) << "Register AndlId " << id << " -> " << it.first->second; + adnl_ids_.push_back(id); + next_peer_id_++; + } + return it.first->second; + } + + void get_peers(td::Promise> promise) { + send_closure(overlays_, &ton::overlay::Overlays::get_overlay_random_peers, adnl_id_, overlay_id_.compute_short_id(), + 30, promise.send_closure(actor_id(this), &PeerManager::got_overlay_random_peers)); + } + + private: + ton::adnl::AdnlNodeIdShort adnl_id_; + ton::overlay::OverlayIdFull overlay_id_; + td::actor::ActorId overlays_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + std::map, td::actor::ActorId> peers_; + std::map> nodes_; + ton::PeerId next_peer_id_{1}; + std::map adnl_to_peer_id_; + std::vector adnl_ids_; + + std::map subscribed_peers_; + + void got_overlay_random_peers(td::Result> r_peers, + td::Promise> promise) { + TRY_RESULT_PROMISE(promise, peers, std::move(r_peers)); + + std::vector res; + for (auto peer : peers) { + res.push_back(register_adnl_id(peer)); + } + + promise.set_value(std::move(res)); + } +}; + +class StorageCli : public td::actor::Actor { + public: + explicit StorageCli(StorageCliOptions options) : options_(std::move(options)) { + } + + private: + StorageCliOptions options_; + td::actor::ActorOwn io_; + //td::actor::ActorOwn dht_server_; + + std::shared_ptr dht_config_; + + td::actor::ActorOwn keyring_; + td::actor::ActorOwn adnl_network_manager_; + td::actor::ActorOwn adnl_; + td::actor::ActorOwn dht_; + td::actor::ActorOwn overlays_; + td::actor::ActorOwn rldp_; + //ton::PublicKeyHash default_dht_node_ = ton::PublicKeyHash::zero(); + ton::PublicKey public_key_; + + bool one_shot_{false}; + bool is_closing_{false}; + td::uint32 ref_cnt_{1}; + + td::Status load_global_config() { + TRY_RESULT_PREFIX(conf_data, td::read_file(options_.config), "failed to read: "); + TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); + + ton::ton_api::config_global conf; + TRY_STATUS_PREFIX(ton::ton_api::from_json(conf, conf_json.get_object()), "json does not fit TL scheme: "); + + // TODO + // add adnl static nodes + //if (conf.adnl_) { + // td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_static_nodes_from_config, + // std::move(conf.adnl_->static_nodes_)); + //} + if (!conf.dht_) { + return td::Status::Error(ton::ErrorCode::error, "does not contain [dht] section"); + } + + TRY_RESULT_PREFIX(dht, ton::dht::Dht::create_global_config(std::move(conf.dht_)), "bad [dht] section: "); + dht_config_ = std::move(dht); + return td::Status::OK(); + } + + void start_up() override { + class Cb : public td::TerminalIO::Callback { + public: + void line_cb(td::BufferSlice line) override { + td::actor::send_closure(id_, &StorageCli::parse_line, std::move(line)); + } + Cb(td::actor::ActorShared id) : id_(std::move(id)) { + } + + private: + td::actor::ActorShared id_; + }; + if (options_.cmd) { + one_shot_ = true; + td::actor::send_closure(actor_id(this), &StorageCli::parse_line, td::BufferSlice(options_.cmd.unwrap())); + } else { + ref_cnt_++; + io_ = td::TerminalIO::create("> ", options_.enable_readline, std::make_unique(actor_shared(this))); + td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); + } + + if (!options_.config.empty()) { + init_network(); + } + } + + void init_network() { + load_global_config().ensure(); + + td::mkdir(options_.db_root).ignore(); + keyring_ = ton::keyring::Keyring::create(options_.db_root + "/keyring"); + adnl_network_manager_ = ton::adnl::AdnlNetworkManager::create(td::narrow_cast(options_.addr.get_port())); + adnl_ = ton::adnl::Adnl::create(options_.db_root, keyring_.get()); + td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_network_manager, adnl_network_manager_.get()); + rldp_ = ton_rldp::Rldp::create(adnl_.get()); + + auto key_path = options_.db_root + "/key.pub"; + auto r_public_key = td::read_file(key_path).move_fmap([](auto raw) { return ton::PublicKey::import(raw); }); + ; + if (r_public_key.is_error()) { + auto private_key = ton::PrivateKey(ton::privkeys::Ed25519::random()); + public_key_ = private_key.compute_public_key(); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(private_key), false, + td::Promise([key_path, str = public_key_.export_as_slice()](auto) { + td::write_file(key_path, str.as_slice()).ensure(); + LOG(INFO) << "New key was saved"; + })); + } else { + public_key_ = r_public_key.move_as_ok(); + } + auto short_id = public_key_.compute_short_id(); + LOG(ERROR) << "Create " << short_id; + + std::set cats; + cats = {0, 1, 2, 3}; + ton::adnl::AdnlCategoryMask cat_mask; + for (auto cat : cats) { + cat_mask[cat] = true; + } + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::add_self_addr, options_.addr, + std::move(cat_mask), cats.size() ? 0 : 1); + + td::uint32 ts = static_cast(td::Clocks::system()); + + std::map addr_lists_; + for (auto cat : cats) { + CHECK(cat >= 0); + ton::adnl::AdnlAddress x = ton::adnl::AdnlAddressImpl::create( + ton::create_tl_object(options_.addr.get_ipv4(), options_.addr.get_port())); + addr_lists_[cat].add_addr(std::move(x)); + addr_lists_[cat].set_version(ts); + addr_lists_[cat].set_reinit_date(ton::adnl::Adnl::adnl_start_time()); + } + + for (auto cat : cats) { + td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{public_key_}, addr_lists_[cat], + static_cast(cat)); + td::actor::send_closure(rldp_, &ton_rldp::Rldp::add_id, + ton::adnl::AdnlNodeIdFull{public_key_}.compute_short_id()); + } + + dht_ = + ton::dht::Dht::create(ton::adnl::AdnlNodeIdShort{short_id}, "" /*NO db for dht! No wrong cache - no problems*/, + dht_config_, keyring_.get(), adnl_.get()) + .move_as_ok(); + + send_closure(adnl_, &ton::Adnl::register_dht_node, dht_.get()); + overlays_ = ton::overlay::Overlays::create(options_.db_root, keyring_.get(), adnl_.get(), dht_.get()); + } + + void exit(td::Result res) { + if (one_shot_) { + td::TerminalIO::out() << "Done, exiting"; + std::_Exit(res.is_ok() ? 0 : 2); + } + } + + void parse_line(td::BufferSlice line) { + if (is_closing_) { + return; + } + td::ConstParser parser(line.as_slice()); + auto cmd = parser.read_word(); + if (cmd.empty()) { + return; + } + + td::Promise cmd_promise = [line = line.clone(), timer = td::PerfWarningTimer(line.as_slice().str()), + cli = actor_id(this)](td::Result res) { + if (res.is_ok()) { + // on_ok + } else { + td::TerminalIO::out() << "Query {" << line.as_slice() << "} FAILED: \n\t" << res.error() << "\n"; + } + send_closure(cli, &StorageCli::exit, std::move(res)); + }; + + if (cmd == "help") { + td::TerminalIO::out() << "help\tThis help\n"; + + td::TerminalIO::out() << "create \tCreate torrent from a directory\n"; + td::TerminalIO::out() << "info \tPrint info about loaded torrent\n"; + td::TerminalIO::out() << "load \tLoad torrent file in memory\n"; + td::TerminalIO::out() << "save \tSave torrent file\n"; + td::TerminalIO::out() << "start \tStart torrent downloading/uploading\n"; + td::TerminalIO::out() << "seed \tStart torrent uploading\n"; + td::TerminalIO::out() << "download \tStart torrent and stop when it is completed\n"; + td::TerminalIO::out() << "stop \tStop torrent downloading\n"; + + td::TerminalIO::out() << "pause \tPause active torrent downloading\n"; + td::TerminalIO::out() << "resume \tResume active torrent downloading\n"; + td::TerminalIO::out() << "priority \tSet file priority(0..254) by file_id, use " + "file_id=* to set priority for all files\n"; + + td::TerminalIO::out() << "exit\tExit\n"; + td::TerminalIO::out() << "quit\tExit\n"; + } else if (cmd == "exit" || cmd == "quit") { + quit(); + } else if (cmd == "create") { + torrent_create(parser.read_all(), std::move(cmd_promise)); + } else if (cmd == "info") { + torrent_info(parser.read_all(), std::move(cmd_promise)); + } else if (cmd == "load") { + cmd_promise.set_result(torrent_load(parser.read_all()).move_map([](auto &&x) { return td::Unit(); })); + } else if (cmd == "save") { + auto id = parser.read_word(); + parser.skip_whitespaces(); + auto file = parser.read_all(); + torrent_save(id, file, std::move(cmd_promise)); + } else if (cmd == "start") { + auto id = parser.read_word(); + torrent_start(id, false, true, std::move(cmd_promise)); + } else if (cmd == "download") { + auto id = parser.read_word(); + torrent_start(id, true, true, std::move(cmd_promise)); + } else if (cmd == "seed") { + auto id = parser.read_word(); + torrent_start(id, false, false, std::move(cmd_promise)); + } else if (cmd == "stop") { + auto id = parser.read_word(); + torrent_stop(id, std::move(cmd_promise)); + } else if (cmd == "pause") { + auto id = parser.read_word(); + torrent_set_should_download(id, false, std::move(cmd_promise)); + } else if (cmd == "resume") { + auto id = parser.read_word(); + torrent_set_should_download(id, true, std::move(cmd_promise)); + } else if (cmd == "priority") { + torrent_set_priority(parser, std::move(cmd_promise)); + } else if (cmd == "get") { + auto name = parser.read_word().str(); + auto key = ton::dht::DhtKey(public_key_.compute_short_id(), name, 0); + send_closure(dht_, &ton::dht::Dht::get_value, std::move(key), cmd_promise.wrap([](auto &&res) { + LOG(ERROR) << to_string(res.tl()); + return td::Unit(); + })); + + } else if (cmd == "set") { + auto name = parser.read_word().str(); + parser.skip_whitespaces(); + auto value = parser.read_all().str(); + + auto key = ton::dht::DhtKey(public_key_.compute_short_id(), name, 0); + auto dht_update_rule = ton::dht::DhtUpdateRuleSignature::create().move_as_ok(); + ton::dht::DhtKeyDescription dht_key_description{key.clone(), public_key_, dht_update_rule, td::BufferSlice()}; + + auto to_sing = dht_key_description.to_sign(); + send_closure(keyring_, &ton::keyring::Keyring::sign_message, public_key_.compute_short_id(), std::move(to_sing), + cmd_promise.send_closure(actor_id(this), &StorageCli::dht_set1, std::move(dht_key_description), + td::BufferSlice(value))); + } else { + cmd_promise.set_error(td::Status::Error(PSLICE() << "Unkwnown query `" << cmd << "`")); + } + if (cmd_promise) { + cmd_promise.set_value(td::Unit()); + } + } + + void dht_set1(ton::dht::DhtKeyDescription dht_key_description, td::BufferSlice value, + td::Result r_signature, td::Promise promise) { + TRY_RESULT_PROMISE(promise, signature, std::move(r_signature)); + dht_key_description.update_signature(std::move(signature)); + dht_key_description.check().ensure(); + + auto ttl = static_cast(td::Clocks::system() + 3600); + ton::dht::DhtValue dht_value{dht_key_description.clone(), std::move(value), ttl, td::BufferSlice("")}; + auto to_sign = dht_value.to_sign(); + send_closure(keyring_, &ton::keyring::Keyring::sign_message, public_key_.compute_short_id(), std::move(to_sign), + promise.send_closure(actor_id(this), &StorageCli::dht_set2, std::move(dht_value))); + } + + void dht_set2(ton::dht::DhtValue dht_value, td::Result r_signature, td::Promise promise) { + TRY_RESULT_PROMISE(promise, signature, std::move(r_signature)); + dht_value.update_signature(std::move(signature)); + dht_value.check().ensure(); + + send_closure(dht_, &ton::dht::Dht::set_value, std::move(dht_value), promise.wrap([](auto &&res) { + LOG(ERROR) << "OK"; + return td::Unit(); + })); + } + + td::uint32 torrent_id_{0}; + struct Info { + td::uint32 id; + td::Bits256 hash; + td::optional torrent; + td::actor::ActorOwn peer_manager; + td::actor::ActorOwn node; + }; + std::map infos_; + + void torrent_create(td::Slice path_raw, td::Promise promise) { + auto path = td::trim(path_raw).str(); + ton::Torrent::Creator::Options options; + options.piece_size = 128 * 1024; + TRY_RESULT_PROMISE(promise, torrent, ton::Torrent::Creator::create_from_path(options, path)); + auto hash = torrent.get_info().header_hash; + for (auto &it : infos_) { + if (it.second.hash == hash) { + promise.set_error(td::Status::Error(PSLICE() << "Torrent already loaded (#" << it.first << ")")); + return; + } + } + td::TerminalIO::out() << "Torrent #" << torrent_id_ << " created\n"; + infos_.emplace(torrent_id_, Info{torrent_id_, hash, std::move(torrent), td::actor::ActorOwn(), + td::actor::ActorOwn()}); + torrent_id_++; + + promise.set_value(td::Unit()); + } + + td::Result to_torrent(td::Slice id_raw) { + TRY_RESULT(id, td::to_integer_safe(td::trim(id_raw))); + auto it = infos_.find(id); + if (it == infos_.end()) { + return td::Status::Error(PSLICE() << "Invalid torrent id <" << id_raw << ">"); + } + if (it->second.torrent) { + return &it->second.torrent.value(); + } + return nullptr; + } + + td::Result to_info(td::Slice id_raw) { + TRY_RESULT(id, td::to_integer_safe(td::trim(id_raw))); + auto it = infos_.find(id); + if (it == infos_.end()) { + return td::Status::Error(PSLICE() << "Invalid torrent id <" << id_raw << ">"); + } + return &it->second; + } + td::Result to_info_or_load(td::Slice id_raw) { + auto r_info = to_info(id_raw); + if (r_info.is_ok()) { + return r_info.ok(); + } + return torrent_load(id_raw); + } + + void torrent_info(td::Slice id_raw, td::Promise promise) { + TRY_RESULT_PROMISE(promise, info, to_info(id_raw)); + if (info->torrent) { + td::TerminalIO::out() << info->torrent.value().get_stats_str(); + promise.set_value(td::Unit()); + } else { + send_closure(info->node, &ton::NodeActor::get_stats_str, promise.wrap([](std::string stats) { + td::TerminalIO::out() << stats; + return td::Unit(); + })); + } + } + + td::actor::ActorOwn create_peer_manager(vm::Cell::Hash hash) { + // create overlay network + td::BufferSlice hash_str(hash.as_slice()); + ton::overlay::OverlayIdFull overlay_id(std::move(hash_str)); + auto adnl_id = ton::adnl::AdnlNodeIdShort{public_key_.compute_short_id()}; + return td::actor::create_actor("PeerManager", adnl_id, std::move(overlay_id), overlays_.get(), + adnl_.get(), rldp_.get()); + } + + void torrent_start(td::Slice id_raw, bool wait_download, bool should_download, td::Promise promise) { + TRY_RESULT_PROMISE(promise, ptr, to_info_or_load(id_raw)); + if (!ptr->torrent) { + promise.set_error(td::Status::Error("torrent is already started")); + return; + } + if (ptr->peer_manager.empty()) { + ptr->peer_manager = create_peer_manager(ptr->torrent.value().get_info().get_hash()); + } + ton::PeerId self_id = 1; + + class Context : public ton::NodeActor::Callback { + public: + Context(td::actor::ActorId peer_manager, td::actor::ActorId storage_cli, + ton::PeerId self_id, td::uint32 torrent_id, td::Promise on_completed) + : peer_manager_(peer_manager) + , storage_cli_(std::move(storage_cli)) + , self_id_(self_id) + , torrent_id_(std::move(torrent_id)) + , on_completed_(std::move(on_completed)) { + } + void get_peers(td::Promise> promise) override { + send_closure(peer_manager_, &PeerManager::get_peers, std::move(promise)); + } + void register_self(td::actor::ActorId self) override { + CHECK(self_.empty()); + self_ = self; + send_closure(peer_manager_, &PeerManager::register_node, self_id_, self_); + } + ~Context() { + if (!self_.empty()) { + send_closure(peer_manager_, &PeerManager::unregister_node, self_id_, self_); + } + } + td::actor::ActorOwn create_peer(ton::PeerId self_id, ton::PeerId peer_id, + td::SharedState state) override { + CHECK(self_id == self_id_); + class PeerCallback : public ton::PeerActor::Callback { + public: + PeerCallback(ton::PeerId self_id, ton::PeerId peer_id, td::actor::ActorId peer_manager) + : self_id_(self_id), peer_id_(peer_id), peer_manager_(std::move(peer_manager)) { + } + void register_self(td::actor::ActorId self) override { + CHECK(self_.empty()); + self_ = std::move(self); + send_closure(peer_manager_, &PeerManager::register_peer, self_id_, peer_id_, self_); + } + void send_query(td::uint64 query_id, td::BufferSlice query) override { + send_closure(peer_manager_, &PeerManager::send_query, self_id_, peer_id_, std::move(query), + promise_send_closure(self_, &ton::PeerActor::on_query_result, query_id)); + } + + ~PeerCallback() { + if (!self_.empty()) { + send_closure(peer_manager_, &PeerManager::unregister_peer, self_id_, peer_id_, self_); + } + } + + private: + td::actor::ActorId self_; + ton::PeerId self_id_; + ton::PeerId peer_id_; + td::actor::ActorId peer_manager_; + }; + + return td::actor::create_actor(PSLICE() << "ton::PeerActor " << self_id << "->" << peer_id, + td::make_unique(self_id, peer_id, peer_manager_), + std::move(state)); + } + + void on_completed() override { + if (on_completed_) { + on_completed_.set_value(td::Unit()); + } + td::TerminalIO::out() << "Torrent #" << torrent_id_ << " completed\n"; + } + + void on_closed(ton::Torrent torrent) override { + send_closure(storage_cli_, &StorageCli::got_torrent, torrent_id_, std::move(torrent)); + } + + private: + td::actor::ActorId peer_manager_; + td::actor::ActorId storage_cli_; + ton::PeerId self_id_; + td::uint32 torrent_id_; + std::vector peers_; + td::Promise on_completed_; + td::actor::ActorId self_; + }; + + td::Promise on_completed; + if (wait_download) { + on_completed = std::move(promise); + } + auto context = + td::make_unique(ptr->peer_manager.get(), actor_id(this), self_id, ptr->id, std::move(on_completed)); + ptr->node = td::actor::create_actor(PSLICE() << "Node#" << self_id, self_id, ptr->torrent.unwrap(), + std::move(context), should_download); + td::TerminalIO::out() << "Torrent #" << ptr->id << " started\n"; + promise.release().release(); + if (promise) { + promise.set_value(td::Unit()); + } + } + + void on_torrent_completed(td::uint32 torrent_id) { + td::TerminalIO::out() << "Torrent #" << torrent_id << " completed\n"; + } + void got_torrent(td::uint32 torrent_id, ton::Torrent &&torrent) { + infos_[torrent_id].torrent = std::move(torrent); + td::TerminalIO::out() << "Torrent #" << torrent_id << " ready to start again\n"; + } + + void torrent_stop(td::Slice id_raw, td::Promise promise) { + TRY_RESULT_PROMISE(promise, ptr, to_info(id_raw)); + ptr->node.reset(); + ptr->peer_manager.reset(); + promise.set_value(td::Unit()); + td::TerminalIO::out() << "Torrent #" << ptr->id << " stopped\n"; + } + + void torrent_set_should_download(td::Slice id_raw, bool should_download, td::Promise promise) { + TRY_RESULT_PROMISE(promise, ptr, to_info(id_raw)); + if (ptr->node.empty()) { + promise.set_error(td::Status::Error("Torrent is not active")); + return; + } + send_closure(ptr->node, &ton::NodeActor::set_should_download, should_download); + promise.set_value(td::Unit()); + } + + void torrent_set_priority(td::ConstParser &parser, td::Promise promise) { + TRY_RESULT_PROMISE(promise, ptr, to_info(parser.read_word())); + if (ptr->node.empty()) { + promise.set_error(td::Status::Error("Torrent is not active")); + return; + } + auto file_id_str = parser.read_word(); + size_t file_id = std::numeric_limits::max(); + if (file_id_str != "*") { + TRY_RESULT_PROMISE_ASSIGN(promise, file_id, td::to_integer_safe(file_id_str)); + } + TRY_RESULT_PROMISE(promise, priority, td::to_integer_safe(parser.read_word())); + if (priority == 255) { + promise.set_error(td::Status::Error("Priority = 255 is reserved")); + return; + } + send_closure(ptr->node, &ton::NodeActor::set_file_priority, file_id, priority); + promise.set_value(td::Unit()); + } + + void torrent_save(td::Slice id_raw, td::Slice path, td::Promise promise) { + TRY_RESULT_PROMISE(promise, ptr, to_torrent(id_raw)); + auto meta = ptr->get_meta(ton::Torrent::GetMetaOptions().with_proof_depth_limit(10)); + TRY_STATUS_PROMISE(promise, td::write_file(path.str(), meta.serialize())); + promise.set_value(td::Unit()); + td::TerminalIO::out() << "Torrent #" << id_raw << " saved\n"; + } + + td::Result torrent_load(td::Slice path) { + TRY_RESULT(data, td::read_file(PSLICE() << td::trim(path))); + TRY_RESULT(meta, ton::TorrentMeta::deserialize(data)); + ton::Torrent::Options options; + options.in_memory = false; + options.root_dir = "."; + options.validate = true; + + TRY_RESULT(torrent, ton::Torrent::open(options, data)); + + auto hash = torrent.get_info().header_hash; + for (auto &it : infos_) { + if (it.second.hash == hash) { + return td::Status::Error(PSLICE() << "Torrent already loaded (#" << it.first << ")"); + } + } + td::TerminalIO::out() << "Torrent #" << torrent_id_ << " created\n"; + auto res = + infos_.emplace(torrent_id_, Info{torrent_id_, hash, std::move(torrent), td::actor::ActorOwn(), + td::actor::ActorOwn()}); + torrent_id_++; + return &res.first->second; + } + + void hangup() override { + quit(); + } + void hangup_shared() override { + CHECK(ref_cnt_ > 0); + ref_cnt_--; + //if (get_link_token() == 1) { + //io_.reset(); + //} + try_stop(); + } + void try_stop() { + if (is_closing_ && ref_cnt_ == 0) { + stop(); + } + } + void quit() { + is_closing_ = true; + io_.reset(); + //client_.reset(); + ref_cnt_--; + try_stop(); + } + + void tear_down() override { + td::actor::SchedulerContext::get()->stop(); + } +}; + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_default_failure_signal_handler(); + + StorageCliOptions options; + td::OptionsParser p; + p.set_description("experimental cli for ton storage"); + p.add_option('h', "help", "prints_help", [&]() { + std::cout << (PSLICE() << p).c_str(); + std::exit(2); + return td::Status::OK(); + }); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + auto verbosity = td::to_integer(arg); + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(FATAL) + verbosity); + return (verbosity >= 0 && verbosity <= 20) ? td::Status::OK() : td::Status::Error("verbosity must be 0..20"); + }); + p.add_option('C', "config", "set ton config", [&](td::Slice arg) { + options.config = arg.str(); + return td::Status::OK(); + }); + p.add_option('D', "db", "root for dbs", [&](td::Slice fname) { + options.db_root = fname.str(); + return td::Status::OK(); + }); + p.add_option('I', "ip", "set ip:port", [&](td::Slice arg) { + td::IPAddress addr; + TRY_STATUS(addr.init_host_port(arg.str())); + options.addr = addr; + return td::Status::OK(); + }); + p.add_option('E', "execute", "execute one command", [&](td::Slice arg) { + options.cmd = arg.str(); + return td::Status::OK(); + }); + p.add_option('d', "dir", "working directory", [&](td::Slice arg) { return td::chdir(arg.str()); }); + + auto S = p.run(argc, argv); + if (S.is_error()) { + std::cerr << S.move_as_error().message().str() << std::endl; + std::_Exit(2); + } + + td::actor::Scheduler scheduler({0}); + scheduler.run_in_context([&] { td::actor::create_actor("console", options).release(); }); + scheduler.run(); + return 0; +} diff --git a/storage/test/storage.cpp b/storage/test/storage.cpp new file mode 100644 index 000000000..c15cab85a --- /dev/null +++ b/storage/test/storage.cpp @@ -0,0 +1,1341 @@ +#include "td/utils/benchmark.h" +#include "td/utils/crypto.h" +#include "td/utils/Container.h" +#include "td/utils/misc.h" +#include "td/utils/optional.h" +#include "td/utils/overloaded.h" +#include "td/utils/Status.h" +#include "td/utils/Span.h" +#include "td/utils/tests.h" +#include "td/utils/Timer.h" +#include "td/utils/Time.h" +#include "td/utils/tl_helpers.h" +#include "td/utils/UInt.h" +#include "td/utils/VectorQueue.h" +#include "td/utils/ThreadSafeCounter.h" + +#include "td/utils/filesystem.h" +#include "td/utils/port/path.h" + +#include "tl-utils/tl-utils.hpp" + +#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" + +#include "td/actor/actor.h" + +#include "td/db/utils/CyclicBuffer.h" + +#include "vm/boc.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "vm/cells/MerkleProof.h" +#include "vm/cells/CellString.h" + +#include "fec/fec.h" + +#include "rldp2/RldpConnection.h" +#include "rldp2/LossSender.h" + +#include "Bitset.h" +#include "PeerState.h" +#include "SharedState.h" +#include "Torrent.h" +#include "TorrentCreator.h" + +#include "NodeActor.h" +#include "PeerActor.h" + +#include "MerkleTree.h" + +constexpr td::uint64 Byte = 1; +constexpr td::uint64 KiloByte = (1 << 10) * Byte; +constexpr td::uint64 MegaByte = (1 << 10) * KiloByte; + +using namespace ton::rldp2; + +extern "C" { +double ndtri(double y0); +double ndtri(double y0); +double nbdtr(int k, int n, double p); +double bdtr(int k, int n, double p); +double pdtr(int k, double y); +double pdtri(int k, double y); +} + +BENCH(Loss, "Loss") { + LossSender sender(0.5, 1e-10); + td::uint64 res = 0; + for (int i = 0; i < n; i++) { + res += sender.send_n(100000); + } + td::do_not_optimize_away(res); +} + +TEST(Rldp, Loss) { + td::bench(LossBench()); + ASSERT_EQ(96, LossSender(0.1, 1e-10).send_n_exact(64)); + ASSERT_EQ(86, LossSender(0.05, 1e-10).send_n_exact(64)); + ASSERT_EQ(75, LossSender(0.01, 1e-10).send_n_exact(64)); + ASSERT_EQ(70, LossSender(0.001, 1e-10).send_n_exact(64)); + + for (auto p1 : {1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10}) { + //CHECK(fabs(ndtri_fast(p1) - ndtri(1 - p1)) < 1e-6); + for (auto loss : {0.5, 0.1, 0.01, 0.001, 0.0001}) { + LossSender sender(loss, p1); + for (auto n : {1, 10, 20, 50, 100, 250, 500, 1000, 2000, 4000, 8000, 16000, 32000}) { + auto exact_m = sender.send_n_exact(n); + auto approx_m = sender.send_n_approx_nbd(n); + CHECK(!sender.has_good_approx() || std::abs(exact_m - approx_m) <= 1); + //std::cerr << "p=" << loss << "\tS1=" << p1 << "\tn=" << n << "\tdiff=" << exact_m - approx_m << "\t" << exact_m + //<< " " << approx_m << std::endl; + } + } + } +} + +TEST(Rldp, sub_or_zero) { + ASSERT_EQ(10u, sub_or_zero(20, 10)); + ASSERT_EQ(0u, sub_or_zero(10, 20)); +} + +TEST(Rldp, RttStats) { + RttStats stats; + ASSERT_TRUE(stats.smoothed_rtt < 0); + + td::Timestamp now; + stats.on_rtt_sample(-1, 0, now); + ASSERT_TRUE(stats.smoothed_rtt < 0); + stats.on_rtt_sample(1, -1, now); + ASSERT_TRUE(stats.smoothed_rtt < 0); + + stats.on_rtt_sample(1, 0, now); + stats.on_rtt_sample(2, 0, now); + stats.on_rtt_sample(1, 0, now); + stats.on_rtt_sample(2, 0, now); + stats.on_rtt_sample(1, 0, now); + stats.on_rtt_sample(2, 0, now); + ASSERT_TRUE(fabs(stats.last_rtt - 2) < 1e-9); + ASSERT_TRUE(fabs(stats.min_rtt - 1) < 1e-9); + ASSERT_TRUE(1 < stats.smoothed_rtt && stats.smoothed_rtt < 2); + ASSERT_TRUE(0.1 < stats.rtt_var && stats.rtt_var < 0.9); +} + +TEST(Rldp, Ack) { + Ack ack; + ASSERT_TRUE(ack.on_got_packet(5)); + ASSERT_TRUE(!ack.on_got_packet(5)); + ASSERT_EQ(5u, ack.max_seqno); + ASSERT_EQ(1u, ack.received_count); + ASSERT_EQ(1u, ack.received_mask); + + ASSERT_TRUE(ack.on_got_packet(3)); + ASSERT_TRUE(!ack.on_got_packet(3)); + ASSERT_EQ(5u, ack.max_seqno); + ASSERT_EQ(2u, ack.received_count); + ASSERT_EQ(5u, ack.received_mask); + + ASSERT_TRUE(ack.on_got_packet(7)); + ASSERT_TRUE(!ack.on_got_packet(7)); + ASSERT_EQ(7u, ack.max_seqno); + ASSERT_EQ(3u, ack.received_count); + ASSERT_EQ(21u, ack.received_mask); + + ASSERT_TRUE(ack.on_got_packet(100)); + ASSERT_TRUE(!ack.on_got_packet(100)); + ASSERT_TRUE(!ack.on_got_packet(8)); + ASSERT_TRUE(!ack.on_got_packet(7)); + ASSERT_EQ(4u, ack.received_count); + ASSERT_EQ(1u, ack.received_mask); +} + +TEST(Rldp, SenderPackets) { + td::Random::Xorshift128plus rnd(123); + + for (int test_i = 0; test_i < 100; test_i++) { + Ack ack; + SenderPackets sender; + std::vector in_flight; + std::set in_flight_set; + std::set received; + std::set dropped; + std::set no_ack; + + td::int32 now = 0; + td::uint32 last_seqno = 0; + + td::uint32 window = rnd.fast(1, 100); + + auto send_query = [&]() { + if (sender.in_flight_count() > window) { + return; + } + last_seqno++; + auto seqno = sender.next_seqno(); + CHECK(seqno == last_seqno); + SenderPackets::Packet packet; + packet.is_in_flight = true; + packet.sent_at = td::Timestamp::at(now); + packet.seqno = seqno; + packet.size = 0; + sender.send(packet); + + in_flight.push_back(seqno); + in_flight_set.insert(seqno); + }; + + auto extract_in_flight_query = [&]() -> td::optional { + if (in_flight_set.empty()) { + return {}; + } + while (true) { + auto position = rnd.fast(0, (int)in_flight.size() - 1); + std::swap(in_flight[position], in_flight.back()); + auto seqno = in_flight.back(); + in_flight.pop_back(); + if (!in_flight_set.count(seqno)) { + continue; + } + in_flight_set.erase(seqno); + return seqno; + } + }; + + auto receive_query = [&]() { + auto o_seqno = extract_in_flight_query(); + if (!o_seqno) { + return; + } + auto seqno = o_seqno.unwrap(); + if (ack.on_got_packet(seqno)) { + received.insert(seqno); + } + no_ack.insert(seqno); + }; + + auto drop_query = [&]() { + auto o_seqno = extract_in_flight_query(); + if (!o_seqno) { + return; + } + auto seqno = o_seqno.unwrap(); + dropped.insert(seqno); + }; + + auto send_ack = [&]() { + sender.on_ack(ack); + no_ack.clear(); + ASSERT_EQ(received.size(), sender.received_count()); + //ASSERT_EQ(no_ack.size() + in_flight_set.size() + dropped.size(), sender.in_flight_count()); + if (!received.empty()) { + ASSERT_EQ(*received.rbegin(), sender.max_packet().seqno); + } + }; + + auto apply_limits = [&]() { + auto till_seqno = sub_or_zero(sender.max_packet().seqno, rnd.fast(3, 31)); + SenderPackets::Limits limits; + limits.sent_at = td::Timestamp::at(0); + limits.seqno = till_seqno; + //ASSERT_EQ(no_ack.size() + in_flight_set.size() + dropped.size(), sender.in_flight_count()); + + in_flight_set.erase(in_flight_set.begin(), in_flight_set.lower_bound(till_seqno)); + dropped.erase(dropped.begin(), dropped.lower_bound(till_seqno)); + no_ack.erase(no_ack.begin(), no_ack.lower_bound(till_seqno)); + + sender.drop_packets(limits); + //LOG(ERROR) << td::tag("max_seqno", sender.max_packet().seqno); + //LOG(ERROR) << td::tag("till_seqno", till_seqno); + //LOG(ERROR) << td::tag("no_ack", no_ack); + //LOG(ERROR) << td::tag("in_flight", in_flight); + //LOG(ERROR) << td::tag("dropped", dropped); + ASSERT_EQ(no_ack.size() + in_flight_set.size() + dropped.size(), sender.in_flight_count()); + }; + + std::vector steps_vec{ + {send_query, 0}, {receive_query, 0}, {drop_query, 0}, {send_ack, 0}, {apply_limits, 0}}; + for (auto &step : steps_vec) { + step.weight = rnd.fast(1, 10); + } + td::RandomSteps steps{std::move(steps_vec)}; + for (int i = 0; i < 1000; i++) { + steps.step(rnd); + } + } +} + +TEST(Rldp, FecHelper) { + FecHelper helper; + td::uint32 x = 5; + td::uint32 y = 5; + td::uint32 n = 10; + helper.symbols_count = n; + ASSERT_EQ(n + x, helper.get_fec_symbols_count()); + ASSERT_EQ(n + x, helper.get_left_fec_symbols_count()); + helper.received_symbols_count = n + 1; + ASSERT_EQ(n + x, helper.get_fec_symbols_count()); + ASSERT_EQ(x - 1, helper.get_left_fec_symbols_count()); + helper.received_symbols_count = n + x; + ASSERT_EQ(n + x + y, helper.get_fec_symbols_count()); + ASSERT_EQ(y, helper.get_left_fec_symbols_count()); + helper.received_symbols_count = n + x + 1; + ASSERT_EQ(n + x + y, helper.get_fec_symbols_count()); + ASSERT_EQ(y - 1, helper.get_left_fec_symbols_count()); + helper.received_symbols_count = n + x + y; + ASSERT_EQ(n + x + 2 * y, helper.get_fec_symbols_count()); + ASSERT_EQ(y, helper.get_left_fec_symbols_count()); +}; + +TEST(Rldp2, Pacer) { + Pacer::Options options; + options.initial_capacity = 0; + options.initial_speed = 100; + options.max_capacity = 1; + options.time_granularity = 0.1; + CHECK(options.initial_speed * options.time_granularity > options.max_capacity * 4); + + Pacer pacer(options); + + // send 1000 packets + auto now = td::Timestamp::at(123); + auto start = now; + for (auto it = 0; it < 1000; it++) { + CHECK(pacer.wakeup_at().is_in_past(now)); + auto o_wakeup_at = pacer.send(1, now); + if (o_wakeup_at) { + now = td::Timestamp::in(td::Random::fast(0.001, 0.1), o_wakeup_at.unwrap()); + } + } + double passed = now.at() - start.at(); + LOG_CHECK(passed > 9.9 && passed < 10.1) << passed; +} + +class Sleep : public td::actor::Actor { + public: + static void put_to_sleep(td::actor::ActorId sleep, td::Timestamp till, td::Promise promise) { + send_closure(sleep, &Sleep::do_put_to_sleep, till, std::move(promise)); + } + + static TD_WARN_UNUSED_RESULT auto create() { + return td::actor::create_actor("Sleep"); + } + + private: + std::multimap> pending_; + + void do_put_to_sleep(td::Timestamp till, td::Promise promise) { + pending_.emplace(till, std::move(promise)); + alarm_timestamp() = pending_.begin()->first; + } + + void loop() override { + while (!pending_.empty() && pending_.begin()->first.is_in_past()) { + pending_.begin()->second.set_value(td::Unit()); + pending_.erase(pending_.begin()); + } + + if (!pending_.empty()) { + alarm_timestamp() = pending_.begin()->first; + } + } +}; + +class NetChannel : public td::actor::Actor { + public: + struct Options { + double loss{0}; + double rtt{0.1}; + + double buffer{128 * KiloByte}; + double speed{1 * MegaByte}; + + double alive_begin = -1; + double sleep_step = 0; + double alive_step = 1; + + static constexpr double eps = 1e-9; + + bool is_sleeping(double now) { + if (sleep_step < eps) { + return false; + } + return alive_begin > now + eps; + } + + double calc_data(double l, double r) { + if (sleep_step < eps) { + return (r - l) * speed; + } + + if (alive_begin < 0) { + alive_begin = l; + } + double res = 0; + while (true) { + double alive_end = alive_begin + alive_step; + if (l < alive_begin) { + l = alive_begin; + } + if (l + eps > r) { + break; + } else if (r < alive_begin + eps) { + break; + } else if (l > alive_end - eps) { + alive_begin += alive_step + sleep_step; + alive_end = alive_begin + alive_step; + } else { + double new_l = td::min(alive_end, r); + res += (new_l - l) * speed; + l = new_l; + } + } + return res; + } + + double calc_wait(double need, double now) { + constexpr double eps = 1e-9; + if (sleep_step < eps) { + return need / speed; + } + if (now < alive_begin) { + return alive_begin - now; + } + return need / speed; + } + + Options with_loss(double loss) { + this->loss = loss; + return *this; + } + Options with_rtt(double rtt) { + this->rtt = rtt; + return *this; + } + Options with_speed(double speed) { + this->speed = speed; + return *this; + } + Options with_buffer(double buffer) { + this->buffer = buffer; + return *this; + } + Options with_sleep_alive(double sleep, double alive) { + this->sleep_step = sleep; + this->alive_step = alive; + return *this; + } + + static Options perfect_net() { + return NetChannel::Options().with_buffer(300 * MegaByte).with_loss(0).with_rtt(0.01).with_speed(100 * MegaByte); + } + static Options lossy_perfect_net() { + return perfect_net().with_loss(0.1); + } + static Options bad_net() { + return NetChannel::Options().with_buffer(128 * KiloByte).with_loss(0.1).with_rtt(0.2).with_speed(128 * KiloByte); + } + }; + + static TD_WARN_UNUSED_RESULT td::actor::ActorOwn create(Options options, + td::actor::ActorId sleep) { + return td::actor::create_actor("NetChannel", options, std::move(sleep)); + } + + NetChannel(Options options, td::actor::ActorId sleep) : options_(options), sleep_(std::move(sleep)) { + } + + td::uint64 total_sent() const { + return total_sent_; + } + + void send(size_t size, td::Promise promise) { + total_sent_ += size; + if (total_size_ + (double)size > options_.buffer) { + LOG(ERROR) << "OVERFLOW"; + promise.set_error(td::Status::Error("buffer overflow")); + return; + } + if (td::Random::fast(0.0, 1.0) < options_.loss) { + //LOG(ERROR) << "LOST"; + promise.set_error(td::Status::Error("lost")); + return; + } + in_cnt_++; + queue_.push(Query{size, std::move(promise)}); + total_size_ += (double)size; + //auto span = queue_.as_mutable_span(); + //std::swap(span[td::Random::fast(0, (int)span.size() - 1)], span.back()); + yield(); + } + + private: + struct Query { + size_t size; + td::Promise promise; + }; + Options options_; + td::VectorQueue queue_; + double total_size_{0}; + + td::uint64 total_sent_{0}; + + td::uint64 in_cnt_{0}; + td::uint64 out_cnt_{0}; + + double got_{0}; + td::Timestamp got_at_{}; + + td::actor::ActorId sleep_; + + void loop() override { + auto now = td::Timestamp::now(); + if (got_at_) { + got_ += options_.calc_data(got_at_.at(), now.at()); + } + got_at_ = now; + + if (options_.is_sleeping(now.at())) { + queue_ = {}; + } + + bool ok = false; + while (!queue_.empty() && (double)queue_.front().size < got_) { + ok = true; + auto query = queue_.pop(); + got_ -= (double)query.size; + total_size_ -= (double)query.size; + out_cnt_++; + Sleep::put_to_sleep(sleep_, td::Timestamp::in(options_.rtt), std::move(query.promise)); + } + + if (queue_.empty()) { + got_at_ = {}; + got_ = 0; + return; + } + + auto wait_bytes = ((double)queue_.front().size - got_); + auto wait_duration = options_.calc_wait(wait_bytes, now.at()); + //LOG(ERROR) << "Wait " << td::format::as_size((td::size_t)wait_bytes) << " " << td::format::as_time(wait_duration) + //<< " " << in_cnt_ << " " << out_cnt_ << " " << ok; + alarm_timestamp() = td::Timestamp::in(wait_duration); + } +}; + +class Rldp : public td::actor::Actor, public ConnectionCallback { + public: + struct Stats { + td::uint64 received_bytes{0}; + td::uint64 sent_bytes{0}; + td::Timestamp last_received_packet_at{}; + td::Timestamp last_sent_packet_at{}; + }; + + void receive_raw(td::BufferSlice raw) { + stats_->received_bytes += raw.size(); + connection_.receive_raw(std::move(raw)); + yield(); + } + + void send(td::BufferSlice data, td::Promise promise) { + TransferId transfer_id; + td::Random::secure_bytes(as_slice(transfer_id)); + connection_.send(transfer_id, std::move(data)); + queries_[transfer_id] = std::move(promise); + yield(); + } + + void add_peer(td::actor::ActorId peer) { + peer_ = peer; + yield(); + } + + void send_raw(td::BufferSlice data) override { + auto size = data.size(); + stats_->sent_bytes += size; + send_closure(net_channel_, &NetChannel::send, size, + [data = std::move(data), peer = peer_](td::Result res) mutable { + if (res.is_ok()) { + send_closure(peer, &Rldp::receive_raw, std::move(data)); + } + }); + } + void receive(TransferId, td::Result data) override { + CHECK(data.is_ok()); + stats_->last_received_packet_at = td::Timestamp::now(); + //LOG(ERROR) << "GOT "; + } + + void on_sent(TransferId query_id, td::Result state) override { + stats_->last_sent_packet_at = td::Timestamp::now(); + //LOG(ERROR) << "SENT " << query_id; + auto it = queries_.find(query_id); + CHECK(queries_.end() != it); + it->second.set_result(std::move(state)); + queries_.erase(it); + } + + explicit Rldp(td::actor::ActorOwn net_channel, Stats *stats) + : net_channel_(std::move(net_channel)), stats_(stats) { + CHECK(stats_); + connection_.set_default_mtu(1 << 31); + } + + private: + RldpConnection connection_; + td::actor::ActorOwn net_channel_; + td::actor::ActorId peer_; + std::map> queries_; + Stats *stats_; + + void loop() override { + alarm_timestamp() = connection_.run(*this); + } +}; + +struct RldpBasicTest { + struct Options { + size_t count{10}; + size_t query_size{1000 * Byte}; + NetChannel::Options net_options; + size_t concurrent_queries{1}; + + Options with_concurrent_queries(size_t concurrent_queries) { + this->concurrent_queries = concurrent_queries; + return *this; + } + + static Options create(size_t count, size_t query_size, NetChannel::Options net_options) { + Options options; + options.count = count; + options.query_size = query_size; + options.net_options = net_options; + return options; + } + }; + + class Test : public td::actor::Actor { + public: + Test(Options options, td::actor::ActorOwn alice, td::actor::ActorOwn bob, + td::actor::ActorOwn sleep, Rldp::Stats *alice_stats, Rldp::Stats *bob_stats) + : options_(options) + , alice_(std::move(alice)) + , bob_(std::move(bob)) + , sleep_(std::move(sleep)) + , alice_stats_(alice_stats) + , bob_stats_(bob_stats) { + } + + private: + Options options_; + td::actor::ActorOwn alice_; + td::actor::ActorOwn bob_; + td::actor::ActorOwn sleep_; + + Rldp::Stats *alice_stats_; + Rldp::Stats *bob_stats_; + td::Timestamp start_at_; + td::Timestamp last_query_at_; + + size_t query_id_{0}; + size_t got_query_id_{0}; + + int cnt_{0}; + void close(td::actor::ActorOwn actor) { + auto actor_copy = actor.get(); + actor.reset(); + send_lambda(actor_copy, + [x = td::create_destructor([self = actor_id(this)] { send_closure(self, &Test::on_closed); })]() {}); + } + void on_closed() { + cnt_--; + if (cnt_ == 0) { + td::actor::SchedulerContext::get()->stop(); + //LOG(ERROR) << "STOP"; + stop(); + } + } + + void start_up() override { + start_at_ = td::Timestamp::now(); + for (size_t i = 0; i < options_.concurrent_queries; i++) { + try_send_query(); + } + } + + void tear_down() override { + td::StringBuilder sb; + sb << "\n"; + sb << "Sent " << options_.count << " * " << td::format::as_size(options_.query_size) << " = " + << td::format::as_size(options_.query_size * options_.count) << "\n"; + sb << "Time: " << td::format::as_time(alice_stats_->last_sent_packet_at.at() - start_at_.at()) << "\n"; + sb << "Extra time: " + << td::format::as_time(alice_stats_->last_sent_packet_at.at() - bob_stats_->last_received_packet_at.at()) + << "\n"; + sb << "Data overhead: " << alice_stats_->sent_bytes - (options_.query_size * options_.count) << "\n"; + sb << "Data overhead: " << (double)alice_stats_->sent_bytes / (double)(options_.query_size * options_.count) + << "\n"; + LOG(ERROR) << sb.as_cslice(); + } + + void try_send_query(td::Result = {}) { + if (query_id_ >= options_.count) { + return; + } + query_id_++; + //LOG(ERROR) << "Create " << query_id_; + last_query_at_ = td::Timestamp::now(); + td::BufferSlice query(options_.query_size); + query.as_slice().fill('A'); + //LOG(ERROR) << "SEND"; + send_closure(alice_, &Rldp::send, std::move(query), + [self = actor_id(this)](auto x) { send_closure(self, &Test::on_query_finished); }); + } + void on_query_finished() { + try_send_query(); + //Sleep::put_to_sleep(sleep_.get(), td::Timestamp::in(20), + //td::promise_send_closure(actor_id(this), &Test::try_send_query)); + got_query_id_++; + //LOG(ERROR) << "Finished " << got_query_id_; + if (got_query_id_ < options_.count) { + return; + } + if (cnt_ == 0) { + cnt_ = 3; + close(std::move(alice_)); + close(std::move(bob_)); + close(std::move(sleep_)); + } + return; + } + }; + + static void run(Options options) { + td::actor::Scheduler scheduler({0}, true); + auto alice_stats = std::make_unique(); + auto bob_stats = std::make_unique(); + + scheduler.run_in_context([&] { + auto sleep = Sleep::create(); + auto alice_to_bob = NetChannel::create(options.net_options, sleep.get()); + auto bob_to_alice = NetChannel::create(options.net_options, sleep.get()); + + auto alice = td::actor::create_actor("Alice", std::move(alice_to_bob), alice_stats.get()); + auto bob = td::actor::create_actor("Bob", std::move(bob_to_alice), bob_stats.get()); + send_closure(alice, &Rldp::add_peer, bob.get()); + send_closure(bob, &Rldp::add_peer, alice.get()); + td::actor::create_actor("Test", options, std::move(alice), std::move(bob), std::move(sleep), + alice_stats.get(), bob_stats.get()) + .release(); + }); + scheduler.run(); + } +}; + +TEST(Rldp, Main) { + using Options = RldpBasicTest::Options; + RldpBasicTest::run(Options::create(10, 10 * MegaByte, NetChannel::Options::perfect_net())); + RldpBasicTest::run(Options::create(10 * 80, 10 * MegaByte / 80, NetChannel::Options::perfect_net())); + RldpBasicTest::run( + Options::create(10 * 80, 10 * MegaByte / 80, NetChannel::Options::perfect_net()).with_concurrent_queries(20)); + return; + + RldpBasicTest::run( + Options::create(10, 10 * MegaByte, NetChannel::Options::perfect_net()).with_concurrent_queries(10)); + RldpBasicTest::run(Options::create(10, 10 * MegaByte, NetChannel::Options::perfect_net())); + return; + RldpBasicTest::run(Options::create(10, 10 * MegaByte, NetChannel::Options::bad_net())); + RldpBasicTest::run(Options::create(10, 10 * MegaByte, NetChannel::Options::bad_net()).with_concurrent_queries(10)); + //RldpBasicTest::run(Options::create(10, 100 * MegaByte, NetChannel::Options::perfect_net().with_sleep_alive(10, 1))); + return; + + RldpBasicTest::run(Options::create(1000, 1 * Byte, NetChannel::Options::lossy_perfect_net())); + RldpBasicTest::run(Options::create(1, 100 * MegaByte, NetChannel::Options::lossy_perfect_net())); + + RldpBasicTest::run(Options::create(100, 1 * MegaByte, NetChannel::Options::bad_net())); + + RldpBasicTest::run(Options::create(1, 1 * Byte, NetChannel::Options::perfect_net())); + RldpBasicTest::run(Options::create(1, 1 * MegaByte, NetChannel::Options::perfect_net())); + + RldpBasicTest::run(Options::create(1, 100 * MegaByte, NetChannel::Options::perfect_net())); +} + +TEST(MerkleTree, Manual) { + td::Random::Xorshift128plus rnd(123); + // create big random file + size_t chunk_size = 768; + // for simplicity numer of chunks in a file is a power of two + size_t chunks_count = (1 << 16) + 1; + size_t file_size = chunk_size * chunks_count; + td::Timer timer; + LOG(INFO) << "Generate random string"; + const auto file = td::rand_string('a', 'z', td::narrow_cast(file_size)); + LOG(INFO) << timer; + + timer = {}; + LOG(INFO) << "Calculate all hashes"; + std::vector hashes(chunks_count); + td::Bits256 bad_hash{}; + for (size_t i = 0; i < chunks_count; i++) { + td::sha256(td::Slice(file).substr(i * chunk_size, chunk_size), hashes[i].as_slice()); + } + LOG(INFO) << timer; + + timer = {}; + LOG(INFO) << "Init merkle tree"; + size_t i = 0; + ton::MerkleTree tree(td::transform(hashes, [&i](auto &x) { return ton::MerkleTree::Chunk{i++, x}; })); + LOG(INFO) << timer; + + auto root_proof = tree.gen_proof(0, chunks_count - 1).move_as_ok(); + auto root_hash = tree.get_root_hash(); + + // first download each chunk one by one + + for (size_t stride : {1 << 6, 1}) { + timer = {}; + LOG(INFO) << "Gen all proofs, stride = " << stride; + for (size_t i = 0; i < chunks_count; i += stride) { + tree.gen_proof(i, i + stride - 1).move_as_ok(); + } + LOG(INFO) << timer; + timer = {}; + LOG(INFO) << "Proof size: " << vm::std_boc_serialize(tree.gen_proof(0, stride - 1).move_as_ok()).ok().size(); + LOG(INFO) << "Download file, stride = " << stride; + { + ton::MerkleTree new_tree(chunks_count, root_hash); + ton::MerkleTree other_new_tree(chunks_count, root_hash); + for (size_t i = 0; i < chunks_count; i += stride) { + new_tree.gen_proof(i, i + stride - 1).ignore(); + new_tree.add_proof(tree.gen_proof(i, i + stride - 1).move_as_ok()).ensure(); + other_new_tree.add_proof(tree.gen_proof(i, i + stride - 1).move_as_ok()).ensure(); + other_new_tree.gen_proof(i, i + stride - 1).ensure(); + other_new_tree.get_root(2); + std::vector chunks; + for (size_t j = 0; j < stride && i + j < chunks_count; j++) { + chunks.push_back({i + j, hashes.at(i + j)}); + } + new_tree.try_add_chunks(chunks).ensure(); + } + + if (stride == 1) { + std::vector chunks; + + for (size_t i = 0; i < chunks_count; i++) { + if (rnd.fast(0, 1) == 1) { + chunks.push_back({i, hashes[i]}); + } else { + chunks.push_back({i, bad_hash}); + } + } + td::Bitset bitmask; + other_new_tree.add_chunks(chunks, bitmask); + for (size_t i = 0; i < chunks_count; i++) { + auto expected = chunks[i].hash == hashes[i]; + auto got = bitmask.get(i); + LOG_CHECK(expected == got) << expected << " " << got << " " << i; + } + } + } + LOG(INFO) << timer; + } +} + +TEST(MerkleTree, Stress) { + td::Random::Xorshift128plus rnd(123); + + for (int t = 0; t < 100; t++) { + td::Bits256 bad_hash{}; + size_t chunks_count = rnd.fast(5, 10); + std::vector hashes(chunks_count); + for (auto &hash : hashes) { + char x = (char)rnd.fast(0, 255); + for (auto &c : hash.as_slice()) { + c = x; + } + } + size_t i = 0; + ton::MerkleTree tree(td::transform(hashes, [&i](auto &x) { return ton::MerkleTree::Chunk{i++, x}; })); + for (int t2 = 0; t2 < 1000; t2++) { + std::vector chunks; + + int mask = rnd.fast(0, (1 << chunks_count) - 1); + for (size_t i = 0; i < chunks_count; i++) { + if ((mask >> i) & 1) { + chunks.push_back({i, hashes[i]}); + } else { + chunks.push_back({i, bad_hash}); + } + } + td::Bitset bitmask_strict; + td::Bitset bitmask; + ton::MerkleTree new_tree(chunks_count, tree.get_root(rnd.fast(1, 5))); + tree.add_chunks(chunks, bitmask_strict); + new_tree.add_chunks(chunks, bitmask); + for (size_t i = 0; i < chunks_count; i++) { + auto expected = chunks[i].hash == hashes[i]; + auto strict_got = bitmask_strict.get(i); + LOG_CHECK(strict_got == expected) << expected << " " << strict_got << " " << i; + auto got = bitmask.get(i); + // got => expected + LOG_CHECK(!got || expected) << expected << " " << got << " " << i; + } + } + } +}; + +struct TorrentMetas { + td::optional torrent; + struct File { + std::string name; + td::BlobView buffer; + }; + std::vector files; +}; + +TorrentMetas create_random_torrent(td::Random::Xorshift128plus &rnd, td::int64 total_size = 0, + td::int32 piece_size = 0) { + ton::Torrent::Creator::Options options; + if (piece_size == 0) { + options.piece_size = rnd.fast(1, 1024); + } else { + options.piece_size = piece_size; + } + if (total_size == 0) { + total_size = rnd.fast(100, 40000); + } + ton::Torrent::Creator creator{options}; + + TorrentMetas res; + auto files_n = rnd.fast(0, 40); + for (int i = 0; i < files_n; i++) { + auto name = PSTRING() << "#" << i << ".txt"; + td::int64 n = 0; + auto left = files_n - i; + if (left == 1) { + n = total_size; + } else { + n = rnd.fast64(total_size / (left * 2), 2 * total_size / left); + } + total_size -= n; + LOG(INFO) << i << "/" << files_n << " " << n; + std::string data; + size_t len = td::min(n, td::int64(1027)); + data.reserve(len); + for (size_t i = 0; i < len; i++) { + data += static_cast(rnd.fast('a', 'z')); + } + res.files.emplace_back(TorrentMetas::File{name, td::CycicBlobView::create(td::BufferSlice(data), n).move_as_ok()}); + creator.add_blob(name, td::CycicBlobView::create(td::BufferSlice(data), n).move_as_ok()).ensure(); + } + LOG(INFO) << "Finalize..."; + res.torrent = creator.finalize().move_as_ok(); + ton::Torrent::GetMetaOptions opt; + LOG(INFO) << "Meta size (full): " << res.torrent.value().get_meta_str(ton::Torrent::GetMetaOptions()).size(); + LOG(INFO) << "Meta size (only proof): " + << res.torrent.value().get_meta_str(ton::Torrent::GetMetaOptions().without_header()).size(); + LOG(INFO) << "Meta size (only small proof): " + << res.torrent.value() + .get_meta_str(ton::Torrent::GetMetaOptions().without_header().with_proof_depth_limit(10)) + .size(); + LOG(INFO) << "Meta size (only header): " + << res.torrent.value().get_meta_str(ton::Torrent::GetMetaOptions().without_proof()).size(); + LOG(INFO) << "Meta size (min): " + << res.torrent.value().get_meta_str(ton::Torrent::GetMetaOptions().without_proof().without_header()).size(); + return res; +} + +TEST(Torrent, Meta) { + td::Random::Xorshift128plus rnd(123); + for (int test_i = 0; test_i < 100; test_i++) { + auto torrent_files = create_random_torrent(rnd); + auto torrent = torrent_files.torrent.unwrap(); + auto files = std::move(torrent_files.files); + + // test TorrentMeta + auto torrent_str = torrent.get_meta_str(); + + auto torrent_file = ton::TorrentMeta::deserialize(torrent_str).move_as_ok(); + CHECK(torrent_file.serialize() == torrent_str); + torrent_str.back()++; + ton::TorrentMeta::deserialize(torrent_str).ensure_error(); + CHECK(torrent.get_info().get_hash() == torrent_file.info.get_hash()); + + ton::Torrent::Options options; + options.in_memory = true; + torrent_file.header = {}; + torrent_file.root_proof = {}; + auto new_torrent = ton::Torrent::open(options, torrent_file).move_as_ok(); + + std::vector order; + for (size_t i = 0; i < torrent.get_info().pieces_count(); i++) { + order.push_back(i); + } + CHECK(!new_torrent.is_completed()); + auto header_parts = + (torrent.get_info().header_size + torrent.get_info().piece_size - 1) / torrent.get_info().piece_size; + random_shuffle(td::MutableSpan(order).substr(header_parts), rnd); + random_shuffle(td::MutableSpan(order).truncate(header_parts + 10), rnd); + for (auto piece_i : order) { + auto piece_data = torrent.get_piece_data(piece_i).move_as_ok(); + auto piece_proof = torrent.get_piece_proof(piece_i).move_as_ok(); + new_torrent.add_piece(piece_i, std::move(piece_data), std::move(piece_proof)).ensure(); + } + CHECK(new_torrent.is_completed()); + new_torrent.validate(); + CHECK(new_torrent.is_completed()); + for (auto &name_data : files) { + ASSERT_EQ(name_data.buffer.to_buffer_slice().move_as_ok(), + new_torrent.read_file(name_data.name).move_as_ok().as_slice()); + } + } +}; + +TEST(Torrent, OneFile) { + td::rmrf("first").ignore(); + td::rmrf("second").ignore(); + + td::mkdir("first").ensure(); + td::mkdir("second").ensure(); + + td::write_file("first/hello.txt", "Hello world!").ensure(); + ton::Torrent::Creator::Options options; + //options.dir_name = "first/"; + options.piece_size = 1024; + auto torrent = ton::Torrent::Creator::create_from_path(options, "first/hello.txt").move_as_ok(); + auto meta = ton::TorrentMeta::deserialize(torrent.get_meta().serialize()).move_as_ok(); + CHECK(torrent.is_completed()); + + { + ton::Torrent::Options options; + options.root_dir = "first/"; + auto other_torrent = ton::Torrent::open(options, meta).move_as_ok(); + CHECK(!other_torrent.is_completed()); + other_torrent.validate(); + CHECK(other_torrent.is_completed()); + CHECK(td::read_file("first/hello.txt").move_as_ok() == "Hello world!"); + } + + { + ton::Torrent::Options options; + options.root_dir = "second/"; + auto other_torrent = ton::Torrent::open(options, meta).move_as_ok(); + CHECK(!other_torrent.is_completed()); + other_torrent.add_piece(0, torrent.get_piece_data(0).move_as_ok(), torrent.get_piece_proof(0).move_as_ok()) + .ensure(); + CHECK(other_torrent.is_completed()); + CHECK(td::read_file("second/hello.txt").move_as_ok() == "Hello world!"); + } +}; + +TEST(Torrent, PartsHelper) { + int parts_count = 100; + ton::PartsHelper parts(parts_count); + + auto a_token = parts.register_peer(1); + auto b_token = parts.register_peer(2); + auto c_token = parts.register_peer(3); + + parts.on_peer_part_ready(a_token, 1); + parts.on_peer_part_ready(a_token, 2); + parts.on_peer_part_ready(a_token, 3); + parts.on_peer_part_ready(b_token, 1); + parts.on_peer_part_ready(b_token, 2); + parts.on_peer_part_ready(c_token, 1); + ASSERT_EQ(0u, parts.get_rarest_parts(10).size()); + + parts.set_peer_limit(a_token, 1); + ASSERT_EQ(1u, parts.get_rarest_parts(10).size()); + parts.set_peer_limit(a_token, 2); + ASSERT_EQ(2u, parts.get_rarest_parts(10).size()); + parts.set_peer_limit(a_token, 3); + ASSERT_EQ(3u, parts.get_rarest_parts(10).size()); +} + +void print_debug(ton::Torrent *torrent) { + LOG(ERROR) << torrent->get_stats_str(); +} + +TEST(Torrent, Peer) { + class PeerManager : public td::actor::Actor { + public: + void send_query(ton::PeerId src, ton::PeerId dst, td::BufferSlice query, td::Promise promise) { + send_closure(get_outbound_channel(src), &NetChannel::send, query.size(), + promise.send_closure(actor_id(this), &PeerManager::do_send_query, src, dst, std::move(query))); + } + + void do_send_query(ton::PeerId src, ton::PeerId dst, td::BufferSlice query, td::Result res, + td::Promise promise) { + TRY_RESULT_PROMISE(promise, x, std::move(res)); + (void)x; + send_closure(get_inbound_channel(dst), &NetChannel::send, query.size(), + promise.send_closure(actor_id(this), &PeerManager::execute_query, src, dst, std::move(query))); + } + + void execute_query(ton::PeerId src, ton::PeerId dst, td::BufferSlice query, td::Result res, + td::Promise promise) { + TRY_RESULT_PROMISE(promise, x, std::move(res)); + (void)x; + promise = promise.send_closure(actor_id(this), &PeerManager::send_response, src, dst); + auto it = peers_.find(std::make_pair(dst, src)); + if (it == peers_.end()) { + LOG(ERROR) << "No such peer"; + auto node_it = nodes_.find(dst); + if (node_it == nodes_.end()) { + LOG(ERROR) << "Unknown query destination"; + promise.set_error(td::Status::Error("Unknown query destination")); + return; + } + send_closure(node_it->second, &ton::NodeActor::start_peer, src, + [promise = std::move(promise), + query = std::move(query)](td::Result> r_peer) mutable { + TRY_RESULT_PROMISE(promise, peer, std::move(r_peer)); + send_closure(peer, &ton::PeerActor::execute_query, std::move(query), std::move(promise)); + }); + return; + } + send_closure(it->second, &ton::PeerActor::execute_query, std::move(query), std::move(promise)); + } + + void send_response(ton::PeerId src, ton::PeerId dst, td::Result r_response, + td::Promise promise) { + TRY_RESULT_PROMISE(promise, response, std::move(r_response)); + send_closure(get_outbound_channel(dst), &NetChannel::send, response.size(), + promise.send_closure(actor_id(this), &PeerManager::do_send_response, src, dst, std::move(response))); + } + + void do_send_response(ton::PeerId src, ton::PeerId dst, td::BufferSlice response, td::Result res, + td::Promise promise) { + TRY_RESULT_PROMISE(promise, x, std::move(res)); + (void)x; + send_closure( + get_inbound_channel(src), &NetChannel::send, response.size(), + promise.send_closure(actor_id(this), &PeerManager::do_execute_response, src, dst, std::move(response))); + } + + void do_execute_response(ton::PeerId src, ton::PeerId dst, td::BufferSlice response, td::Result res, + td::Promise promise) { + TRY_RESULT_PROMISE(promise, x, std::move(res)); + (void)x; + promise.set_value(std::move(response)); + } + + void register_peer(ton::PeerId src, ton::PeerId dst, td::actor::ActorId peer) { + peers_[std::make_pair(src, dst)] = std::move(peer); + } + + void register_node(ton::PeerId src, td::actor::ActorId node) { + nodes_[src] = std::move(node); + } + ~PeerManager() { + for (auto &it : inbound_channel_) { + LOG(ERROR) << it.first << " received " << td::format::as_size(it.second.get_actor_unsafe().total_sent()); + } + for (auto &it : outbound_channel_) { + LOG(ERROR) << it.first << " sent " << td::format::as_size(it.second.get_actor_unsafe().total_sent()); + } + } + + private: + std::map, td::actor::ActorId> peers_; + std::map> nodes_; + std::map> inbound_channel_; + std::map> outbound_channel_; + + td::actor::ActorOwn sleep_; + void start_up() override { + sleep_ = Sleep::create(); + } + + td::actor::ActorId get_outbound_channel(ton::PeerId peer_id) { + auto &res = outbound_channel_[peer_id]; + if (res.empty()) { + NetChannel::Options options; + options.speed = 1000 * MegaByte; + options.buffer = 1000 * MegaByte; + options.rtt = 0; + res = NetChannel::create(options, sleep_.get()); + } + return res.get(); + } + td::actor::ActorId get_inbound_channel(ton::PeerId peer_id) { + auto &res = inbound_channel_[peer_id]; + if (res.empty()) { + NetChannel::Options options; + options.speed = 1000 * MegaByte; + options.buffer = 1000 * MegaByte; + options.rtt = 0; + res = NetChannel::create(options, sleep_.get()); + } + return res.get(); + } + }; + + class PeerCreator : public ton::NodeActor::Callback { + public: + PeerCreator(td::actor::ActorId peer_manager, ton::PeerId self_id, std::vector peers, + std::shared_ptr stop_watcher, std::shared_ptr complete_watcher) + : peer_manager_(std::move(peer_manager)) + , peers_(std::move(peers)) + , self_id_(self_id) + , stop_watcher_(stop_watcher) + , complete_watcher_(complete_watcher) { + } + void get_peers(td::Promise> promise) override { + auto peers = peers_; + promise.set_value(std::move(peers)); + } + void register_self(td::actor::ActorId self) override { + self_ = self; + send_closure(peer_manager_, &PeerManager::register_node, self_id_, self_); + } + td::actor::ActorOwn create_peer(ton::PeerId self_id, ton::PeerId peer_id, + td::SharedState state) override { + class PeerCallback : public ton::PeerActor::Callback { + public: + PeerCallback(ton::PeerId self_id, ton::PeerId peer_id, td::actor::ActorId peer_manager) + : self_id_{self_id}, peer_id_{peer_id}, peer_manager_(peer_manager) { + } + void register_self(td::actor::ActorId self) override { + self_ = std::move(self); + send_closure(peer_manager_, &PeerManager::register_peer, self_id_, peer_id_, self_); + } + void send_query(td::uint64 query_id, td::BufferSlice query) override { + CHECK(!self_.empty()); + class X : public td::actor::Actor { + public: + void start_up() override { + //LOG(ERROR) << "start"; + alarm_timestamp() = td::Timestamp::in(4); + } + void tear_down() override { + //LOG(ERROR) << "finish"; + } + void alarm() override { + //LOG(FATAL) << "WTF?"; + alarm_timestamp() = td::Timestamp::in(4); + } + }; + send_closure( + peer_manager_, &PeerManager::send_query, self_id_, peer_id_, std::move(query), + [self = self_, query_id, + tmp = td::actor::create_actor(PSLICE() << self_id_ << "->" << peer_id_ << " : " << query_id)]( + auto x) { promise_send_closure(self, &ton::PeerActor::on_query_result, query_id)(std::move(x)); }); + } + + private: + ton::PeerId self_id_; + ton::PeerId peer_id_; + td::actor::ActorId self_; + td::actor::ActorId peer_manager_; + }; + + return td::actor::create_actor(PSLICE() << "ton::PeerActor " << self_id << "->" << peer_id, + td::make_unique(self_id, peer_id, peer_manager_), + std::move(state)); + } + + void on_completed() override { + complete_watcher_.reset(); + } + + void on_closed(ton::Torrent torrent) override { + CHECK(torrent.is_completed()); + //TODO: validate torrent + stop_watcher_.reset(); + } + + private: + td::actor::ActorId peer_manager_; + std::vector peers_; + ton::PeerId self_id_; + std::shared_ptr stop_watcher_; + std::shared_ptr complete_watcher_; + td::actor::ActorId self_; + }; + + size_t peers_n = 20; + td::uint64 file_size = 200 * MegaByte; + td::Random::Xorshift128plus rnd(123); + LOG(INFO) << "Start create random_torrent of size " << file_size; + auto torrent = create_random_torrent(rnd, file_size, 128 * KiloByte).torrent.unwrap(); + LOG(INFO) << "Random torrent is created"; + + std::vector peers; + for (size_t i = 1; i <= peers_n; i++) { + peers.push_back(i); + } + auto gen_peers = [&](size_t self_id, size_t n) { + std::vector peers; + if (n > peers_n - 1) { + n = peers_n - 1; + } + while (n != 0) { + size_t id = rnd.fast(1, td::narrow_cast(peers_n)); + if (id == self_id) { + continue; + } + if (std::find(peers.begin(), peers.end(), id) != peers.end()) { + continue; + } + n--; + peers.push_back(id); + } + return peers; + }; + + struct StatsActor : public td::actor::Actor { + public: + StatsActor(td::actor::ActorId node_actor) : node_actor_(node_actor) { + } + + private: + td::actor::ActorId node_actor_; + void start_up() override { + alarm_timestamp() = td::Timestamp::in(1); + } + void alarm() override { + send_closure(node_actor_, &ton::NodeActor::with_torrent, [](td::Result r_torrent) { + if (r_torrent.is_error()) { + return; + } + auto torrent = r_torrent.move_as_ok(); + print_debug(torrent); + }); + alarm_timestamp() = td::Timestamp::in(4); + } + }; + + auto info = torrent.get_info(); + + auto stop_watcher = td::create_shared_destructor([] { td::actor::SchedulerContext::get()->stop(); }); + auto guard = std::make_shared>>(); + auto complete_watcher = td::create_shared_destructor([guard] {}); + + td::actor::Scheduler scheduler({0}, true); + + scheduler.run_in_context([&] { + auto peer_manager = td::actor::create_actor("PeerManager"); + guard->push_back(td::actor::create_actor( + "Node#1", 1, std::move(torrent), + td::make_unique(peer_manager.get(), 1, gen_peers(1, 2), stop_watcher, complete_watcher))); + for (size_t i = 2; i <= peers_n; i++) { + ton::Torrent::Options options; + options.in_memory = true; + auto other_torrent = ton::Torrent::open(options, ton::TorrentMeta(info)).move_as_ok(); + auto node_actor = td::actor::create_actor( + PSLICE() << "Node#" << i, i, std::move(other_torrent), + td::make_unique(peer_manager.get(), i, gen_peers(i, 2), stop_watcher, complete_watcher)); + + if (i == 3) { + td::actor::create_actor("StatsActor", node_actor.get()).release(); + } + guard->push_back(std::move(node_actor)); + } + guard->push_back(std::move(peer_manager)); + }); + stop_watcher.reset(); + guard.reset(); + complete_watcher.reset(); + scheduler.run(); +} diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index eacc6192d..d38589747 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -106,7 +106,8 @@ class Scheduler { }; enum Mode { Running, Paused }; - Scheduler(std::vector infos, Mode mode = Paused) : infos_(std::move(infos)) { + Scheduler(std::vector infos, bool skip_timeouts = false, Mode mode = Paused) + : infos_(std::move(infos)), skip_timeouts_(skip_timeouts) { init(); if (mode == Running) { start(); @@ -184,6 +185,7 @@ class Scheduler { std::shared_ptr group_info_; std::vector> schedulers_; bool is_started_{false}; + bool skip_timeouts_{false}; void init() { CHECK(infos_.size() < 256); @@ -191,7 +193,8 @@ class Scheduler { group_info_ = std::make_shared(infos_.size()); td::uint8 id = 0; for (const auto &info : infos_) { - schedulers_.emplace_back(td::make_unique(group_info_, core::SchedulerId{id}, info.cpu_threads_)); + schedulers_.emplace_back( + td::make_unique(group_info_, core::SchedulerId{id}, info.cpu_threads_, skip_timeouts_)); id++; } } @@ -250,7 +253,7 @@ T ¤t_actor() { inline void send_message(core::ActorInfo &actor_info, core::ActorMessage message) { auto scheduler_context_ptr = core::SchedulerContext::get(); if (scheduler_context_ptr == nullptr) { - LOG(ERROR) << "send to actor is silently ignored"; + //LOG(ERROR) << "send to actor is silently ignored"; return; } auto &scheduler_context = *scheduler_context_ptr; @@ -263,10 +266,11 @@ inline void send_message(ActorRef actor_ref, core::ActorMessage message) { message.set_link_token(actor_ref.link_token); send_message(actor_ref.actor_info, std::move(message)); } + inline void send_message_later(core::ActorInfo &actor_info, core::ActorMessage message) { auto scheduler_context_ptr = core::SchedulerContext::get(); if (scheduler_context_ptr == nullptr) { - LOG(ERROR) << "send to actor is silently ignored"; + //LOG(ERROR) << "send to actor is silently ignored"; return; } auto &scheduler_context = *scheduler_context_ptr; @@ -285,7 +289,7 @@ template void send_immediate(ActorRef actor_ref, ExecuteF &&execute, ToMessageF &&to_message) { auto scheduler_context_ptr = core::SchedulerContext::get(); if (scheduler_context_ptr == nullptr) { - LOG(ERROR) << "send to actor is silently ignored"; + //LOG(ERROR) << "send to actor is silently ignored"; return; } auto &scheduler_context = *scheduler_context_ptr; @@ -368,28 +372,26 @@ void send_closure_later(ActorRef actor_ref, ArgsT &&... args) { inline void send_signals(ActorRef actor_ref, ActorSignals signals) { auto scheduler_context_ptr = core::SchedulerContext::get(); if (scheduler_context_ptr == nullptr) { - LOG(ERROR) << "send to actor is silently ignored"; + //LOG(ERROR) << "send to actor is silently ignored"; return; } auto &scheduler_context = *scheduler_context_ptr; - core::ActorExecutor executor(actor_ref.actor_info, scheduler_context, - core::ActorExecutor::Options().with_has_poll(scheduler_context.has_poll())); - if (executor.can_send_immediate()) { - return executor.send_immediate(signals.raw()); - } - executor.send(signals.raw()); + core::ActorExecutor executor( + actor_ref.actor_info, scheduler_context, + core::ActorExecutor::Options().with_has_poll(scheduler_context.has_poll()).with_signals(signals.raw())); } inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { auto scheduler_context_ptr = core::SchedulerContext::get(); if (scheduler_context_ptr == nullptr) { - LOG(ERROR) << "send to actor is silently ignored"; + //LOG(ERROR) << "send to actor is silently ignored"; return; } auto &scheduler_context = *scheduler_context_ptr; core::ActorExecutor executor(actor_ref.actor_info, scheduler_context, - core::ActorExecutor::Options().with_has_poll(scheduler_context.has_poll())); - executor.send((signals | ActorSignals::pause()).raw()); + core::ActorExecutor::Options() + .with_has_poll(scheduler_context.has_poll()) + .with_signals((signals | ActorSignals::pause()).raw())); } inline void register_actor_info_ptr(core::ActorInfoPtr actor_info_ptr) { diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 8c5fde622..267758d5e 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -81,7 +81,7 @@ void ActorExecutor::start() noexcept { return; } - ActorSignals signals; + ActorSignals signals{options_.signals}; SCOPE_EXIT { pending_signals_.add_signals(signals); }; @@ -209,6 +209,7 @@ bool ActorExecutor::flush_one_signal(ActorSignals &signals) { case ActorSignals::Alarm: if (actor_execute_context_.get_alarm_timestamp() && actor_execute_context_.get_alarm_timestamp().is_in_past()) { actor_execute_context_.alarm_timestamp() = Timestamp::never(); + actor_info_.set_alarm_timestamp(Timestamp::never()); actor_info_.actor().alarm(); } break; diff --git a/tdactor/td/actor/core/ActorExecutor.h b/tdactor/td/actor/core/ActorExecutor.h index 9a2c5d29e..366cb754d 100644 --- a/tdactor/td/actor/core/ActorExecutor.h +++ b/tdactor/td/actor/core/ActorExecutor.h @@ -42,8 +42,14 @@ class ActorExecutor { this->has_poll = new_has_poll; return *this; } + Options &with_signals(ActorSignals signals) { + this->signals = signals; + return *this; + } + bool from_queue{false}; bool has_poll{false}; + ActorSignals signals; }; ActorExecutor(ActorInfo &actor_info, SchedulerDispatcher &dispatcher, Options options) diff --git a/tdactor/td/actor/core/IoWorker.cpp b/tdactor/td/actor/core/IoWorker.cpp index ee3576bdf..f4d2dfcdd 100644 --- a/tdactor/td/actor/core/IoWorker.cpp +++ b/tdactor/td/actor/core/IoWorker.cpp @@ -37,7 +37,7 @@ void IoWorker::tear_down() { #endif } -bool IoWorker::run_once(double timeout) { +bool IoWorker::run_once(double timeout, bool skip_timeouts) { auto &dispatcher = *SchedulerContext::get(); #if TD_PORT_POSIX auto &poll = SchedulerContext::get()->get_poll(); @@ -87,6 +87,14 @@ bool IoWorker::run_once(double timeout) { if (timeout_ms < 0) { timeout_ms = 0; } + + if (timeout_ms > 0 && skip_timeouts) { + timeout_ms = 0; + //auto *heap_node = heap.top(); + //auto *actor_info = ActorInfo::from_heap_node(heap_node); + //LOG(ERROR) << "Jump: " << wakeup_timestamp.at() << " " << actor_info->get_name(); + Time::jump_in_future(wakeup_timestamp.at() + 1e-9); + } //const int thirty_seconds = 30 * 1000; //if (timeout_ms > thirty_seconds) { //timeout_ms = thirty_seconds; diff --git a/tdactor/td/actor/core/IoWorker.h b/tdactor/td/actor/core/IoWorker.h index 529a01f81..6d630cc57 100644 --- a/tdactor/td/actor/core/IoWorker.h +++ b/tdactor/td/actor/core/IoWorker.h @@ -33,7 +33,7 @@ class IoWorker { void start_up(); void tear_down(); - bool run_once(double timeout); + bool run_once(double timeout, bool skip_timeouts = false); private: MpscPollableQueue &queue_; diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index f79620d9f..40f1e72cd 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -24,7 +24,6 @@ namespace td { namespace actor { namespace core { - std::atomic debug; void set_debug(bool flag) { debug = flag; @@ -34,8 +33,11 @@ bool need_debug() { return debug.load(std::memory_order_relaxed); } -Scheduler::Scheduler(std::shared_ptr scheduler_group_info, SchedulerId id, size_t cpu_threads_count) - : scheduler_group_info_(std::move(scheduler_group_info)), cpu_threads_(cpu_threads_count) { +Scheduler::Scheduler(std::shared_ptr scheduler_group_info, SchedulerId id, size_t cpu_threads_count, + bool skip_timeouts) + : scheduler_group_info_(std::move(scheduler_group_info)) + , cpu_threads_(cpu_threads_count) + , skip_timeouts_(skip_timeouts) { scheduler_group_info_->active_scheduler_count++; info_ = &scheduler_group_info_->schedulers.at(id.value()); info_->id = id; @@ -101,7 +103,7 @@ bool Scheduler::run(double timeout) { if (SchedulerContext::get()->is_stop_requested()) { res = false; } else { - res = io_worker_->run_once(timeout); + res = io_worker_->run_once(timeout, skip_timeouts_); } if (!res) { if (!is_stopped_) { @@ -331,6 +333,7 @@ void Scheduler::close_scheduler_group(SchedulerGroupInfo &group_info) { worker->actor_info_creator.clear(); } } + //for (auto &scheduler : group_info.schedulers) { //scheduler.io_worker->actor_info_creator.ensure_empty(); //for (auto &worker : scheduler.cpu_workers) { diff --git a/tdactor/td/actor/core/Scheduler.h b/tdactor/td/actor/core/Scheduler.h index 377d835f1..3de519e09 100644 --- a/tdactor/td/actor/core/Scheduler.h +++ b/tdactor/td/actor/core/Scheduler.h @@ -200,7 +200,8 @@ class Scheduler { return thread_id; } - Scheduler(std::shared_ptr scheduler_group_info, SchedulerId id, size_t cpu_threads_count); + Scheduler(std::shared_ptr scheduler_group_info, SchedulerId id, size_t cpu_threads_count, + bool skip_timeouts = false); Scheduler(const Scheduler &) = delete; Scheduler &operator=(const Scheduler &) = delete; @@ -241,6 +242,7 @@ class Scheduler { Poll poll_; KHeap heap_; std::unique_ptr io_worker_; + bool skip_timeouts_{false}; class ContextImpl : public SchedulerContext { public: diff --git a/tddb/CMakeLists.txt b/tddb/CMakeLists.txt index 960703b6f..1acd54202 100644 --- a/tddb/CMakeLists.txt +++ b/tddb/CMakeLists.txt @@ -1,13 +1,8 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) #SOURCE SETS -set(TDDB_SOURCE - td/db/MemoryKeyValue.cpp - - td/db/KeyValue.h - td/db/KeyValueAsync.h - td/db/MemoryKeyValue.h - +set(TDDB_UTILS_SOURCE + td/db/utils/BlobView.cpp td/db/utils/ChainBuffer.cpp td/db/utils/CyclicBuffer.cpp td/db/utils/FileSyncState.cpp @@ -15,12 +10,21 @@ set(TDDB_SOURCE td/db/utils/StreamToFileActor.cpp td/db/utils/FileToStreamActor.cpp + td/db/utils/BlobView.h td/db/utils/ChainBuffer.h td/db/utils/CyclicBuffer.h td/db/utils/FileSyncState.h td/db/utils/StreamInterface.h td/db/utils/StreamToFileActor.h td/db/utils/FileToStreamActor.h +) + +set(TDDB_SOURCE + td/db/MemoryKeyValue.cpp + + td/db/KeyValue.h + td/db/KeyValueAsync.h + td/db/MemoryKeyValue.h td/db/binlog/Binlog.cpp td/db/binlog/BinlogReaderHelper.cpp @@ -44,10 +48,13 @@ set(TDDB_TEST_SOURCE #RULES #LIBRARIES +add_library(tddb_utils STATIC ${TDDB_UTILS_SOURCE}) +target_include_directories(tddb_utils PUBLIC $) +target_link_libraries(tddb_utils PUBLIC tdutils tdactor) add_library(tddb STATIC ${TDDB_SOURCE}) target_include_directories(tddb PUBLIC $) -target_link_libraries(tddb PUBLIC tdutils tdactor) +target_link_libraries(tddb PUBLIC tdutils tdactor tddb_utils) if (TDDB_USE_ROCKSDB) target_sources(tddb PRIVATE ${TDDB_ROCKSDB_SOURCE}) @@ -63,7 +70,7 @@ target_link_libraries(io-bench tdutils tdactor tddb) #add_subdirectory(benchmark) # END-INTERNAL -install(TARGETS tddb EXPORT TdTargets +install(TARGETS tddb tddb_utils EXPORT TdTargets LIBRARY DESTINATION lib ARCHIVE DESTINATION lib RUNTIME DESTINATION bin diff --git a/tddb/td/db/utils/BlobView.cpp b/tddb/td/db/utils/BlobView.cpp new file mode 100644 index 000000000..213b73610 --- /dev/null +++ b/tddb/td/db/utils/BlobView.cpp @@ -0,0 +1,339 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include "td/db/utils/BlobView.h" + +#include "td/utils/port/FileFd.h" +#include "td/utils/HashMap.h" + +#include "td/utils/format.h" +#include "td/utils/port/RwMutex.h" +#include "td/utils/port/MemoryMapping.h" + +#include +#include + +namespace td { + +class BlobViewImpl { + public: + virtual ~BlobViewImpl() = default; + td::Result view(td::MutableSlice slice, td::uint64 offset); + td::Result view_copy(td::MutableSlice slice, td::uint64 offset); + td::Result to_buffer_slice(); + td::Result write(td::Slice data, td::uint64 offset); + virtual td::Status sync() { + return td::Status::OK(); + } + virtual td::uint64 size() = 0; + + private: + virtual td::Result view_impl(td::MutableSlice slice, td::uint64 offset) = 0; + virtual td::Result write_impl(td::Slice data, td::uint64 offset) { + return td::Status::Error("Read only blob"); + } +}; + +BlobView::BlobView() = default; + +BlobView::BlobView(std::unique_ptr impl) : impl_(std::move(impl)) { +} + +BlobView::BlobView(BlobView &&) = default; + +BlobView &BlobView::operator=(BlobView &&) = default; + +BlobView::~BlobView() { +} + +td::Result BlobView::view(td::MutableSlice slice, td::uint64 offset) { + CHECK(impl_); + return impl_->view(slice, offset); +} +td::Result BlobView::write(td::Slice data, td::uint64 offset) { + CHECK(impl_); + return impl_->write(data, offset); +} + +td::Result BlobView::to_buffer_slice() { + td::BufferSlice res(size()); + TRY_RESULT(read_size, view_copy(res.as_slice(), 0)); + if (read_size != res.size()) { + return td::Status::Error("Can't view the whole blob"); + } + return std::move(res); +} + +td::Result BlobView::view_copy(td::MutableSlice slice, td::uint64 offset) { + TRY_RESULT(res, view(slice, offset)); + if (res.begin() != slice.begin()) { + slice.copy_from(res); + } + return res.size(); +} + +td::uint64 BlobView::size() { + CHECK(impl_); + return impl_->size(); +} + +td::Result BlobViewImpl::view(td::MutableSlice slice, td::uint64 offset) { + if (offset > size() || slice.size() > size() - offset) { + return td::Status::Error(PSLICE() << "BlobView: invalid range requested " << td::tag("slice offset", offset) + << td::tag("slice size", slice.size()) << td::tag("blob size", size())); + } + return view_impl(slice, offset); +} + +td::Result BlobViewImpl::write(td::Slice slice, td::uint64 offset) { + if (offset > size() || slice.size() > size() - offset) { + return td::Status::Error(PSLICE() << "BlobView: invalid range requested " << td::tag("slice offset", offset) + << td::tag("slice size", slice.size()) << td::tag("blob size", size())); + } + return write_impl(slice, offset); +} + +namespace { +class BufferSliceBlobViewImpl : public BlobViewImpl { + public: + BufferSliceBlobViewImpl(td::BufferSlice slice) : slice_(std::move(slice)) { + } + td::Result view_impl(td::MutableSlice slice, td::uint64 offset) override { + // optimize anyway + if (offset > std::numeric_limits::max()) { + return td::Slice(); + } + return slice_.as_slice().substr(static_cast(offset), slice.size()); + } + + td::Result write_impl(td::Slice data, td::uint64 offset) override { + slice_.as_slice().substr(offset).copy_from(data); + return data.size(); + } + + td::uint64 size() override { + return slice_.size(); + } + + private: + td::BufferSlice slice_; +}; +} // namespace + +BlobView BufferSliceBlobView::create(td::BufferSlice slice) { + return BlobView(std::make_unique(std::move(slice))); +} + +class FileBlobViewImpl : public BlobViewImpl { + public: + FileBlobViewImpl(td::FileFd fd, td::uint64 file_size) : fd_(std::move(fd)), file_size_(file_size) { + } + + td::uint64 size() override { + return file_size_; + } + td::Result view_impl(td::MutableSlice slice, td::uint64 offset) override { + CHECK(offset < size()); + CHECK(size() - offset >= slice.size()); + slice.truncate(file_size_ - offset); + auto first_page = offset / page_size; + auto last_page = (offset + slice.size() - 1) / page_size; + td::uint64 res_offset = 0; + for (auto page_i = first_page; page_i <= last_page; page_i++) { + auto page_offset = page_i * page_size; + auto from = td::max(page_offset, offset); + auto till = td::min(page_offset + page_size, offset + slice.size()); + CHECK(from < till); + TRY_RESULT(page, load_page(page_i)); + auto len = till - from; + slice.substr(res_offset, len).copy_from(page.substr(from - page_offset, len)); + res_offset += len; + } + CHECK(slice.size() == res_offset); + total_view_size_ += slice.size(); + return slice; + } + ~FileBlobViewImpl() { + //LOG(ERROR) << "LOADED " << pages_.size() << " " << total_view_size_; + } + + private: + td::FileFd fd_; + td::uint64 file_size_; + const td::uint64 page_size = 4096; + td::uint64 total_view_size_{0}; + + td::RwMutex pages_rw_mutex_; + td::HashMap pages_; + + std::mutex fd_mutex_; + + td::Result load_page(td::uint64 page_i) { + { + auto pages_guard = pages_rw_mutex_.lock_read(); + auto it = pages_.find(page_i); + if (it != pages_.end()) { + return it->second.as_slice(); + } + } + + std::lock_guard fd_guard(fd_mutex_); + { + auto pages_guard = pages_rw_mutex_.lock_read(); + auto it = pages_.find(page_i); + if (it != pages_.end()) { + return it->second.as_slice(); + } + } + auto offset = page_i * page_size; + + auto size = td::min(file_size_ - offset, page_size); + auto buffer_slice = td::BufferSlice(size); + TRY_RESULT(read_size, fd_.pread(buffer_slice.as_slice(), offset)); + if (read_size != buffer_slice.size()) { + return td::Status::Error("not enough data in file"); + } + + auto pages_guard = pages_rw_mutex_.lock_write(); + auto &res = pages_[page_i]; + res = std::move(buffer_slice); + return res.as_slice(); + } +}; + +td::Result FileBlobView::create(td::CSlice file_path, td::uint64 file_size) { + TRY_RESULT(fd, td::FileFd::open(file_path, td::FileFd::Flags::Read)); + TRY_RESULT(stat, fd.stat()); + if (file_size == 0) { + file_size = stat.size_; + } else if (file_size != (td::uint64)stat.size_) { + return td::Status::Error(PSLICE() << "Wrong file size (1) expected:" << file_size << " got:" << stat.size_); + } + return BlobView(std::make_unique(std::move(fd), file_size)); +} + +class FileNoCacheBlobViewImpl : public BlobViewImpl { + public: + FileNoCacheBlobViewImpl(td::FileFd fd, td::uint64 file_size) : fd_(std::move(fd)), file_size_(file_size) { + } + + td::uint64 size() override { + return file_size_; + } + td::Result view_impl(td::MutableSlice slice, td::uint64 offset) override { + CHECK(offset < size()); + CHECK(size() - offset >= slice.size()); + slice.truncate(file_size_ - offset); + TRY_RESULT(read_size, fd_.pread(slice, offset)); + slice.truncate(read_size); + return slice; + } + + td::Result write_impl(td::Slice data, td::uint64 offset) override { + return fd_.pwrite(data, offset); + } + + ~FileNoCacheBlobViewImpl() { + } + + private: + td::FileFd fd_; + td::uint64 file_size_; +}; + +td::Result FileNoCacheBlobView::create(td::CSlice file_path, td::uint64 file_size, bool may_write) { + td::int32 flags = td::FileFd::Flags::Read; + if (may_write) { + flags |= td::FileFd::Flags::Create | td::FileFd::Flags::Write; + } + TRY_RESULT(fd, td::FileFd::open(file_path, flags)); + TRY_RESULT(stat, fd.stat()); + if (file_size == 0) { + file_size = stat.size_; + } else if (file_size != (td::uint64)stat.size_) { + if (stat.size_ == 0) { + TRY_STATUS(fd.seek(file_size)); + TRY_STATUS(fd.truncate_to_current_position(file_size)); + TRY_STATUS(fd.seek(0)); + } else { + LOG(ERROR) << file_path; + return td::Status::Error(PSLICE() << "Wrong file size (2) expected:" << file_size << " got:" << stat.size_); + } + } + return BlobView(std::make_unique(std::move(fd), file_size)); +} + +class FileMemoryMappingBlobViewImpl : public BlobViewImpl { + public: + FileMemoryMappingBlobViewImpl(td::MemoryMapping mapping) : mapping_(std::move(mapping)) { + } + td::Result view_impl(td::MutableSlice slice, td::uint64 offset) override { + // optimize anyway + return mapping_.as_slice().substr(offset, slice.size()); + } + td::uint64 size() override { + return mapping_.as_slice().size(); + } + + private: + td::MemoryMapping mapping_; +}; + +td::Result FileMemoryMappingBlobView::create(td::CSlice file_path, td::uint64 file_size) { + TRY_RESULT(fd, td::FileFd::open(file_path, td::FileFd::Flags::Read)); + TRY_RESULT(stat, fd.stat()); + if (file_size == 0) { + file_size = stat.size_; + } else if (file_size != (td::uint64)stat.size_) { + return td::Status::Error(PSLICE() << "Wrong file size (3) expected:" << file_size << " got:" << stat.size_); + } + + TRY_RESULT(mapping, td::MemoryMapping::create_from_file(fd)); + + return BlobView(std::make_unique(std::move(mapping))); +} + +class CyclicBlobViewImpl : public BlobViewImpl { + public: + CyclicBlobViewImpl(td::BufferSlice data, td::uint64 total_size) : data_(std::move(data)), total_size_(total_size) { + } + td::Result view_impl(td::MutableSlice slice, td::uint64 offset) override { + auto res = slice; + offset %= data_.size(); + while (!slice.empty()) { + auto from = data_.as_slice().substr(offset).truncate(slice.size()); + slice.copy_from(from); + slice.remove_prefix(from.size()); + offset = 0; + } + return res; + } + td::uint64 size() override { + return total_size_; + } + + private: + td::BufferSlice data_; + td::uint64 total_size_; +}; + +td::Result CycicBlobView::create(td::BufferSlice data, td::uint64 total_size) { + return BlobView(std::make_unique(std::move(data), total_size)); +} + +} // namespace td diff --git a/tddb/td/db/utils/BlobView.h b/tddb/td/db/utils/BlobView.h new file mode 100644 index 000000000..5ae69b69a --- /dev/null +++ b/tddb/td/db/utils/BlobView.h @@ -0,0 +1,67 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2019 Telegram Systems LLP +*/ +#pragma once +#include "td/utils/buffer.h" + +namespace td { +class BlobViewImpl; + +class BlobView { + public: + BlobView(); + explicit BlobView(std::unique_ptr impl); + BlobView(BlobView &&); + BlobView &operator=(BlobView &&); + ~BlobView(); + td::Result to_buffer_slice(); + td::Result view(td::MutableSlice slice, td::uint64 offset); + td::Result view_copy(td::MutableSlice slice, td::uint64 offset); + td::Result write(td::Slice data, td::uint64 offset); + td::uint64 size(); + + explicit operator bool() const { + return bool(impl_); + } + + private: + std::unique_ptr impl_; +}; + +class BufferSliceBlobView { + public: + static BlobView create(td::BufferSlice slice); +}; +class FileBlobView { + public: + static td::Result create(td::CSlice file_path, td::uint64 file_size = 0); +}; +class FileNoCacheBlobView { + public: + static td::Result create(td::CSlice file_path, td::uint64 file_size = 0, bool may_write = false); +}; +class FileMemoryMappingBlobView { + public: + static td::Result create(td::CSlice file_path, td::uint64 file_size = 0); +}; + +// For testing purposes +struct CycicBlobView { + static td::Result create(td::BufferSlice data, td::uint64 total_size); +}; +} // namespace td diff --git a/tddb/td/db/utils/CyclicBuffer.cpp b/tddb/td/db/utils/CyclicBuffer.cpp index ea3b2c6c3..d8f7e079b 100644 --- a/tddb/td/db/utils/CyclicBuffer.cpp +++ b/tddb/td/db/utils/CyclicBuffer.cpp @@ -61,7 +61,7 @@ class CyclicBuffer : public StreamWriterInterface, public StreamReaderInterface } Span prepare_readv() override { reader_.io_slice_ = as_io_slice(prepare_read()); - return Span(&reader_.io_slice_, 1); + return span_one(reader_.io_slice_); } void confirm_read(size_t size) override { reader_.pos_.store(reader_.pos_.load(std::memory_order_relaxed) + size); diff --git a/tdnet/td/net/UdpServer.cpp b/tdnet/td/net/UdpServer.cpp index a1c53aa63..ba28c5cfc 100644 --- a/tdnet/td/net/UdpServer.cpp +++ b/tdnet/td/net/UdpServer.cpp @@ -53,6 +53,7 @@ class UdpServerImpl : public UdpServer { }; void UdpServerImpl::send(td::UdpMessage &&message) { + //LOG(WARNING) << "TO: " << message.address; fd_.send(std::move(message)); loop(); // TODO: some yield logic } @@ -107,6 +108,7 @@ void UdpServerImpl::loop() { if (!o_message) { return Status::OK(); } + //LOG(WARNING) << "FROM" << o_message.value().address; callback_->on_udp_message(std::move(*o_message)); } return Status::OK(); diff --git a/tdutils/td/utils/Heap.h b/tdutils/td/utils/Heap.h index 37be5e07f..fb4925199 100644 --- a/tdutils/td/utils/Heap.h +++ b/tdutils/td/utils/Heap.h @@ -49,6 +49,10 @@ class KHeap { return array_[0].key_; } + HeapNode *top() const { + return array_[0].node_; + } + HeapNode *pop() { CHECK(!empty()); HeapNode *result = array_[0].node_; diff --git a/tdutils/td/utils/PathView.h b/tdutils/td/utils/PathView.h index f131dc79d..e7a31da59 100644 --- a/tdutils/td/utils/PathView.h +++ b/tdutils/td/utils/PathView.h @@ -54,6 +54,9 @@ class PathView { Slice parent_dir() const { return path_.substr(0, last_slash_ + 1); } + Slice parent_dir_noslash() const { + return last_slash_ <= 0 ? td::Slice(".") : path_.substr(0, last_slash_); + } Slice extension() const { if (last_dot_ == static_cast(path_.size())) { diff --git a/tdutils/td/utils/Random.cpp b/tdutils/td/utils/Random.cpp index 827628a01..a7fd0b9d4 100644 --- a/tdutils/td/utils/Random.cpp +++ b/tdutils/td/utils/Random.cpp @@ -192,5 +192,8 @@ uint64 Random::Xorshift128plus::operator()() { int Random::Xorshift128plus::fast(int min, int max) { return static_cast((*this)() % (max - min + 1) + min); } +int64 Random::Xorshift128plus::fast64(int64 min, int64 max) { + return static_cast((*this)() % (max - min + 1) + min); +} } // namespace td diff --git a/tdutils/td/utils/Random.h b/tdutils/td/utils/Random.h index 96761c70f..aa7f70289 100644 --- a/tdutils/td/utils/Random.h +++ b/tdutils/td/utils/Random.h @@ -20,6 +20,7 @@ #include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/Span.h" namespace td { @@ -45,16 +46,31 @@ class Random { static int fast(int min, int max); static double fast(double min, double max); + class Fast { + public: + uint64 operator()() { + return fast_uint64(); + } + }; class Xorshift128plus { public: explicit Xorshift128plus(uint64 seed); Xorshift128plus(uint64 seed_a, uint64 seed_b); uint64 operator()(); int fast(int min, int max); + int64 fast64(int64 min, int64 max); private: uint64 seed_[2]; }; }; +template +void random_shuffle(td::MutableSpan v, R &rnd) { + for (std::size_t i = 1; i < v.size(); i++) { + auto pos = static_cast(rnd() % (i + 1)); + std::swap(v[i], v[pos]); + } +} + } // namespace td diff --git a/tdutils/td/utils/Span.h b/tdutils/td/utils/Span.h index a4a8edffa..3fd49033a 100644 --- a/tdutils/td/utils/Span.h +++ b/tdutils/td/utils/Span.h @@ -86,6 +86,26 @@ class SpanImpl { return data_[i]; } + InnerT &back() { + DCHECK(!empty()); + return data_[size() - 1]; + } + + const InnerT &back() const { + DCHECK(!empty()); + return data_[size() - 1]; + } + + InnerT &front() { + DCHECK(!empty()); + return data_[0]; + } + + const InnerT &front() const { + DCHECK(!empty()); + return data_[0]; + } + InnerT *data() const { return data_; } @@ -109,8 +129,9 @@ class SpanImpl { } SpanImpl &truncate(size_t size) { - CHECK(size <= size_); - size_ = size; + if (size < size_) { + size_ = size; + } return *this; } @@ -136,9 +157,45 @@ template Span span(const T *ptr, size_t size) { return Span(ptr, size); } +template +Span span(const std::vector &vec) { + return Span(vec); +} + template MutableSpan mutable_span(T *ptr, size_t size) { return MutableSpan(ptr, size); } +template +MutableSpan mutable_span(std::vector &vec) { + return MutableSpan(vec); +} + +template +Span span_one(const T &value) { + return td::Span(&value, 1); +} +template +MutableSpan mutable_span_one(T &value) { + return td::MutableSpan(&value, 1); +} + +template +Span as_span(Span span) { + return span; +} +template +Span as_span(const std::vector &vec) { + return Span(vec); +} + +template +MutableSpan as_mutable_span(MutableSpan span) { + return span; +} +template +MutableSpan as_mutable_span(std::vector &vec) { + return MutableSpan(vec); +} } // namespace td diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index a9879996d..f23c4ce1e 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -582,6 +582,22 @@ class Result { *this = Result(); } + template + td::Result()(std::declval()))> move_map(F &&f) { + if (is_error()) { + return move_as_error(); + } + return f(move_as_ok()); + } + + template + decltype(std::declval()(std::declval())) move_fmap(F &&f) { + if (is_error()) { + return move_as_error(); + } + return f(move_as_ok()); + } + private: Status status_; union { diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index 4df488683..57b183349 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -38,6 +38,13 @@ class ThreadSafeMultiCounter { tls_.for_each([&](auto &value) { res += value[index].load(); }); return res; } + void clear() { + tls_.for_each([&](auto &value) { + for (auto &x : value) { + x = 0; + } + }); + } private: ThreadLocalStorage, N>> tls_; @@ -108,6 +115,11 @@ class NamedThreadSafeCounter { } } + void clear() { + std::unique_lock guard(mutex_); + counter_.clear(); + } + friend StringBuilder &operator<<(StringBuilder &sb, const NamedThreadSafeCounter &counter) { counter.for_each([&sb](Slice name, int64 cnt) { sb << name << ": " << cnt << "\n"; }); return sb; diff --git a/tdutils/td/utils/Time.cpp b/tdutils/td/utils/Time.cpp index ca4a5e46c..c792dd24d 100644 --- a/tdutils/td/utils/Time.cpp +++ b/tdutils/td/utils/Time.cpp @@ -19,14 +19,36 @@ #include "td/utils/Time.h" #include +#include namespace td { bool operator==(Timestamp a, Timestamp b) { return std::abs(a.at() - b.at()) < 1e-6; } +namespace { +std::atomic time_diff; +} double Time::now() { + return now_unadjusted() + time_diff.load(std::memory_order_relaxed); +} + +double Time::now_unadjusted() { return Clocks::monotonic(); } +void Time::jump_in_future(double at) { + auto old_time_diff = time_diff.load(); + + while (true) { + auto diff = at - now(); + if (diff < 0) { + return; + } + if (time_diff.compare_exchange_strong(old_time_diff, old_time_diff + diff)) { + return; + } + } +} + } // namespace td diff --git a/tdutils/td/utils/Time.h b/tdutils/td/utils/Time.h index 42fef730a..3dc5a53e7 100644 --- a/tdutils/td/utils/Time.h +++ b/tdutils/td/utils/Time.h @@ -40,6 +40,10 @@ class Time { // As an alternative we may say that now_cached is a thread local copy of now return now(); } + static double now_unadjusted(); + + // Used for testing. After jump_in_future(at) is called, now() >= at. + static void jump_in_future(double at); }; inline void relax_timeout_at(double *timeout, double new_timeout) { @@ -70,12 +74,15 @@ class Timestamp { return Timestamp{timeout - td::Clocks::system() + Time::now()}; } - static Timestamp in(double timeout) { - return Timestamp{Time::now_cached() + timeout}; + static Timestamp in(double timeout, td::Timestamp now = td::Timestamp::now_cached()) { + return Timestamp{now.at() + timeout}; } + bool is_in_past(td::Timestamp now) const { + return at_ <= now.at(); + } bool is_in_past() const { - return at_ <= Time::now_cached(); + return is_in_past(now_cached()); } explicit operator bool() const { @@ -111,6 +118,10 @@ class Timestamp { } }; +inline bool operator<(const Timestamp &a, const Timestamp &b) { + return a.at() < b.at(); +} + template void store(const Timestamp ×tamp, StorerT &storer) { storer.store_binary(timestamp.at() - Time::now() + Clocks::system()); diff --git a/tdutils/td/utils/TimedStat.h b/tdutils/td/utils/TimedStat.h index 599ab68f7..6e9498827 100644 --- a/tdutils/td/utils/TimedStat.h +++ b/tdutils/td/utils/TimedStat.h @@ -19,6 +19,7 @@ #pragma once #include "td/utils/common.h" +#include "td/utils/optional.h" #include @@ -80,4 +81,27 @@ class TimedStat { } }; +template +struct MinMaxStat { + public: + using Event = T; + void on_event(Event event) { + if (!best_ || Cmp()(event, best_.value())) { + best_ = event; + } + } + td::optional get_stat() const { + return best_.copy(); + } + + private: + td::optional best_; +}; + +template +using MinStat = MinMaxStat>; + +template +using MaxStat = MinMaxStat>; + } // namespace td diff --git a/tdutils/td/utils/VectorQueue.h b/tdutils/td/utils/VectorQueue.h index 549d23043..79bd9b058 100644 --- a/tdutils/td/utils/VectorQueue.h +++ b/tdutils/td/utils/VectorQueue.h @@ -50,6 +50,12 @@ class VectorQueue { T &back() { return vector_.back(); } + const T &front() const { + return vector_[read_pos_]; + } + const T &back() const { + return vector_.back(); + } bool empty() const { return size() == 0; } diff --git a/tdutils/td/utils/bits.h b/tdutils/td/utils/bits.h index 888168100..54e73535d 100644 --- a/tdutils/td/utils/bits.h +++ b/tdutils/td/utils/bits.h @@ -270,4 +270,40 @@ inline int32 count_bits64(uint64 x) { #endif +struct BitsRange { + td::uint64 bits{0}; + mutable td::int32 pos{-1}; + + explicit BitsRange(td::uint64 bits = 0) : bits{bits}, pos{-1} { + } + + BitsRange begin() const { + return *this; + } + + BitsRange end() const { + return BitsRange{}; + } + + td::int32 operator*() const { + if (pos == -1) { + pos = td::count_trailing_zeroes64(bits); + } + return pos; + } + + bool operator!=(const BitsRange &other) const { + return bits != other.bits; + } + + BitsRange &operator++() { + auto i = **this; + if (i != 64) { + bits ^= 1ull << i; + } + pos = -1; + return *this; + } +}; + } // namespace td diff --git a/tdutils/td/utils/format.h b/tdutils/td/utils/format.h index dda678698..e8319411a 100644 --- a/tdutils/td/utils/format.h +++ b/tdutils/td/utils/format.h @@ -25,6 +25,7 @@ #include #include +#include namespace td { namespace format { @@ -333,5 +334,9 @@ template StringBuilder &operator<<(StringBuilder &stream, const vector &vec) { return stream << format::as_array(vec); } +template +StringBuilder &operator<<(StringBuilder &stream, const std::set &vec) { + return stream << format::as_array(vec); +} } // namespace td diff --git a/tdutils/td/utils/optional.h b/tdutils/td/utils/optional.h index 3eb2d5182..44575948c 100644 --- a/tdutils/td/utils/optional.h +++ b/tdutils/td/utils/optional.h @@ -59,9 +59,11 @@ class optional { return impl_.is_ok(); } T &value() { + DCHECK(*this); return impl_.ok_ref(); } const T &value() const { + DCHECK(*this); return impl_.ok_ref(); } T &operator*() { diff --git a/tdutils/td/utils/tests.cpp b/tdutils/td/utils/tests.cpp index 6263c60b9..01e14e7a9 100644 --- a/tdutils/td/utils/tests.cpp +++ b/tdutils/td/utils/tests.cpp @@ -204,6 +204,7 @@ bool TestsRunner::run_all_step() { } LOG(ERROR) << "Run test " << tag("name", name); state_.start = Time::now(); + state_.start_unadjusted = Time::now_unadjusted(); state_.is_running = true; } @@ -211,7 +212,13 @@ bool TestsRunner::run_all_step() { break; } - LOG(ERROR) << format::as_time(Time::now() - state_.start); + auto passed = Time::now() - state_.start; + auto real_passed = Time::now_unadjusted() - state_.start_unadjusted; + if (real_passed + 1e-9 > passed) { + LOG(ERROR) << format::as_time(passed); + } else { + LOG(ERROR) << format::as_time(passed) << " real[" << format::as_time(real_passed) << "]"; + } if (regression_tester_) { regression_tester_->save_db(); } diff --git a/tdutils/td/utils/tests.h b/tdutils/td/utils/tests.h index d7a6e3431..dbdd3ce6e 100644 --- a/tdutils/td/utils/tests.h +++ b/tdutils/td/utils/tests.h @@ -25,6 +25,7 @@ #include "td/utils/port/thread.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" +#include "td/utils/Span.h" #include "td/utils/Status.h" #include @@ -117,6 +118,7 @@ class TestsRunner : public TestContext { size_t it{0}; bool is_running = false; double start{0}; + double start_unadjusted{0}; size_t end{0}; }; bool stress_flag_{false}; diff --git a/tdutils/test/heap.cpp b/tdutils/test/heap.cpp index 8a20216bb..390b08834 100644 --- a/tdutils/test/heap.cpp +++ b/tdutils/test/heap.cpp @@ -38,8 +38,8 @@ TEST(Heap, sort_random_perm) { for (int i = 0; i < n; i++) { v[i] = i; } - std::srand(123); - std::random_shuffle(v.begin(), v.end()); + td::Random::Xorshift128plus rnd(123); + td::random_shuffle(as_mutable_span(v), rnd); std::vector nodes(n); KHeap kheap; for (int i = 0; i < n; i++) { diff --git a/tdutils/test/misc.cpp b/tdutils/test/misc.cpp index 768aebd31..6d24809d5 100644 --- a/tdutils/test/misc.cpp +++ b/tdutils/test/misc.cpp @@ -710,6 +710,39 @@ TEST(Misc, Bits) { ASSERT_EQ(4, count_bits64((1ull << 63) | 7)); } +TEST(Misc, BitsRange) { + auto to_vec_a = [](td::uint64 x) { + std::vector bits; + for (auto i : td::BitsRange(x)) { + bits.push_back(i); + } + return bits; + }; + + auto to_vec_b = [](td::uint64 x) { + std::vector bits; + td::int32 pos = 0; + while (x != 0) { + if ((x & 1) != 0) { + bits.push_back(pos); + } + x >>= 1; + pos++; + } + return bits; + }; + + auto do_check = [](std::vector a, std::vector b) { ASSERT_EQ(b, a); }; + auto check = [&](td::uint64 x) { do_check(to_vec_a(x), to_vec_b(x)); }; + + do_check(to_vec_a(21), {0, 2, 4}); + for (int x = 0; x < 100; x++) { + check(x); + check(std::numeric_limits::max() - x); + check(std::numeric_limits::max() - x); + } +} + #if !TD_THREAD_UNSUPPORTED TEST(Misc, Time) { Stage run; diff --git a/test/test-rldp2.cpp b/test/test-rldp2.cpp new file mode 100644 index 000000000..0ab7fc663 --- /dev/null +++ b/test/test-rldp2.cpp @@ -0,0 +1,203 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2019 Telegram Systems LLP +*/ +#include "adnl/adnl-network-manager.h" +#include "adnl/adnl-test-loopback-implementation.h" +#include "adnl/adnl.h" +#include "rldp2/rldp.h" + +#include "td/utils/port/signals.h" +#include "td/utils/port/path.h" +#include "td/utils/Random.h" + +#include +#include + +int main() { + SET_VERBOSITY_LEVEL(verbosity_INFO); + + std::string db_root_ = "tmp-ee"; + td::rmrf(db_root_).ignore(); + td::mkdir(db_root_).ensure(); + + td::set_default_failure_signal_handler().ensure(); + + td::actor::ActorOwn keyring; + td::actor::ActorOwn network_manager; + td::actor::ActorOwn adnl; + td::actor::ActorOwn rldp; + + ton::adnl::AdnlNodeIdShort src; + ton::adnl::AdnlNodeIdShort dst; + + td::actor::Scheduler scheduler({0}); + + scheduler.run_in_context([&] { + keyring = ton::keyring::Keyring::create(db_root_); + network_manager = td::actor::create_actor("test net"); + adnl = ton::adnl::Adnl::create(db_root_, keyring.get()); + rldp = ton::rldp2::Rldp::create(adnl.get()); + td::actor::send_closure(adnl, &ton::adnl::Adnl::register_network_manager, network_manager.get()); + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + src = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + dst = ton::adnl::AdnlNodeIdShort{pub2.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, td::uint8(0)); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub2}, addr, td::uint8(0)); + td::actor::send_closure(rldp, &ton::rldp2::Rldp::add_id, src); + td::actor::send_closure(rldp, &ton::rldp2::Rldp::add_id, dst); + + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, src, ton::adnl::AdnlNodeIdFull{pub2}, addr); + + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, src, true, true); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, dst, true, true); + }); + + auto send_packet = [&](td::uint32 i) { + td::BufferSlice d{5}; + d.as_slice()[0] = '1'; + d.as_slice().remove_prefix(1).copy_from(td::Slice{reinterpret_cast(&i), 4}); + + return d; + }; + + std::atomic remaining{0}; + scheduler.run_in_context([&] { + class Callback : public ton::adnl::Adnl::Callback { + public: + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + CHECK(src == src); + CHECK(dst == dst); + if (data.size() >= 5) { + CHECK(td::crc32c(data.as_slice().truncate(data.size() - 4)) == + *reinterpret_cast(data.as_slice().remove_prefix(data.size() - 4).begin())); + } + CHECK(remaining_ > 0); + remaining_--; + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + CHECK(data.size() == 5); + + td::uint32 s = *reinterpret_cast(data.as_slice().remove_prefix(1).begin()); + + td::BufferSlice d{s}; + if (s >= 4) { + td::Random::secure_bytes(d.as_slice().truncate(s - 4)); + auto x = td::crc32c(d.as_slice().truncate(d.size() - 4)); + + d.as_slice().remove_prefix(d.size() - 4).copy_from(td::Slice{reinterpret_cast(&x), 4}); + } else { + td::Random::secure_bytes(d.as_slice()); + } + + promise.set_value(std::move(d)); + } + Callback(std::atomic &remaining) : remaining_(remaining) { + } + + private: + std::atomic &remaining_; + }; + td::actor::send_closure(adnl, &ton::adnl::Adnl::subscribe, dst, "1", std::make_unique(remaining)); + }); + + std::vector sizes{1, 1024, 1 << 20, 2 << 20, 3 << 20, 10 << 20, 16 << 20}; + + for (auto &size : sizes) { + LOG(ERROR) << "testing delivering of packet of size " << size; + + auto f = td::Clocks::system(); + scheduler.run_in_context([&] { + remaining++; + td::actor::send_closure(rldp, &ton::rldp2::Rldp::send_query_ex, src, dst, std::string("t"), + td::PromiseCreator::lambda([&](td::Result R) { + R.ensure(); + remaining--; + }), + td::Timestamp::in(1024.0), send_packet(size), size + 1024); + }); + + auto t = td::Timestamp::in(1024.0); + while (scheduler.run(16)) { + if (!remaining) { + break; + } + if (t.is_in_past()) { + LOG(FATAL) << "failed to receive packets: remaining=" << remaining; + } + } + + LOG(ERROR) << "success. Time=" << (td::Clocks::system() - f); + } + + scheduler.run_in_context([&] { + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::set_loss_probability, 0.1); + }); + LOG(ERROR) << "set loss to 10%"; + + for (auto &size : sizes) { + LOG(ERROR) << "testing delivering of packet of size " << size; + + auto f = td::Clocks::system(); + scheduler.run_in_context([&] { + remaining++; + td::actor::send_closure(rldp, &ton::rldp2::Rldp::send_query_ex, src, dst, std::string("t"), + td::PromiseCreator::lambda([&](td::Result R) { + R.ensure(); + remaining--; + }), + td::Timestamp::in(1024.0), send_packet(size), size + 1024); + }); + + auto t = td::Timestamp::in(1024.0); + while (scheduler.run(16)) { + if (!remaining) { + break; + } + if (t.is_in_past()) { + LOG(FATAL) << "failed to receive packets: remaining=" << remaining; + } + } + + LOG(ERROR) << "success. Time=" << (td::Clocks::system() - f); + } + + td::rmrf(db_root_).ensure(); + std::_Exit(0); + return 0; +} diff --git a/tl-utils/common-utils.hpp b/tl-utils/common-utils.hpp index fbda82cea..d61bd79c0 100644 --- a/tl-utils/common-utils.hpp +++ b/tl-utils/common-utils.hpp @@ -192,17 +192,20 @@ td::Result fetch_result(const td::BufferSlice &message, template td::BufferSlice create_serialize_tl_object(Args &&... args) { - return serialize_tl_object(create_tl_object(std::forward(args)...), true); + Type object(std::forward(args)...); + return serialize_tl_object(&object, true); } template td::BufferSlice create_serialize_tl_object_suffix(td::Slice suffix, Args &&... args) { - return serialize_tl_object(create_tl_object(std::forward(args)...), true, suffix); + Type object(std::forward(args)...); + return serialize_tl_object(&object, true, suffix); } template auto create_hash_tl_object(Args &&... args) { - return get_tl_object_sha_bits256(create_tl_object(std::forward(args)...)); + Type object(std::forward(args)...); + return get_tl_object_sha_bits256(&object); } } // namespace ton diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5debeb4d6..3d14ac656 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -150,6 +150,10 @@ adnl.db.node.value date:int id:PublicKey addr_list:adnl.addressList priority_add ---types--- +rldp2.messagePart transfer_id:int256 fec_type:fec.Type part:int total_size:long seqno:int data:bytes = rldp2.MessagePart; +rldp2.confirm transfer_id:int256 part:int max_seqno:int received_mask:int received_count:int = rldp2.MessagePart; +rldp2.complete transfer_id:int256 part:int = rldp2.MessagePart; + rldp.messagePart transfer_id:int256 fec_type:fec.Type part:int total_size:long seqno:int data:bytes = rldp.MessagePart; rldp.confirm transfer_id:int256 part:int seqno:int = rldp.MessagePart; rldp.complete transfer_id:int256 part:int = rldp.MessagePart; @@ -655,6 +659,26 @@ engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus; engine.validator.controlQuery data:bytes = Object; +---types--- + +storage.pong = storage.Pong; +storage.ok = Ok; + +storage.state will_upload:Bool want_download:Bool = storage.State; +storage.piece proof:bytes data:bytes = storage.Piece; + +storage.updateInit have_pieces:bytes state:storage.State = storage.Update; +storage.updateHavePieces piece_id:(vector int) = storage.Update; +storage.updateState state:storage.State = storage.Update; + +---functions--- + +storage.ping session_id:long = storage.Pong; +storage.addUpdate session_id:long seqno:int update:storage.Update = Ok; + +storage.getPiece piece_id:int = storage.Piece; + +storage.queryPrefix id:int256 = Object; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9e1dec6ead0fcf41faa5287f02cee650f652ede3..65ec36c53d48ab476a36fa4c205c5f714ac65a37 100644 GIT binary patch delta 1202 zcmaFykom$#X5L4$^{p77;Lt|i$&Bo*GTMik{Ie(TXO`T&fKfn7(qxu6i(pYsN`aA{ zZ)$OIVtQ&oVo?bL1M@^jaY5c+TN-$aOY)0=!g>LjsmYTUa!d0pJ@z$+2P!)GfwY2P zSAnoGTv>2QVhKo@sLCVp06v)J(1Mi2l2ov%n>R>@urr!&W>t3Q)d0DN3#KSPn*qeX zc}&O$=f= zUWo6K^YhX&i*gwlKx#l%uud*q=sx*?9xF?3VnzI9MNN5L-lEjx)XcKfl=$4l;>jDe zrTO{bBFXusc_owcHDoo=Es^KgX~qY$B)1?Z73fj0i*go*V7Le+NDF{Ln##Zc5(Bx2 zX|m!FUtyhqrmkyK!iJ=D*NDNsYN~{zEV+CjvNDOQfP@(j*`pORmm2a%Yputvreq!-!z8{gvdgkVuu3XWgTyvz~? zP`snqk&#%I8V~krF~j6Udx?6eW1touL^c=}=O9CnE%Q|PVJ(4bnFr8VP?}6F1}cPE z$_QkGJjMaGFFq6OE1(!C9f=;0SlCe z_yO73u&@H<53m9t0ZQ}<&#D>(V78^FmVg5ZCH} zQIZ|Z1Z4Nh>!2sV_P3I6;6Ykgnp#vDP?VaMS+T&UKy&hiaEZwV4kD9#-V1DYd9XwP E0H$AiCIA2c delta 63 zcmccck@>|#X5L4$^{p77V9!S0$&8y1FbYU*-XIgg&ZxbaRmGimbAw?^;bxWrEwL2 diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 1d1b14889..4d928c18d 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -57,9 +57,6 @@ raw.transactions transactions:vector previous_transaction_id:in pchan.config alice_public_key:string alice_address:accountAddress bob_public_key:string bob_address:accountAddress init_timeout:int32 close_timeout:int32 channel_id:int64 = pchan.Config; raw.initialAccountState code:bytes data:bytes = InitialAccountState; -testGiver.initialAccountState = InitialAccountState; -testWallet.initialAccountState public_key:string = InitialAccountState; -wallet.initialAccountState public_key:string = InitialAccountState; wallet.v3.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; wallet.highload.v1.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; wallet.highload.v2.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; @@ -72,12 +69,9 @@ dns.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; pchan.initialAccountState config:pchan.config = InitialAccountState; raw.accountState code:bytes data:bytes frozen_hash:bytes = AccountState; -testWallet.accountState seqno:int32 = AccountState; -wallet.accountState seqno:int32 = AccountState; wallet.v3.accountState wallet_id:int64 seqno:int32 = AccountState; wallet.highload.v1.accountState wallet_id:int64 seqno:int32 = AccountState; wallet.highload.v2.accountState wallet_id:int64 = AccountState; -testGiver.accountState seqno:int32 = AccountState; dns.accountState wallet_id:int64 = AccountState; rwallet.accountState wallet_id:int64 seqno:int32 unlocked_balance:int64 config:rwallet.config = AccountState; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 278fac757ad7eb76184f62631d197e015409c1d4..0add8dd63c4348b7fa9354f16a243edec32eaf57 100644 GIT binary patch delta 419 zcmZoT&3NJ%yv&l! z#2jQbT$>x2XLB;vZa$GT?mnOH0m!uY#geT_Yq?YI< zB3T1cd-Ir(598!S9c@E&oe}2`ybvxYpbKL04?YbDV*`H(gi*t9fF{Br02Q$jFu`!; zs>h;SQXfAT46RzyMMN3O|s- zB4;)o4%@ba=o(Y@Uo(&chyUdHvXWeIJ&cnJ<(cfg(bad_config.str()))).ensure_error(); auto address = sync_send(client, make_object( - make_object(), 0, -1)) + make_object( + "Pubjns2gp7DGCnEH7EOWeCnb6Lw1akm538YYaz6sdLVHfRB2", 123), + 0, -1)) .move_as_ok(); sync_send(client, make_object(std::move(address))).ensure_error(); sync_send(client, make_object()).ensure(); diff --git a/tonlib/test/online.cpp b/tonlib/test/online.cpp index 1ab6ab71b..3b8518530 100644 --- a/tonlib/test/online.cpp +++ b/tonlib/test/online.cpp @@ -39,8 +39,6 @@ #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/MultisigWallet.h" -#include "smc-envelope/TestGiver.h" -#include "smc-envelope/TestWallet.h" #include "tonlib/LastBlock.h" #include "tonlib/ExtClient.h" #include "tonlib/utils.h" @@ -182,7 +180,6 @@ AccountState get_account_state(Client& client, std::string address) { res.type = AccountState::Empty; break; case tonlib_api::wallet_v3_accountState::ID: - case tonlib_api::wallet_accountState::ID: res.type = AccountState::Wallet; break; case tonlib_api::dns_accountState::ID: diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 9afc00190..74ed3561a 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -30,12 +30,9 @@ #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" -#include "smc-envelope/TestWallet.h" -#include "smc-envelope/Wallet.h" #include "smc-envelope/WalletV3.h" #include "smc-envelope/HighloadWallet.h" #include "smc-envelope/HighloadWalletV2.h" -#include "smc-envelope/TestGiver.h" #include "smc-envelope/PaymentChannel.h" #include "smc-envelope/SmartContractCode.h" @@ -277,21 +274,6 @@ class AccountState { raw().frozen_hash, get_sync_time()); } - td::Result> to_testWallet_accountState() const { - if (wallet_type_ != SimpleWallet) { - return TonlibError::AccountTypeUnexpected("TestWallet"); - } - TRY_RESULT(seqno, ton::TestWallet(get_smc_state()).get_seqno()); - return tonlib_api::make_object(static_cast(seqno)); - } - - td::Result> to_wallet_accountState() const { - if (wallet_type_ != Wallet) { - return TonlibError::AccountTypeUnexpected("Wallet"); - } - TRY_RESULT(seqno, ton::Wallet(get_smc_state()).get_seqno()); - return tonlib_api::make_object(static_cast(seqno)); - } td::Result> to_wallet_v3_accountState() const { if (wallet_type_ != WalletV3) { return TonlibError::AccountTypeUnexpected("WalletV3"); @@ -372,13 +354,6 @@ class AccountState { std::move(tl_state), info.description); } - td::Result> to_testGiver_accountState() const { - if (wallet_type_ != Giver) { - return TonlibError::AccountTypeUnexpected("TestGiver"); - } - TRY_RESULT(seqno, ton::TestGiver(get_smc_state()).get_seqno()); - return tonlib_api::make_object(static_cast(seqno)); - } td::Result> to_dns_accountState() const { if (wallet_type_ != ManualDns) { return TonlibError::AccountTypeUnexpected("ManualDns"); @@ -398,12 +373,6 @@ class AccountState { return to_uninited_accountState(); case Unknown: return f(to_raw_accountState()); - case Giver: - return f(to_testGiver_accountState()); - case SimpleWallet: - return f(to_testWallet_accountState()); - case Wallet: - return f(to_wallet_accountState()); case WalletV3: return f(to_wallet_v3_accountState()); case HighloadWalletV1: @@ -432,9 +401,6 @@ class AccountState { enum WalletType { Empty, Unknown, - Giver, - SimpleWallet, - Wallet, WalletV3, HighloadWalletV1, HighloadWalletV2, @@ -455,9 +421,6 @@ class AccountState { case AccountState::ManualDns: case AccountState::PaymentChannel: return false; - case AccountState::Giver: - case AccountState::SimpleWallet: - case AccountState::Wallet: case AccountState::WalletV3: case AccountState::HighloadWalletV1: case AccountState::HighloadWalletV2: @@ -474,12 +437,6 @@ class AccountState { case AccountState::ManualDns: case AccountState::PaymentChannel: return {}; - case AccountState::Giver: - return td::make_unique(get_smc_state()); - case AccountState::SimpleWallet: - return td::make_unique(get_smc_state()); - case AccountState::Wallet: - return td::make_unique(get_smc_state()); case AccountState::WalletV3: return td::make_unique(get_smc_state()); case AccountState::HighloadWalletV1: @@ -570,28 +527,27 @@ class AccountState { if (wallet_type_ != WalletType::Empty) { return wallet_type_; } - auto wallet_id = address_.workchain + wallet_id_; - auto o_revision = ton::WalletV3::guess_revision(address_, key, wallet_id); + auto wallet_id = static_cast(address_.workchain + wallet_id_); + ton::WalletV3::InitData init_data{key.as_octet_string(), wallet_id}; + auto o_revision = ton::WalletV3::guess_revision(address_, init_data); if (o_revision) { wallet_type_ = WalletType::WalletV3; wallet_revision_ = o_revision.value(); - set_new_state({ton::WalletV3::get_init_code(wallet_revision_), ton::WalletV3::get_init_data(key, wallet_id)}); + set_new_state(ton::WalletV3::get_init_state(wallet_revision_, init_data)); return wallet_type_; } - o_revision = ton::HighloadWalletV2::guess_revision(address_, key, wallet_id + address_.workchain); + o_revision = ton::HighloadWalletV2::guess_revision(address_, init_data); if (o_revision) { wallet_type_ = WalletType::HighloadWalletV2; wallet_revision_ = o_revision.value(); - set_new_state({ton::HighloadWalletV2::get_init_code(wallet_revision_), - ton::HighloadWalletV2::get_init_data(key, wallet_id + address_.workchain)}); + set_new_state(ton::HighloadWallet::get_init_state(wallet_revision_, init_data)); return wallet_type_; } - o_revision = ton::HighloadWallet::guess_revision(address_, key, wallet_id); + o_revision = ton::HighloadWallet::guess_revision(address_, init_data); if (o_revision) { wallet_type_ = WalletType::HighloadWalletV1; wallet_revision_ = o_revision.value(); - set_new_state( - {ton::HighloadWallet::get_init_code(wallet_revision_), ton::HighloadWallet::get_init_data(key, wallet_id)}); + set_new_state(ton::HighloadWallet::get_init_state(wallet_revision_, init_data)); return wallet_type_; } o_revision = ton::ManualDns::guess_revision(address_, key, wallet_id); @@ -602,15 +558,6 @@ class AccountState { set_new_state(dns->get_state()); return wallet_type_; } - if (ton::GenericAccount::get_address(address_.workchain, ton::TestWallet::get_init_state(key)).addr == - address_.addr) { - set_new_state({ton::TestWallet::get_init_code(), ton::TestWallet::get_init_data(key)}); - wallet_type_ = WalletType::SimpleWallet; - } else if (ton::GenericAccount::get_address(address_.workchain, ton::Wallet::get_init_state(key)).addr == - address_.addr) { - set_new_state({ton::Wallet::get_init_code(), ton::Wallet::get_init_data(key)}); - wallet_type_ = WalletType::Wallet; - } return wallet_type_; } @@ -618,7 +565,8 @@ class AccountState { if (wallet_type_ != WalletType::Empty) { return wallet_type_; } - set_new_state({ton::WalletV3::get_init_code(), ton::WalletV3::get_init_data(key, wallet_id_)}); + ton::WalletV3::InitData init_data(key.as_octet_string(), wallet_id_ + address_.workchain); + set_new_state(ton::WalletV3::get_init_state(0, init_data)); wallet_type_ = WalletType::WalletV3; return wallet_type_; } @@ -696,16 +644,8 @@ class AccountState { return wallet_type_; } - if (code_hash == ton::TestGiver::get_init_code_hash()) { - wallet_type_ = WalletType::Giver; - } else if (code_hash == ton::TestWallet::get_init_code_hash()) { - wallet_type_ = WalletType::SimpleWallet; - } else if (code_hash == ton::Wallet::get_init_code_hash()) { - wallet_type_ = WalletType::Wallet; - } else { - LOG(WARNING) << "Unknown code hash: " << td::base64_encode(code_hash.as_slice()); - wallet_type_ = WalletType::Unknown; - } + LOG(WARNING) << "Unknown code hash: " << td::base64_encode(code_hash.as_slice()); + wallet_type_ = WalletType::Unknown; return wallet_type_; } }; @@ -1601,51 +1541,27 @@ td::Result get_account_address(const tonlib_api::raw_initialA ton::GenericAccount::get_init_state(std::move(code), std::move(data))); } -td::Result get_account_address(const tonlib_api::testGiver_initialAccountState& test_wallet_state, - td::int32 revision, ton::WorkchainId workchain_id) { - return ton::TestGiver::address(); -} - -td::Result get_account_address(const tonlib_api::testWallet_initialAccountState& test_wallet_state, - td::int32 revision, ton::WorkchainId workchain_id) { - TRY_RESULT(key_bytes, get_public_key(test_wallet_state.public_key_)); - auto key = td::Ed25519::PublicKey(td::SecureString(key_bytes.key)); - return ton::GenericAccount::get_address(workchain_id, ton::TestWallet::get_init_state(key, revision)); -} - -td::Result get_account_address(const tonlib_api::wallet_initialAccountState& wallet_state, - td::int32 revision, ton::WorkchainId workchain_id) { - TRY_RESULT(key_bytes, get_public_key(wallet_state.public_key_)); - auto key = td::Ed25519::PublicKey(td::SecureString(key_bytes.key)); - return ton::GenericAccount::get_address(workchain_id, ton::Wallet::get_init_state(key, revision)); -} td::Result get_account_address(const tonlib_api::wallet_v3_initialAccountState& test_wallet_state, td::int32 revision, ton::WorkchainId workchain_id) { TRY_RESULT(key_bytes, get_public_key(test_wallet_state.public_key_)); - auto key = td::Ed25519::PublicKey(td::SecureString(key_bytes.key)); - return ton::GenericAccount::get_address( - workchain_id, - ton::WalletV3::get_init_state(key, static_cast(test_wallet_state.wallet_id_), revision)); + return ton::WalletV3::create({key_bytes.key, static_cast(test_wallet_state.wallet_id_)}, revision) + ->get_address(workchain_id); } td::Result get_account_address( const tonlib_api::wallet_highload_v1_initialAccountState& test_wallet_state, td::int32 revision, ton::WorkchainId workchain_id) { TRY_RESULT(key_bytes, get_public_key(test_wallet_state.public_key_)); - auto key = td::Ed25519::PublicKey(td::SecureString(key_bytes.key)); - return ton::GenericAccount::get_address( - workchain_id, - ton::HighloadWallet::get_init_state(key, static_cast(test_wallet_state.wallet_id_), revision)); + return ton::HighloadWallet::create({key_bytes.key, static_cast(test_wallet_state.wallet_id_)}, revision) + ->get_address(workchain_id); } td::Result get_account_address( const tonlib_api::wallet_highload_v2_initialAccountState& test_wallet_state, td::int32 revision, ton::WorkchainId workchain_id) { TRY_RESULT(key_bytes, get_public_key(test_wallet_state.public_key_)); - auto key = td::Ed25519::PublicKey(td::SecureString(key_bytes.key)); - return ton::GenericAccount::get_address( - workchain_id, - ton::HighloadWalletV2::get_init_state(key, static_cast(test_wallet_state.wallet_id_), revision)); + return ton::HighloadWalletV2::create({key_bytes.key, static_cast(test_wallet_state.wallet_id_)}, revision) + ->get_address(workchain_id); } td::Result get_account_address(const tonlib_api::dns_initialAccountState& dns_state, @@ -1678,9 +1594,6 @@ static td::optional get_wallet_type(tonlib_api::In state, td::overloaded( [](const tonlib_api::raw_initialAccountState&) { return td::optional(); }, - [](const tonlib_api::testGiver_initialAccountState&) { return td::optional(); }, - [](const tonlib_api::testWallet_initialAccountState&) { return ton::SmartContractCode::WalletV1; }, - [](const tonlib_api::wallet_initialAccountState&) { return ton::SmartContractCode::WalletV2; }, [](const tonlib_api::wallet_v3_initialAccountState&) { return ton::SmartContractCode::WalletV3; }, [](const tonlib_api::wallet_highload_v1_initialAccountState&) { return ton::SmartContractCode::HighloadWalletV1; @@ -1999,7 +1912,7 @@ const MasterConfig& get_default_master_config() { "root_hash": "VCSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=", "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" }, - "init_block": + "init_block": {"workchain":-1,"shard":-9223372036854775808,"seqno":870721,"root_hash":"jYKhSQ1xeSPprzgjqiUOnAWwc2yqs7nCVAU21k922s4=","file_hash":"kHidF02CZpaz2ia9jtXUJLp0AiWMWwfzprTUIsddHSo="} } })abc"); @@ -2636,7 +2549,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { return td::Status::OK(); })); // Use this limit as a preventive check - if (res.message.size() > ton::Wallet::max_message_size) { + if (res.message.size() > ton::WalletV3Traits::max_message_size) { return TonlibError::MessageTooLong(); } TRY_STATUS(std::move(status)); @@ -3116,11 +3029,6 @@ class GenericCreateSendGrams : public TonlibQueryActor { return td::Status::OK(); }; - if (source_->get_wallet_type() == AccountState::Giver) { - valid_until = 0; - private_key_ = td::Ed25519::PrivateKey(td::SecureString(std::string(32, '\0'))); - } - return with_wallet(*source_->get_wallet()); } }; // namespace tonlib diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 4fec3db61..ffd2052ad 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -177,8 +177,8 @@ class TonlibCli : public td::actor::Actor { private: td::actor::ActorShared id_; }; - ref_cnt_++; if (!options_.one_shot) { + ref_cnt_++; io_ = td::TerminalIO::create("> ", options_.enable_readline, std::make_unique(actor_shared(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); } @@ -1587,12 +1587,6 @@ class TonlibCli : public td::actor::Actor { template auto with_account_state(int version, std::string public_key, td::uint32 wallet_id, F&& f) { - if (version == 1) { - return f(make_object(public_key)); - } - if (version == 2) { - return f(make_object(public_key)); - } if (version == 4) { return f(make_object(public_key, wallet_id)); } @@ -1654,17 +1648,6 @@ class TonlibCli : public td::actor::Actor { return std::move(res); } } - if (key == "giver") { - auto obj = tonlib::TonlibClient::static_request( - make_object(make_object(), 0, -1)); - if (obj->get_id() != tonlib_api::error::ID) { - Address res; - res.address = ton::move_tl_object_as(obj); - return std::move(res); - } else { - LOG(ERROR) << "Unexpected error during testGiver_getAccountAddress: " << to_string(obj); - } - } if (!need_private_key) { auto r_addr = to_account_address(key); if (r_addr.is_ok()) { @@ -2073,7 +2056,7 @@ int main(int argc, char* argv[]) { td::OptionsParser p; TonlibCli::Options options; - p.set_description("console for validator for TON Blockchain"); + p.set_description("cli wrapper around tonlib"); p.add_option('h', "help", "prints_help", [&]() { std::cout << (PSLICE() << p).c_str(); std::exit(2); diff --git a/validator/impl/block.cpp b/validator/impl/block.cpp index 8a8433f1d..b581404af 100644 --- a/validator/impl/block.cpp +++ b/validator/impl/block.cpp @@ -42,7 +42,7 @@ td::Status BlockQ::init() { } vm::StaticBagOfCellsDbLazy::Options options; options.check_crc32c = true; - auto res = vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(data_.clone()), options); + auto res = vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(data_.clone()), options); if (res.is_error()) { return res.move_as_error(); } diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 5f1c6b438..3b8a45492 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -18,7 +18,7 @@ */ #include "collator-impl.h" #include "vm/boc.h" -#include "vm/db/BlobView.h" +#include "td/db/utils/BlobView.h" #include "vm/db/StaticBagOfCellsDb.h" #include "block/mc-config.h" #include "block/block.h" @@ -28,7 +28,7 @@ #include "crypto/openssl/rand.hpp" #include "ton/ton-shard.h" #include "adnl/utils.hpp" -#include +#include #include #include "fabric.h" #include "validator-set.hpp" diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 9ccdd2634..b3c0f3eb5 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1939,7 +1939,7 @@ bool LiteQuery::construct_proof_link_forward_cont(ton::BlockIdExt cur, ton::Bloc // for zero state, lazily deserialize buffer_ instead vm::StaticBagOfCellsDbLazy::Options options; options.check_crc32c = true; - auto res = vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(std::move(buffer_)), options); + auto res = vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(std::move(buffer_)), options); if (res.is_error()) { return fatal_error(res.move_as_error()); } diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index 225a82de8..033a1ab12 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -106,7 +106,7 @@ td::Result ProofLinkQ::get_virtual_root(bool lazy) if (lazy) { vm::StaticBagOfCellsDbLazy::Options options; options.check_crc32c = true; - auto res = vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(data_.clone()), options); + auto res = vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(data_.clone()), options); if (res.is_error()) { return res.move_as_error(); } diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 6931ce124..18f71d32c 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -20,7 +20,7 @@ #include "message-queue.hpp" #include "validator-set.hpp" #include "vm/boc.h" -#include "vm/db/BlobView.h" +#include "td/db/utils/BlobView.h" #include "vm/db/StaticBagOfCellsDb.h" #include "vm/cellslice.h" #include "vm/cells/MerkleUpdate.h" @@ -86,7 +86,7 @@ td::Status ShardStateQ::init() { #if LAZY_STATE_DESERIALIZE vm::StaticBagOfCellsDbLazy::Options options; options.check_crc32c = true; - auto res = vm::StaticBagOfCellsDbLazy::create(vm::BufferSliceBlobView::create(data.clone()), options); + auto res = vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(data.clone()), options); if (res.is_error()) { return res.move_as_error(); } From dbde9c1c40c1fae11fd20d6e4dbd4223eec47a8d Mon Sep 17 00:00:00 2001 From: ton Date: Wed, 3 Jun 2020 21:47:47 +0400 Subject: [PATCH 2/2] minor bugfix --- crypto/func/test/a12_1.fc | 3 ++- crypto/func/test/a12_2.fc | 3 ++- crypto/func/test/a12_3.fc | 1 + crypto/func/test/a12_4.fc | 1 + crypto/func/test/a12_5.fc | 1 + crypto/func/test/a12_6.fc | 1 + crypto/func/test/a12_8.fc | 4 ++++ crypto/func/test/a12_9.fc | 4 ++++ validator/manager.cpp | 16 ++++++++-------- 9 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 crypto/func/test/a12_8.fc create mode 100644 crypto/func/test/a12_9.fc diff --git a/crypto/func/test/a12_1.fc b/crypto/func/test/a12_1.fc index 45c97ee5f..2f59e07e7 100644 --- a/crypto/func/test/a12_1.fc +++ b/crypto/func/test/a12_1.fc @@ -1,3 +1,4 @@ _ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j) { - return (f, j, a, b, i, g, d, e, h, c); + return (f, j, a, b, i, g, d, e, h, c); ;; optimal code is 3 ops, 6 bytes + ;; s6 s0 s4 XCHG3 s4 s8 s9 XCHG3 8 0 REVERSE } diff --git a/crypto/func/test/a12_2.fc b/crypto/func/test/a12_2.fc index 8ee93d273..7c60c6fd5 100644 --- a/crypto/func/test/a12_2.fc +++ b/crypto/func/test/a12_2.fc @@ -1,3 +1,4 @@ _ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j) { - return (f, d, j, a, i, g, b, e, h, c); + return (f, d, j, a, i, g, b, e, h, c); ;; optimal code is 4 ops, 6 bytes + ;; s1 s8 XCHG s1 s6 XCHG 7 3 BLKSWAP s9 s4 XCHG2 } diff --git a/crypto/func/test/a12_3.fc b/crypto/func/test/a12_3.fc index faa5c9c20..86b3409a9 100644 --- a/crypto/func/test/a12_3.fc +++ b/crypto/func/test/a12_3.fc @@ -1,3 +1,4 @@ _ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) { return (e, h, d, a, c, g, f, b, i, j, k); ;; optimal code is 4 ops, 8 bytes + ;; s9 s9 XCHG2 s6 s6 s8 XCHG3 s10 s7 s3 XCHG3 10 0 REVERSE } diff --git a/crypto/func/test/a12_4.fc b/crypto/func/test/a12_4.fc index bf0d797eb..a4132917c 100644 --- a/crypto/func/test/a12_4.fc +++ b/crypto/func/test/a12_4.fc @@ -1,3 +1,4 @@ _ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) { return (b, a, c, e, g, d, f, k, i, j, h); ;; optimal code is 4 ops, 8 bytes + ;; s5 s0 s8 XCHG3 2 9 BLKSWAP s8 s9 s10 XCHG3 s7 s0 s5 XCHG3 } diff --git a/crypto/func/test/a12_5.fc b/crypto/func/test/a12_5.fc index 4c7235801..277308a17 100644 --- a/crypto/func/test/a12_5.fc +++ b/crypto/func/test/a12_5.fc @@ -1,3 +1,4 @@ _ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) { return (c, g, d, k, a, f, e, h, i, j, b); ;; optimal code is 6 ops, 6 bytes + ;; s0 s7 XCHG s0 s8 XCHG s0 s10 XCHG s0 s6 XCHG s0 s4 XCHG s0 s9 XCHG } diff --git a/crypto/func/test/a12_6.fc b/crypto/func/test/a12_6.fc index 49982541d..01f7a5862 100644 --- a/crypto/func/test/a12_6.fc +++ b/crypto/func/test/a12_6.fc @@ -1,3 +1,4 @@ _ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k) { return (h, e, d, j, k, f, i, a, b, c, g); ;; optimal code is 3 ops, 6 bytes + ;; s3 s10 XCHG s6 s7 s6 XCHG3 s9 s8 s4 XCHG3 } diff --git a/crypto/func/test/a12_8.fc b/crypto/func/test/a12_8.fc new file mode 100644 index 000000000..054ae81d5 --- /dev/null +++ b/crypto/func/test/a12_8.fc @@ -0,0 +1,4 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o, int p) { + return (l, i, o, h, a, m, j, d, p, g, n, b, k, e, c, f); ;; optimal code is 5 ops, 10 bytes + ;; 8 0 REVERSE s3 s9 s14 XCHG3 s15 s6 s4 XCHG3 s8 s12 XCHG s11 s13 s10 XCHG3 +} diff --git a/crypto/func/test/a12_9.fc b/crypto/func/test/a12_9.fc new file mode 100644 index 000000000..cd5789e24 --- /dev/null +++ b/crypto/func/test/a12_9.fc @@ -0,0 +1,4 @@ +_ f(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, int k, int l, int m, int n, int o, int p) { + return (g, j, n, f, o, i, e, l, d, h, b, k, m, c, p, a); ;; optimal code is 5 ops, 10 bytes + ;; s7 s5 s14 XCHG3 s10 s4 s5 XCHG3 s12 s8 s11 XCHG3 s7 s6 s9 XCHG3 s13 s14 s15 XCHG3 +} diff --git a/validator/manager.cpp b/validator/manager.cpp index 3d711550c..dcd3f89ba 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1663,14 +1663,6 @@ void ValidatorManagerImpl::completed_prestart_sync() { } void ValidatorManagerImpl::new_masterchain_block() { - update_shards(); - update_shard_blocks(); - - if (!shard_client_.empty()) { - td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, - last_masterchain_block_handle_, last_masterchain_state_); - } - if (last_masterchain_seqno_ > 0 && last_masterchain_block_handle_->is_key_block()) { last_key_block_handle_ = last_masterchain_block_handle_; if (last_key_block_handle_->id().seqno() > last_known_key_block_handle_->id().seqno()) { @@ -1679,6 +1671,14 @@ void ValidatorManagerImpl::new_masterchain_block() { } } + update_shards(); + update_shard_blocks(); + + if (!shard_client_.empty()) { + td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, + last_masterchain_block_handle_, last_masterchain_state_); + } + if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; }