New archive format and importing archive slices

SpyCheese · Jun 5, 2024 · 001b95d · 001b95d
1 parent eb4c876
commit 001b95d
Show file tree

Hide file tree

Showing 27 changed files with 568 additions and 261 deletions.
diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl
@@ -456,6 +456,7 @@ tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
 tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo;
+tonNode.getShardArchiveInfo masterchain_seqno:int shard_prefix:tonNode.shardId = tonNode.ArchiveInfo;
 tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data;
 tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt)
     limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof;

diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo
diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp
@@ -600,8 +600,8 @@ void ArchiveManager::load_package(PackageId id) {
     }
   }
 
-  desc.file =
-      td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_);
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, 0, db_root_,
+                                                    archive_lru_.get(), statistics_);
 
   m.emplace(id, std::move(desc));
   update_permanent_slices();
@@ -635,8 +635,9 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull
   FileDescription new_desc{id, false};
   td::mkdir(db_root_ + id.path()).ensure();
   std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
-  new_desc.file =
-      td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_);
+  new_desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false,
+                                                        id.key || id.temp ? 0 : cur_shard_split_depth_, db_root_,
+                                                        archive_lru_.get(), statistics_);
   const FileDescription &desc = f.emplace(id, std::move(new_desc));
   if (!id.temp) {
     update_desc(f, desc, shard, seqno, ts, lt);
@@ -1091,14 +1092,16 @@ PackageId ArchiveManager::get_package_id_force(BlockSeqno masterchain_seqno, Sha
   return it->first;
 }
 
-void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) {
+void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix,
+                                    td::Promise<td::uint64> promise) {
   auto F = get_file_desc_by_seqno(ShardIdFull{masterchainId}, masterchain_seqno, false);
   if (!F) {
     promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found"));
     return;
   }
 
-  td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, std::move(promise));
+  td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, shard_prefix,
+                          std::move(promise));
 }
 
 void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,

diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp
@@ -65,7 +65,7 @@ class ArchiveManager : public td::actor::Actor {
   void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise<ConstBlockHandle> promise);
   void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise<ConstBlockHandle> promise);
 
-  void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise);
+  void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise<td::uint64> promise);
   void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit,
                          td::Promise<td::BufferSlice> promise);
 
@@ -75,6 +75,10 @@ class ArchiveManager : public td::actor::Actor {
   void commit_transaction();
   void set_async_mode(bool mode, td::Promise<td::Unit> promise);
 
+  void set_current_shard_split_depth(td::uint32 value) {
+    cur_shard_split_depth_ = value;
+  }
+
   static constexpr td::uint32 archive_size() {
     return 20000;
   }
@@ -173,6 +177,7 @@ class ArchiveManager : public td::actor::Actor {
   bool async_mode_ = false;
   bool huge_transaction_started_ = false;
   td::uint32 huge_transaction_size_ = 0;
+  td::uint32 cur_shard_split_depth_ = 0;
 
   DbStatistics statistics_;