BitTorrentClient related refactorings (#2279)

erigontech · Sep 1, 2024 · c6f3cb3 · c6f3cb3
1 parent b7bcc1e
commit c6f3cb3
Show file tree

Hide file tree

Showing 47 changed files with 176 additions and 245 deletions.
diff --git a/cmd/capi/execute.cpp b/cmd/capi/execute.cpp
@@ -34,7 +34,7 @@
 #include <silkworm/db/access_layer.hpp>
 #include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/snapshot_bundle_factory_impl.hpp>
-#include <silkworm/db/snapshots/repository.hpp>
+#include <silkworm/db/snapshots/snapshot_repository.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/rpc/daemon.hpp>

diff --git a/cmd/common/settings.hpp b/cmd/common/settings.hpp
@@ -16,7 +16,7 @@
 
 #pragma once
 
-#include <silkworm/db/snapshots/settings.hpp>
+#include <silkworm/db/snapshots/snapshot_settings.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/node/settings.hpp>
 #include <silkworm/rpc/settings.hpp>

diff --git a/cmd/common/snapshot_options.hpp b/cmd/common/snapshot_options.hpp
@@ -18,7 +18,7 @@
 
 #include <CLI/CLI.hpp>
 
-#include <silkworm/db/snapshots/settings.hpp>
+#include <silkworm/db/snapshots/snapshot_settings.hpp>
 
 namespace silkworm::cmd::common {
 

diff --git a/cmd/dev/check_changes.cpp b/cmd/dev/check_changes.cpp
@@ -26,7 +26,7 @@
 #include <silkworm/db/access_layer.hpp>
 #include <silkworm/db/buffer.hpp>
 #include <silkworm/db/snapshot_bundle_factory_impl.hpp>
-#include <silkworm/db/snapshots/repository.hpp>
+#include <silkworm/db/snapshots/snapshot_repository.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>

diff --git a/cmd/dev/db_toolbox.cpp b/cmd/dev/db_toolbox.cpp
@@ -46,8 +46,8 @@
 #include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/prune_mode.hpp>
 #include <silkworm/db/snapshot_bundle_factory_impl.hpp>
-#include <silkworm/db/snapshots/repository.hpp>
-#include <silkworm/db/snapshots/settings.hpp>
+#include <silkworm/db/snapshots/snapshot_repository.hpp>
+#include <silkworm/db/snapshots/snapshot_settings.hpp>
 #include <silkworm/db/stage_scheduler.hpp>
 #include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/decoding_exception.hpp>

diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp
@@ -43,9 +43,9 @@
 #include <silkworm/db/snapshot_sync.hpp>
 #include <silkworm/db/snapshots/bittorrent/client.hpp>
 #include <silkworm/db/snapshots/bittorrent/web_seed_client.hpp>
-#include <silkworm/db/snapshots/repository.hpp>
 #include <silkworm/db/snapshots/seg/seg_zip.hpp>
 #include <silkworm/db/snapshots/snapshot_reader.hpp>
+#include <silkworm/db/snapshots/snapshot_repository.hpp>
 #include <silkworm/db/transactions/txn_index.hpp>
 #include <silkworm/db/transactions/txn_queries.hpp>
 #include <silkworm/db/transactions/txn_to_block_index.hpp>
@@ -80,6 +80,7 @@ struct DownloadSettings : public bittorrent::BitTorrentSettings {
     ChainId chain_id{kMainnetConfig.chain_id};
     std::string url_seed;
     bool download_web_seed_torrents{false};
+    std::optional<std::string> magnet_uri;
 };
 
 static const auto kTorrentRepoPath{bittorrent::BitTorrentSettings::kDefaultTorrentRepoPath};
@@ -142,8 +143,6 @@ void parse_command_line(int argc, char* argv[], CLI::App& app, SnapshotToolboxSe
     auto& snapshot_settings = settings.snapshot_settings;
     auto& bittorrent_settings = settings.download_settings;
 
-    bittorrent_settings.magnets_file_path = ".magnet_links";
-
     add_logging_options(app, log_settings);
 
     std::map<SnapshotTool, CLI::App*> commands;
@@ -182,9 +181,7 @@ void parse_command_line(int argc, char* argv[], CLI::App& app, SnapshotToolboxSe
         add_option_chain(*cmd, bittorrent_settings.chain_id);
         cmd->add_option("--torrent_dir", bittorrent_settings.repository_path, "Path to torrent file repository")
             ->capture_default_str();
-        cmd->add_option("--magnet_file",
-                        bittorrent_settings.magnets_file_path,
-                        "File containing magnet links to download")
+        cmd->add_option("--magnet", bittorrent_settings.magnet_uri, "Magnet link to download")
             ->capture_default_str();
         cmd->add_option("--url_seed", bittorrent_settings.url_seed, "URL seed to download from")
             ->capture_default_str();
@@ -431,7 +428,7 @@ static void download_bittorrent(bittorrent::BitTorrentClient& client) {
     });
     std::thread scheduler_thread{[&scheduler]() { scheduler.run(); }};
 
-    client.execute_loop();
+    client.execution_loop();
 
     scheduler_thread.join();
 }
@@ -452,12 +449,16 @@ void download(const DownloadSettings& settings) {
             }
             download_bittorrent(client);
         }
-    } else {
-        // Download the target files by using the magnet links contained in input file (i.e. settings.magnets_file_path)
+    } else if (settings.magnet_uri) {
+        // Download the magnet link
         bittorrent::BitTorrentClient client{settings};  // NOLINT(cppcoreguidelines-slicing)
-        SILK_INFO << "Bittorrent async download started for magnet file: " << *settings.magnets_file_path;
+        SILK_INFO << "Bittorrent async download started for magnet file: " << *settings.magnet_uri;
+        client.add_magnet_uri(*settings.magnet_uri);
         download_bittorrent(client);
-        SILK_INFO << "Bittorrent async download completed for magnet file: " << *settings.magnets_file_path;
+        SILK_INFO << "Bittorrent async download completed for magnet file: " << *settings.magnet_uri;
+    } else {
+        SILK_WARN << "No download source. Pass either --url_seed or --magnet";
+        return;
     }
 
     std::chrono::duration elapsed{std::chrono::steady_clock::now() - start};

diff --git a/silkworm/capi/instance.hpp b/silkworm/capi/instance.hpp
@@ -22,7 +22,7 @@
 
 #include <boost/asio/cancellation_signal.hpp>
 
-#include <silkworm/db/snapshots/repository.hpp>
+#include <silkworm/db/snapshots/snapshot_repository.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/context_pool_settings.hpp>
 #include <silkworm/node/stagedsync/execution_engine.hpp>

diff --git a/silkworm/db/access_layer.cpp b/silkworm/db/access_layer.cpp
@@ -27,7 +27,7 @@
 #include <silkworm/db/blocks/headers/header_queries.hpp>
 #include <silkworm/db/mdbx/bitmap.hpp>
 #include <silkworm/db/receipt_cbor.hpp>
-#include <silkworm/db/snapshots/repository.hpp>
+#include <silkworm/db/snapshots/snapshot_repository.hpp>
 #include <silkworm/db/tables.hpp>
 #include <silkworm/db/transactions/txn_queries.hpp>
 #include <silkworm/infra/common/decoding_exception.hpp>

diff --git a/silkworm/db/blocks/bodies/body_index.hpp b/silkworm/db/blocks/bodies/body_index.hpp
@@ -22,7 +22,7 @@
 
 #include <silkworm/core/common/bytes.hpp>
 #include <silkworm/db/snapshots/index_builder.hpp>
-#include <silkworm/db/snapshots/path.hpp>
+#include <silkworm/db/snapshots/snapshot_path.hpp>
 #include <silkworm/infra/common/memory_mapped_file.hpp>
 
 namespace silkworm::snapshots {

diff --git a/silkworm/db/blocks/headers/header_index.hpp b/silkworm/db/blocks/headers/header_index.hpp
@@ -22,7 +22,7 @@
 
 #include <silkworm/core/common/bytes.hpp>
 #include <silkworm/db/snapshots/index_builder.hpp>
-#include <silkworm/db/snapshots/path.hpp>
+#include <silkworm/db/snapshots/snapshot_path.hpp>
 #include <silkworm/infra/common/memory_mapped_file.hpp>
 
 namespace silkworm::snapshots {

diff --git a/silkworm/db/freezer.cpp b/silkworm/db/freezer.cpp
@@ -30,8 +30,8 @@
 #include "blocks/headers/header_snapshot_freezer.hpp"
 #include "prune_mode.hpp"
 #include "snapshot_freezer.hpp"
-#include "snapshots/path.hpp"
 #include "snapshots/snapshot_bundle.hpp"
+#include "snapshots/snapshot_path.hpp"
 #include "snapshots/snapshot_writer.hpp"
 #include "transactions/txn_snapshot_freezer.hpp"
 

diff --git a/silkworm/db/freezer.hpp b/silkworm/db/freezer.hpp
@@ -18,7 +18,7 @@
 
 #include "data_migration.hpp"
 #include "mdbx/mdbx.hpp"
-#include "snapshots/repository.hpp"
+#include "snapshots/snapshot_repository.hpp"
 #include "stage_scheduler.hpp"
 
 namespace silkworm::db {

diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp
@@ -20,7 +20,7 @@
 
 #include <silkworm/db/blocks/bodies/body_index.hpp>
 #include <silkworm/db/blocks/headers/header_index.hpp>
-#include <silkworm/db/snapshots/path.hpp>
+#include <silkworm/db/snapshots/snapshot_path.hpp>
 #include <silkworm/db/transactions/txn_index.hpp>
 #include <silkworm/db/transactions/txn_to_block_index.hpp>
 

diff --git a/silkworm/db/snapshot_merger.cpp b/silkworm/db/snapshot_merger.cpp
@@ -24,9 +24,9 @@
 #include <silkworm/infra/common/filesystem.hpp>
 #include <silkworm/infra/common/log.hpp>
 
-#include "snapshots/path.hpp"
 #include "snapshots/seg/compressor.hpp"
 #include "snapshots/snapshot_bundle.hpp"
+#include "snapshots/snapshot_path.hpp"
 #include "snapshots/snapshot_writer.hpp"
 
 namespace silkworm::db {

diff --git a/silkworm/db/snapshot_merger.hpp b/silkworm/db/snapshot_merger.hpp
@@ -17,7 +17,7 @@
 #pragma once
 
 #include "data_migration.hpp"
-#include "snapshots/repository.hpp"
+#include "snapshots/snapshot_repository.hpp"
 
 namespace silkworm::db {
 

diff --git a/silkworm/db/snapshot_recompress.cpp b/silkworm/db/snapshot_recompress.cpp
@@ -21,7 +21,7 @@
 
 #include "blocks/bodies/body_snapshot.hpp"
 #include "blocks/headers/header_snapshot.hpp"
-#include "snapshots/path.hpp"
+#include "snapshots/snapshot_path.hpp"
 #include "transactions/txn_snapshot.hpp"
 
 namespace silkworm::snapshots {

diff --git a/silkworm/db/snapshot_repository_test.cpp b/silkworm/db/snapshot_repository_test.cpp
@@ -23,7 +23,7 @@
 #include <silkworm/db/blocks/headers/header_index.hpp>
 #include <silkworm/db/snapshot_bundle_factory_impl.hpp>
 #include <silkworm/db/snapshots/index_builder.hpp>
-#include <silkworm/db/snapshots/repository.hpp>
+#include <silkworm/db/snapshots/snapshot_repository.hpp>
 #include <silkworm/db/test_util/temp_snapshots.hpp>
 #include <silkworm/db/transactions/txn_index.hpp>
 #include <silkworm/db/transactions/txn_queries.hpp>

diff --git a/silkworm/db/snapshot_sync.cpp b/silkworm/db/snapshot_sync.cpp
@@ -27,7 +27,7 @@
 #include <silkworm/db/mdbx/etl_mdbx_collector.hpp>
 #include <silkworm/db/snapshots/config.hpp>
 #include <silkworm/db/snapshots/index_builder.hpp>
-#include <silkworm/db/snapshots/path.hpp>
+#include <silkworm/db/snapshots/snapshot_path.hpp>
 #include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/ensure.hpp>
 #include <silkworm/infra/common/environment.hpp>
@@ -93,18 +93,6 @@ bool SnapshotSync::download_snapshots(const std::vector<std::string>& snapshot_f
         SILK_WARN << "SnapshotSync: downloading missing snapshots";
     }
 
-    for (auto [block_from, block_to] : missing_block_ranges) {
-        for (const auto type : magic_enum::enum_values<SnapshotType>()) {
-            auto snapshot_path = SnapshotPath::from(repository_->path(), kSnapshotV1, block_from, block_to, type);
-            if (!snapshot_path.torrent_file_needed()) {
-                continue;
-            }
-            SILK_INFO << "SnapshotSync: seeding a new snapshot [DOING NOTHING NOW]";
-            // TODO(canepat) create torrent file
-            // client.add_torrent(snapshot_path.create_torrent_file());
-        }
-    }
-
     const auto snapshot_config = Config::lookup_known_config(config_.chain_id, snapshot_file_names);
     if (snapshot_config.preverified_snapshots().empty()) {
         SILK_ERROR << "SnapshotSync: no preverified snapshots found";
@@ -156,9 +144,9 @@ bool SnapshotSync::download_snapshots(const std::vector<std::string>& snapshot_f
     client_thread_ = std::thread([&]() {
         log::set_thread_name("bit-torrent");
         try {
-            client_.execute_loop();
+            client_.execution_loop();
         } catch (const std::exception& ex) {
-            SILK_CRIT << "SnapshotSync: BitTorrentClient execute_loop exception: " << ex.what();
+            SILK_CRIT << "SnapshotSync: BitTorrentClient execution_loop exception: " << ex.what();
             std::terminate();
         }
     });

diff --git a/silkworm/db/snapshot_sync.hpp b/silkworm/db/snapshot_sync.hpp
@@ -23,8 +23,8 @@
 #include <silkworm/core/chain/config.hpp>
 #include <silkworm/db/access_layer.hpp>
 #include <silkworm/db/snapshots/bittorrent/client.hpp>
-#include <silkworm/db/snapshots/repository.hpp>
-#include <silkworm/db/snapshots/settings.hpp>
+#include <silkworm/db/snapshots/snapshot_repository.hpp>
+#include <silkworm/db/snapshots/snapshot_settings.hpp>
 #include <silkworm/infra/concurrency/stoppable.hpp>
 
 namespace silkworm::db {