From ffb2efcd616554a49af075be80b85caa771f2902 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Tue, 5 Nov 2024 12:13:39 +0100 Subject: [PATCH 1/7] rename snapshot_repository variables to repository --- cmd/capi/execute.cpp | 10 +++---- cmd/dev/snapshots.cpp | 48 ++++++++++++++++---------------- silkworm/capi/fork_validator.cpp | 2 +- silkworm/capi/instance.hpp | 2 +- silkworm/capi/rpcdaemon.cpp | 2 +- silkworm/capi/silkworm.cpp | 18 ++++++------ silkworm/rpc/daemon.cpp | 6 ++-- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/cmd/capi/execute.cpp b/cmd/capi/execute.cpp index e020d4cbd7..465364e772 100644 --- a/cmd/capi/execute.cpp +++ b/cmd/capi/execute.cpp @@ -145,12 +145,12 @@ const char* make_path(const snapshots::SnapshotPath& p) { return path; } -std::vector collect_all_snapshots(const SnapshotRepository& snapshot_repository) { +std::vector collect_all_snapshots(const SnapshotRepository& repository) { std::vector headers_snapshot_sequence; std::vector bodies_snapshot_sequence; std::vector transactions_snapshot_sequence; - for (const auto& bundle_ptr : snapshot_repository.view_bundles()) { + for (const auto& bundle_ptr : repository.view_bundles()) { const auto& bundle = *bundle_ptr; { { @@ -206,9 +206,9 @@ std::vector collect_all_snapshots(const SnapshotRepositor } } - ensure(headers_snapshot_sequence.size() == snapshot_repository.bundles_count(), "invalid header snapshot count"); - ensure(bodies_snapshot_sequence.size() == snapshot_repository.bundles_count(), "invalid body snapshot count"); - ensure(transactions_snapshot_sequence.size() == snapshot_repository.bundles_count(), "invalid tx snapshot count"); + ensure(headers_snapshot_sequence.size() == repository.bundles_count(), "invalid header snapshot count"); + ensure(bodies_snapshot_sequence.size() == repository.bundles_count(), "invalid body snapshot count"); + ensure(transactions_snapshot_sequence.size() == repository.bundles_count(), "invalid tx snapshot count"); std::vector snapshot_sequence; snapshot_sequence.reserve(headers_snapshot_sequence.size()); diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index f546c2ea39..0b6b52a932 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -324,11 +324,11 @@ BodyCounters count_bodies_in_one(const SnapshotSubcommandSettings& settings, con } BodyCounters count_bodies_in_all(const SnapshotSubcommandSettings& settings) { - auto snapshot_repository = make_repository(settings.settings); - snapshot_repository.reopen_folder(); + auto repository = make_repository(settings.settings); + repository.reopen_folder(); int num_bodies = 0; uint64_t num_txns = 0; - for (const auto& bundle_ptr : snapshot_repository.view_bundles()) { + for (const auto& bundle_ptr : repository.view_bundles()) { const auto& bundle = *bundle_ptr; const auto [body_count, txn_count] = count_bodies_in_one(settings, bundle.body_segment); num_bodies += body_count; @@ -374,10 +374,10 @@ int count_headers_in_one(const SnapshotSubcommandSettings& settings, const Segme } int count_headers_in_all(const SnapshotSubcommandSettings& settings) { - auto snapshot_repository = make_repository(settings.settings); - snapshot_repository.reopen_folder(); + auto repository = make_repository(settings.settings); + repository.reopen_folder(); int num_headers{0}; - for (const auto& bundle_ptr : snapshot_repository.view_bundles()) { + for (const auto& bundle_ptr : repository.view_bundles()) { const auto& bundle = *bundle_ptr; const auto header_count = count_headers_in_one(settings, bundle.header_segment); num_headers += header_count; @@ -721,9 +721,9 @@ void lookup_header_by_hash(const SnapshotSubcommandSettings& settings) { std::optional matching_snapshot_path; std::optional matching_header; - auto snapshot_repository = make_repository(settings.settings); - snapshot_repository.reopen_folder(); - for (const auto& bundle_ptr : snapshot_repository.view_bundles_reverse()) { + auto repository = make_repository(settings.settings); + repository.reopen_folder(); + for (const auto& bundle_ptr : repository.view_bundles_reverse()) { const auto& bundle = *bundle_ptr; auto segment_and_index = bundle.segment_and_index(SnapshotType::headers); const auto header = HeaderFindByHashQuery{segment_and_index}.exec(*hash); @@ -751,9 +751,9 @@ void lookup_header_by_number(const SnapshotSubcommandSettings& settings) { SILK_INFO << "Lookup header number: " << block_number; std::chrono::time_point start{std::chrono::steady_clock::now()}; - auto snapshot_repository = make_repository(settings.settings); - snapshot_repository.reopen_folder(); - const auto [segment_and_index, _] = snapshot_repository.find_segment(SnapshotType::headers, block_number); + auto repository = make_repository(settings.settings); + repository.reopen_folder(); + const auto [segment_and_index, _] = repository.find_segment(SnapshotType::headers, block_number); if (segment_and_index) { const auto header = HeaderFindByBlockNumQuery{*segment_and_index}.exec(block_number); ensure(header.has_value(), @@ -811,11 +811,11 @@ void lookup_body_in_one(const SnapshotSubcommandSettings& settings, BlockNum blo } void lookup_body_in_all(const SnapshotSubcommandSettings& settings, BlockNum block_number) { - auto snapshot_repository = make_repository(settings.settings); - snapshot_repository.reopen_folder(); + auto repository = make_repository(settings.settings); + repository.reopen_folder(); std::chrono::time_point start{std::chrono::steady_clock::now()}; - const auto [segment_and_index, _] = snapshot_repository.find_segment(SnapshotType::bodies, block_number); + const auto [segment_and_index, _] = repository.find_segment(SnapshotType::bodies, block_number); if (segment_and_index) { const auto body = BodyFindByBlockNumQuery{*segment_and_index}.exec(block_number); ensure(body.has_value(), @@ -918,12 +918,12 @@ void lookup_txn_by_hash_in_one(const SnapshotSubcommandSettings& settings, const } void lookup_txn_by_hash_in_all(const SnapshotSubcommandSettings& settings, const Hash& hash) { - auto snapshot_repository = make_repository(settings.settings); - snapshot_repository.reopen_folder(); + auto repository = make_repository(settings.settings); + repository.reopen_folder(); std::optional matching_snapshot_path; std::chrono::time_point start{std::chrono::steady_clock::now()}; - for (const auto& bundle_ptr : snapshot_repository.view_bundles_reverse()) { + for (const auto& bundle_ptr : repository.view_bundles_reverse()) { const auto& bundle = *bundle_ptr; auto segment_and_index = bundle.segment_and_index(SnapshotType::transactions); const auto transaction = TransactionFindByHashQuery{segment_and_index}.exec(hash); @@ -983,12 +983,12 @@ void lookup_txn_by_id_in_one(const SnapshotSubcommandSettings& settings, uint64_ } void lookup_txn_by_id_in_all(const SnapshotSubcommandSettings& settings, uint64_t txn_id) { - auto snapshot_repository = make_repository(settings.settings); - snapshot_repository.reopen_folder(); + auto repository = make_repository(settings.settings); + repository.reopen_folder(); std::optional matching_snapshot_path; std::chrono::time_point start{std::chrono::steady_clock::now()}; - for (const auto& bundle_ptr : snapshot_repository.view_bundles_reverse()) { + for (const auto& bundle_ptr : repository.view_bundles_reverse()) { const auto& bundle = *bundle_ptr; auto segment_and_index = bundle.segment_and_index(SnapshotType::transactions); const auto transaction = TransactionFindByIdQuery{segment_and_index}.exec(txn_id); @@ -1029,10 +1029,10 @@ void lookup_transaction(const SnapshotSubcommandSettings& settings) { } void merge(const SnapshotSettings& settings) { - auto snapshot_repository = make_repository(settings); - snapshot_repository.reopen_folder(); + auto repository = make_repository(settings); + repository.reopen_folder(); TemporaryDirectory tmp_dir; - db::SnapshotMerger merger{snapshot_repository, tmp_dir.path()}; + db::SnapshotMerger merger{repository, tmp_dir.path()}; test_util::TaskRunner runner; runner.run(merger.exec()); } diff --git a/silkworm/capi/fork_validator.cpp b/silkworm/capi/fork_validator.cpp index 96232c68cf..85b3da8d38 100644 --- a/silkworm/capi/fork_validator.cpp +++ b/silkworm/capi/fork_validator.cpp @@ -126,7 +126,7 @@ SILKWORM_EXPORT int silkworm_start_fork_validator(SilkwormHandle handle, MDBX_en silkworm::db::EnvUnmanaged unmanaged_env{mdbx_env}; silkworm::db::RWAccess rw_access{unmanaged_env}; - silkworm::db::DataStoreRef data_store{rw_access, *handle->snapshot_repository}; + silkworm::db::DataStoreRef data_store{rw_access, *handle->repository}; silkworm::db::DataModelFactory data_model_factory{std::move(data_store)}; handle->execution_engine = std::make_unique( diff --git a/silkworm/capi/instance.hpp b/silkworm/capi/instance.hpp index e547184f33..6a9bb53f92 100644 --- a/silkworm/capi/instance.hpp +++ b/silkworm/capi/instance.hpp @@ -33,7 +33,7 @@ struct SilkwormInstance { silkworm::concurrency::ContextPoolSettings context_pool_settings; std::filesystem::path data_dir_path; silkworm::NodeSettings node_settings; - std::unique_ptr snapshot_repository; + std::unique_ptr repository; std::unique_ptr rpcdaemon; std::unique_ptr execution_engine; diff --git a/silkworm/capi/rpcdaemon.cpp b/silkworm/capi/rpcdaemon.cpp index 4647fac918..63cb7f0762 100644 --- a/silkworm/capi/rpcdaemon.cpp +++ b/silkworm/capi/rpcdaemon.cpp @@ -103,7 +103,7 @@ SILKWORM_EXPORT int silkworm_start_rpcdaemon(SilkwormHandle handle, MDBX_env* en auto daemon_settings = make_daemon_settings(handle, *settings); db::DataStoreRef data_store{ db::RWAccess{db::EnvUnmanaged{env}}, - *handle->snapshot_repository, + *handle->repository, }; // Create the one-and-only Silkrpc daemon diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 8a41333969..a204c678cf 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -224,7 +224,7 @@ SILKWORM_EXPORT int silkworm_init(SilkwormHandle* handle, const struct SilkwormS log::Info{"Silkworm build info", log_args_for_version()}; // NOLINT(*-unused-raii) auto data_dir_path = parse_path(settings->data_dir_path); - auto snapshot_repository = db::blocks::make_blocks_repository( + auto repository = db::blocks::make_blocks_repository( DataDirectory{data_dir_path}.snapshots().path()); // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) @@ -235,7 +235,7 @@ SILKWORM_EXPORT int silkworm_init(SilkwormHandle* handle, const struct SilkwormS }, .data_dir_path = std::move(data_dir_path), .node_settings = {}, - .snapshot_repository = std::make_unique(std::move(snapshot_repository)), + .repository = std::make_unique(std::move(repository)), .rpcdaemon = {}, .execution_engine = {}, .sentry_thread = {}, @@ -337,7 +337,7 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc } SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSnapshot* snapshot) SILKWORM_NOEXCEPT { - if (!handle || !handle->snapshot_repository) { + if (!handle || !handle->repository) { return SILKWORM_INVALID_HANDLE; } if (!snapshot) { @@ -391,7 +391,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn .idx_txn_hash_2_block = std::move(idx_txn_hash_2_block), }, }; - handle->snapshot_repository->add_snapshot_bundle(std::move(bundle)); + handle->repository->add_snapshot_bundle(std::move(bundle)); return SILKWORM_OK; } @@ -489,7 +489,7 @@ int silkworm_execute_blocks_ephemeral(SilkwormHandle handle, MDBX_txn* mdbx_txn, try { auto txn = db::RWTxnUnmanaged{mdbx_txn}; - db::Buffer state_buffer{txn, std::make_unique(db::DataModel{txn, *handle->snapshot_repository})}; + db::Buffer state_buffer{txn, std::make_unique(db::DataModel{txn, *handle->repository})}; state_buffer.set_memory_limit(batch_size); const size_t max_batch_size{batch_size}; @@ -498,7 +498,7 @@ int silkworm_execute_blocks_ephemeral(SilkwormHandle handle, MDBX_txn* mdbx_txn, BlockNum block_number{start_block}; BlockNum batch_start_block_number{start_block}; BlockNum last_block_number = 0; - db::DataModel da_layer{txn, *handle->snapshot_repository}; + db::DataModel da_layer{txn, *handle->repository}; AnalysisCache analysis_cache{execution::block::BlockExecutor::kDefaultAnalysisCacheSize}; execution::block::BlockExecutor block_executor{*chain_info, write_receipts, write_call_traces, write_change_sets}; @@ -609,13 +609,13 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, auto txn = rw_access.start_rw_tx(); const auto env_path = unmanaged_env.get_path(); - db::Buffer state_buffer{txn, std::make_unique(db::DataModel{txn, *handle->snapshot_repository})}; + db::Buffer state_buffer{txn, std::make_unique(db::DataModel{txn, *handle->repository})}; state_buffer.set_memory_limit(batch_size); BoundedBuffer> block_buffer{kMaxBlockBufferSize}; [[maybe_unused]] auto _ = gsl::finally([&block_buffer] { block_buffer.terminate_and_release_all(); }); - db::DataStoreRef data_store{rw_access, *handle->snapshot_repository}; + db::DataStoreRef data_store{rw_access, *handle->repository}; db::DataModelFactory data_model_factory{std::move(data_store)}; BlockProvider block_provider{ @@ -718,7 +718,7 @@ SILKWORM_EXPORT int silkworm_fini(SilkwormHandle handle) SILKWORM_NOEXCEPT { if (!handle) { return SILKWORM_INVALID_HANDLE; } - if (!handle->snapshot_repository) { + if (!handle->repository) { return SILKWORM_INVALID_HANDLE; } delete handle; diff --git a/silkworm/rpc/daemon.cpp b/silkworm/rpc/daemon.cpp index 6c1ce1cbec..efb24173c2 100644 --- a/silkworm/rpc/daemon.cpp +++ b/silkworm/rpc/daemon.cpp @@ -137,12 +137,12 @@ int Daemon::run(const DaemonSettings& settings) { data_dir.snapshots().path(), }); - auto& snapshot_repository = data_store->ref().repository; - snapshot_repository.reopen_folder(); + auto& repository = data_store->ref().repository; + repository.reopen_folder(); // At startup check that chain configuration is valid db::ROTxnManaged ro_txn = data_store->chaindata().start_ro_tx(); - db::DataModel data_access{ro_txn, snapshot_repository}; + db::DataModel data_access{ro_txn, repository}; if (const auto chain_config{data_access.read_chain_config()}; !chain_config) { throw std::runtime_error{"invalid chain configuration"}; } From 4eee245953d27f59afec9c6c63b71b27fcf0a240 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Tue, 5 Nov 2024 12:16:30 +0100 Subject: [PATCH 2/7] move db/transactions into db/blocks --- cmd/dev/snapshots.cpp | 6 +++--- silkworm/capi/silkworm.cpp | 4 ++-- silkworm/capi/silkworm_test.cpp | 4 ++-- silkworm/db/access_layer.cpp | 2 +- silkworm/db/{ => blocks}/transactions/txn_index.cpp | 0 silkworm/db/{ => blocks}/transactions/txn_index.hpp | 0 silkworm/db/{ => blocks}/transactions/txn_queries.hpp | 0 silkworm/db/{ => blocks}/transactions/txn_segment.hpp | 0 .../{ => blocks}/transactions/txn_segment_collation.cpp | 0 .../{ => blocks}/transactions/txn_segment_collation.hpp | 0 .../{ => blocks}/transactions/txn_segment_word_codec.cpp | 0 .../{ => blocks}/transactions/txn_segment_word_codec.hpp | 0 .../db/{ => blocks}/transactions/txn_to_block_index.cpp | 0 .../db/{ => blocks}/transactions/txn_to_block_index.hpp | 0 .../db/{ => blocks}/transactions/txs_and_bodies_query.cpp | 0 .../db/{ => blocks}/transactions/txs_and_bodies_query.hpp | 0 silkworm/db/freezer.cpp | 2 +- silkworm/db/snapshot_benchmark.cpp | 4 ++-- silkworm/db/snapshot_bundle_factory_impl.cpp | 4 ++-- silkworm/db/snapshot_index_builder_test.cpp | 4 ++-- silkworm/db/snapshot_recompress.cpp | 2 +- silkworm/db/snapshot_repository_test.cpp | 6 +++--- silkworm/db/snapshot_sync_test.cpp | 4 ++-- silkworm/db/snapshot_test.cpp | 8 ++++---- 24 files changed, 25 insertions(+), 25 deletions(-) rename silkworm/db/{ => blocks}/transactions/txn_index.cpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_index.hpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_queries.hpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_segment.hpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_segment_collation.cpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_segment_collation.hpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_segment_word_codec.cpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_segment_word_codec.hpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_to_block_index.cpp (100%) rename silkworm/db/{ => blocks}/transactions/txn_to_block_index.hpp (100%) rename silkworm/db/{ => blocks}/transactions/txs_and_bodies_query.cpp (100%) rename silkworm/db/{ => blocks}/transactions/txs_and_bodies_query.hpp (100%) diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index 0b6b52a932..ddc13281bf 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -39,6 +39,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -52,9 +55,6 @@ #include #include #include -#include -#include -#include #include #include #include diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index a204c678cf..62e412b9de 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -37,12 +37,12 @@ #include #include #include +#include +#include #include #include #include #include -#include -#include #include #include #include diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index bd24554fa7..f3964b884f 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -25,12 +25,12 @@ #include #include #include +#include +#include #include #include #include #include -#include -#include #include #include #include diff --git a/silkworm/db/access_layer.cpp b/silkworm/db/access_layer.cpp index c09ea8f237..1afd569289 100644 --- a/silkworm/db/access_layer.cpp +++ b/silkworm/db/access_layer.cpp @@ -25,11 +25,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include diff --git a/silkworm/db/transactions/txn_index.cpp b/silkworm/db/blocks/transactions/txn_index.cpp similarity index 100% rename from silkworm/db/transactions/txn_index.cpp rename to silkworm/db/blocks/transactions/txn_index.cpp diff --git a/silkworm/db/transactions/txn_index.hpp b/silkworm/db/blocks/transactions/txn_index.hpp similarity index 100% rename from silkworm/db/transactions/txn_index.hpp rename to silkworm/db/blocks/transactions/txn_index.hpp diff --git a/silkworm/db/transactions/txn_queries.hpp b/silkworm/db/blocks/transactions/txn_queries.hpp similarity index 100% rename from silkworm/db/transactions/txn_queries.hpp rename to silkworm/db/blocks/transactions/txn_queries.hpp diff --git a/silkworm/db/transactions/txn_segment.hpp b/silkworm/db/blocks/transactions/txn_segment.hpp similarity index 100% rename from silkworm/db/transactions/txn_segment.hpp rename to silkworm/db/blocks/transactions/txn_segment.hpp diff --git a/silkworm/db/transactions/txn_segment_collation.cpp b/silkworm/db/blocks/transactions/txn_segment_collation.cpp similarity index 100% rename from silkworm/db/transactions/txn_segment_collation.cpp rename to silkworm/db/blocks/transactions/txn_segment_collation.cpp diff --git a/silkworm/db/transactions/txn_segment_collation.hpp b/silkworm/db/blocks/transactions/txn_segment_collation.hpp similarity index 100% rename from silkworm/db/transactions/txn_segment_collation.hpp rename to silkworm/db/blocks/transactions/txn_segment_collation.hpp diff --git a/silkworm/db/transactions/txn_segment_word_codec.cpp b/silkworm/db/blocks/transactions/txn_segment_word_codec.cpp similarity index 100% rename from silkworm/db/transactions/txn_segment_word_codec.cpp rename to silkworm/db/blocks/transactions/txn_segment_word_codec.cpp diff --git a/silkworm/db/transactions/txn_segment_word_codec.hpp b/silkworm/db/blocks/transactions/txn_segment_word_codec.hpp similarity index 100% rename from silkworm/db/transactions/txn_segment_word_codec.hpp rename to silkworm/db/blocks/transactions/txn_segment_word_codec.hpp diff --git a/silkworm/db/transactions/txn_to_block_index.cpp b/silkworm/db/blocks/transactions/txn_to_block_index.cpp similarity index 100% rename from silkworm/db/transactions/txn_to_block_index.cpp rename to silkworm/db/blocks/transactions/txn_to_block_index.cpp diff --git a/silkworm/db/transactions/txn_to_block_index.hpp b/silkworm/db/blocks/transactions/txn_to_block_index.hpp similarity index 100% rename from silkworm/db/transactions/txn_to_block_index.hpp rename to silkworm/db/blocks/transactions/txn_to_block_index.hpp diff --git a/silkworm/db/transactions/txs_and_bodies_query.cpp b/silkworm/db/blocks/transactions/txs_and_bodies_query.cpp similarity index 100% rename from silkworm/db/transactions/txs_and_bodies_query.cpp rename to silkworm/db/blocks/transactions/txs_and_bodies_query.cpp diff --git a/silkworm/db/transactions/txs_and_bodies_query.hpp b/silkworm/db/blocks/transactions/txs_and_bodies_query.hpp similarity index 100% rename from silkworm/db/transactions/txs_and_bodies_query.hpp rename to silkworm/db/blocks/transactions/txs_and_bodies_query.hpp diff --git a/silkworm/db/freezer.cpp b/silkworm/db/freezer.cpp index c55cddf68b..047084661b 100644 --- a/silkworm/db/freezer.cpp +++ b/silkworm/db/freezer.cpp @@ -29,12 +29,12 @@ #include "blocks/bodies/body_queries.hpp" #include "blocks/bodies/body_segment_collation.hpp" #include "blocks/headers/header_segment_collation.hpp" +#include "blocks/transactions/txn_segment_collation.hpp" #include "datastore/segment_collation.hpp" #include "datastore/snapshots/common/snapshot_path.hpp" #include "datastore/snapshots/segment/segment_writer.hpp" #include "datastore/snapshots/snapshot_bundle.hpp" #include "prune_mode.hpp" -#include "transactions/txn_segment_collation.hpp" namespace silkworm::db { diff --git a/silkworm/db/snapshot_benchmark.cpp b/silkworm/db/snapshot_benchmark.cpp index 9f9a770c75..f7a7450c2b 100644 --- a/silkworm/db/snapshot_benchmark.cpp +++ b/silkworm/db/snapshot_benchmark.cpp @@ -20,11 +20,11 @@ #include #include #include +#include +#include #include #include #include -#include -#include #include #include #include diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp index aa57f4d4bd..c711fecfa1 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.cpp +++ b/silkworm/db/snapshot_bundle_factory_impl.cpp @@ -19,9 +19,9 @@ #include #include #include +#include +#include #include -#include -#include namespace silkworm::db { diff --git a/silkworm/db/snapshot_index_builder_test.cpp b/silkworm/db/snapshot_index_builder_test.cpp index 11c41faa12..002c7be136 100644 --- a/silkworm/db/snapshot_index_builder_test.cpp +++ b/silkworm/db/snapshot_index_builder_test.cpp @@ -19,10 +19,10 @@ #include #include +#include +#include #include #include -#include -#include #include #include #include diff --git a/silkworm/db/snapshot_recompress.cpp b/silkworm/db/snapshot_recompress.cpp index ff24d64974..1f0e99a328 100644 --- a/silkworm/db/snapshot_recompress.cpp +++ b/silkworm/db/snapshot_recompress.cpp @@ -21,8 +21,8 @@ #include "blocks/bodies/body_segment.hpp" #include "blocks/headers/header_segment.hpp" +#include "blocks/transactions/txn_segment.hpp" #include "datastore/snapshots/common/snapshot_path.hpp" -#include "transactions/txn_segment.hpp" namespace silkworm::snapshots { diff --git a/silkworm/db/snapshot_repository_test.cpp b/silkworm/db/snapshot_repository_test.cpp index 2b2ccf5e55..f5d8aabcea 100644 --- a/silkworm/db/snapshot_repository_test.cpp +++ b/silkworm/db/snapshot_repository_test.cpp @@ -22,12 +22,12 @@ #include #include #include +#include +#include +#include #include #include #include -#include -#include -#include #include #include #include diff --git a/silkworm/db/snapshot_sync_test.cpp b/silkworm/db/snapshot_sync_test.cpp index 2097ee5bca..673948ccd1 100644 --- a/silkworm/db/snapshot_sync_test.cpp +++ b/silkworm/db/snapshot_sync_test.cpp @@ -21,10 +21,10 @@ #include #include #include +#include +#include #include #include -#include -#include #include #include #include diff --git a/silkworm/db/snapshot_test.cpp b/silkworm/db/snapshot_test.cpp index 5ff110120d..2e7bf120b0 100644 --- a/silkworm/db/snapshot_test.cpp +++ b/silkworm/db/snapshot_test.cpp @@ -27,13 +27,13 @@ #include "blocks/bodies/body_queries.hpp" #include "blocks/headers/header_index.hpp" #include "blocks/headers/header_queries.hpp" +#include "blocks/transactions/txn_index.hpp" +#include "blocks/transactions/txn_queries.hpp" +#include "blocks/transactions/txn_segment_word_codec.hpp" +#include "blocks/transactions/txn_to_block_index.hpp" #include "datastore/snapshots/index_builder.hpp" #include "datastore/snapshots/segment/segment_reader.hpp" #include "test_util/temp_snapshots.hpp" -#include "transactions/txn_index.hpp" -#include "transactions/txn_queries.hpp" -#include "transactions/txn_segment_word_codec.hpp" -#include "transactions/txn_to_block_index.hpp" namespace silkworm::snapshots { From f4c817b511a640f7f2d9a4ceffa4ea2a223d1042 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Tue, 5 Nov 2024 12:47:26 +0100 Subject: [PATCH 3/7] make opened bundles by default --- silkworm/capi/silkworm.cpp | 1 + silkworm/db/datastore/snapshot_merger.cpp | 5 ++--- .../db/datastore/snapshots/snapshot_bundle.cpp | 2 +- .../db/datastore/snapshots/snapshot_bundle.hpp | 8 +++++--- .../snapshots/snapshot_bundle_factory.hpp | 3 ++- .../datastore/snapshots/snapshot_repository.cpp | 7 ++----- silkworm/db/freezer.cpp | 5 ++--- silkworm/db/snapshot_bundle_factory_impl.cpp | 15 +++++++++++++-- silkworm/db/snapshot_bundle_factory_impl.hpp | 3 ++- silkworm/db/snapshot_sync.cpp | 2 +- silkworm/db/snapshot_sync_test.cpp | 1 + 11 files changed, 32 insertions(+), 20 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 62e412b9de..60ba6215a5 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -390,6 +390,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn .idx_txn_hash = std::move(idx_txn_hash), .idx_txn_hash_2_block = std::move(idx_txn_hash_2_block), }, + true, }; handle->repository->add_snapshot_bundle(std::move(bundle)); return SILKWORM_OK; diff --git a/silkworm/db/datastore/snapshot_merger.cpp b/silkworm/db/datastore/snapshot_merger.cpp index 31c1a35714..d5d2967778 100644 --- a/silkworm/db/datastore/snapshot_merger.cpp +++ b/silkworm/db/datastore/snapshot_merger.cpp @@ -95,9 +95,8 @@ std::shared_ptr SnapshotMerger::migrate(std::unique_ptr SnapshotBundle::files() { return files; } -std::vector SnapshotBundle::snapshot_paths() { +std::vector SnapshotBundle::segment_paths() { std::vector paths; paths.reserve(kSnapshotsCount); diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp index d222e63d2a..f10fdf3fc0 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp @@ -50,9 +50,11 @@ struct SnapshotBundleData { }; struct SnapshotBundle : public SnapshotBundleData { - explicit SnapshotBundle(StepRange step_range, SnapshotBundleData bundle) + explicit SnapshotBundle(StepRange step_range, SnapshotBundleData bundle, bool open) : SnapshotBundleData{std::move(bundle)}, - step_range_{step_range} {} + step_range_{step_range} { + if (open) reopen(); + } virtual ~SnapshotBundle(); SnapshotBundle(SnapshotBundle&&) = default; @@ -128,7 +130,7 @@ struct SnapshotBundle : public SnapshotBundleData { StepRange step_range() const { return step_range_; } std::vector files(); - std::vector snapshot_paths(); + std::vector segment_paths(); void reopen(); void close(); diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp index b335cdf7c8..155a302bb1 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp @@ -31,8 +31,9 @@ struct SnapshotBundleFactory { virtual ~SnapshotBundleFactory() = default; using PathByTypeProvider = std::function; - virtual SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const = 0; + virtual SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path, bool open) const = 0; virtual SnapshotBundle make(const std::filesystem::path& dir_path, StepRange range) const = 0; + virtual SnapshotBundle make_paths(const std::filesystem::path& dir_path, StepRange range) const = 0; virtual std::vector> index_builders(const SnapshotPath& segment_path) const = 0; virtual std::vector> index_builders(const SnapshotPathList& segment_paths) const = 0; diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index fc085e5267..4732c44639 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -42,8 +42,6 @@ void SnapshotRepository::add_snapshot_bundle(SnapshotBundle bundle) { } void SnapshotRepository::replace_snapshot_bundles(SnapshotBundle bundle) { - bundle.reopen(); - std::scoped_lock lock(*bundles_mutex_); // copy bundles prior to modification auto bundles = std::make_shared(*bundles_); @@ -148,8 +146,7 @@ void SnapshotRepository::reopen_folder() { auto index_path = [&](SnapshotType type) { return all_index_paths[groups[num][true][type]]; }; - SnapshotBundle bundle = bundle_factory_->make(snapshot_path, index_path); - bundle.reopen(); + SnapshotBundle bundle = bundle_factory_->make(snapshot_path, index_path, true); bundles->insert_or_assign(num, std::make_shared(std::move(bundle))); } @@ -244,7 +241,7 @@ void SnapshotRepository::remove_stale_indexes() const { } void SnapshotRepository::build_indexes(SnapshotBundle& bundle) const { - for (auto& builder : bundle_factory_->index_builders(bundle.snapshot_paths())) { + for (auto& builder : bundle_factory_->index_builders(bundle.segment_paths())) { builder->build(); } } diff --git a/silkworm/db/freezer.cpp b/silkworm/db/freezer.cpp index 047084661b..d85ba9fe4e 100644 --- a/silkworm/db/freezer.cpp +++ b/silkworm/db/freezer.cpp @@ -110,9 +110,8 @@ std::shared_ptr Freezer::migrate(std::unique_ptr> SnapshotBundleFactoryImpl::index_builders(const SnapshotPath& segment_path) const { diff --git a/silkworm/db/snapshot_bundle_factory_impl.hpp b/silkworm/db/snapshot_bundle_factory_impl.hpp index f742264017..0f5ff2585d 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.hpp +++ b/silkworm/db/snapshot_bundle_factory_impl.hpp @@ -23,8 +23,9 @@ namespace silkworm::db { struct SnapshotBundleFactoryImpl : public snapshots::SnapshotBundleFactory { ~SnapshotBundleFactoryImpl() override = default; - snapshots::SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const override; + snapshots::SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path, bool open) const override; snapshots::SnapshotBundle make(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; + snapshots::SnapshotBundle make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; std::vector> index_builders(const snapshots::SnapshotPath& segment_path) const override; std::vector> index_builders(const snapshots::SnapshotPathList& segment_paths) const override; }; diff --git a/silkworm/db/snapshot_sync.cpp b/silkworm/db/snapshot_sync.cpp index 2d7f9d90cd..c030a58e6d 100644 --- a/silkworm/db/snapshot_sync.cpp +++ b/silkworm/db/snapshot_sync.cpp @@ -297,7 +297,7 @@ void SnapshotSync::seed_frozen_bundle(StepRange range) { } void SnapshotSync::seed_bundle(SnapshotBundle& bundle) { - for (auto& path : bundle.snapshot_paths()) { + for (auto& path : bundle.segment_paths()) { seed_snapshot(path); } } diff --git a/silkworm/db/snapshot_sync_test.cpp b/silkworm/db/snapshot_sync_test.cpp index 673948ccd1..37abe170cd 100644 --- a/silkworm/db/snapshot_sync_test.cpp +++ b/silkworm/db/snapshot_sync_test.cpp @@ -153,6 +153,7 @@ TEST_CASE("SnapshotSync::update_block_headers", "[db][snapshot][sync]") { .idx_txn_hash = std::move(idx_txn_hash), .idx_txn_hash_2_block = std::move(idx_txn_hash_2_block), }, + true, }; auto& repository = snapshot_sync.repository(); repository.add_snapshot_bundle(std::move(bundle)); From 2a2c63d1f52c6893ffb6f640b8eaa61225114fa4 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Tue, 5 Nov 2024 13:24:56 +0100 Subject: [PATCH 4/7] open repository by default --- cmd/capi/execute.cpp | 1 - cmd/dev/check_changes.cpp | 4 +--- cmd/dev/db_toolbox.cpp | 11 +++++++---- cmd/dev/snapshots.cpp | 8 -------- silkworm/capi/silkworm.cpp | 3 ++- silkworm/db/blocks/schema_config.cpp | 3 ++- silkworm/db/blocks/schema_config.hpp | 2 +- .../datastore/snapshots/snapshot_repository.cpp | 5 ++++- .../datastore/snapshots/snapshot_repository.hpp | 1 + silkworm/db/snapshot_benchmark.cpp | 12 +----------- silkworm/db/snapshot_repository_test.cpp | 15 +++++++-------- silkworm/infra/common/directories.cpp | 3 ++- .../node/stagedsync/stages/stage_bodies_test.cpp | 2 ++ .../node/stagedsync/stages/stage_headers_test.cpp | 4 +++- silkworm/node/stagedsync/stages_test.cpp | 3 +-- silkworm/rpc/daemon.cpp | 5 +---- silkworm/sync/internals/header_retrieval_test.cpp | 2 ++ silkworm/sync/messages/internal_message_test.cpp | 2 ++ silkworm/sync/sync_pos_test.cpp | 2 ++ 19 files changed, 41 insertions(+), 47 deletions(-) diff --git a/cmd/capi/execute.cpp b/cmd/capi/execute.cpp index 465364e772..888ca8e56d 100644 --- a/cmd/capi/execute.cpp +++ b/cmd/capi/execute.cpp @@ -294,7 +294,6 @@ int execute_blocks(SilkwormHandle handle, ExecuteBlocksSettings settings, const }; auto& repository = data_store.ref().repository; - repository.reopen_folder(); // Collect all snapshots auto all_chain_snapshots{collect_all_snapshots(repository)}; diff --git a/cmd/dev/check_changes.cpp b/cmd/dev/check_changes.cpp index 8a2f208ce0..a336b57c22 100644 --- a/cmd/dev/check_changes.cpp +++ b/cmd/dev/check_changes.cpp @@ -115,9 +115,7 @@ int main(int argc, char* argv[]) { throw std::runtime_error("Unable to retrieve chain config"); } - auto& repository = data_store.ref().repository; - repository.reopen_folder(); - db::DataModel access_layer{txn, repository}; + db::DataModel access_layer = db::DataModelFactory{data_store.ref()}(txn); AnalysisCache analysis_cache{/*max_size=*/5'000}; std::vector receipts; diff --git a/cmd/dev/db_toolbox.cpp b/cmd/dev/db_toolbox.cpp index defaa4b002..de8f3dcc69 100644 --- a/cmd/dev/db_toolbox.cpp +++ b/cmd/dev/db_toolbox.cpp @@ -2267,10 +2267,13 @@ void do_freeze(EnvConfig& config, const DataDirectory& data_dir, bool keep_block }; StageSchedulerAdapter stage_scheduler{data_store.chaindata_rw()}; - auto& repository = data_store.ref().repository; - repository.reopen_folder(); - - Freezer freezer{data_store.chaindata(), repository, stage_scheduler, data_dir.temp().path(), keep_blocks}; + Freezer freezer{ + data_store.chaindata(), + data_store.ref().repository, + stage_scheduler, + data_dir.temp().path(), + keep_blocks, + }; test_util::TaskRunner runner; runner.run(freezer.exec() || stage_scheduler.async_run("StageSchedulerAdapter")); diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index ddc13281bf..678ba11873 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -325,7 +325,6 @@ BodyCounters count_bodies_in_one(const SnapshotSubcommandSettings& settings, con BodyCounters count_bodies_in_all(const SnapshotSubcommandSettings& settings) { auto repository = make_repository(settings.settings); - repository.reopen_folder(); int num_bodies = 0; uint64_t num_txns = 0; for (const auto& bundle_ptr : repository.view_bundles()) { @@ -375,7 +374,6 @@ int count_headers_in_one(const SnapshotSubcommandSettings& settings, const Segme int count_headers_in_all(const SnapshotSubcommandSettings& settings) { auto repository = make_repository(settings.settings); - repository.reopen_folder(); int num_headers{0}; for (const auto& bundle_ptr : repository.view_bundles()) { const auto& bundle = *bundle_ptr; @@ -722,7 +720,6 @@ void lookup_header_by_hash(const SnapshotSubcommandSettings& settings) { std::optional matching_snapshot_path; std::optional matching_header; auto repository = make_repository(settings.settings); - repository.reopen_folder(); for (const auto& bundle_ptr : repository.view_bundles_reverse()) { const auto& bundle = *bundle_ptr; auto segment_and_index = bundle.segment_and_index(SnapshotType::headers); @@ -752,7 +749,6 @@ void lookup_header_by_number(const SnapshotSubcommandSettings& settings) { std::chrono::time_point start{std::chrono::steady_clock::now()}; auto repository = make_repository(settings.settings); - repository.reopen_folder(); const auto [segment_and_index, _] = repository.find_segment(SnapshotType::headers, block_number); if (segment_and_index) { const auto header = HeaderFindByBlockNumQuery{*segment_and_index}.exec(block_number); @@ -812,7 +808,6 @@ void lookup_body_in_one(const SnapshotSubcommandSettings& settings, BlockNum blo void lookup_body_in_all(const SnapshotSubcommandSettings& settings, BlockNum block_number) { auto repository = make_repository(settings.settings); - repository.reopen_folder(); std::chrono::time_point start{std::chrono::steady_clock::now()}; const auto [segment_and_index, _] = repository.find_segment(SnapshotType::bodies, block_number); @@ -919,7 +914,6 @@ void lookup_txn_by_hash_in_one(const SnapshotSubcommandSettings& settings, const void lookup_txn_by_hash_in_all(const SnapshotSubcommandSettings& settings, const Hash& hash) { auto repository = make_repository(settings.settings); - repository.reopen_folder(); std::optional matching_snapshot_path; std::chrono::time_point start{std::chrono::steady_clock::now()}; @@ -984,7 +978,6 @@ void lookup_txn_by_id_in_one(const SnapshotSubcommandSettings& settings, uint64_ void lookup_txn_by_id_in_all(const SnapshotSubcommandSettings& settings, uint64_t txn_id) { auto repository = make_repository(settings.settings); - repository.reopen_folder(); std::optional matching_snapshot_path; std::chrono::time_point start{std::chrono::steady_clock::now()}; @@ -1030,7 +1023,6 @@ void lookup_transaction(const SnapshotSubcommandSettings& settings) { void merge(const SnapshotSettings& settings) { auto repository = make_repository(settings); - repository.reopen_folder(); TemporaryDirectory tmp_dir; db::SnapshotMerger merger{repository, tmp_dir.path()}; test_util::TaskRunner runner; diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 60ba6215a5..46ad97ef5c 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -225,7 +225,8 @@ SILKWORM_EXPORT int silkworm_init(SilkwormHandle* handle, const struct SilkwormS auto data_dir_path = parse_path(settings->data_dir_path); auto repository = db::blocks::make_blocks_repository( - DataDirectory{data_dir_path}.snapshots().path()); + DataDirectory{data_dir_path}.snapshots().path(), + /* open = */ false); // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) *handle = new SilkwormInstance{ diff --git a/silkworm/db/blocks/schema_config.cpp b/silkworm/db/blocks/schema_config.cpp index 405033e296..399bb4caad 100644 --- a/silkworm/db/blocks/schema_config.cpp +++ b/silkworm/db/blocks/schema_config.cpp @@ -20,9 +20,10 @@ namespace silkworm::db::blocks { -snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path) { +snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path, bool open) { return snapshots::SnapshotRepository{ std::move(dir_path), + open, std::make_unique(), std::make_unique(), }; diff --git a/silkworm/db/blocks/schema_config.hpp b/silkworm/db/blocks/schema_config.hpp index 37abb6a971..8f3cd23b46 100644 --- a/silkworm/db/blocks/schema_config.hpp +++ b/silkworm/db/blocks/schema_config.hpp @@ -23,6 +23,6 @@ namespace silkworm::db::blocks { inline constexpr datastore::EntityName kBlocksRepositoryName{"Blocks"}; -snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path); +snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path, bool open = true); } // namespace silkworm::db::blocks diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 4732c44639..770d53d888 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -29,13 +29,16 @@ namespace fs = std::filesystem; SnapshotRepository::SnapshotRepository( std::filesystem::path dir_path, + bool open, std::unique_ptr step_converter, std::unique_ptr bundle_factory) : dir_path_(std::move(dir_path)), step_converter_(std::move(step_converter)), bundle_factory_(std::move(bundle_factory)), bundles_(std::make_shared()), - bundles_mutex_(std::make_unique()) {} + bundles_mutex_(std::make_unique()) { + if (open) reopen_folder(); +} void SnapshotRepository::add_snapshot_bundle(SnapshotBundle bundle) { replace_snapshot_bundles(std::move(bundle)); diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index d7a166f4e0..4e9df6f95d 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -47,6 +47,7 @@ class SnapshotRepository { public: SnapshotRepository( std::filesystem::path dir_path, + bool open, std::unique_ptr step_converter, std::unique_ptr bundle_factory); diff --git a/silkworm/db/snapshot_benchmark.cpp b/silkworm/db/snapshot_benchmark.cpp index f7a7450c2b..b026da115b 100644 --- a/silkworm/db/snapshot_benchmark.cpp +++ b/silkworm/db/snapshot_benchmark.cpp @@ -69,19 +69,12 @@ static void open_snapshot(benchmark::State& state) { } BENCHMARK(open_snapshot); -static SnapshotRepository make_repository(std::filesystem::path dir_path) { - return db::blocks::make_blocks_repository(std::move(dir_path)); -} - static void build_header_index(benchmark::State& state) { TemporaryDirectory tmp_dir; - auto repository = make_repository(tmp_dir.path()); // These sample snapshot files just contain data for block range [1'500'012, 1'500'013], hence current snapshot // file name format is not sufficient to support them (see checks commented out below) test::SampleHeaderSnapshotFile header_segment{tmp_dir.path()}; - test::SampleBodySnapshotFile body_segment{tmp_dir.path()}; - test::SampleTransactionSnapshotFile txn_segment{tmp_dir.path()}; for ([[maybe_unused]] auto _ : state) { auto header_index = HeaderIndex::make(header_segment.path()); @@ -93,7 +86,6 @@ BENCHMARK(build_header_index); static void build_body_index(benchmark::State& state) { TemporaryDirectory tmp_dir; - auto repository = make_repository(tmp_dir.path()); // These sample snapshot files just contain data for block range [1'500'012, 1'500'013], hence current snapshot // file name format is not sufficient to support them (see checks commented out below) @@ -109,7 +101,6 @@ BENCHMARK(build_body_index); static void build_tx_index(benchmark::State& state) { TemporaryDirectory tmp_dir; - auto repository = make_repository(tmp_dir.path()); // These sample snapshot files just contain data for block range [1'500'012, 1'500'013], hence current snapshot // file name format is not sufficient to support them (see checks commented out below) @@ -134,7 +125,6 @@ BENCHMARK(build_tx_index); static void reopen_folder(benchmark::State& state) { SetLogVerbosityGuard guard{log::Level::kNone}; TemporaryDirectory tmp_dir; - auto repository = make_repository(tmp_dir.path()); // These sample snapshot files just contain data for block range [1'500'012, 1'500'013], hence current snapshot // file name format is not sufficient to support them (see checks commented out below) @@ -158,7 +148,7 @@ static void reopen_folder(benchmark::State& state) { tx_index_hash_to_block.build(); for ([[maybe_unused]] auto _ : state) { - repository.reopen_folder(); + [[maybe_unused]] auto repository = db::blocks::make_blocks_repository(tmp_dir.path()); } } BENCHMARK(reopen_folder); diff --git a/silkworm/db/snapshot_repository_test.cpp b/silkworm/db/snapshot_repository_test.cpp index f5d8aabcea..5e9de36b1d 100644 --- a/silkworm/db/snapshot_repository_test.cpp +++ b/silkworm/db/snapshot_repository_test.cpp @@ -58,7 +58,6 @@ TEST_CASE("SnapshotRepository::reopen_folder.partial_bundle", "[silkworm][node][ test::TemporarySnapshotFile tmp_snapshot_2{tmp_dir.path(), "v1-011500-012000-bodies.seg"}; test::TemporarySnapshotFile tmp_snapshot_3{tmp_dir.path(), "v1-015000-015500-transactions.seg"}; auto repository = make_repository(tmp_dir.path()); - repository.reopen_folder(); CHECK(repository.bundles_count() == 0); CHECK(repository.max_block_available() == 0); } @@ -67,10 +66,8 @@ TEST_CASE("SnapshotRepository::view", "[silkworm][node][snapshot]") { SetLogVerbosityGuard guard{log::Level::kNone}; TemporaryDirectory tmp_dir; - auto repository = make_repository(tmp_dir.path()); - SECTION("no snapshots") { - repository.reopen_folder(); + auto repository = make_repository(tmp_dir.path()); CHECK_FALSE_FIRST(repository.find_segment(SnapshotType::headers, 14'500'000)); CHECK_FALSE_FIRST(repository.find_segment(SnapshotType::bodies, 11'500'000)); @@ -91,7 +88,8 @@ TEST_CASE("SnapshotRepository::view", "[silkworm][node][snapshot]") { test::TemporarySnapshotFile tmp_snapshot_1{tmp_dir.path(), "v1-014500-015000-headers.seg"}; test::TemporarySnapshotFile tmp_snapshot_2{tmp_dir.path(), "v1-011500-012000-bodies.seg"}; test::TemporarySnapshotFile tmp_snapshot_3{tmp_dir.path(), "v1-015000-015500-transactions.seg"}; - repository.reopen_folder(); + + auto repository = make_repository(tmp_dir.path()); CHECK_FALSE_FIRST(repository.find_segment(SnapshotType::headers, 14'500'000)); CHECK_FALSE_FIRST(repository.find_segment(SnapshotType::bodies, 11'500'000)); @@ -114,6 +112,7 @@ TEST_CASE("SnapshotRepository::view", "[silkworm][node][snapshot]") { test::SampleBodySnapshotFile tmp_snapshot_2{tmp_dir.path()}; test::SampleTransactionSnapshotFile tmp_snapshot_3{tmp_dir.path()}; + auto repository = make_repository(tmp_dir.path()); for (auto& index_builder : repository.missing_indexes()) { index_builder->build(); } @@ -141,7 +140,6 @@ TEST_CASE("SnapshotRepository::view", "[silkworm][node][snapshot]") { TEST_CASE("SnapshotRepository::find_segment", "[silkworm][node][snapshot]") { SetLogVerbosityGuard guard{log::Level::kNone}; TemporaryDirectory tmp_dir; - auto repository = make_repository(tmp_dir.path()); // These sample snapshot files just contain data for block range [1'500'012, 1'500'013], hence current snapshot // file name format is not sufficient to support them (see checks commented out below) @@ -149,6 +147,8 @@ TEST_CASE("SnapshotRepository::find_segment", "[silkworm][node][snapshot]") { test::SampleBodySnapshotFile body_segment{tmp_dir.path()}; test::SampleTransactionSnapshotFile txn_segment{tmp_dir.path()}; + auto repository = make_repository(tmp_dir.path()); + SECTION("header w/o index") { CHECK_FALSE_FIRST(repository.find_segment(SnapshotType::headers, 1'500'011)); CHECK_FALSE_FIRST(repository.find_segment(SnapshotType::headers, 1'500'012)); @@ -207,7 +207,6 @@ TEST_CASE("SnapshotRepository::find_segment", "[silkworm][node][snapshot]") { TEST_CASE("SnapshotRepository::find_block_number", "[silkworm][node][snapshot]") { SetLogVerbosityGuard guard{log::Level::kNone}; TemporaryDirectory tmp_dir; - auto repository = make_repository(tmp_dir.path()); // These sample snapshot files just contain data for block range [1'500'012, 1'500'013], hence current snapshot // file name format is not sufficient to support them (see checks commented out below) @@ -226,7 +225,7 @@ TEST_CASE("SnapshotRepository::find_block_number", "[silkworm][node][snapshot]") REQUIRE_NOTHROW(TransactionIndex::make(body_segment_path, txn_segment_path).build()); REQUIRE_NOTHROW(TransactionToBlockIndex::make(body_segment_path, txn_segment_path, txn_segment.block_num_range().start).build()); - REQUIRE_NOTHROW(repository.reopen_folder()); + auto repository = make_repository(tmp_dir.path()); TransactionBlockNumByTxnHashRepoQuery query{repository.view_bundles_reverse()}; diff --git a/silkworm/infra/common/directories.cpp b/silkworm/infra/common/directories.cpp index cc53b9f9e6..e436dfa745 100644 --- a/silkworm/infra/common/directories.cpp +++ b/silkworm/infra/common/directories.cpp @@ -177,9 +177,10 @@ std::filesystem::path silkworm::DataDirectory::get_default_storage_path() { void DataDirectory::deploy() { Directory::create(); chaindata_.create(); + nodes_.create(); + snapshots_.create(); temp_.create(); temp_.clear(); - nodes_.create(); } std::filesystem::path TemporaryDirectory::get_os_temporary_path() { return std::filesystem::temp_directory_path(); } diff --git a/silkworm/node/stagedsync/stages/stage_bodies_test.cpp b/silkworm/node/stagedsync/stages/stage_bodies_test.cpp index d05cd2cd33..0d32d2c163 100644 --- a/silkworm/node/stagedsync/stages/stage_bodies_test.cpp +++ b/silkworm/node/stagedsync/stages/stage_bodies_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include namespace silkworm { @@ -34,6 +35,7 @@ class BodiesStageForTest : public stagedsync::BodiesStage { using BodyDataModelForTest = BodiesStageForTest::BodyDataModel; TEST_CASE("BodiesStage - data model") { + silkworm::test_util::SetLogVerbosityGuard log_guard{log::Level::kNone}; db::test_util::TempChainDataStore context; context.add_genesis_data(); context.commit_txn(); diff --git a/silkworm/node/stagedsync/stages/stage_headers_test.cpp b/silkworm/node/stagedsync/stages/stage_headers_test.cpp index 5c66a21f0d..28efb752be 100644 --- a/silkworm/node/stagedsync/stages/stage_headers_test.cpp +++ b/silkworm/node/stagedsync/stages/stage_headers_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace silkworm { @@ -34,7 +35,8 @@ class HeadersStageForTest : public stagedsync::HeadersStage { using HeaderDataModelForTest = HeadersStageForTest::HeaderDataModel; TEST_CASE("HeadersStage - data model") { - test_util::TempChainDataStore context; + silkworm::test_util::SetLogVerbosityGuard log_guard{log::Level::kNone}; + db::test_util::TempChainDataStore context; context.add_genesis_data(); context.commit_txn(); diff --git a/silkworm/node/stagedsync/stages_test.cpp b/silkworm/node/stagedsync/stages_test.cpp index 7d6ddb6d8f..6d947528be 100644 --- a/silkworm/node/stagedsync/stages_test.cpp +++ b/silkworm/node/stagedsync/stages_test.cpp @@ -70,6 +70,7 @@ static stagedsync::CallTraceIndex make_call_traces_stage( } TEST_CASE("Sync Stages") { + silkworm::test_util::SetLogVerbosityGuard log_guard{log::Level::kNone}; TemporaryDirectory tmp_dir; NodeSettings node_settings{}; node_settings.data_directory = std::make_unique(tmp_dir.path()); @@ -85,8 +86,6 @@ TEST_CASE("Sync Stages") { std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt); - silkworm::test_util::SetLogVerbosityGuard log_guard{log::Level::kNone}; - db::DataStore data_store{ node_settings.chaindata_env_config, node_settings.data_directory->snapshots().path(), diff --git a/silkworm/rpc/daemon.cpp b/silkworm/rpc/daemon.cpp index efb24173c2..8a66c58744 100644 --- a/silkworm/rpc/daemon.cpp +++ b/silkworm/rpc/daemon.cpp @@ -137,12 +137,9 @@ int Daemon::run(const DaemonSettings& settings) { data_dir.snapshots().path(), }); - auto& repository = data_store->ref().repository; - repository.reopen_folder(); - // At startup check that chain configuration is valid db::ROTxnManaged ro_txn = data_store->chaindata().start_ro_tx(); - db::DataModel data_access{ro_txn, repository}; + db::DataModel data_access = db::DataModelFactory{data_store->ref()}(ro_txn); if (const auto chain_config{data_access.read_chain_config()}; !chain_config) { throw std::runtime_error{"invalid chain configuration"}; } diff --git a/silkworm/sync/internals/header_retrieval_test.cpp b/silkworm/sync/internals/header_retrieval_test.cpp index 823abd37fe..a8238d55bb 100644 --- a/silkworm/sync/internals/header_retrieval_test.cpp +++ b/silkworm/sync/internals/header_retrieval_test.cpp @@ -19,10 +19,12 @@ #include #include +#include namespace silkworm { TEST_CASE("HeaderRetrieval") { + test_util::SetLogVerbosityGuard log_guard{log::Level::kNone}; db::test_util::TempChainDataStore context; context.add_genesis_data(); context.commit_txn(); diff --git a/silkworm/sync/messages/internal_message_test.cpp b/silkworm/sync/messages/internal_message_test.cpp index 28bfcbcd57..f2d0f3079a 100644 --- a/silkworm/sync/messages/internal_message_test.cpp +++ b/silkworm/sync/messages/internal_message_test.cpp @@ -19,12 +19,14 @@ #include #include +#include #include namespace silkworm { // Switch off the null sanitizer because nullptr SentryClient is formally dereferenced in command->execute. [[clang::no_sanitize("null")]] TEST_CASE("internal message") { + silkworm::test_util::SetLogVerbosityGuard log_guard{log::Level::kNone}; db::test_util::TempChainDataStore context; // not used in the test execution db::DataStoreRef data_store = context->ref(); diff --git a/silkworm/sync/sync_pos_test.cpp b/silkworm/sync/sync_pos_test.cpp index 28ac3b6ff6..c419e89937 100644 --- a/silkworm/sync/sync_pos_test.cpp +++ b/silkworm/sync/sync_pos_test.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -117,6 +118,7 @@ static rpc::NewPayloadRequest make_payload_request_v3() { } TEST_CASE_METHOD(PoSSyncTest, "PoSSync::new_payload timeout") { + silkworm::test_util::SetLogVerbosityGuard log_guard{log::Level::kNone}; using namespace std::chrono_literals; using testing::_; using testing::InvokeWithoutArgs; From abcd971c794c89f524d9877b6617a4c2a7a2dfb5 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Tue, 5 Nov 2024 15:05:03 +0100 Subject: [PATCH 5/7] separate bundle paths, always open SnapshotBundle --- silkworm/capi/silkworm.cpp | 1 - silkworm/db/datastore/snapshot_merger.cpp | 11 ++++--- .../datastore/snapshots/snapshot_bundle.cpp | 20 +++++++++++++ .../datastore/snapshots/snapshot_bundle.hpp | 30 +++++++++++++++++-- .../snapshots/snapshot_bundle_factory.hpp | 4 +-- .../snapshots/snapshot_repository.cpp | 4 +-- .../snapshots/snapshot_repository.hpp | 2 +- silkworm/db/freezer.cpp | 11 ++++--- silkworm/db/snapshot_bundle_factory_impl.cpp | 23 ++++++++++---- silkworm/db/snapshot_bundle_factory_impl.hpp | 4 +-- silkworm/db/snapshot_sync_test.cpp | 1 - 11 files changed, 83 insertions(+), 28 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 46ad97ef5c..7d1372e25e 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -391,7 +391,6 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn .idx_txn_hash = std::move(idx_txn_hash), .idx_txn_hash_2_block = std::move(idx_txn_hash_2_block), }, - true, }; handle->repository->add_snapshot_bundle(std::move(bundle)); return SILKWORM_OK; diff --git a/silkworm/db/datastore/snapshot_merger.cpp b/silkworm/db/datastore/snapshot_merger.cpp index d5d2967778..c6b96687d5 100644 --- a/silkworm/db/datastore/snapshot_merger.cpp +++ b/silkworm/db/datastore/snapshot_merger.cpp @@ -47,10 +47,10 @@ struct SnapshotMergerCommand : public DataMigrationCommand { }; struct SnapshotMergerResult : public DataMigrationResult { - SnapshotBundle bundle; + SnapshotBundlePaths bundle_paths; - explicit SnapshotMergerResult(SnapshotBundle bundle1) - : bundle(std::move(bundle1)) {} + explicit SnapshotMergerResult(SnapshotBundlePaths bundle_paths1) + : bundle_paths(std::move(bundle_paths1)) {} ~SnapshotMergerResult() override = default; }; @@ -114,8 +114,7 @@ std::shared_ptr SnapshotMerger::migrate(std::unique_ptr result) { auto& merger_result = dynamic_cast(*result); - auto& bundle = merger_result.bundle; - snapshots_.build_indexes(bundle); + snapshots_.build_indexes(merger_result.bundle_paths); } static void schedule_bundle_cleanup(SnapshotBundle& bundle) { @@ -128,7 +127,7 @@ static void schedule_bundle_cleanup(SnapshotBundle& bundle) { void SnapshotMerger::commit(std::shared_ptr result) { auto& freezer_result = dynamic_cast(*result); - auto& bundle = freezer_result.bundle; + auto& bundle = freezer_result.bundle_paths; auto merged_bundles = snapshots_.bundles_in_range(bundle.step_range()); move_files(bundle.files(), snapshots_.path()); diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp index 518d868b81..ffd745bc56 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp @@ -71,4 +71,24 @@ std::vector SnapshotBundle::segment_paths() { return paths; } +std::vector SnapshotBundlePaths::segment_paths() const { + return { + header_segment_path, + body_segment_path, + txn_segment_path, + }; +} + +std::vector SnapshotBundlePaths::files() const { + return std::vector{ + header_segment_path.path(), + idx_header_hash_path.path(), + body_segment_path.path(), + idx_body_number_path.path(), + txn_segment_path.path(), + idx_txn_hash_path.path(), + idx_txn_hash_2_block_path.path(), + }; +} + } // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp index f10fdf3fc0..39841e777c 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp @@ -49,11 +49,37 @@ struct SnapshotBundleData { static constexpr size_t kIndexesCount = 4; }; +struct SnapshotBundlePathsData { + SnapshotPath header_segment_path; + SnapshotPath idx_header_hash_path; + + SnapshotPath body_segment_path; + SnapshotPath idx_body_number_path; + + SnapshotPath txn_segment_path; + SnapshotPath idx_txn_hash_path; + SnapshotPath idx_txn_hash_2_block_path; +}; + +struct SnapshotBundlePaths : public SnapshotBundlePathsData { + SnapshotBundlePaths(StepRange step_range, SnapshotBundlePathsData bundle) + : SnapshotBundlePathsData{std::move(bundle)}, + step_range_{step_range} {} + + StepRange step_range() const { return step_range_; } + + std::vector files() const; + std::vector segment_paths() const; + + private: + StepRange step_range_; +}; + struct SnapshotBundle : public SnapshotBundleData { - explicit SnapshotBundle(StepRange step_range, SnapshotBundleData bundle, bool open) + explicit SnapshotBundle(StepRange step_range, SnapshotBundleData bundle) : SnapshotBundleData{std::move(bundle)}, step_range_{step_range} { - if (open) reopen(); + reopen(); } virtual ~SnapshotBundle(); diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp index 155a302bb1..b888b4984e 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp @@ -31,9 +31,9 @@ struct SnapshotBundleFactory { virtual ~SnapshotBundleFactory() = default; using PathByTypeProvider = std::function; - virtual SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path, bool open) const = 0; + virtual SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const = 0; virtual SnapshotBundle make(const std::filesystem::path& dir_path, StepRange range) const = 0; - virtual SnapshotBundle make_paths(const std::filesystem::path& dir_path, StepRange range) const = 0; + virtual SnapshotBundlePaths make_paths(const std::filesystem::path& dir_path, StepRange range) const = 0; virtual std::vector> index_builders(const SnapshotPath& segment_path) const = 0; virtual std::vector> index_builders(const SnapshotPathList& segment_paths) const = 0; diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 770d53d888..3f73dd7ed8 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -149,7 +149,7 @@ void SnapshotRepository::reopen_folder() { auto index_path = [&](SnapshotType type) { return all_index_paths[groups[num][true][type]]; }; - SnapshotBundle bundle = bundle_factory_->make(snapshot_path, index_path, true); + SnapshotBundle bundle = bundle_factory_->make(snapshot_path, index_path); bundles->insert_or_assign(num, std::make_shared(std::move(bundle))); } @@ -243,7 +243,7 @@ void SnapshotRepository::remove_stale_indexes() const { } } -void SnapshotRepository::build_indexes(SnapshotBundle& bundle) const { +void SnapshotRepository::build_indexes(SnapshotBundlePaths& bundle) const { for (auto& builder : bundle_factory_->index_builders(bundle.segment_paths())) { builder->build(); } diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index 4e9df6f95d..8e3c12fd2e 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -75,7 +75,7 @@ class SnapshotRepository { std::vector> missing_indexes() const; void remove_stale_indexes() const; - void build_indexes(SnapshotBundle& bundle) const; + void build_indexes(SnapshotBundlePaths& bundle) const; using Bundles = std::map>; diff --git a/silkworm/db/freezer.cpp b/silkworm/db/freezer.cpp index d85ba9fe4e..9b1e801ebb 100644 --- a/silkworm/db/freezer.cpp +++ b/silkworm/db/freezer.cpp @@ -41,10 +41,10 @@ namespace silkworm::db { using namespace silkworm::snapshots; struct FreezerResult : public DataMigrationResult { - SnapshotBundle bundle; + SnapshotBundlePaths bundle_paths; - explicit FreezerResult(SnapshotBundle bundle1) - : bundle(std::move(bundle1)) {} + explicit FreezerResult(SnapshotBundlePaths bundle_paths1) + : bundle_paths(std::move(bundle_paths1)) {} ~FreezerResult() override = default; }; @@ -126,13 +126,12 @@ std::shared_ptr Freezer::migrate(std::unique_ptr result) { auto& freezer_result = dynamic_cast(*result); - auto& bundle = freezer_result.bundle; - snapshots_.build_indexes(bundle); + snapshots_.build_indexes(freezer_result.bundle_paths); } void Freezer::commit(std::shared_ptr result) { auto& freezer_result = dynamic_cast(*result); - auto& bundle = freezer_result.bundle; + auto& bundle = freezer_result.bundle_paths; move_files(bundle.files(), snapshots_.path()); auto final_bundle = snapshots_.bundle_factory().make(snapshots_.path(), bundle.step_range()); diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp index 1cd8e60be5..373509124d 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.cpp +++ b/silkworm/db/snapshot_bundle_factory_impl.cpp @@ -27,7 +27,7 @@ namespace silkworm::db { using namespace snapshots; -SnapshotBundle SnapshotBundleFactoryImpl::make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path, bool open) const { +SnapshotBundle SnapshotBundleFactoryImpl::make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const { return SnapshotBundle{ snapshot_path(SnapshotType::headers).step_range(), { @@ -41,7 +41,6 @@ SnapshotBundle SnapshotBundleFactoryImpl::make(PathByTypeProvider snapshot_path, .idx_txn_hash = Index(index_path(SnapshotType::transactions)), .idx_txn_hash_2_block = Index(index_path(SnapshotType::transactions_to_block)), }, - open, }; } @@ -52,17 +51,31 @@ SnapshotBundle SnapshotBundleFactoryImpl::make(const std::filesystem::path& dir_ PathByTypeProvider index_path = [&](silkworm::snapshots::SnapshotType type) { return SnapshotPath::make(dir_path, kSnapshotV1, range, type, kIdxExtension); }; - return make(std::move(snapshot_path), std::move(index_path), true); + return make(std::move(snapshot_path), std::move(index_path)); } -SnapshotBundle SnapshotBundleFactoryImpl::make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const { +SnapshotBundlePaths SnapshotBundleFactoryImpl::make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const { PathByTypeProvider snapshot_path = [&](silkworm::snapshots::SnapshotType type) { return SnapshotPath::make(dir_path, kSnapshotV1, range, type); }; PathByTypeProvider index_path = [&](silkworm::snapshots::SnapshotType type) { return SnapshotPath::make(dir_path, kSnapshotV1, range, type, kIdxExtension); }; - return make(std::move(snapshot_path), std::move(index_path), false); + + return SnapshotBundlePaths{ + range, + SnapshotBundlePathsData{ + .header_segment_path = snapshot_path(SnapshotType::headers), + .idx_header_hash_path = index_path(SnapshotType::headers), + + .body_segment_path = snapshot_path(SnapshotType::bodies), + .idx_body_number_path = index_path(SnapshotType::bodies), + + .txn_segment_path = snapshot_path(SnapshotType::transactions), + .idx_txn_hash_path = index_path(SnapshotType::transactions), + .idx_txn_hash_2_block_path = index_path(SnapshotType::transactions_to_block), + }, + }; } std::vector> SnapshotBundleFactoryImpl::index_builders(const SnapshotPath& segment_path) const { diff --git a/silkworm/db/snapshot_bundle_factory_impl.hpp b/silkworm/db/snapshot_bundle_factory_impl.hpp index 0f5ff2585d..e4ca587b99 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.hpp +++ b/silkworm/db/snapshot_bundle_factory_impl.hpp @@ -23,9 +23,9 @@ namespace silkworm::db { struct SnapshotBundleFactoryImpl : public snapshots::SnapshotBundleFactory { ~SnapshotBundleFactoryImpl() override = default; - snapshots::SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path, bool open) const override; + snapshots::SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const override; snapshots::SnapshotBundle make(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; - snapshots::SnapshotBundle make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; + snapshots::SnapshotBundlePaths make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; std::vector> index_builders(const snapshots::SnapshotPath& segment_path) const override; std::vector> index_builders(const snapshots::SnapshotPathList& segment_paths) const override; }; diff --git a/silkworm/db/snapshot_sync_test.cpp b/silkworm/db/snapshot_sync_test.cpp index 37abe170cd..673948ccd1 100644 --- a/silkworm/db/snapshot_sync_test.cpp +++ b/silkworm/db/snapshot_sync_test.cpp @@ -153,7 +153,6 @@ TEST_CASE("SnapshotSync::update_block_headers", "[db][snapshot][sync]") { .idx_txn_hash = std::move(idx_txn_hash), .idx_txn_hash_2_block = std::move(idx_txn_hash_2_block), }, - true, }; auto& repository = snapshot_sync.repository(); repository.add_snapshot_bundle(std::move(bundle)); From 2c5f89681109176fa65dd9d0cdf7f3208ca548f9 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Tue, 5 Nov 2024 17:00:12 +0100 Subject: [PATCH 6/7] schema --- silkworm/db/blocks/schema_config.cpp | 15 +++- silkworm/db/blocks/schema_config.hpp | 21 ++++- silkworm/db/data_store.cpp | 35 +++++++++ silkworm/db/data_store.hpp | 3 + silkworm/db/datastore/data_store.hpp | 8 +- silkworm/db/datastore/schema.hpp | 27 +++++++ silkworm/db/datastore/snapshots/schema.cpp | 76 +++++++++++++++++++ silkworm/db/datastore/snapshots/schema.hpp | 63 +++++++++++++++ .../datastore/snapshots/snapshot_bundle.cpp | 22 ++---- .../datastore/snapshots/snapshot_bundle.hpp | 23 ++---- silkworm/db/snapshot_bundle_factory_impl.cpp | 20 +---- silkworm/db/snapshot_bundle_factory_impl.hpp | 6 ++ silkworm/db/state/schema_config.cpp | 26 +++++++ silkworm/db/state/schema_config.hpp | 3 + 14 files changed, 298 insertions(+), 50 deletions(-) create mode 100644 silkworm/db/data_store.cpp create mode 100644 silkworm/db/datastore/schema.hpp create mode 100644 silkworm/db/datastore/snapshots/schema.cpp create mode 100644 silkworm/db/datastore/snapshots/schema.hpp create mode 100644 silkworm/db/state/schema_config.cpp diff --git a/silkworm/db/blocks/schema_config.cpp b/silkworm/db/blocks/schema_config.cpp index 399bb4caad..987193e6eb 100644 --- a/silkworm/db/blocks/schema_config.cpp +++ b/silkworm/db/blocks/schema_config.cpp @@ -20,12 +20,25 @@ namespace silkworm::db::blocks { +snapshots::Schema::RepositoryDef make_blocks_repository_schema() { + snapshots::Schema::RepositoryDef schema; + schema + .segment(kHeaderSegmentName) + .rec_split_index(kIdxHeaderHashName) + .segment(kBodySegmentName) + .rec_split_index(kIdxBodyNumberName) + .segment(kTxnSegmentName) + .rec_split_index(kIdxTxnHashName) + .rec_split_index(kIdxTxnHash2BlockName); + return schema; +} + snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path, bool open) { return snapshots::SnapshotRepository{ std::move(dir_path), open, std::make_unique(), - std::make_unique(), + std::make_unique(make_blocks_repository_schema()), }; } diff --git a/silkworm/db/blocks/schema_config.hpp b/silkworm/db/blocks/schema_config.hpp index 8f3cd23b46..bcd485f9d7 100644 --- a/silkworm/db/blocks/schema_config.hpp +++ b/silkworm/db/blocks/schema_config.hpp @@ -17,12 +17,31 @@ #pragma once #include "../datastore/common/entity_name.hpp" +#include "../datastore/snapshots/schema.hpp" #include "../datastore/snapshots/snapshot_repository.hpp" namespace silkworm::db::blocks { inline constexpr datastore::EntityName kBlocksRepositoryName{"Blocks"}; -snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path, bool open = true); +snapshots::Schema::RepositoryDef make_blocks_repository_schema(); + +snapshots::SnapshotRepository make_blocks_repository( + std::filesystem::path dir_path, + bool open = true); + +inline constexpr datastore::EntityName kHeaderSegmentName{"headers"}; +//! Index header_hash -> block_num -> headers_segment_offset +inline constexpr datastore::EntityName kIdxHeaderHashName{"headers"}; + +inline constexpr datastore::EntityName kBodySegmentName{"bodies"}; +//! Index block_num -> bodies_segment_offset +inline constexpr datastore::EntityName kIdxBodyNumberName{"bodies"}; + +inline constexpr datastore::EntityName kTxnSegmentName{"transactions"}; +//! Index transaction_hash -> txn_id -> transactions_segment_offset +inline constexpr datastore::EntityName kIdxTxnHashName{"transactions"}; +//! Index transaction_hash -> block_num +inline constexpr datastore::EntityName kIdxTxnHash2BlockName{"transactions_to_block"}; } // namespace silkworm::db::blocks diff --git a/silkworm/db/data_store.cpp b/silkworm/db/data_store.cpp new file mode 100644 index 0000000000..9fa7652a84 --- /dev/null +++ b/silkworm/db/data_store.cpp @@ -0,0 +1,35 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "data_store.hpp" + +namespace silkworm::db { + +datastore::Schema DataStore::make_schema( + bool enabled_state_repository) { + snapshots::Schema snapshots; + snapshots.repository(blocks::kBlocksRepositoryName) = blocks::make_blocks_repository_schema(); + + if (enabled_state_repository) { + snapshots.repository(state::kStateRepositoryName) = state::make_state_repository_schema(); + } + + return { + std::move(snapshots), + }; +} + +} // namespace silkworm::db diff --git a/silkworm/db/data_store.hpp b/silkworm/db/data_store.hpp index 3c6b0ab529..5b2e2d48c4 100644 --- a/silkworm/db/data_store.hpp +++ b/silkworm/db/data_store.hpp @@ -37,6 +37,7 @@ class DataStore { snapshots::SnapshotRepository blocks_repository, std::optional state_repository = std::nullopt) : store_{ + make_schema(state_repository.has_value()), std::move(chaindata_env), make_repositories_map(std::move(blocks_repository), std::move(state_repository)), } {} @@ -61,6 +62,8 @@ class DataStore { db::RWAccess chaindata_rw() const { return store_.chaindata_rw(); } private: + static datastore::Schema make_schema(bool enabled_state_repository); + static std::map> make_repositories_map( snapshots::SnapshotRepository blocks_repository, std::optional state_repository) { diff --git a/silkworm/db/datastore/data_store.hpp b/silkworm/db/datastore/data_store.hpp index f04c083d04..5b953c9a19 100644 --- a/silkworm/db/datastore/data_store.hpp +++ b/silkworm/db/datastore/data_store.hpp @@ -21,6 +21,7 @@ #include "common/entity_name.hpp" #include "mdbx/mdbx.hpp" +#include "schema.hpp" #include "snapshots/snapshot_repository.hpp" namespace silkworm::datastore { @@ -28,9 +29,11 @@ namespace silkworm::datastore { class DataStore { public: DataStore( + Schema schema, mdbx::env_managed chaindata_env, std::map> repositories) - : chaindata_env_{std::move(chaindata_env)}, + : schema_{std::move(schema)}, + chaindata_env_{std::move(chaindata_env)}, repositories_{std::move(repositories)} {} void close() { @@ -39,11 +42,14 @@ class DataStore { entry.second->close(); } + const Schema& schema() const { return schema_; } db::ROAccess chaindata() const { return db::ROAccess{chaindata_env_}; } db::RWAccess chaindata_rw() const { return db::RWAccess{chaindata_env_}; } + snapshots::SnapshotRepository& repository(const EntityName& name) const { return *repositories_.at(name); } private: + Schema schema_; mdbx::env_managed chaindata_env_; std::map> repositories_; }; diff --git a/silkworm/db/datastore/schema.hpp b/silkworm/db/datastore/schema.hpp new file mode 100644 index 0000000000..5cf37cf8bb --- /dev/null +++ b/silkworm/db/datastore/schema.hpp @@ -0,0 +1,27 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "snapshots/schema.hpp" + +namespace silkworm::datastore { + +struct Schema { + snapshots::Schema snapshots; +}; + +} // namespace silkworm::datastore diff --git a/silkworm/db/datastore/snapshots/schema.cpp b/silkworm/db/datastore/snapshots/schema.cpp new file mode 100644 index 0000000000..08c1df00c9 --- /dev/null +++ b/silkworm/db/datastore/snapshots/schema.cpp @@ -0,0 +1,76 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "schema.hpp" + +#include + +namespace silkworm::snapshots { + +std::map Schema::RepositoryDef::make_segment_paths( + const std::filesystem::path& dir_path, + StepRange range) const { + std::map results; + for (auto& entry : segment_defs_) { + auto type = *magic_enum::enum_cast(entry.first.name); + results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, type)); + } + return results; +} + +std::map Schema::RepositoryDef::make_segments( + const std::filesystem::path& dir_path, + StepRange range) const { + std::map results; + for (auto& entry : make_segment_paths(dir_path, range)) { + results.emplace(entry.first, SegmentFileReader{entry.second}); + } + return results; +} + +std::map Schema::RepositoryDef::make_rec_split_index_paths( + const std::filesystem::path& dir_path, + StepRange range) const { + std::map results; + for (auto& entry : rec_split_index_defs_) { + auto type = *magic_enum::enum_cast(entry.first.name); + results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, type, kIdxExtension)); + } + return results; +} + +std::map Schema::RepositoryDef::make_rec_split_indexes( + const std::filesystem::path& dir_path, + StepRange range) const { + std::map results; + for (auto& entry : make_rec_split_index_paths(dir_path, range)) { + results.emplace(entry.first, Index{entry.second}); + } + return results; +} + +std::vector Schema::RepositoryDef::make_all_paths( + const std::filesystem::path& dir_path, + StepRange range) const { + std::vector results; + for (auto& entry : make_segment_paths(dir_path, range)) + results.push_back(std::move(entry.second)); + for (auto& entry : make_rec_split_index_paths(dir_path, range)) + results.push_back(std::move(entry.second)); + return results; +} + +} // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/schema.hpp b/silkworm/db/datastore/snapshots/schema.hpp new file mode 100644 index 0000000000..154c094446 --- /dev/null +++ b/silkworm/db/datastore/snapshots/schema.hpp @@ -0,0 +1,63 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include + +#include "../common/entity_name.hpp" +#include "common/snapshot_path.hpp" +#include "rec_split_index/index.hpp" +#include "segment/segment_reader.hpp" + +namespace silkworm::snapshots { + +class Schema { + public: + class RepositoryDef { + public: + RepositoryDef& segment(datastore::EntityName name) { + segment_defs_.try_emplace(name); + return *this; + } + + RepositoryDef& rec_split_index(datastore::EntityName name) { + rec_split_index_defs_.try_emplace(name); + return *this; + } + + std::map make_segment_paths(const std::filesystem::path& dir_path, StepRange range) const; + std::map make_segments(const std::filesystem::path& dir_path, StepRange range) const; + std::map make_rec_split_index_paths(const std::filesystem::path& dir_path, StepRange range) const; + std::map make_rec_split_indexes(const std::filesystem::path& dir_path, StepRange range) const; + std::vector make_all_paths(const std::filesystem::path& dir_path, StepRange range) const; + + private: + std::map segment_defs_; + std::map rec_split_index_defs_; + }; + + RepositoryDef& repository(datastore::EntityName name) { + return repository_defs_[name]; + } + + private: + std::map repository_defs_; +}; + +} // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp index ffd745bc56..2347a53910 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp @@ -72,23 +72,17 @@ std::vector SnapshotBundle::segment_paths() { } std::vector SnapshotBundlePaths::segment_paths() const { - return { - header_segment_path, - body_segment_path, - txn_segment_path, - }; + std::vector results; + for (auto& entry : schema_.make_segment_paths(dir_path_, step_range_)) + results.push_back(std::move(entry.second)); + return results; } std::vector SnapshotBundlePaths::files() const { - return std::vector{ - header_segment_path.path(), - idx_header_hash_path.path(), - body_segment_path.path(), - idx_body_number_path.path(), - txn_segment_path.path(), - idx_txn_hash_path.path(), - idx_txn_hash_2_block_path.path(), - }; + std::vector results; + for (auto& path : schema_.make_all_paths(dir_path_, step_range_)) + results.push_back(path.path()); + return results; } } // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp index 39841e777c..7d60c1bc19 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp @@ -24,7 +24,9 @@ #include #include "common/snapshot_path.hpp" +#include "common/util/iterator/map_values_view.hpp" #include "rec_split_index/index.hpp" +#include "schema.hpp" #include "segment/segment_reader.hpp" #include "segment_and_index.hpp" @@ -49,21 +51,10 @@ struct SnapshotBundleData { static constexpr size_t kIndexesCount = 4; }; -struct SnapshotBundlePathsData { - SnapshotPath header_segment_path; - SnapshotPath idx_header_hash_path; - - SnapshotPath body_segment_path; - SnapshotPath idx_body_number_path; - - SnapshotPath txn_segment_path; - SnapshotPath idx_txn_hash_path; - SnapshotPath idx_txn_hash_2_block_path; -}; - -struct SnapshotBundlePaths : public SnapshotBundlePathsData { - SnapshotBundlePaths(StepRange step_range, SnapshotBundlePathsData bundle) - : SnapshotBundlePathsData{std::move(bundle)}, +struct SnapshotBundlePaths { + SnapshotBundlePaths(Schema::RepositoryDef schema, std::filesystem::path dir_path, StepRange step_range) + : schema_{std::move(schema)}, + dir_path_{std::move(dir_path)}, step_range_{step_range} {} StepRange step_range() const { return step_range_; } @@ -72,6 +63,8 @@ struct SnapshotBundlePaths : public SnapshotBundlePathsData { std::vector segment_paths() const; private: + Schema::RepositoryDef schema_; + std::filesystem::path dir_path_; StepRange step_range_; }; diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp index 373509124d..64a8d355c0 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.cpp +++ b/silkworm/db/snapshot_bundle_factory_impl.cpp @@ -55,26 +55,10 @@ SnapshotBundle SnapshotBundleFactoryImpl::make(const std::filesystem::path& dir_ } SnapshotBundlePaths SnapshotBundleFactoryImpl::make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const { - PathByTypeProvider snapshot_path = [&](silkworm::snapshots::SnapshotType type) { - return SnapshotPath::make(dir_path, kSnapshotV1, range, type); - }; - PathByTypeProvider index_path = [&](silkworm::snapshots::SnapshotType type) { - return SnapshotPath::make(dir_path, kSnapshotV1, range, type, kIdxExtension); - }; - return SnapshotBundlePaths{ + schema_, + dir_path, range, - SnapshotBundlePathsData{ - .header_segment_path = snapshot_path(SnapshotType::headers), - .idx_header_hash_path = index_path(SnapshotType::headers), - - .body_segment_path = snapshot_path(SnapshotType::bodies), - .idx_body_number_path = index_path(SnapshotType::bodies), - - .txn_segment_path = snapshot_path(SnapshotType::transactions), - .idx_txn_hash_path = index_path(SnapshotType::transactions), - .idx_txn_hash_2_block_path = index_path(SnapshotType::transactions_to_block), - }, }; } diff --git a/silkworm/db/snapshot_bundle_factory_impl.hpp b/silkworm/db/snapshot_bundle_factory_impl.hpp index e4ca587b99..fe99a496f8 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.hpp +++ b/silkworm/db/snapshot_bundle_factory_impl.hpp @@ -16,11 +16,14 @@ #pragma once +#include #include namespace silkworm::db { struct SnapshotBundleFactoryImpl : public snapshots::SnapshotBundleFactory { + SnapshotBundleFactoryImpl(snapshots::Schema::RepositoryDef schema) + : schema_{std::move(schema)} {} ~SnapshotBundleFactoryImpl() override = default; snapshots::SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const override; @@ -28,6 +31,9 @@ struct SnapshotBundleFactoryImpl : public snapshots::SnapshotBundleFactory { snapshots::SnapshotBundlePaths make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; std::vector> index_builders(const snapshots::SnapshotPath& segment_path) const override; std::vector> index_builders(const snapshots::SnapshotPathList& segment_paths) const override; + + private: + snapshots::Schema::RepositoryDef schema_; }; } // namespace silkworm::db diff --git a/silkworm/db/state/schema_config.cpp b/silkworm/db/state/schema_config.cpp new file mode 100644 index 0000000000..866dc61c99 --- /dev/null +++ b/silkworm/db/state/schema_config.cpp @@ -0,0 +1,26 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "schema_config.hpp" + +namespace silkworm::db::state { + +snapshots::Schema::RepositoryDef make_state_repository_schema() { + snapshots::Schema::RepositoryDef schema; + return schema; +} + +} // namespace silkworm::db::state diff --git a/silkworm/db/state/schema_config.hpp b/silkworm/db/state/schema_config.hpp index 483f1395d7..b6536820c5 100644 --- a/silkworm/db/state/schema_config.hpp +++ b/silkworm/db/state/schema_config.hpp @@ -17,9 +17,12 @@ #pragma once #include "../datastore/common/entity_name.hpp" +#include "../datastore/snapshots/schema.hpp" namespace silkworm::db::state { inline constexpr datastore::EntityName kStateRepositoryName{"State"}; +snapshots::Schema::RepositoryDef make_state_repository_schema(); + } // namespace silkworm::db::state From 8741a7639ced3bb3485c7867a9d5203f61a8b65f Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 10:01:24 +0100 Subject: [PATCH 7/7] SnapshotBundle uses schema --- cmd/capi/execute.cpp | 44 ++++---- cmd/dev/snapshots.cpp | 8 +- silkworm/capi/silkworm.cpp | 28 +++-- silkworm/db/blocks/schema_config.hpp | 14 +++ .../db/blocks/transactions/txn_queries.hpp | 9 +- .../snapshots/common/snapshot_path.cpp | 45 +++++--- .../snapshots/common/snapshot_path.hpp | 1 + .../datastore/snapshots/snapshot_bundle.cpp | 62 +++++++---- .../datastore/snapshots/snapshot_bundle.hpp | 102 ++++-------------- .../snapshots/snapshot_bundle_factory.hpp | 2 - .../snapshots/snapshot_repository.cpp | 80 +++++++------- .../snapshots/snapshot_repository.hpp | 3 +- silkworm/db/snapshot_bundle_factory_impl.cpp | 26 +---- silkworm/db/snapshot_bundle_factory_impl.hpp | 1 - silkworm/db/snapshot_sync.cpp | 8 +- silkworm/db/snapshot_sync_test.cpp | 22 +--- 16 files changed, 209 insertions(+), 246 deletions(-) diff --git a/cmd/capi/execute.cpp b/cmd/capi/execute.cpp index 888ca8e56d..5bd8f5d1d6 100644 --- a/cmd/capi/execute.cpp +++ b/cmd/capi/execute.cpp @@ -151,19 +151,19 @@ std::vector collect_all_snapshots(const SnapshotRepositor std::vector transactions_snapshot_sequence; for (const auto& bundle_ptr : repository.view_bundles()) { - const auto& bundle = *bundle_ptr; + db::blocks::BundleDataRef bundle{**bundle_ptr}; { { SilkwormHeadersSnapshot raw_headers_snapshot{ .segment{ - .file_path = make_path(bundle.header_segment.path()), - .memory_address = bundle.header_segment.memory_file_region().data(), - .memory_length = bundle.header_segment.memory_file_region().size(), + .file_path = make_path(bundle.header_segment().path()), + .memory_address = bundle.header_segment().memory_file_region().data(), + .memory_length = bundle.header_segment().memory_file_region().size(), }, .header_hash_index{ - .file_path = make_path(bundle.idx_header_hash.path()), - .memory_address = bundle.idx_header_hash.memory_file_region().data(), - .memory_length = bundle.idx_header_hash.memory_file_region().size(), + .file_path = make_path(bundle.idx_header_hash().path()), + .memory_address = bundle.idx_header_hash().memory_file_region().data(), + .memory_length = bundle.idx_header_hash().memory_file_region().size(), }, }; headers_snapshot_sequence.push_back(raw_headers_snapshot); @@ -171,14 +171,14 @@ std::vector collect_all_snapshots(const SnapshotRepositor { SilkwormBodiesSnapshot raw_bodies_snapshot{ .segment{ - .file_path = make_path(bundle.body_segment.path()), - .memory_address = bundle.body_segment.memory_file_region().data(), - .memory_length = bundle.body_segment.memory_file_region().size(), + .file_path = make_path(bundle.body_segment().path()), + .memory_address = bundle.body_segment().memory_file_region().data(), + .memory_length = bundle.body_segment().memory_file_region().size(), }, .block_num_index{ - .file_path = make_path(bundle.idx_body_number.path()), - .memory_address = bundle.idx_body_number.memory_file_region().data(), - .memory_length = bundle.idx_body_number.memory_file_region().size(), + .file_path = make_path(bundle.idx_body_number().path()), + .memory_address = bundle.idx_body_number().memory_file_region().data(), + .memory_length = bundle.idx_body_number().memory_file_region().size(), }, }; bodies_snapshot_sequence.push_back(raw_bodies_snapshot); @@ -186,19 +186,19 @@ std::vector collect_all_snapshots(const SnapshotRepositor { SilkwormTransactionsSnapshot raw_transactions_snapshot{ .segment{ - .file_path = make_path(bundle.txn_segment.path()), - .memory_address = bundle.txn_segment.memory_file_region().data(), - .memory_length = bundle.txn_segment.memory_file_region().size(), + .file_path = make_path(bundle.txn_segment().path()), + .memory_address = bundle.txn_segment().memory_file_region().data(), + .memory_length = bundle.txn_segment().memory_file_region().size(), }, .tx_hash_index{ - .file_path = make_path(bundle.idx_txn_hash.path()), - .memory_address = bundle.idx_txn_hash.memory_file_region().data(), - .memory_length = bundle.idx_txn_hash.memory_file_region().size(), + .file_path = make_path(bundle.idx_txn_hash().path()), + .memory_address = bundle.idx_txn_hash().memory_file_region().data(), + .memory_length = bundle.idx_txn_hash().memory_file_region().size(), }, .tx_hash_2_block_index{ - .file_path = make_path(bundle.idx_txn_hash_2_block.path()), - .memory_address = bundle.idx_txn_hash_2_block.memory_file_region().data(), - .memory_length = bundle.idx_txn_hash_2_block.memory_file_region().size(), + .file_path = make_path(bundle.idx_txn_hash_2_block().path()), + .memory_address = bundle.idx_txn_hash_2_block().memory_file_region().data(), + .memory_length = bundle.idx_txn_hash_2_block().memory_file_region().size(), }, }; transactions_snapshot_sequence.push_back(raw_transactions_snapshot); diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index 678ba11873..f0cfdb064e 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -328,8 +328,8 @@ BodyCounters count_bodies_in_all(const SnapshotSubcommandSettings& settings) { int num_bodies = 0; uint64_t num_txns = 0; for (const auto& bundle_ptr : repository.view_bundles()) { - const auto& bundle = *bundle_ptr; - const auto [body_count, txn_count] = count_bodies_in_one(settings, bundle.body_segment); + db::blocks::BundleDataRef bundle{**bundle_ptr}; + const auto [body_count, txn_count] = count_bodies_in_one(settings, bundle.body_segment()); num_bodies += body_count; num_txns += txn_count; } @@ -376,8 +376,8 @@ int count_headers_in_all(const SnapshotSubcommandSettings& settings) { auto repository = make_repository(settings.settings); int num_headers{0}; for (const auto& bundle_ptr : repository.view_bundles()) { - const auto& bundle = *bundle_ptr; - const auto header_count = count_headers_in_one(settings, bundle.header_segment); + db::blocks::BundleDataRef bundle{**bundle_ptr}; + const auto header_count = count_headers_in_one(settings, bundle.header_segment()); num_headers += header_count; } return num_headers; diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 7d1372e25e..641dd18f80 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -378,19 +378,25 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn snapshots::Index idx_txn_hash{transactions_segment_path->related_path(snapshots::SnapshotType::transactions, snapshots::kIdxExtension), make_region(ts.tx_hash_index)}; snapshots::Index idx_txn_hash_2_block{transactions_segment_path->related_path(snapshots::SnapshotType::transactions_to_block, snapshots::kIdxExtension), make_region(ts.tx_hash_2_block_index)}; - snapshots::SnapshotBundle bundle{ - headers_segment_path->step_range(), - { - .header_segment = std::move(header_segment), - .idx_header_hash = std::move(idx_header_hash), + snapshots::SnapshotBundleData bundle_data = [&]() { + snapshots::SnapshotBundleData data; - .body_segment = std::move(body_segment), - .idx_body_number = std::move(idx_body_number), + data.segments.emplace(db::blocks::kHeaderSegmentName, std::move(header_segment)); + data.rec_split_indexes.emplace(db::blocks::kIdxHeaderHashName, std::move(idx_header_hash)); - .txn_segment = std::move(txn_segment), - .idx_txn_hash = std::move(idx_txn_hash), - .idx_txn_hash_2_block = std::move(idx_txn_hash_2_block), - }, + data.segments.emplace(db::blocks::kBodySegmentName, std::move(body_segment)); + data.rec_split_indexes.emplace(db::blocks::kIdxBodyNumberName, std::move(idx_body_number)); + + data.segments.emplace(db::blocks::kTxnSegmentName, std::move(txn_segment)); + data.rec_split_indexes.emplace(db::blocks::kIdxTxnHashName, std::move(idx_txn_hash)); + data.rec_split_indexes.emplace(db::blocks::kIdxTxnHash2BlockName, std::move(idx_txn_hash_2_block)); + + return data; + }(); + + snapshots::SnapshotBundle bundle{ + headers_segment_path->step_range(), + std::move(bundle_data), }; handle->repository->add_snapshot_bundle(std::move(bundle)); return SILKWORM_OK; diff --git a/silkworm/db/blocks/schema_config.hpp b/silkworm/db/blocks/schema_config.hpp index bcd485f9d7..2dd909b396 100644 --- a/silkworm/db/blocks/schema_config.hpp +++ b/silkworm/db/blocks/schema_config.hpp @@ -44,4 +44,18 @@ inline constexpr datastore::EntityName kIdxTxnHashName{"transactions"}; //! Index transaction_hash -> block_num inline constexpr datastore::EntityName kIdxTxnHash2BlockName{"transactions_to_block"}; +struct BundleDataRef { + const snapshots::SnapshotBundleData& data; + + const snapshots::SegmentFileReader& header_segment() const { return data.segments.at(kHeaderSegmentName); } + const snapshots::Index& idx_header_hash() const { return data.rec_split_indexes.at(kIdxHeaderHashName); } + + const snapshots::SegmentFileReader& body_segment() const { return data.segments.at(kBodySegmentName); } + const snapshots::Index& idx_body_number() const { return data.rec_split_indexes.at(kIdxBodyNumberName); } + + const snapshots::SegmentFileReader& txn_segment() const { return data.segments.at(kTxnSegmentName); } + const snapshots::Index& idx_txn_hash() const { return data.rec_split_indexes.at(kIdxTxnHashName); } + const snapshots::Index& idx_txn_hash_2_block() const { return data.rec_split_indexes.at(kIdxTxnHash2BlockName); } +}; + } // namespace silkworm::db::blocks diff --git a/silkworm/db/blocks/transactions/txn_queries.hpp b/silkworm/db/blocks/transactions/txn_queries.hpp index 21cb5e8352..428a6c21d9 100644 --- a/silkworm/db/blocks/transactions/txn_queries.hpp +++ b/silkworm/db/blocks/transactions/txn_queries.hpp @@ -22,6 +22,7 @@ #include #include +#include "../schema_config.hpp" #include "txn_segment.hpp" namespace silkworm::snapshots { @@ -59,10 +60,10 @@ class TransactionBlockNumByTxnHashRepoQuery { std::optional exec(const Hash& hash) { for (const TBundle& bundle_ptr : bundles_) { - const auto& bundle = *bundle_ptr; - const SegmentFileReader& segment = bundle.txn_segment; - const Index& idx_txn_hash = bundle.idx_txn_hash; - const Index& idx_txn_hash_2_block = bundle.idx_txn_hash_2_block; + db::blocks::BundleDataRef bundle{**bundle_ptr}; + const SegmentFileReader& segment = bundle.txn_segment(); + const Index& idx_txn_hash = bundle.idx_txn_hash(); + const Index& idx_txn_hash_2_block = bundle.idx_txn_hash_2_block(); TransactionFindByHashQuery cross_check_query{{segment, idx_txn_hash}}; TransactionBlockNumByTxnHashQuery query{idx_txn_hash_2_block, cross_check_query}; diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index 3e9f4e6c54..3569dcc8ba 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -32,7 +32,7 @@ namespace silkworm::snapshots { namespace fs = std::filesystem; -std::optional SnapshotPath::parse(fs::path path) { +std::optional SnapshotPath::parse_step_range(const fs::path& path) { const std::string filename_no_ext = path.stem().string(); // Expected stem format: -<6_digit_block_from>-<6_digit_block_to>- @@ -43,17 +43,6 @@ std::optional SnapshotPath::parse(fs::path path) { const auto [ver, from, to, tag] = std::tie(tokens[0], tokens[1], tokens[2], tokens[3]); - // Expected version format: v (hence check length, check first char and parse w/ offset by one) - if (ver.empty() || ver[0] != 'v') { - return std::nullopt; - } - - uint8_t ver_num = 0; - const auto ver_result = std::from_chars(ver.data() + 1, ver.data() + ver.size(), ver_num); - if (ver_result.ec == std::errc::invalid_argument) { - return std::nullopt; - } - // Expected scaled block format: if (from.size() != 6 || to.size() != 6) { return std::nullopt; @@ -76,6 +65,36 @@ std::optional SnapshotPath::parse(fs::path path) { return std::nullopt; } + return StepRange{step_from, step_to}; +} + +std::optional SnapshotPath::parse(fs::path path) { + const std::string filename_no_ext = path.stem().string(); + + // Expected stem format: -<6_digit_block_from>-<6_digit_block_to>- + const std::vector tokens = absl::StrSplit(filename_no_ext, absl::MaxSplits('-', 3)); + if (tokens.size() != 4) { + return std::nullopt; + } + + const auto [ver, from, to, tag] = std::tie(tokens[0], tokens[1], tokens[2], tokens[3]); + + // Expected version format: v (hence check length, check first char and parse w/ offset by one) + if (ver.empty() || ver[0] != 'v') { + return std::nullopt; + } + + uint8_t ver_num = 0; + const auto ver_result = std::from_chars(ver.data() + 1, ver.data() + ver.size(), ver_num); + if (ver_result.ec == std::errc::invalid_argument) { + return std::nullopt; + } + + auto step_range = parse_step_range(path); + if (!step_range) { + return std::nullopt; + } + // Expected tag format: headers|bodies|transactions|transactions-to-block // parsing relies on magic_enum, so SnapshotType items must match exactly std::string tag_str{tag.data(), tag.size()}; @@ -85,7 +104,7 @@ std::optional SnapshotPath::parse(fs::path path) { return std::nullopt; } - return SnapshotPath{std::move(path), ver_num, {step_from, step_to}, *type}; + return SnapshotPath{std::move(path), ver_num, *step_range, *type}; } SnapshotPath SnapshotPath::make( diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index bf10ce7053..9df9d6f9a4 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -36,6 +36,7 @@ inline constexpr uint8_t kSnapshotV1{1}; class SnapshotPath { public: static std::optional parse(std::filesystem::path path); + static std::optional parse_step_range(const std::filesystem::path& path); static SnapshotPath make( const std::filesystem::path& dir, diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp index 2347a53910..e50c001ef9 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp @@ -16,57 +16,79 @@ #include "snapshot_bundle.hpp" +#include + #include namespace silkworm::snapshots { +SnapshotBundleData make_bundle_data( + const Schema::RepositoryDef& schema, + const std::filesystem::path& dir_path, + StepRange step_range) { + return { + schema.make_segments(dir_path, step_range), + schema.make_rec_split_indexes(dir_path, step_range), + }; +} + SnapshotBundle::~SnapshotBundle() { close(); } void SnapshotBundle::reopen() { - for (auto& segment_ref : segments()) { - segment_ref.get().reopen_segment(); - ensure(!segment_ref.get().empty(), [&]() { - return "invalid empty snapshot " + segment_ref.get().fs_path().string(); + for (auto& entry : data_.segments) { + SegmentFileReader& segment = entry.second; + segment.reopen_segment(); + ensure(!segment.empty(), [&]() { + return "invalid empty snapshot " + segment.fs_path().string(); }); } - for (auto& index_ref : indexes()) { - index_ref.get().reopen_index(); + for (auto& entry : data_.rec_split_indexes) { + Index& index = entry.second; + index.reopen_index(); } } void SnapshotBundle::close() { - for (auto& index_ref : indexes()) { - index_ref.get().close_index(); + for (auto& entry : data_.rec_split_indexes) { + Index& index = entry.second; + index.close_index(); } - for (auto& segment_ref : segments()) { - segment_ref.get().close(); + for (auto& entry : data_.segments) { + SegmentFileReader& segment = entry.second; + segment.close(); } if (on_close_callback_) { on_close_callback_(*this); } } +const SegmentFileReader& SnapshotBundle::segment(SnapshotType type) const { + datastore::EntityName name{magic_enum::enum_name(type)}; + return data_.segments.at(name); +} + +const Index& SnapshotBundle::index(SnapshotType type) const { + datastore::EntityName name{magic_enum::enum_name(type)}; + return data_.rec_split_indexes.at(name); +} + std::vector SnapshotBundle::files() { std::vector files; - files.reserve(kSnapshotsCount + kIndexesCount); - - for (auto& segment_ref : segments()) { - files.push_back(segment_ref.get().path().path()); + for (const SegmentFileReader& segment : segments()) { + files.push_back(segment.path().path()); } - for (auto& index_ref : indexes()) { - files.push_back(index_ref.get().path().path()); + for (const Index& index : rec_split_indexes()) { + files.push_back(index.path().path()); } return files; } std::vector SnapshotBundle::segment_paths() { std::vector paths; - paths.reserve(kSnapshotsCount); - - for (auto& segment_ref : segments()) { - paths.push_back(segment_ref.get().path()); + for (const SegmentFileReader& segment : segments()) { + paths.push_back(segment.path()); } return paths; } diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp index 7d60c1bc19..702ed81a17 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp @@ -33,24 +33,15 @@ namespace silkworm::snapshots { struct SnapshotBundleData { - SegmentFileReader header_segment; - //! Index header_hash -> block_num -> headers_segment_offset - Index idx_header_hash; - - SegmentFileReader body_segment; - //! Index block_num -> bodies_segment_offset - Index idx_body_number; - - SegmentFileReader txn_segment; - //! Index transaction_hash -> txn_id -> transactions_segment_offset - Index idx_txn_hash; - //! Index transaction_hash -> block_num - Index idx_txn_hash_2_block; - - static constexpr size_t kSnapshotsCount = 3; - static constexpr size_t kIndexesCount = 4; + std::map segments; + std::map rec_split_indexes; }; +SnapshotBundleData make_bundle_data( + const Schema::RepositoryDef& schema, + const std::filesystem::path& dir_path, + StepRange step_range); + struct SnapshotBundlePaths { SnapshotBundlePaths(Schema::RepositoryDef schema, std::filesystem::path dir_path, StepRange step_range) : schema_{std::move(schema)}, @@ -68,10 +59,10 @@ struct SnapshotBundlePaths { StepRange step_range_; }; -struct SnapshotBundle : public SnapshotBundleData { - explicit SnapshotBundle(StepRange step_range, SnapshotBundleData bundle) - : SnapshotBundleData{std::move(bundle)}, - step_range_{step_range} { +struct SnapshotBundle { + SnapshotBundle(StepRange step_range, SnapshotBundleData data) + : step_range_{step_range}, + data_{std::move(data)} { reopen(); } virtual ~SnapshotBundle(); @@ -79,69 +70,14 @@ struct SnapshotBundle : public SnapshotBundleData { SnapshotBundle(SnapshotBundle&&) = default; SnapshotBundle& operator=(SnapshotBundle&&) noexcept = default; - std::array, kSnapshotsCount> segments() { - return { - header_segment, - body_segment, - txn_segment, - }; - } - - std::array, kIndexesCount> indexes() { - return { - idx_header_hash, - idx_body_number, - idx_txn_hash, - idx_txn_hash_2_block, - }; - } - - std::array snapshot_types() { - return { - SnapshotType::headers, - SnapshotType::bodies, - SnapshotType::transactions, - }; - } - - std::array index_types() { - return { - SnapshotType::headers, - SnapshotType::bodies, - SnapshotType::transactions, - SnapshotType::transactions_to_block, - }; - } - - const SegmentFileReader& segment(SnapshotType type) const { - switch (type) { - case headers: - return header_segment; - case bodies: - return body_segment; - case transactions: - case transactions_to_block: - return txn_segment; - } - SILKWORM_ASSERT(false); - return header_segment; + auto segments() { + return make_map_values_view(data_.segments); } - - const Index& index(SnapshotType type) const { - switch (type) { - case headers: - return idx_header_hash; - case bodies: - return idx_body_number; - case transactions: - return idx_txn_hash; - case transactions_to_block: - return idx_txn_hash_2_block; - } - SILKWORM_ASSERT(false); - return idx_header_hash; + auto rec_split_indexes() { + return make_map_values_view(data_.rec_split_indexes); } - + const SegmentFileReader& segment(SnapshotType type) const; + const Index& index(SnapshotType type) const; SegmentAndIndex segment_and_index(SnapshotType type) const { return {segment(type), index(type)}; } @@ -158,8 +94,12 @@ struct SnapshotBundle : public SnapshotBundleData { on_close_callback_ = std::move(callback); } + const SnapshotBundleData& operator*() const { return data_; } + const SnapshotBundleData* operator->() const { return &data_; } + private: StepRange step_range_; + SnapshotBundleData data_; std::function on_close_callback_; }; diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp index b888b4984e..0195de4b58 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp @@ -30,8 +30,6 @@ namespace silkworm::snapshots { struct SnapshotBundleFactory { virtual ~SnapshotBundleFactory() = default; - using PathByTypeProvider = std::function; - virtual SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const = 0; virtual SnapshotBundle make(const std::filesystem::path& dir_path, StepRange range) const = 0; virtual SnapshotBundlePaths make_paths(const std::filesystem::path& dir_path, StepRange range) const = 0; diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 3f73dd7ed8..8c0419d848 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -113,62 +113,42 @@ std::vector> SnapshotRepository::missing_indexes() void SnapshotRepository::reopen_folder() { SILK_INFO << "Reopen snapshot repository folder: " << dir_path_.string(); - SnapshotPathList all_snapshot_paths = get_segment_files(); - SnapshotPathList all_index_paths = get_idx_files(); - std::map>> groups; + auto file_ranges = list_dir_file_ranges(); + if (file_ranges.empty()) return; - for (size_t i = 0; i < all_snapshot_paths.size(); ++i) { - auto& path = all_snapshot_paths[i]; - auto& group = groups[path.step_range().start][false]; - group[path.type()] = i; - } - - for (size_t i = 0; i < all_index_paths.size(); ++i) { - auto& path = all_index_paths[i]; - auto& group = groups[path.step_range().start][true]; - group[path.type()] = i; - } - - Step num{0}; - if (!groups.empty()) { - num = groups.begin()->first; - } + // sort file_ranges by range.start + std::sort(file_ranges.begin(), file_ranges.end(), [](const StepRange& r1, const StepRange& r2) -> bool { + return r1.start < r2.start; + }); std::unique_lock lock(*bundles_mutex_); // copy bundles prior to modification auto bundles = std::make_shared(*bundles_); - while (groups.contains(num) && - (groups[num][false].size() == SnapshotBundle::kSnapshotsCount) && - (groups[num][true].size() == SnapshotBundle::kIndexesCount)) { + Step num = file_ranges[0].start; + for (const auto& range : file_ranges) { + // avoid gaps/overlaps + if (range.start != num) continue; + if (range.size() == 0) continue; + if (!bundles->contains(num)) { - auto snapshot_path = [&](SnapshotType type) { - return all_snapshot_paths[groups[num][false][type]]; - }; - auto index_path = [&](SnapshotType type) { - return all_index_paths[groups[num][true][type]]; - }; - SnapshotBundle bundle = bundle_factory_->make(snapshot_path, index_path); - - bundles->insert_or_assign(num, std::make_shared(std::move(bundle))); + SnapshotBundlePaths bundle_paths = bundle_factory_->make_paths(dir_path_, range); + // if all bundle paths exist + if (std::ranges::all_of(bundle_paths.files(), [](const fs::path& p) { return fs::exists(p); })) { + SnapshotBundle bundle = bundle_factory_->make(dir_path_, range); + bundles->insert_or_assign(num, std::make_shared(std::move(bundle))); + } } - auto& bundle = *bundles->at(num); - - if (num < bundle.step_range().end) { - num = bundle.step_range().end; - } else { - break; - } + // avoid gaps/overlaps + num = range.end; } bundles_ = bundles; lock.unlock(); SILK_INFO << "Total reopened bundles: " << bundles_count() - << " segments: " << total_segments_count() - << " indexes: " << total_indexes_count() << " max block available: " << max_block_available(); } @@ -221,6 +201,26 @@ SnapshotPathList SnapshotRepository::get_files(const std::string& ext) const { return snapshot_files; } +std::vector SnapshotRepository::list_dir_file_ranges() const { + ensure(fs::exists(dir_path_), + [&]() { return "SnapshotRepository: " + dir_path_.string() + " does not exist"; }); + ensure(fs::is_directory(dir_path_), + [&]() { return "SnapshotRepository: " + dir_path_.string() + " is a not folder"; }); + + std::vector results; + for (const auto& file : fs::directory_iterator{dir_path_}) { + if (!fs::is_regular_file(file.path())) { + continue; + } + const auto range = SnapshotPath::parse_step_range(file.path()); + if (range) { + results.push_back(*range); + } + } + + return results; +} + bool is_stale_index_path(const SnapshotPath& index_path) { SnapshotType snapshot_type = (index_path.type() == SnapshotType::transactions_to_block) ? SnapshotType::transactions diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index 8e3c12fd2e..ede6698f13 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -66,8 +66,6 @@ class SnapshotRepository { void replace_snapshot_bundles(SnapshotBundle bundle); size_t bundles_count() const; - size_t total_segments_count() const { return bundles_count() * SnapshotBundle::kSnapshotsCount; } - size_t total_indexes_count() const { return bundles_count() * SnapshotBundle::kIndexesCount; } //! All types of .seg and .idx files are available up to this block number BlockNum max_block_available() const; @@ -123,6 +121,7 @@ class SnapshotRepository { } SnapshotPathList get_files(const std::string& ext) const; + std::vector list_dir_file_ranges() const; SnapshotPathList stale_index_paths() const; diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp index 64a8d355c0..ca6aa0674c 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.cpp +++ b/silkworm/db/snapshot_bundle_factory_impl.cpp @@ -27,31 +27,11 @@ namespace silkworm::db { using namespace snapshots; -SnapshotBundle SnapshotBundleFactoryImpl::make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const { - return SnapshotBundle{ - snapshot_path(SnapshotType::headers).step_range(), - { - .header_segment = SegmentFileReader(snapshot_path(SnapshotType::headers)), - .idx_header_hash = Index(index_path(SnapshotType::headers)), - - .body_segment = SegmentFileReader(snapshot_path(SnapshotType::bodies)), - .idx_body_number = Index(index_path(SnapshotType::bodies)), - - .txn_segment = SegmentFileReader(snapshot_path(SnapshotType::transactions)), - .idx_txn_hash = Index(index_path(SnapshotType::transactions)), - .idx_txn_hash_2_block = Index(index_path(SnapshotType::transactions_to_block)), - }, - }; -} - SnapshotBundle SnapshotBundleFactoryImpl::make(const std::filesystem::path& dir_path, snapshots::StepRange range) const { - PathByTypeProvider snapshot_path = [&](silkworm::snapshots::SnapshotType type) { - return SnapshotPath::make(dir_path, kSnapshotV1, range, type); - }; - PathByTypeProvider index_path = [&](silkworm::snapshots::SnapshotType type) { - return SnapshotPath::make(dir_path, kSnapshotV1, range, type, kIdxExtension); + return SnapshotBundle{ + range, + make_bundle_data(schema_, dir_path, range), }; - return make(std::move(snapshot_path), std::move(index_path)); } SnapshotBundlePaths SnapshotBundleFactoryImpl::make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const { diff --git a/silkworm/db/snapshot_bundle_factory_impl.hpp b/silkworm/db/snapshot_bundle_factory_impl.hpp index fe99a496f8..0af8dc5895 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.hpp +++ b/silkworm/db/snapshot_bundle_factory_impl.hpp @@ -26,7 +26,6 @@ struct SnapshotBundleFactoryImpl : public snapshots::SnapshotBundleFactory { : schema_{std::move(schema)} {} ~SnapshotBundleFactoryImpl() override = default; - snapshots::SnapshotBundle make(PathByTypeProvider snapshot_path, PathByTypeProvider index_path) const override; snapshots::SnapshotBundle make(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; snapshots::SnapshotBundlePaths make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; std::vector> index_builders(const snapshots::SnapshotPath& segment_path) const override; diff --git a/silkworm/db/snapshot_sync.cpp b/silkworm/db/snapshot_sync.cpp index c030a58e6d..2159173bee 100644 --- a/silkworm/db/snapshot_sync.cpp +++ b/silkworm/db/snapshot_sync.cpp @@ -279,9 +279,9 @@ void SnapshotSync::seed_frozen_local_snapshots() { auto& bundle = *bundle_ptr; auto block_range = bundle.step_range().to_block_num_range(); bool is_frozen = block_range.size() >= kMaxMergerSnapshotSize; - const auto first_snapshot = bundle.segments()[0]; + const SegmentFileReader& first_snapshot = *bundle.segments().begin(); // assume that if one snapshot in the bundle is preverified, then all of them are - bool is_preverified = snapshots_config_.contains_file_name(first_snapshot.get().path().filename()); + bool is_preverified = snapshots_config_.contains_file_name(first_snapshot.path().filename()); if (is_frozen && !is_preverified) { seed_bundle(bundle); } @@ -336,8 +336,8 @@ void SnapshotSync::update_block_headers(RWTxn& txn, BlockNum max_block_available uint64_t block_count{0}; for (const auto& bundle_ptr : repository_.view_bundles()) { - const auto& bundle = *bundle_ptr; - for (const BlockHeader& header : HeaderSegmentReader{bundle.header_segment}) { + db::blocks::BundleDataRef bundle{**bundle_ptr}; + for (const BlockHeader& header : HeaderSegmentReader{bundle.header_segment()}) { SILK_TRACE << "SnapshotSync: header number=" << header.number << " hash=" << Hash{header.hash()}.to_hex(); const auto block_number = header.number; if (block_number > max_block_available) continue; diff --git a/silkworm/db/snapshot_sync_test.cpp b/silkworm/db/snapshot_sync_test.cpp index 673948ccd1..a482de524f 100644 --- a/silkworm/db/snapshot_sync_test.cpp +++ b/silkworm/db/snapshot_sync_test.cpp @@ -115,44 +115,28 @@ TEST_CASE("SnapshotSync::update_block_headers", "[db][snapshot][sync]") { // Create a sample Header snapshot+index snapshots::test_util::SampleHeaderSnapshotFile header_segment_file{tmp_dir_path}; auto& header_segment_path = header_segment_file.path(); - SegmentFileReader header_segment{header_segment_path}; auto header_index_builder = HeaderIndex::make(header_segment_path); header_index_builder.set_base_data_id(header_segment_file.block_num_range().start); REQUIRE_NOTHROW(header_index_builder.build()); - Index idx_header_hash{header_segment_path.index_file()}; // Create a sample Body snapshot+index snapshots::test_util::SampleBodySnapshotFile body_segment_file{tmp_dir_path}; auto& body_segment_path = body_segment_file.path(); - SegmentFileReader body_segment{body_segment_path}; auto body_index_builder = BodyIndex::make(body_segment_path); body_index_builder.set_base_data_id(body_segment_file.block_num_range().start); REQUIRE_NOTHROW(body_index_builder.build()); - Index idx_body_number{body_segment_path.index_file()}; // Create a sample Transaction snapshot+indexes snapshots::test_util::SampleTransactionSnapshotFile txn_segment_file{tmp_dir_path}; auto& txn_segment_path = txn_segment_file.path(); - SegmentFileReader txn_segment{txn_segment_path}; REQUIRE_NOTHROW(TransactionIndex::make(body_segment_path, txn_segment_path).build()); REQUIRE_NOTHROW(TransactionToBlockIndex::make(body_segment_path, txn_segment_path, txn_segment_file.block_num_range().start).build()); - Index idx_txn_hash{txn_segment_path.related_path(SnapshotType::transactions, kIdxExtension)}; - Index idx_txn_hash_2_block{txn_segment_path.related_path(SnapshotType::transactions_to_block, kIdxExtension)}; // Add a sample Snapshot bundle to the repository + auto step_range = StepRange::from_block_num_range(snapshots::test_util::kSampleSnapshotBlockRange); SnapshotBundle bundle{ - header_segment_path.step_range(), - { - .header_segment = std::move(header_segment), - .idx_header_hash = std::move(idx_header_hash), - - .body_segment = std::move(body_segment), - .idx_body_number = std::move(idx_body_number), - - .txn_segment = std::move(txn_segment), - .idx_txn_hash = std::move(idx_txn_hash), - .idx_txn_hash_2_block = std::move(idx_txn_hash_2_block), - }, + step_range, + make_bundle_data(blocks::make_blocks_repository_schema(), tmp_dir_path, step_range), }; auto& repository = snapshot_sync.repository(); repository.add_snapshot_bundle(std::move(bundle));