From 600b7fd1ed19cb3aa2e9dbef3de0a6ab5fca97aa Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 15:56:58 +0100 Subject: [PATCH 01/15] refactor dev/snapshots to use bundle factory --- cmd/dev/snapshots.cpp | 28 +++------------------------- silkworm/db/blocks/schema_config.cpp | 6 +++++- silkworm/db/blocks/schema_config.hpp | 4 ++++ 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index f0cfdb064e..2c97969eed 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -344,7 +344,6 @@ void count_bodies(const SnapshotSubcommandSettings& settings, int repetitions) { if (settings.segment_file_name) { const auto snapshot_path{SnapshotPath::parse(std::filesystem::path{*settings.segment_file_name})}; ensure(snapshot_path.has_value(), "count_bodies: invalid snapshot_file path format"); - ensure(snapshot_path->type() == SnapshotType::bodies, "count_bodies: snapshot_file must point to body segment"); SegmentFileReader body_segment{*snapshot_path}; body_segment.reopen_segment(); std::tie(num_bodies, num_txns) = count_bodies_in_one(settings, body_segment); @@ -390,7 +389,6 @@ void count_headers(const SnapshotSubcommandSettings& settings, int repetitions) if (settings.segment_file_name) { const auto snapshot_path{SnapshotPath::parse(std::filesystem::path{*settings.segment_file_name})}; ensure(snapshot_path.has_value(), "count_headers: invalid snapshot_file path format"); - ensure(snapshot_path->type() == SnapshotType::headers, "count_headers: snapshot_file must point to header segment"); SegmentFileReader header_segment{*snapshot_path}; header_segment.reopen_segment(); num_headers = count_headers_in_one(settings, header_segment); @@ -407,32 +405,12 @@ void create_index(const SnapshotSubcommandSettings& settings, int repetitions) { ensure(settings.segment_file_name.has_value(), "create_index: --snapshot_file must be specified"); SILK_INFO << "Create index for snapshot: " << *settings.segment_file_name; std::chrono::time_point start{std::chrono::steady_clock::now()}; + auto bundle_factory = db::blocks::make_blocks_bundle_factory(); const auto snapshot_path = SnapshotPath::parse(std::filesystem::path{*settings.segment_file_name}); if (snapshot_path) { for (int i{0}; i < repetitions; ++i) { - switch (snapshot_path->type()) { - case SnapshotType::headers: { - auto index = HeaderIndex::make(*snapshot_path); - index.build(); - break; - } - case SnapshotType::bodies: { - auto index = BodyIndex::make(*snapshot_path); - index.build(); - break; - } - case SnapshotType::transactions: { - auto bodies_segment_path = snapshot_path->related_path(SnapshotType::bodies, kSegmentExtension); - auto index = TransactionIndex::make(bodies_segment_path, *snapshot_path); - index.build(); - - auto index_hash_to_block = TransactionToBlockIndex::make(bodies_segment_path, *snapshot_path); - index_hash_to_block.build(); - break; - } - default: { - SILKWORM_ASSERT(false); - } + for (auto& builder : bundle_factory->index_builders(*snapshot_path)) { + builder->build(); } } } else { diff --git a/silkworm/db/blocks/schema_config.cpp b/silkworm/db/blocks/schema_config.cpp index 987193e6eb..978d41c48e 100644 --- a/silkworm/db/blocks/schema_config.cpp +++ b/silkworm/db/blocks/schema_config.cpp @@ -33,12 +33,16 @@ snapshots::Schema::RepositoryDef make_blocks_repository_schema() { return schema; } +std::unique_ptr make_blocks_bundle_factory() { + return std::make_unique(make_blocks_repository_schema()); +} + snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path, bool open) { return snapshots::SnapshotRepository{ std::move(dir_path), open, std::make_unique(), - std::make_unique(make_blocks_repository_schema()), + make_blocks_bundle_factory(), }; } diff --git a/silkworm/db/blocks/schema_config.hpp b/silkworm/db/blocks/schema_config.hpp index 2dd909b396..0266d85666 100644 --- a/silkworm/db/blocks/schema_config.hpp +++ b/silkworm/db/blocks/schema_config.hpp @@ -16,9 +16,11 @@ #pragma once +#include #include "../datastore/common/entity_name.hpp" #include "../datastore/snapshots/schema.hpp" #include "../datastore/snapshots/snapshot_repository.hpp" +#include "../datastore/snapshots/snapshot_bundle_factory.hpp" namespace silkworm::db::blocks { @@ -26,6 +28,8 @@ inline constexpr datastore::EntityName kBlocksRepositoryName{"Blocks"}; snapshots::Schema::RepositoryDef make_blocks_repository_schema(); +std::unique_ptr make_blocks_bundle_factory(); + snapshots::SnapshotRepository make_blocks_repository( std::filesystem::path dir_path, bool open = true); From 35a9d08300bc81b5dfec29c3f38f3566cc30e87e Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 16:27:11 +0100 Subject: [PATCH 02/15] use EntityName in freezer/merger --- silkworm/db/blocks/schema_config.hpp | 3 +- silkworm/db/datastore/snapshot_merger.cpp | 6 ++-- .../snapshots/common/snapshot_path.cpp | 4 --- .../snapshots/common/snapshot_path.hpp | 1 - .../datastore/snapshots/snapshot_bundle.cpp | 15 ++++++--- .../datastore/snapshots/snapshot_bundle.hpp | 7 +++- .../snapshots/snapshot_repository.cpp | 8 +++-- .../snapshots/snapshot_repository.hpp | 2 +- silkworm/db/freezer.cpp | 33 +++++++++---------- 9 files changed, 44 insertions(+), 35 deletions(-) diff --git a/silkworm/db/blocks/schema_config.hpp b/silkworm/db/blocks/schema_config.hpp index 0266d85666..cc8dd00ef9 100644 --- a/silkworm/db/blocks/schema_config.hpp +++ b/silkworm/db/blocks/schema_config.hpp @@ -17,10 +17,11 @@ #pragma once #include + #include "../datastore/common/entity_name.hpp" #include "../datastore/snapshots/schema.hpp" -#include "../datastore/snapshots/snapshot_repository.hpp" #include "../datastore/snapshots/snapshot_bundle_factory.hpp" +#include "../datastore/snapshots/snapshot_repository.hpp" namespace silkworm::db::blocks { diff --git a/silkworm/db/datastore/snapshot_merger.cpp b/silkworm/db/datastore/snapshot_merger.cpp index c6b96687d5..23bb77cfcd 100644 --- a/silkworm/db/datastore/snapshot_merger.cpp +++ b/silkworm/db/datastore/snapshot_merger.cpp @@ -96,13 +96,13 @@ std::shared_ptr SnapshotMerger::migrate(std::unique_ptr reader{bundle.segment(path.type())}; + SegmentReader reader{bundle.segment(name)}; std::copy(reader.begin(), reader.end(), compressor.add_word_iterator()); } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index 3569dcc8ba..3036a5f91b 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -117,10 +117,6 @@ SnapshotPath SnapshotPath::make( return SnapshotPath{dir / filename, version, step_range, type}; } -std::string SnapshotPath::type_string() const { - return std::string{magic_enum::enum_name(type_)}; -} - fs::path SnapshotPath::make_filename( uint8_t version, StepRange step_range, diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index 9df9d6f9a4..e39f63b1b6 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -51,7 +51,6 @@ class SnapshotPath { uint8_t version() const { return version_; } StepRange step_range() const { return step_range_; } SnapshotType type() const { return type_; } - std::string type_string() const; bool exists() const { return std::filesystem::exists(path_); } SnapshotPath related_path(SnapshotType type, const char* ext) const; diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp index e50c001ef9..8bd929c1fa 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp @@ -74,6 +74,14 @@ const Index& SnapshotBundle::index(SnapshotType type) const { return data_.rec_split_indexes.at(name); } +const SegmentFileReader& SnapshotBundle::segment(datastore::EntityName name) const { + return data_.segments.at(name); +} + +const Index& SnapshotBundle::index(datastore::EntityName name) const { + return data_.rec_split_indexes.at(name); +} + std::vector SnapshotBundle::files() { std::vector files; for (const SegmentFileReader& segment : segments()) { @@ -93,11 +101,8 @@ std::vector SnapshotBundle::segment_paths() { return paths; } -std::vector SnapshotBundlePaths::segment_paths() const { - std::vector results; - for (auto& entry : schema_.make_segment_paths(dir_path_, step_range_)) - results.push_back(std::move(entry.second)); - return results; +std::map SnapshotBundlePaths::segment_paths() const { + return schema_.make_segment_paths(dir_path_, step_range_); } std::vector SnapshotBundlePaths::files() const { diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp index 702ed81a17..f34cdaa65d 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp @@ -51,7 +51,7 @@ struct SnapshotBundlePaths { StepRange step_range() const { return step_range_; } std::vector files() const; - std::vector segment_paths() const; + std::map segment_paths() const; private: Schema::RepositoryDef schema_; @@ -81,6 +81,11 @@ struct SnapshotBundle { SegmentAndIndex segment_and_index(SnapshotType type) const { return {segment(type), index(type)}; } + const SegmentFileReader& segment(datastore::EntityName name) const; + const Index& index(datastore::EntityName name) const; + SegmentAndIndex segment_and_index(datastore::EntityName name) const { + return {segment(name), index(name)}; + } StepRange step_range() const { return step_range_; } diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 8c0419d848..5e6ee8a774 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -243,8 +243,12 @@ void SnapshotRepository::remove_stale_indexes() const { } } -void SnapshotRepository::build_indexes(SnapshotBundlePaths& bundle) const { - for (auto& builder : bundle_factory_->index_builders(bundle.segment_paths())) { +void SnapshotRepository::build_indexes(const SnapshotBundlePaths& bundle) const { + std::vector segment_paths; + for (auto& entry : bundle.segment_paths()) + segment_paths.push_back(std::move(entry.second)); + + for (auto& builder : bundle_factory_->index_builders(segment_paths)) { builder->build(); } } diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index ede6698f13..7b1de77a4b 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -73,7 +73,7 @@ class SnapshotRepository { std::vector> missing_indexes() const; void remove_stale_indexes() const; - void build_indexes(SnapshotBundlePaths& bundle) const; + void build_indexes(const SnapshotBundlePaths& bundle) const; using Bundles = std::map>; diff --git a/silkworm/db/freezer.cpp b/silkworm/db/freezer.cpp index 9b1e801ebb..a85efd289e 100644 --- a/silkworm/db/freezer.cpp +++ b/silkworm/db/freezer.cpp @@ -29,6 +29,7 @@ #include "blocks/bodies/body_queries.hpp" #include "blocks/bodies/body_segment_collation.hpp" #include "blocks/headers/header_segment_collation.hpp" +#include "blocks/schema_config.hpp" #include "blocks/transactions/txn_segment_collation.hpp" #include "datastore/segment_collation.hpp" #include "datastore/snapshots/common/snapshot_path.hpp" @@ -87,22 +88,20 @@ std::unique_ptr Freezer::next_command() { return {}; } -static const SegmentCollation& get_collation(SnapshotType type) { +static const SegmentCollation& get_collation(datastore::EntityName name) { static HeaderSegmentCollation header_collation; static BodySegmentCollation body_collation; static TransactionSegmentCollation txn_collation; - switch (type) { - case SnapshotType::headers: - return header_collation; - case SnapshotType::bodies: - return body_collation; - case SnapshotType::transactions: - return txn_collation; - default: - SILKWORM_ASSERT(false); - throw std::runtime_error("invalid type"); - } + if (name == blocks::kHeaderSegmentName) + return header_collation; + if (name == blocks::kBodySegmentName) + return body_collation; + if (name == blocks::kTxnSegmentName) + return txn_collation; + + SILKWORM_ASSERT(false); + throw std::runtime_error("invalid name"); } std::shared_ptr Freezer::migrate(std::unique_ptr command) { @@ -111,11 +110,11 @@ std::shared_ptr Freezer::migrate(std::unique_ptr Freezer::cleanup() { } void Freezer::prune_collations(RWTxn& db_tx, BlockNumRange range) const { - get_collation(SnapshotType::transactions).prune(db_tx, range); - get_collation(SnapshotType::bodies).prune(db_tx, range); - get_collation(SnapshotType::headers).prune(db_tx, range); + get_collation(blocks::kHeaderSegmentName).prune(db_tx, range); + get_collation(blocks::kBodySegmentName).prune(db_tx, range); + get_collation(blocks::kTxnSegmentName).prune(db_tx, range); } } // namespace silkworm::db From d204e265230070bcb5053cea882c0360bbf6cf22 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 16:44:53 +0100 Subject: [PATCH 03/15] use string tag in SnapshotPath test --- .../db/datastore/snapshots/common/snapshot_path.cpp | 7 +++++-- .../db/datastore/snapshots/common/snapshot_path.hpp | 3 +++ .../snapshots/common/snapshot_path_test.cpp | 12 ++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index 3036a5f91b..d95cf59127 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -104,7 +104,7 @@ std::optional SnapshotPath::parse(fs::path path) { return std::nullopt; } - return SnapshotPath{std::move(path), ver_num, *step_range, *type}; + return SnapshotPath{std::move(path), ver_num, *step_range, std::move(tag_str), *type}; } SnapshotPath SnapshotPath::make( @@ -114,7 +114,8 @@ SnapshotPath SnapshotPath::make( SnapshotType type, const char* ext) { const auto filename = SnapshotPath::make_filename(version, step_range, type, ext); - return SnapshotPath{dir / filename, version, step_range, type}; + std::string tag{magic_enum::enum_name(type)}; + return SnapshotPath{dir / filename, version, step_range, std::move(tag), type}; } fs::path SnapshotPath::make_filename( @@ -141,10 +142,12 @@ SnapshotPath::SnapshotPath( fs::path path, uint8_t version, StepRange step_range, + std::string tag, SnapshotType type) : path_{std::move(path)}, version_{version}, step_range_{step_range}, + tag_{std::move(tag)}, type_{type} { } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index e39f63b1b6..f4bd2f44e5 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -50,6 +50,7 @@ class SnapshotPath { std::string extension() const { return path_.extension().string(); } uint8_t version() const { return version_; } StepRange step_range() const { return step_range_; } + const std::string& tag() const { return tag_; } SnapshotType type() const { return type_; } bool exists() const { return std::filesystem::exists(path_); } @@ -72,11 +73,13 @@ class SnapshotPath { std::filesystem::path path, uint8_t version, StepRange step_range, + std::string tag, SnapshotType type); std::filesystem::path path_; uint8_t version_{0}; StepRange step_range_; + std::string tag_; SnapshotType type_; }; diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp index 335e6e0eb3..3aa1c806f7 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp @@ -54,12 +54,12 @@ TEST_CASE("SnapshotPath::parse", "[silkworm][node][snapshot]") { struct ValidFilenameExpectation { const char* filename; BlockNumRange block_num_range; - SnapshotType type; + std::string tag; }; const ValidFilenameExpectation valid_filenames[]{ - {"v1-014500-015000-headers.seg", {14'500'000, 15'000'000}, SnapshotType::headers}, - {"v1-011500-012000-bodies.seg", {11'500'000, 12'000'000}, SnapshotType::bodies}, - {"v1-018300-018400-transactions.seg", {18'300'000, 18'400'000}, SnapshotType::transactions}, + {"v1-014500-015000-headers.seg", {14'500'000, 15'000'000}, "headers"}, + {"v1-011500-012000-bodies.seg", {11'500'000, 12'000'000}, "bodies"}, + {"v1-018300-018400-transactions.seg", {18'300'000, 18'400'000}, "transactions"}, }; for (const auto& filename_expectation : valid_filenames) { const auto snapshot_file = SnapshotPath::parse(filename_expectation.filename); @@ -68,13 +68,13 @@ TEST_CASE("SnapshotPath::parse", "[silkworm][node][snapshot]") { CHECK(snapshot_file->path() == filename_expectation.filename); CHECK(snapshot_file->version() == 1); CHECK(snapshot_file->step_range() == StepRange::from_block_num_range(filename_expectation.block_num_range)); - CHECK(snapshot_file->type() == filename_expectation.type); + CHECK(snapshot_file->tag() == filename_expectation.tag); const SnapshotPath index_file = snapshot_file->index_file(); CHECK(index_file.path().stem() == snapshot_file->path().stem()); CHECK(index_file.path().extension() == kIdxExtension); CHECK(index_file.version() == 1); CHECK(index_file.step_range() == StepRange::from_block_num_range(filename_expectation.block_num_range)); - CHECK(index_file.type() == filename_expectation.type); + CHECK(index_file.tag() == filename_expectation.tag); } } } From 1b7cea8997fc511a0b940e8ac43019bd1b7e5db1 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 16:57:02 +0100 Subject: [PATCH 04/15] refactor SnapshotRepository::is_stale_index_path --- .../snapshots/snapshot_bundle_factory.hpp | 2 ++ .../datastore/snapshots/snapshot_repository.cpp | 15 ++++++++------- .../datastore/snapshots/snapshot_repository.hpp | 1 + silkworm/db/snapshot_bundle_factory_impl.cpp | 8 ++++++++ silkworm/db/snapshot_bundle_factory_impl.hpp | 1 + 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp index 0195de4b58..269b519b35 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle_factory.hpp @@ -35,6 +35,8 @@ struct SnapshotBundleFactory { virtual std::vector> index_builders(const SnapshotPath& segment_path) const = 0; virtual std::vector> index_builders(const SnapshotPathList& segment_paths) const = 0; + + virtual SnapshotPathList index_dependency_paths(const SnapshotPath& index_path) const = 0; }; } // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 5e6ee8a774..27648d0bae 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -221,18 +221,19 @@ std::vector SnapshotRepository::list_dir_file_ranges() const { return results; } -bool is_stale_index_path(const SnapshotPath& index_path) { - SnapshotType snapshot_type = (index_path.type() == SnapshotType::transactions_to_block) - ? SnapshotType::transactions - : index_path.type(); - SnapshotPath snapshot_path = index_path.related_path(snapshot_type, kSegmentExtension); - return (fs::last_write_time(index_path.path()) < fs::last_write_time(snapshot_path.path())); +bool SnapshotRepository::is_stale_index_path(const SnapshotPath& index_path) const { + return std::ranges::any_of( + bundle_factory_->index_dependency_paths(index_path), + [&](const SnapshotPath& dep_path) { return fs::last_write_time(index_path.path()) < fs::last_write_time(dep_path.path()); }); } SnapshotPathList SnapshotRepository::stale_index_paths() const { SnapshotPathList results; auto all_files = this->get_idx_files(); - std::copy_if(all_files.begin(), all_files.end(), std::back_inserter(results), is_stale_index_path); + std::copy_if( + all_files.begin(), all_files.end(), + std::back_inserter(results), + [this](const SnapshotPath& index_path) { return this->is_stale_index_path(index_path); }); return results; } diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index 7b1de77a4b..c0c7443504 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -123,6 +123,7 @@ class SnapshotRepository { SnapshotPathList get_files(const std::string& ext) const; std::vector list_dir_file_ranges() const; + bool is_stale_index_path(const SnapshotPath& index_path) const; SnapshotPathList stale_index_paths() const; //! Path to the snapshots directory diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp index ca6aa0674c..c3ed6c1e65 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.cpp +++ b/silkworm/db/snapshot_bundle_factory_impl.cpp @@ -71,4 +71,12 @@ std::vector> SnapshotBundleFactoryImpl::index_buil return all_builders; } +SnapshotPathList SnapshotBundleFactoryImpl::index_dependency_paths(const SnapshotPath& index_path) const { + SnapshotType snapshot_type = (index_path.type() == SnapshotType::transactions_to_block) + ? SnapshotType::transactions + : index_path.type(); + SnapshotPath snapshot_path = index_path.related_path(snapshot_type, kSegmentExtension); + return {snapshot_path}; +} + } // namespace silkworm::db diff --git a/silkworm/db/snapshot_bundle_factory_impl.hpp b/silkworm/db/snapshot_bundle_factory_impl.hpp index 0af8dc5895..7e08ef1874 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.hpp +++ b/silkworm/db/snapshot_bundle_factory_impl.hpp @@ -30,6 +30,7 @@ struct SnapshotBundleFactoryImpl : public snapshots::SnapshotBundleFactory { snapshots::SnapshotBundlePaths make_paths(const std::filesystem::path& dir_path, snapshots::StepRange range) const override; std::vector> index_builders(const snapshots::SnapshotPath& segment_path) const override; std::vector> index_builders(const snapshots::SnapshotPathList& segment_paths) const override; + snapshots::SnapshotPathList index_dependency_paths(const snapshots::SnapshotPath& index_path) const override; private: snapshots::Schema::RepositoryDef schema_; From 5c6d62483c0a5acc138b78b815fab39b1a3513c4 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 17:04:29 +0100 Subject: [PATCH 05/15] use EntityName in snapshot_file_recompress --- silkworm/db/snapshot_recompress.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/silkworm/db/snapshot_recompress.cpp b/silkworm/db/snapshot_recompress.cpp index 1f0e99a328..449f6a9ced 100644 --- a/silkworm/db/snapshot_recompress.cpp +++ b/silkworm/db/snapshot_recompress.cpp @@ -21,6 +21,7 @@ #include "blocks/bodies/body_segment.hpp" #include "blocks/headers/header_segment.hpp" +#include "blocks/schema_config.hpp" #include "blocks/transactions/txn_segment.hpp" #include "datastore/snapshots/common/snapshot_path.hpp" @@ -45,17 +46,16 @@ void snapshot_file_recompress(const std::filesystem::path& path) { TemporaryDirectory tmp_dir; SegmentFileWriter file_writer{*SnapshotPath::parse(out_path), tmp_dir.path()}; - switch (path_opt->type()) { - case SnapshotType::headers: + const auto& tag = path_opt->tag(); + datastore::EntityName name{tag}; + { + if (name == db::blocks::kHeaderSegmentName) copy_reader_to_writer(file_reader, file_writer); - break; - case SnapshotType::bodies: + else if (name == db::blocks::kBodySegmentName) copy_reader_to_writer(file_reader, file_writer); - break; - case SnapshotType::transactions: + else if (name == db::blocks::kTxnSegmentName) copy_reader_to_writer(file_reader, file_writer); - break; - default: + else throw std::runtime_error{"invalid snapshot type"}; } From 5d8ecb9769837f8a0e964f2ec158ec6ebed93951 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 17:32:25 +0100 Subject: [PATCH 06/15] use EntityName in SnapshotBundleFactoryImpl --- .../snapshots/common/snapshot_path.cpp | 22 ++++++++++--- .../snapshots/common/snapshot_path.hpp | 10 +++++- silkworm/db/snapshot_bundle_factory_impl.cpp | 33 ++++++++++++------- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index d95cf59127..ca84f69ac3 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -107,33 +107,47 @@ std::optional SnapshotPath::parse(fs::path path) { return SnapshotPath{std::move(path), ver_num, *step_range, std::move(tag_str), *type}; } +SnapshotPath SnapshotPath::make( + const fs::path& dir, + uint8_t version, + StepRange step_range, + std::string tag, + const char* ext) { + const auto filename = SnapshotPath::make_filename(version, step_range, tag, ext); + auto type = *magic_enum::enum_cast(tag); + return SnapshotPath{dir / filename, version, step_range, std::move(tag), type}; +} + SnapshotPath SnapshotPath::make( const fs::path& dir, uint8_t version, StepRange step_range, SnapshotType type, const char* ext) { - const auto filename = SnapshotPath::make_filename(version, step_range, type, ext); std::string tag{magic_enum::enum_name(type)}; + const auto filename = SnapshotPath::make_filename(version, step_range, tag, ext); return SnapshotPath{dir / filename, version, step_range, std::move(tag), type}; } fs::path SnapshotPath::make_filename( uint8_t version, StepRange step_range, - SnapshotType type, + std::string tag, const char* ext) { - std::string snapshot_type_name{magic_enum::enum_name(type)}; std::string filename = absl::StrFormat( "v%d-%06d-%06d-%s%s", version, step_range.start.value, step_range.end.value, - absl::StrReplaceAll(snapshot_type_name, {{"_", "-"}}), + absl::StrReplaceAll(tag, {{"_", "-"}}), ext); return fs::path{filename}; } +SnapshotPath SnapshotPath::related_path(std::string tag, const char* ext) const { + return SnapshotPath::make(path_.parent_path(), version_, step_range_, std::move(tag), ext); +} + SnapshotPath SnapshotPath::related_path(SnapshotType type, const char* ext) const { return SnapshotPath::make(path_.parent_path(), version_, step_range_, type, ext); } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index f4bd2f44e5..5474e57fd3 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -38,6 +38,13 @@ class SnapshotPath { static std::optional parse(std::filesystem::path path); static std::optional parse_step_range(const std::filesystem::path& path); + static SnapshotPath make( + const std::filesystem::path& dir, + uint8_t version, + StepRange step_range, + std::string tag, + const char* ext = kSegmentExtension); + static SnapshotPath make( const std::filesystem::path& dir, uint8_t version, @@ -54,6 +61,7 @@ class SnapshotPath { SnapshotType type() const { return type_; } bool exists() const { return std::filesystem::exists(path_); } + SnapshotPath related_path(std::string tag, const char* ext) const; SnapshotPath related_path(SnapshotType type, const char* ext) const; SnapshotPath index_file() const { return related_path(type_, kIdxExtension); @@ -66,7 +74,7 @@ class SnapshotPath { static std::filesystem::path make_filename( uint8_t version, StepRange step_range, - SnapshotType type, + std::string tag, const char* ext); SnapshotPath( diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp index c3ed6c1e65..54951b00d8 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.cpp +++ b/silkworm/db/snapshot_bundle_factory_impl.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -43,12 +44,13 @@ SnapshotBundlePaths SnapshotBundleFactoryImpl::make_paths(const std::filesystem: } std::vector> SnapshotBundleFactoryImpl::index_builders(const SnapshotPath& segment_path) const { - switch (segment_path.type()) { - case SnapshotType::headers: + datastore::EntityName name{segment_path.tag()}; + { + if (name == db::blocks::kHeaderSegmentName) return {std::make_shared(HeaderIndex::make(segment_path))}; - case SnapshotType::bodies: + if (name == db::blocks::kBodySegmentName) return {std::make_shared(BodyIndex::make(segment_path))}; - case SnapshotType::transactions: { + if (name == db::blocks::kTxnSegmentName) { auto bodies_segment_path = segment_path.related_path(SnapshotType::bodies, kSegmentExtension); if (!bodies_segment_path.exists()) return {}; return { @@ -56,9 +58,8 @@ std::vector> SnapshotBundleFactoryImpl::index_buil std::make_shared(TransactionToBlockIndex::make(bodies_segment_path, segment_path)), }; } - default: - SILKWORM_ASSERT(false); - return {}; + SILKWORM_ASSERT(false); + return {}; } } @@ -72,10 +73,20 @@ std::vector> SnapshotBundleFactoryImpl::index_buil } SnapshotPathList SnapshotBundleFactoryImpl::index_dependency_paths(const SnapshotPath& index_path) const { - SnapshotType snapshot_type = (index_path.type() == SnapshotType::transactions_to_block) - ? SnapshotType::transactions - : index_path.type(); - SnapshotPath snapshot_path = index_path.related_path(snapshot_type, kSegmentExtension); + datastore::EntityName name{index_path.tag()}; + datastore::EntityName segment_name = [name]() -> datastore::EntityName { + if (name == db::blocks::kIdxHeaderHashName) + return db::blocks::kHeaderSegmentName; + if (name == db::blocks::kIdxBodyNumberName) + return db::blocks::kBodySegmentName; + if (name == db::blocks::kIdxTxnHashName) + return db::blocks::kTxnSegmentName; + if (name == db::blocks::kIdxTxnHash2BlockName) + return db::blocks::kTxnSegmentName; + SILKWORM_ASSERT(false); + std::abort(); + }(); + SnapshotPath snapshot_path = index_path.related_path(segment_name.to_string(), kSegmentExtension); return {snapshot_path}; } From 3b5022d82783c51ab9cc74cd73e670fc0490d355 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 17:39:00 +0100 Subject: [PATCH 07/15] use EntityName in silkworm_build_recsplit_indexes --- silkworm/capi/silkworm.cpp | 24 +++++++------------ .../snapshots/common/snapshot_path.cpp | 4 ++-- .../snapshots/common/snapshot_path.hpp | 3 +-- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 641dd18f80..e12c91e631 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -265,19 +265,15 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc return SILKWORM_INVALID_PATH; } - std::shared_ptr index; - switch (snapshot_path->type()) { - case snapshots::SnapshotType::headers: { - index = std::make_shared(snapshots::HeaderIndex::make(*snapshot_path, segment_region)); + datastore::EntityName name{snapshot_path->tag()}; + { + if (name == db::blocks::kHeaderSegmentName) { + auto index = std::make_shared(snapshots::HeaderIndex::make(*snapshot_path, segment_region)); needed_indexes.push_back(index); - break; - } - case snapshots::SnapshotType::bodies: { - index = std::make_shared(snapshots::BodyIndex::make(*snapshot_path, segment_region)); + } else if (name == db::blocks::kBodySegmentName) { + auto index = std::make_shared(snapshots::BodyIndex::make(*snapshot_path, segment_region)); needed_indexes.push_back(index); - break; - } - case snapshots::SnapshotType::transactions: { + } else if (name == db::blocks::kTxnSegmentName) { auto bodies_segment_path = snapshot_path->related_path(snapshots::SnapshotType::bodies, snapshots::kSegmentExtension); auto bodies_file = std::find_if(segments, segments + len, [&](SilkwormMemoryMappedFile* file) -> bool { return snapshots::SnapshotPath::parse(file->file_path) == bodies_segment_path; @@ -286,7 +282,7 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc if (bodies_file < segments + len) { auto bodies_segment_region = make_region(**bodies_file); - index = std::make_shared(snapshots::TransactionIndex::make( + auto index = std::make_shared(snapshots::TransactionIndex::make( bodies_segment_path, bodies_segment_region, *snapshot_path, segment_region)); needed_indexes.push_back(index); @@ -294,9 +290,7 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc bodies_segment_path, bodies_segment_region, *snapshot_path, segment_region)); needed_indexes.push_back(index); } - break; - } - default: { + } else { SILKWORM_ASSERT(false); } } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index ca84f69ac3..ce0dfed7c7 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -175,8 +175,8 @@ bool operator<(const SnapshotPath& lhs, const SnapshotPath& rhs) { if (lhs.step_range_.end != rhs.step_range_.end) { return lhs.step_range_.end < rhs.step_range_.end; } - if (lhs.type_ != rhs.type_) { - return lhs.type_ < rhs.type_; + if (lhs.tag_ != rhs.tag_) { + return lhs.tag_ < rhs.tag_; } return lhs.path_.extension() < rhs.path_.extension(); } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index 5474e57fd3..c5f6ec233a 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -58,13 +58,12 @@ class SnapshotPath { uint8_t version() const { return version_; } StepRange step_range() const { return step_range_; } const std::string& tag() const { return tag_; } - SnapshotType type() const { return type_; } bool exists() const { return std::filesystem::exists(path_); } SnapshotPath related_path(std::string tag, const char* ext) const; SnapshotPath related_path(SnapshotType type, const char* ext) const; SnapshotPath index_file() const { - return related_path(type_, kIdxExtension); + return related_path(tag_, kIdxExtension); } friend bool operator<(const SnapshotPath& lhs, const SnapshotPath& rhs); From 0d58cc8306c314e98df6710cd388571f8dcd5735 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 17:50:19 +0100 Subject: [PATCH 08/15] replace SnapshotPath::make, related_path --- silkworm/capi/silkworm.cpp | 6 +++--- .../db/blocks/transactions/txn_to_block_index.hpp | 3 ++- .../datastore/snapshots/common/snapshot_path.cpp | 15 --------------- .../datastore/snapshots/common/snapshot_path.hpp | 8 -------- .../snapshots/common/snapshot_path_test.cpp | 2 +- .../snapshots/kv_segment/kv_segment_test.cpp | 2 +- silkworm/db/datastore/snapshots/schema.cpp | 8 ++++---- .../datastore/snapshots/segment/segment_test.cpp | 2 +- silkworm/db/snapshot_bundle_factory_impl.cpp | 2 +- silkworm/db/snapshot_test.cpp | 4 ++-- 10 files changed, 15 insertions(+), 37 deletions(-) diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index e12c91e631..771c35c54d 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -274,7 +274,7 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc auto index = std::make_shared(snapshots::BodyIndex::make(*snapshot_path, segment_region)); needed_indexes.push_back(index); } else if (name == db::blocks::kTxnSegmentName) { - auto bodies_segment_path = snapshot_path->related_path(snapshots::SnapshotType::bodies, snapshots::kSegmentExtension); + auto bodies_segment_path = snapshot_path->related_path(db::blocks::kBodySegmentName.to_string(), snapshots::kSegmentExtension); auto bodies_file = std::find_if(segments, segments + len, [&](SilkwormMemoryMappedFile* file) -> bool { return snapshots::SnapshotPath::parse(file->file_path) == bodies_segment_path; }); @@ -369,8 +369,8 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn return SILKWORM_INVALID_PATH; } snapshots::SegmentFileReader txn_segment{*transactions_segment_path, make_region(ts.segment)}; - snapshots::Index idx_txn_hash{transactions_segment_path->related_path(snapshots::SnapshotType::transactions, snapshots::kIdxExtension), make_region(ts.tx_hash_index)}; - snapshots::Index idx_txn_hash_2_block{transactions_segment_path->related_path(snapshots::SnapshotType::transactions_to_block, snapshots::kIdxExtension), make_region(ts.tx_hash_2_block_index)}; + snapshots::Index idx_txn_hash{transactions_segment_path->index_file(), make_region(ts.tx_hash_index)}; + snapshots::Index idx_txn_hash_2_block{transactions_segment_path->related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), snapshots::kIdxExtension), make_region(ts.tx_hash_2_block_index)}; snapshots::SnapshotBundleData bundle_data = [&]() { snapshots::SnapshotBundleData data; diff --git a/silkworm/db/blocks/transactions/txn_to_block_index.hpp b/silkworm/db/blocks/transactions/txn_to_block_index.hpp index 9521bf878e..20064ddafc 100644 --- a/silkworm/db/blocks/transactions/txn_to_block_index.hpp +++ b/silkworm/db/blocks/transactions/txn_to_block_index.hpp @@ -25,6 +25,7 @@ #include #include +#include "../schema_config.hpp" #include "txn_index.hpp" #include "txs_and_bodies_query.hpp" @@ -95,7 +96,7 @@ class TransactionToBlockIndex { private: static IndexDescriptor make_descriptor(const SnapshotPath& segment_path, BlockNum first_block_num, uint64_t first_tx_id) { return { - .index_file = segment_path.related_path(SnapshotType::transactions_to_block, kIdxExtension), + .index_file = segment_path.related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), kIdxExtension), .key_factory = std::make_unique(first_tx_id), .base_data_id = first_block_num, .double_enum_index = false, diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index ce0dfed7c7..60183b51a2 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -118,17 +118,6 @@ SnapshotPath SnapshotPath::make( return SnapshotPath{dir / filename, version, step_range, std::move(tag), type}; } -SnapshotPath SnapshotPath::make( - const fs::path& dir, - uint8_t version, - StepRange step_range, - SnapshotType type, - const char* ext) { - std::string tag{magic_enum::enum_name(type)}; - const auto filename = SnapshotPath::make_filename(version, step_range, tag, ext); - return SnapshotPath{dir / filename, version, step_range, std::move(tag), type}; -} - fs::path SnapshotPath::make_filename( uint8_t version, StepRange step_range, @@ -148,10 +137,6 @@ SnapshotPath SnapshotPath::related_path(std::string tag, const char* ext) const return SnapshotPath::make(path_.parent_path(), version_, step_range_, std::move(tag), ext); } -SnapshotPath SnapshotPath::related_path(SnapshotType type, const char* ext) const { - return SnapshotPath::make(path_.parent_path(), version_, step_range_, type, ext); -} - SnapshotPath::SnapshotPath( fs::path path, uint8_t version, diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index c5f6ec233a..33e1af226d 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -45,13 +45,6 @@ class SnapshotPath { std::string tag, const char* ext = kSegmentExtension); - static SnapshotPath make( - const std::filesystem::path& dir, - uint8_t version, - StepRange step_range, - SnapshotType type, - const char* ext = kSegmentExtension); - std::string filename() const { return path_.filename().string(); } const std::filesystem::path& path() const { return path_; } std::string extension() const { return path_.extension().string(); } @@ -61,7 +54,6 @@ class SnapshotPath { bool exists() const { return std::filesystem::exists(path_); } SnapshotPath related_path(std::string tag, const char* ext) const; - SnapshotPath related_path(SnapshotType type, const char* ext) const; SnapshotPath index_file() const { return related_path(tag_, kIdxExtension); } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp index 3aa1c806f7..db6f2eb99b 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp @@ -82,7 +82,7 @@ TEST_CASE("SnapshotPath::parse", "[silkworm][node][snapshot]") { TEST_CASE("SnapshotPath::from", "[silkworm][node][snapshot]") { SECTION("invalid") { - CHECK_THROWS_AS(SnapshotPath::make(std::filesystem::path{}, kSnapshotV1, StepRange{Step{1'000}, Step{999}}, SnapshotType::headers), + CHECK_THROWS_AS(SnapshotPath::make(std::filesystem::path{}, kSnapshotV1, StepRange{Step{1'000}, Step{999}}, "headers"), std::logic_error); } } diff --git a/silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp b/silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp index 51fef0617c..cad86ee2f4 100644 --- a/silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp +++ b/silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp @@ -43,7 +43,7 @@ struct CharCodec : public Encoder, public Decoder { TEST_CASE("KVSegmentFile") { TemporaryDirectory tmp_dir; - auto path = SnapshotPath::make(tmp_dir.path(), kSnapshotV1, StepRange{Step{0}, Step{1}}, SnapshotType::headers); + auto path = SnapshotPath::make(tmp_dir.path(), kSnapshotV1, StepRange{Step{0}, Step{1}}, "headers"); static constexpr seg::CompressionKind kCompressionKind = seg::CompressionKind::kKeys; std::vector> entries = { diff --git a/silkworm/db/datastore/snapshots/schema.cpp b/silkworm/db/datastore/snapshots/schema.cpp index 08c1df00c9..c0d6ddef3b 100644 --- a/silkworm/db/datastore/snapshots/schema.cpp +++ b/silkworm/db/datastore/snapshots/schema.cpp @@ -25,8 +25,8 @@ std::map Schema::RepositoryDef::make_segmen StepRange range) const { std::map results; for (auto& entry : segment_defs_) { - auto type = *magic_enum::enum_cast(entry.first.name); - results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, type)); + auto tag = entry.first.to_string(); + results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, std::move(tag))); } return results; } @@ -46,8 +46,8 @@ std::map Schema::RepositoryDef::make_rec_sp StepRange range) const { std::map results; for (auto& entry : rec_split_index_defs_) { - auto type = *magic_enum::enum_cast(entry.first.name); - results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, type, kIdxExtension)); + auto tag = entry.first.to_string(); + results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, std::move(tag), kIdxExtension)); } return results; } diff --git a/silkworm/db/datastore/snapshots/segment/segment_test.cpp b/silkworm/db/datastore/snapshots/segment/segment_test.cpp index f73ed6b417..672cdfea01 100644 --- a/silkworm/db/datastore/snapshots/segment/segment_test.cpp +++ b/silkworm/db/datastore/snapshots/segment/segment_test.cpp @@ -26,7 +26,7 @@ namespace silkworm::snapshots { TEST_CASE("SegmentFile") { TemporaryDirectory tmp_dir; - auto path = SnapshotPath::make(tmp_dir.path(), kSnapshotV1, StepRange{Step{0}, Step{1}}, SnapshotType::headers); + auto path = SnapshotPath::make(tmp_dir.path(), kSnapshotV1, StepRange{Step{0}, Step{1}}, "headers"); std::vector items = { "first", diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp index 54951b00d8..af1e95448f 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.cpp +++ b/silkworm/db/snapshot_bundle_factory_impl.cpp @@ -51,7 +51,7 @@ std::vector> SnapshotBundleFactoryImpl::index_buil if (name == db::blocks::kBodySegmentName) return {std::make_shared(BodyIndex::make(segment_path))}; if (name == db::blocks::kTxnSegmentName) { - auto bodies_segment_path = segment_path.related_path(SnapshotType::bodies, kSegmentExtension); + auto bodies_segment_path = segment_path.related_path(db::blocks::kBodySegmentName.to_string(), kSegmentExtension); if (!bodies_segment_path.exists()) return {}; return { std::make_shared(TransactionIndex::make(bodies_segment_path, segment_path)), diff --git a/silkworm/db/snapshot_test.cpp b/silkworm/db/snapshot_test.cpp index 2e7bf120b0..6771994cb4 100644 --- a/silkworm/db/snapshot_test.cpp +++ b/silkworm/db/snapshot_test.cpp @@ -50,7 +50,7 @@ class SnapshotPathForTest : public SnapshotPath { tmp_dir, kSnapshotV1, step_range, - SnapshotType::headers), + "headers"), } {} }; @@ -236,7 +236,7 @@ TEST_CASE("TransactionSnapshot::block_num_by_txn_hash OK", "[silkworm][node][sna idx_txn_hash.reopen_index(); TransactionFindByIdQuery txn_by_id{{txn_segment, idx_txn_hash}}; - Index idx_txn_hash_2_block{txn_segment_path.related_path(SnapshotType::transactions_to_block, kIdxExtension)}; + Index idx_txn_hash_2_block{txn_segment_path.related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), kIdxExtension)}; idx_txn_hash_2_block.reopen_index(); TransactionBlockNumByTxnHashQuery block_num_by_txn_hash{idx_txn_hash_2_block, TransactionFindByHashQuery{{txn_segment, idx_txn_hash}}}; From 4c80eaf15bb607a00474a611bdc77cf518f5e8b4 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 20:58:32 +0100 Subject: [PATCH 09/15] remove SnapshotPath::type_ --- .../snapshots/common/snapshot_path.cpp | 18 ++++-------------- .../snapshots/common/snapshot_path.hpp | 5 +---- .../db/datastore/snapshots/snapshot_bundle.hpp | 3 +-- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index 60183b51a2..230429ea67 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -18,13 +18,10 @@ #include #include -#include -#include #include #include #include -#include #include @@ -99,12 +96,8 @@ std::optional SnapshotPath::parse(fs::path path) { // parsing relies on magic_enum, so SnapshotType items must match exactly std::string tag_str{tag.data(), tag.size()}; std::replace(tag_str.begin(), tag_str.end(), '-', '_'); - const auto type = magic_enum::enum_cast(tag_str); - if (!type) { - return std::nullopt; - } - return SnapshotPath{std::move(path), ver_num, *step_range, std::move(tag_str), *type}; + return SnapshotPath{std::move(path), ver_num, *step_range, std::move(tag_str)}; } SnapshotPath SnapshotPath::make( @@ -114,8 +107,7 @@ SnapshotPath SnapshotPath::make( std::string tag, const char* ext) { const auto filename = SnapshotPath::make_filename(version, step_range, tag, ext); - auto type = *magic_enum::enum_cast(tag); - return SnapshotPath{dir / filename, version, step_range, std::move(tag), type}; + return SnapshotPath{dir / filename, version, step_range, std::move(tag)}; } fs::path SnapshotPath::make_filename( @@ -141,13 +133,11 @@ SnapshotPath::SnapshotPath( fs::path path, uint8_t version, StepRange step_range, - std::string tag, - SnapshotType type) + std::string tag) : path_{std::move(path)}, version_{version}, step_range_{step_range}, - tag_{std::move(tag)}, - type_{type} { + tag_{std::move(tag)} { } bool operator<(const SnapshotPath& lhs, const SnapshotPath& rhs) { diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index 33e1af226d..ac4db65e18 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -22,7 +22,6 @@ #include #include -#include "snapshot_type.hpp" #include "step.hpp" namespace silkworm::snapshots { @@ -72,14 +71,12 @@ class SnapshotPath { std::filesystem::path path, uint8_t version, StepRange step_range, - std::string tag, - SnapshotType type); + std::string tag); std::filesystem::path path_; uint8_t version_{0}; StepRange step_range_; std::string tag_; - SnapshotType type_; }; using SnapshotPathList = std::vector; diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp index f34cdaa65d..ed3107ee2c 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp @@ -21,9 +21,8 @@ #include #include -#include - #include "common/snapshot_path.hpp" +#include "common/snapshot_type.hpp" #include "common/util/iterator/map_values_view.hpp" #include "rec_split_index/index.hpp" #include "schema.hpp" From afcc9b0c370d2ee677d43aa0beb47f52f77051ed Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 21:20:22 +0100 Subject: [PATCH 10/15] use EntityName for find_segment --- cmd/dev/snapshots.cpp | 10 +++--- silkworm/db/access_layer.cpp | 12 +++---- silkworm/db/blocks/bodies/body_queries.hpp | 3 +- .../snapshots/common/snapshot_type.hpp | 35 ------------------- .../datastore/snapshots/snapshot_bundle.cpp | 12 ------- .../datastore/snapshots/snapshot_bundle.hpp | 6 ---- .../snapshots/snapshot_repository.cpp | 6 ++-- .../snapshots/snapshot_repository.hpp | 3 +- silkworm/db/snapshot_repository_test.cpp | 8 +++++ silkworm/db/snapshot_sync.cpp | 2 +- silkworm/db/test_util/temp_snapshots.hpp | 1 - 11 files changed, 28 insertions(+), 70 deletions(-) delete mode 100644 silkworm/db/datastore/snapshots/common/snapshot_type.hpp diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index 2c97969eed..cbe0a468ed 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -700,7 +700,7 @@ void lookup_header_by_hash(const SnapshotSubcommandSettings& settings) { auto repository = make_repository(settings.settings); for (const auto& bundle_ptr : repository.view_bundles_reverse()) { const auto& bundle = *bundle_ptr; - auto segment_and_index = bundle.segment_and_index(SnapshotType::headers); + auto segment_and_index = bundle.segment_and_index(db::blocks::kHeaderSegmentName); const auto header = HeaderFindByHashQuery{segment_and_index}.exec(*hash); if (header) { matching_header = header; @@ -727,7 +727,7 @@ void lookup_header_by_number(const SnapshotSubcommandSettings& settings) { std::chrono::time_point start{std::chrono::steady_clock::now()}; auto repository = make_repository(settings.settings); - const auto [segment_and_index, _] = repository.find_segment(SnapshotType::headers, block_number); + const auto [segment_and_index, _] = repository.find_segment(db::blocks::kHeaderSegmentName, block_number); if (segment_and_index) { const auto header = HeaderFindByBlockNumQuery{*segment_and_index}.exec(block_number); ensure(header.has_value(), @@ -788,7 +788,7 @@ void lookup_body_in_all(const SnapshotSubcommandSettings& settings, BlockNum blo auto repository = make_repository(settings.settings); std::chrono::time_point start{std::chrono::steady_clock::now()}; - const auto [segment_and_index, _] = repository.find_segment(SnapshotType::bodies, block_number); + const auto [segment_and_index, _] = repository.find_segment(db::blocks::kBodySegmentName, block_number); if (segment_and_index) { const auto body = BodyFindByBlockNumQuery{*segment_and_index}.exec(block_number); ensure(body.has_value(), @@ -897,7 +897,7 @@ void lookup_txn_by_hash_in_all(const SnapshotSubcommandSettings& settings, const std::chrono::time_point start{std::chrono::steady_clock::now()}; for (const auto& bundle_ptr : repository.view_bundles_reverse()) { const auto& bundle = *bundle_ptr; - auto segment_and_index = bundle.segment_and_index(SnapshotType::transactions); + auto segment_and_index = bundle.segment_and_index(db::blocks::kTxnSegmentName); const auto transaction = TransactionFindByHashQuery{segment_and_index}.exec(hash); if (transaction) { matching_snapshot_path = segment_and_index.segment.path(); @@ -961,7 +961,7 @@ void lookup_txn_by_id_in_all(const SnapshotSubcommandSettings& settings, uint64_ std::chrono::time_point start{std::chrono::steady_clock::now()}; for (const auto& bundle_ptr : repository.view_bundles_reverse()) { const auto& bundle = *bundle_ptr; - auto segment_and_index = bundle.segment_and_index(SnapshotType::transactions); + auto segment_and_index = bundle.segment_and_index(db::blocks::kTxnSegmentName); const auto transaction = TransactionFindByIdQuery{segment_and_index}.exec(txn_id); if (transaction) { matching_snapshot_path = segment_and_index.segment.path(); diff --git a/silkworm/db/access_layer.cpp b/silkworm/db/access_layer.cpp index 1afd569289..f1a9794f2b 100644 --- a/silkworm/db/access_layer.cpp +++ b/silkworm/db/access_layer.cpp @@ -1256,7 +1256,7 @@ bool DataModel::read_block_from_snapshot(BlockNum height, Block& block) const { std::optional DataModel::read_header_from_snapshot(BlockNum height) const { std::optional block_header; // We know the header snapshot in advance: find it based on target block number - const auto [segment_and_index, _] = repository_.find_segment(SnapshotType::headers, height); + const auto [segment_and_index, _] = repository_.find_segment(blocks::kHeaderSegmentName, height); if (segment_and_index) { block_header = HeaderFindByBlockNumQuery{*segment_and_index}.exec(height); } @@ -1268,7 +1268,7 @@ std::optional DataModel::read_header_from_snapshot(const Hash& hash // We don't know the header snapshot in advance: search for block hash in each header snapshot in reverse order for (const auto& bundle_ptr : repository_.view_bundles_reverse()) { const auto& bundle = *bundle_ptr; - auto segment_and_index = bundle.segment_and_index(SnapshotType::headers); + auto segment_and_index = bundle.segment_and_index(blocks::kHeaderSegmentName); block_header = HeaderFindByHashQuery{segment_and_index}.exec(hash); if (block_header) break; } @@ -1299,7 +1299,7 @@ bool DataModel::read_body_from_snapshot(BlockNum height, BlockBody& body) const bool DataModel::is_body_in_snapshot(BlockNum height) const { // We know the body snapshot in advance: find it based on target block number - const auto [segment_and_index, _] = repository_.find_segment(SnapshotType::bodies, height); + const auto [segment_and_index, _] = repository_.find_segment(blocks::kBodySegmentName, height); if (segment_and_index) { const auto stored_body = BodyFindByBlockNumQuery{*segment_and_index}.exec(height); return stored_body.has_value(); @@ -1313,7 +1313,7 @@ bool DataModel::read_transactions_from_snapshot(BlockNum height, uint64_t base_t return true; } - const auto [segment_and_index, _] = repository_.find_segment(SnapshotType::transactions, height); + const auto [segment_and_index, _] = repository_.find_segment(blocks::kTxnSegmentName, height); if (!segment_and_index) return false; txs = TransactionRangeFromIdQuery{*segment_and_index}.exec_into_vector(base_txn_id, txn_count); @@ -1322,7 +1322,7 @@ bool DataModel::read_transactions_from_snapshot(BlockNum height, uint64_t base_t } bool DataModel::read_rlp_transactions_from_snapshot(BlockNum height, std::vector& rlp_txs) const { - const auto [body_segment_and_index, _] = repository_.find_segment(SnapshotType::bodies, height); + const auto [body_segment_and_index, _] = repository_.find_segment(blocks::kBodySegmentName, height); if (body_segment_and_index) { auto stored_body = BodyFindByBlockNumQuery{*body_segment_and_index}.exec(height); if (!stored_body) return false; @@ -1333,7 +1333,7 @@ bool DataModel::read_rlp_transactions_from_snapshot(BlockNum height, std::vector if (txn_count == 0) return true; - const auto [tx_segment_and_index, _2] = repository_.find_segment(SnapshotType::transactions, height); + const auto [tx_segment_and_index, _2] = repository_.find_segment(blocks::kTxnSegmentName, height); if (!tx_segment_and_index) return false; rlp_txs = TransactionPayloadRlpRangeFromIdQuery{*tx_segment_and_index}.exec_into_vector(base_txn_id, txn_count); diff --git a/silkworm/db/blocks/bodies/body_queries.hpp b/silkworm/db/blocks/bodies/body_queries.hpp index d594b92a85..b8b25bc95a 100644 --- a/silkworm/db/blocks/bodies/body_queries.hpp +++ b/silkworm/db/blocks/bodies/body_queries.hpp @@ -19,6 +19,7 @@ #include #include +#include "../schema_config.hpp" #include "body_segment.hpp" namespace silkworm::snapshots { @@ -32,7 +33,7 @@ class BodyFindByBlockNumMultiQuery { : repository_{repository} {} std::optional exec(BlockNum block_num) { - const auto [segment_and_index, _] = repository_.find_segment(SnapshotType::bodies, block_num); + const auto [segment_and_index, _] = repository_.find_segment(db::blocks::kBodySegmentName, block_num); if (!segment_and_index) return std::nullopt; return BodyFindByBlockNumQuery{*segment_and_index}.exec(block_num); } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_type.hpp b/silkworm/db/datastore/snapshots/common/snapshot_type.hpp deleted file mode 100644 index 23422d90af..0000000000 --- a/silkworm/db/datastore/snapshots/common/snapshot_type.hpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright 2024 The Silkworm Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#pragma once - -#include - -namespace silkworm::snapshots { - -//! The snapshot category corresponding to the snapshot file type -//! @remark item names do NOT follow Google style to obtain the tag used in file names from magic_enum::enum_name -//! @see SnapshotPath#make_filename -// NOLINTBEGIN(readability-identifier-naming) -enum SnapshotType : uint8_t { - headers = 0, - bodies = 1, - transactions = 2, - transactions_to_block = 3, -}; -// NOLINTEND(readability-identifier-naming) - -} // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp index 8bd929c1fa..91746543ca 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.cpp @@ -16,8 +16,6 @@ #include "snapshot_bundle.hpp" -#include - #include namespace silkworm::snapshots { @@ -64,16 +62,6 @@ void SnapshotBundle::close() { } } -const SegmentFileReader& SnapshotBundle::segment(SnapshotType type) const { - datastore::EntityName name{magic_enum::enum_name(type)}; - return data_.segments.at(name); -} - -const Index& SnapshotBundle::index(SnapshotType type) const { - datastore::EntityName name{magic_enum::enum_name(type)}; - return data_.rec_split_indexes.at(name); -} - const SegmentFileReader& SnapshotBundle::segment(datastore::EntityName name) const { return data_.segments.at(name); } diff --git a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp index ed3107ee2c..0b99c3e2bb 100644 --- a/silkworm/db/datastore/snapshots/snapshot_bundle.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_bundle.hpp @@ -22,7 +22,6 @@ #include #include "common/snapshot_path.hpp" -#include "common/snapshot_type.hpp" #include "common/util/iterator/map_values_view.hpp" #include "rec_split_index/index.hpp" #include "schema.hpp" @@ -75,11 +74,6 @@ struct SnapshotBundle { auto rec_split_indexes() { return make_map_values_view(data_.rec_split_indexes); } - const SegmentFileReader& segment(SnapshotType type) const; - const Index& index(SnapshotType type) const; - SegmentAndIndex segment_and_index(SnapshotType type) const { - return {segment(type), index(type)}; - } const SegmentFileReader& segment(datastore::EntityName name) const; const Index& index(datastore::EntityName name) const; SegmentAndIndex segment_and_index(datastore::EntityName name) const { diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 27648d0bae..48d1d3fe3d 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -93,10 +93,12 @@ Step SnapshotRepository::max_end_step() const { return bundle.step_range().end; } -std::pair, std::shared_ptr> SnapshotRepository::find_segment(SnapshotType type, Timestamp t) const { +std::pair, std::shared_ptr> SnapshotRepository::find_segment( + datastore::EntityName name, + Timestamp t) const { auto bundle = find_bundle(step_converter_->step_from_timestamp(t)); if (bundle) { - return {bundle->segment_and_index(type), bundle}; + return {bundle->segment_and_index(name), bundle}; } return {std::nullopt, {}}; } diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index c0c7443504..d8e4bbf78d 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -27,6 +27,7 @@ #include #include +#include "../common/entity_name.hpp" #include "common/snapshot_path.hpp" #include "common/util/iterator/map_values_view.hpp" #include "index_builder.hpp" @@ -104,7 +105,7 @@ class SnapshotRepository { return BundlesView{std::ranges::reverse_view(make_map_values_view(*bundles_)), bundles_}; } - std::pair, std::shared_ptr> find_segment(SnapshotType type, Timestamp t) const; + std::pair, std::shared_ptr> find_segment(datastore::EntityName name, Timestamp t) const; std::shared_ptr find_bundle(Step step) const; std::vector> bundles_in_range(StepRange range) const; diff --git a/silkworm/db/snapshot_repository_test.cpp b/silkworm/db/snapshot_repository_test.cpp index 5e9de36b1d..ef2aa18f8f 100644 --- a/silkworm/db/snapshot_repository_test.cpp +++ b/silkworm/db/snapshot_repository_test.cpp @@ -44,6 +44,14 @@ static SnapshotRepository make_repository(std::filesystem::path dir_path) { return db::blocks::make_blocks_repository(std::move(dir_path)); } +// NOLINTBEGIN(readability-identifier-naming) +struct SnapshotType { + static constexpr datastore::EntityName headers{db::blocks::kHeaderSegmentName}; + static constexpr datastore::EntityName bodies{db::blocks::kBodySegmentName}; + static constexpr datastore::EntityName transactions{db::blocks::kTxnSegmentName}; +}; +// NOLINTEND(readability-identifier-naming) + TEST_CASE("SnapshotRepository::SnapshotRepository", "[silkworm][node][snapshot]") { SetLogVerbosityGuard guard{log::Level::kNone}; TemporaryDirectory tmp_dir; diff --git a/silkworm/db/snapshot_sync.cpp b/silkworm/db/snapshot_sync.cpp index 2159173bee..b3ed99a775 100644 --- a/silkworm/db/snapshot_sync.cpp +++ b/silkworm/db/snapshot_sync.cpp @@ -390,7 +390,7 @@ void SnapshotSync::update_block_bodies(RWTxn& txn, BlockNum max_block_available) } // Reset sequence for kBlockTransactions table - const auto [txn_segment, _] = repository_.find_segment(SnapshotType::transactions, max_block_available); + const auto [txn_segment, _] = repository_.find_segment(db::blocks::kTxnSegmentName, max_block_available); ensure(txn_segment.has_value(), "SnapshotSync: snapshots max block not found in any snapshot"); const auto last_tx_id = txn_segment->index.base_data_id() + txn_segment->segment.item_count(); reset_map_sequence(txn, table::kBlockTransactions.name, last_tx_id + 1); diff --git a/silkworm/db/test_util/temp_snapshots.hpp b/silkworm/db/test_util/temp_snapshots.hpp index 757131630e..8119d19fde 100644 --- a/silkworm/db/test_util/temp_snapshots.hpp +++ b/silkworm/db/test_util/temp_snapshots.hpp @@ -28,7 +28,6 @@ namespace silkworm::snapshots::test_util { using snapshots::SnapshotPath; -using snapshots::SnapshotType; //! Big-endian encoder inline size_t encode_big_endian(uint64_t value, Bytes& output) { From 384f4b62e271b5499bd1a5be10886815cf117ee9 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 21:34:51 +0100 Subject: [PATCH 11/15] dash in kIdxTxnHash2BlockName to eliminate tag conversions --- silkworm/db/blocks/schema_config.hpp | 2 +- .../db/datastore/snapshots/common/snapshot_path.cpp | 12 +++--------- .../db/datastore/snapshots/common/snapshot_path.hpp | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/silkworm/db/blocks/schema_config.hpp b/silkworm/db/blocks/schema_config.hpp index cc8dd00ef9..db1b3146ca 100644 --- a/silkworm/db/blocks/schema_config.hpp +++ b/silkworm/db/blocks/schema_config.hpp @@ -47,7 +47,7 @@ inline constexpr datastore::EntityName kTxnSegmentName{"transactions"}; //! Index transaction_hash -> txn_id -> transactions_segment_offset inline constexpr datastore::EntityName kIdxTxnHashName{"transactions"}; //! Index transaction_hash -> block_num -inline constexpr datastore::EntityName kIdxTxnHash2BlockName{"transactions_to_block"}; +inline constexpr datastore::EntityName kIdxTxnHash2BlockName{"transactions-to-block"}; struct BundleDataRef { const snapshots::SnapshotBundleData& data; diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index 230429ea67..fae920559a 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include @@ -92,12 +91,7 @@ std::optional SnapshotPath::parse(fs::path path) { return std::nullopt; } - // Expected tag format: headers|bodies|transactions|transactions-to-block - // parsing relies on magic_enum, so SnapshotType items must match exactly - std::string tag_str{tag.data(), tag.size()}; - std::replace(tag_str.begin(), tag_str.end(), '-', '_'); - - return SnapshotPath{std::move(path), ver_num, *step_range, std::move(tag_str)}; + return SnapshotPath{std::move(path), ver_num, *step_range, std::string{tag}}; } SnapshotPath SnapshotPath::make( @@ -113,14 +107,14 @@ SnapshotPath SnapshotPath::make( fs::path SnapshotPath::make_filename( uint8_t version, StepRange step_range, - std::string tag, + std::string_view tag, const char* ext) { std::string filename = absl::StrFormat( "v%d-%06d-%06d-%s%s", version, step_range.start.value, step_range.end.value, - absl::StrReplaceAll(tag, {{"_", "-"}}), + tag, ext); return fs::path{filename}; } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index ac4db65e18..16fc51a82d 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -64,7 +64,7 @@ class SnapshotPath { static std::filesystem::path make_filename( uint8_t version, StepRange step_range, - std::string tag, + std::string_view tag, const char* ext); SnapshotPath( From edcd88f25057699a2b8ab1282d979611db3d21b2 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 21:39:08 +0100 Subject: [PATCH 12/15] remove unused magic_enum --- silkworm/db/datastore/snapshots/CMakeLists.txt | 2 -- silkworm/db/datastore/snapshots/config/config_test.cpp | 1 - silkworm/db/datastore/snapshots/schema.cpp | 2 -- 3 files changed, 5 deletions(-) diff --git a/silkworm/db/datastore/snapshots/CMakeLists.txt b/silkworm/db/datastore/snapshots/CMakeLists.txt index c11adc1b7a..68e093fb8c 100644 --- a/silkworm/db/datastore/snapshots/CMakeLists.txt +++ b/silkworm/db/datastore/snapshots/CMakeLists.txt @@ -21,7 +21,6 @@ add_subdirectory(seg) find_package(absl REQUIRED COMPONENTS strings) find_package(Boost REQUIRED COMPONENTS headers url) # headers for signals2 -find_package(magic_enum REQUIRED) find_package(Microsoft.GSL REQUIRED) find_package(OpenSSL REQUIRED) @@ -29,7 +28,6 @@ find_package(OpenSSL REQUIRED) set(LIBS_PRIVATE absl::strings Boost::headers - magic_enum::magic_enum OpenSSL::Crypto silkworm_snapshots_seg ) diff --git a/silkworm/db/datastore/snapshots/config/config_test.cpp b/silkworm/db/datastore/snapshots/config/config_test.cpp index 5c62318852..7cb18d51a1 100644 --- a/silkworm/db/datastore/snapshots/config/config_test.cpp +++ b/silkworm/db/datastore/snapshots/config/config_test.cpp @@ -19,7 +19,6 @@ #include #include -#include namespace silkworm::snapshots { diff --git a/silkworm/db/datastore/snapshots/schema.cpp b/silkworm/db/datastore/snapshots/schema.cpp index c0d6ddef3b..2b19c2bdce 100644 --- a/silkworm/db/datastore/snapshots/schema.cpp +++ b/silkworm/db/datastore/snapshots/schema.cpp @@ -16,8 +16,6 @@ #include "schema.hpp" -#include - namespace silkworm::snapshots { std::map Schema::RepositoryDef::make_segment_paths( From dfbf52c4a8b8fa9e4f56fea199d7bdf764f5d893 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 21:48:32 +0100 Subject: [PATCH 13/15] rename index_file to related_path_ext --- cmd/dev/snapshots.cpp | 8 ++++---- silkworm/capi/silkworm.cpp | 6 +++--- silkworm/capi/silkworm_test.cpp | 6 +++--- silkworm/db/blocks/bodies/body_index.hpp | 2 +- silkworm/db/blocks/headers/header_index.hpp | 2 +- silkworm/db/blocks/transactions/txn_index.hpp | 2 +- .../db/datastore/snapshots/common/snapshot_path.hpp | 4 ++-- .../snapshots/common/snapshot_path_test.cpp | 2 +- silkworm/db/snapshot_test.cpp | 12 ++++++------ 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index cbe0a468ed..4ce1312376 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -426,7 +426,7 @@ void open_index(const SnapshotSubcommandSettings& settings) { SILK_INFO << "Open index for snapshot: " << segment_file_path; const auto snapshot_path{snapshots::SnapshotPath::parse(segment_file_path)}; ensure(snapshot_path.has_value(), [&]() { return "open_index: invalid snapshot file " + segment_file_path.filename().string(); }); - const auto index_path{snapshot_path->index_file()}; + const auto index_path{snapshot_path->related_path_ext(kIdxExtension)}; SILK_INFO << "Index file: " << index_path.path(); std::chrono::time_point start{std::chrono::steady_clock::now()}; rec_split::RecSplitIndex idx{index_path.path()}; @@ -768,7 +768,7 @@ void lookup_body_in_one(const SnapshotSubcommandSettings& settings, BlockNum blo SegmentFileReader body_segment{*snapshot_path}; body_segment.reopen_segment(); - Index idx_body_number{snapshot_path->index_file()}; + Index idx_body_number{snapshot_path->related_path_ext(kIdxExtension)}; idx_body_number.reopen_index(); const auto body = BodyFindByBlockNumQuery{{body_segment, idx_body_number}}.exec(block_number); @@ -873,7 +873,7 @@ void lookup_txn_by_hash_in_one(const SnapshotSubcommandSettings& settings, const txn_segment.reopen_segment(); { - Index idx_txn_hash{snapshot_path->index_file()}; + Index idx_txn_hash{snapshot_path->related_path_ext(kIdxExtension)}; idx_txn_hash.reopen_index(); const auto transaction = TransactionFindByHashQuery{{txn_segment, idx_txn_hash}}.exec(hash); @@ -937,7 +937,7 @@ void lookup_txn_by_id_in_one(const SnapshotSubcommandSettings& settings, uint64_ txn_segment.reopen_segment(); { - Index idx_txn_hash{snapshot_path->index_file()}; + Index idx_txn_hash{snapshot_path->related_path_ext(kIdxExtension)}; idx_txn_hash.reopen_index(); const auto transaction = TransactionFindByIdQuery{{txn_segment, idx_txn_hash}}.exec(txn_id); diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 771c35c54d..bc9d9584d7 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -347,7 +347,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn return SILKWORM_INVALID_PATH; } snapshots::SegmentFileReader header_segment{*headers_segment_path, make_region(hs.segment)}; - snapshots::Index idx_header_hash{headers_segment_path->index_file(), make_region(hs.header_hash_index)}; + snapshots::Index idx_header_hash{headers_segment_path->related_path_ext(snapshots::kIdxExtension), make_region(hs.header_hash_index)}; const SilkwormBodiesSnapshot& bs = snapshot->bodies; if (!bs.segment.file_path || !bs.block_num_index.file_path) { @@ -358,7 +358,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn return SILKWORM_INVALID_PATH; } snapshots::SegmentFileReader body_segment{*bodies_segment_path, make_region(bs.segment)}; - snapshots::Index idx_body_number{bodies_segment_path->index_file(), make_region(bs.block_num_index)}; + snapshots::Index idx_body_number{bodies_segment_path->related_path_ext(snapshots::kIdxExtension), make_region(bs.block_num_index)}; const SilkwormTransactionsSnapshot& ts = snapshot->transactions; if (!ts.segment.file_path || !ts.tx_hash_index.file_path || !ts.tx_hash_2_block_index.file_path) { @@ -369,7 +369,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn return SILKWORM_INVALID_PATH; } snapshots::SegmentFileReader txn_segment{*transactions_segment_path, make_region(ts.segment)}; - snapshots::Index idx_txn_hash{transactions_segment_path->index_file(), make_region(ts.tx_hash_index)}; + snapshots::Index idx_txn_hash{transactions_segment_path->related_path_ext(snapshots::kIdxExtension), make_region(ts.tx_hash_index)}; snapshots::Index idx_txn_hash_2_block{transactions_segment_path->related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), snapshots::kIdxExtension), make_region(ts.tx_hash_2_block_index)}; snapshots::SnapshotBundleData bundle_data = [&]() { diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index f3964b884f..274e0986f4 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -821,7 +821,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") { REQUIRE_NOTHROW(header_index_builder.build()); snapshots::SegmentFileReader header_segment{header_segment_path}; header_segment.reopen_segment(); - snapshots::Index idx_header_hash{header_segment_path.index_file()}; + snapshots::Index idx_header_hash{header_segment_path.related_path_ext(snapshots::kIdxExtension)}; idx_header_hash.reopen_index(); auto body_index_builder = snapshots::BodyIndex::make(body_segment_path); @@ -829,7 +829,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") { REQUIRE_NOTHROW(body_index_builder.build()); snapshots::SegmentFileReader body_segment{body_segment_path}; body_segment.reopen_segment(); - snapshots::Index idx_body_number{body_segment_path.index_file()}; + snapshots::Index idx_body_number{body_segment_path.related_path_ext(snapshots::kIdxExtension)}; idx_body_number.reopen_index(); auto tx_index_builder = snapshots::TransactionIndex::make(body_segment_path, txn_segment_path); @@ -838,7 +838,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") { tx_index_hash_to_block_builder.build(); snapshots::SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - snapshots::Index idx_txn_hash{txn_segment_path.index_file()}; + snapshots::Index idx_txn_hash{txn_segment_path.related_path_ext(snapshots::kIdxExtension)}; idx_txn_hash.reopen_index(); snapshots::Index idx_txn_hash_2_block{tx_index_hash_to_block_builder.path()}; idx_txn_hash_2_block.reopen_index(); diff --git a/silkworm/db/blocks/bodies/body_index.hpp b/silkworm/db/blocks/bodies/body_index.hpp index 6785ba3ea5..b525d8e22b 100644 --- a/silkworm/db/blocks/bodies/body_index.hpp +++ b/silkworm/db/blocks/bodies/body_index.hpp @@ -43,7 +43,7 @@ class BodyIndex { private: static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) { return { - .index_file = segment_path.index_file(), + .index_file = segment_path.related_path_ext(kIdxExtension), .key_factory = std::make_unique(), .base_data_id = segment_path.step_range().to_block_num_range().start, }; diff --git a/silkworm/db/blocks/headers/header_index.hpp b/silkworm/db/blocks/headers/header_index.hpp index 072e48f49d..59500ca986 100644 --- a/silkworm/db/blocks/headers/header_index.hpp +++ b/silkworm/db/blocks/headers/header_index.hpp @@ -43,7 +43,7 @@ class HeaderIndex { private: static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) { return { - .index_file = segment_path.index_file(), + .index_file = segment_path.related_path_ext(kIdxExtension), .key_factory = std::make_unique(), .base_data_id = segment_path.step_range().to_block_num_range().start, }; diff --git a/silkworm/db/blocks/transactions/txn_index.hpp b/silkworm/db/blocks/transactions/txn_index.hpp index b0ebad5624..562a5f08be 100644 --- a/silkworm/db/blocks/transactions/txn_index.hpp +++ b/silkworm/db/blocks/transactions/txn_index.hpp @@ -66,7 +66,7 @@ class TransactionIndex { private: static IndexDescriptor make_descriptor(const SnapshotPath& segment_path, uint64_t first_tx_id) { return { - .index_file = segment_path.index_file(), + .index_file = segment_path.related_path_ext(kIdxExtension), .key_factory = std::make_unique(first_tx_id), .base_data_id = first_tx_id, .less_false_positives = true, diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index 16fc51a82d..ac7241b78c 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -53,8 +53,8 @@ class SnapshotPath { bool exists() const { return std::filesystem::exists(path_); } SnapshotPath related_path(std::string tag, const char* ext) const; - SnapshotPath index_file() const { - return related_path(tag_, kIdxExtension); + SnapshotPath related_path_ext(const char* ext) const { + return related_path(tag_, ext); } friend bool operator<(const SnapshotPath& lhs, const SnapshotPath& rhs); diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp index db6f2eb99b..6dff58815e 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp @@ -69,7 +69,7 @@ TEST_CASE("SnapshotPath::parse", "[silkworm][node][snapshot]") { CHECK(snapshot_file->version() == 1); CHECK(snapshot_file->step_range() == StepRange::from_block_num_range(filename_expectation.block_num_range)); CHECK(snapshot_file->tag() == filename_expectation.tag); - const SnapshotPath index_file = snapshot_file->index_file(); + const SnapshotPath index_file = snapshot_file->related_path_ext(kIdxExtension); CHECK(index_file.path().stem() == snapshot_file->path().stem()); CHECK(index_file.path().extension() == kIdxExtension); CHECK(index_file.version() == 1); diff --git a/silkworm/db/snapshot_test.cpp b/silkworm/db/snapshot_test.cpp index 6771994cb4..7181efa4b2 100644 --- a/silkworm/db/snapshot_test.cpp +++ b/silkworm/db/snapshot_test.cpp @@ -131,7 +131,7 @@ TEST_CASE("HeaderSnapshot::header_by_number OK", "[silkworm][node][snapshot][ind SegmentFileReader header_segment{header_segment_path}; header_segment.reopen_segment(); - Index idx_header_hash{header_segment_path.index_file()}; + Index idx_header_hash{header_segment_path.related_path_ext(kIdxExtension)}; idx_header_hash.reopen_index(); HeaderFindByBlockNumQuery header_by_number{{header_segment, idx_header_hash}}; @@ -174,7 +174,7 @@ TEST_CASE("BodySnapshot::body_by_number OK", "[silkworm][node][snapshot][index]" SegmentFileReader body_segment{body_segment_path}; body_segment.reopen_segment(); - Index idx_body_number{body_segment_path.index_file()}; + Index idx_body_number{body_segment_path.related_path_ext(kIdxExtension)}; idx_body_number.reopen_index(); BodyFindByBlockNumQuery body_by_number{{body_segment, idx_body_number}}; @@ -203,7 +203,7 @@ TEST_CASE("TransactionSnapshot::txn_by_id OK", "[silkworm][node][snapshot][index SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - Index idx_txn_hash{txn_segment_path.index_file()}; + Index idx_txn_hash{txn_segment_path.related_path_ext(kIdxExtension)}; idx_txn_hash.reopen_index(); TransactionFindByIdQuery txn_by_id{{txn_segment, idx_txn_hash}}; @@ -232,7 +232,7 @@ TEST_CASE("TransactionSnapshot::block_num_by_txn_hash OK", "[silkworm][node][sna SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - Index idx_txn_hash{txn_segment_path.index_file()}; + Index idx_txn_hash{txn_segment_path.related_path_ext(kIdxExtension)}; idx_txn_hash.reopen_index(); TransactionFindByIdQuery txn_by_id{{txn_segment, idx_txn_hash}}; @@ -274,7 +274,7 @@ TEST_CASE("TransactionSnapshot::txn_range OK", "[silkworm][node][snapshot][index SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - Index idx_txn_hash{txn_segment_path.index_file()}; + Index idx_txn_hash{txn_segment_path.related_path_ext(kIdxExtension)}; idx_txn_hash.reopen_index(); TransactionRangeFromIdQuery query{{txn_segment, idx_txn_hash}}; @@ -306,7 +306,7 @@ TEST_CASE("TransactionSnapshot::txn_rlp_range OK", "[silkworm][node][snapshot][i SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - Index idx_txn_hash{txn_segment_path.index_file()}; + Index idx_txn_hash{txn_segment_path.related_path_ext(kIdxExtension)}; idx_txn_hash.reopen_index(); TransactionPayloadRlpRangeFromIdQuery query{{txn_segment, idx_txn_hash}}; From 49a30e62f362dc2571e8c8b9662cd096ad059270 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 22:00:40 +0100 Subject: [PATCH 14/15] make kSegmentExtension non-optional --- silkworm/db/datastore/snapshots/common/snapshot_path.hpp | 2 +- silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp | 2 +- silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp | 2 +- silkworm/db/datastore/snapshots/schema.cpp | 2 +- silkworm/db/datastore/snapshots/segment/segment_test.cpp | 2 +- silkworm/db/snapshot_test.cpp | 3 ++- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index ac7241b78c..3d0662d6c6 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -42,7 +42,7 @@ class SnapshotPath { uint8_t version, StepRange step_range, std::string tag, - const char* ext = kSegmentExtension); + const char* ext); std::string filename() const { return path_.filename().string(); } const std::filesystem::path& path() const { return path_; } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp index 6dff58815e..800eca9828 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp @@ -82,7 +82,7 @@ TEST_CASE("SnapshotPath::parse", "[silkworm][node][snapshot]") { TEST_CASE("SnapshotPath::from", "[silkworm][node][snapshot]") { SECTION("invalid") { - CHECK_THROWS_AS(SnapshotPath::make(std::filesystem::path{}, kSnapshotV1, StepRange{Step{1'000}, Step{999}}, "headers"), + CHECK_THROWS_AS(SnapshotPath::make(std::filesystem::path{}, kSnapshotV1, StepRange{Step{1'000}, Step{999}}, "headers", ".seg"), std::logic_error); } } diff --git a/silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp b/silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp index cad86ee2f4..957d9d1a1f 100644 --- a/silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp +++ b/silkworm/db/datastore/snapshots/kv_segment/kv_segment_test.cpp @@ -43,7 +43,7 @@ struct CharCodec : public Encoder, public Decoder { TEST_CASE("KVSegmentFile") { TemporaryDirectory tmp_dir; - auto path = SnapshotPath::make(tmp_dir.path(), kSnapshotV1, StepRange{Step{0}, Step{1}}, "headers"); + auto path = SnapshotPath::make(tmp_dir.path(), kSnapshotV1, StepRange{Step{0}, Step{1}}, "headers", ".seg"); static constexpr seg::CompressionKind kCompressionKind = seg::CompressionKind::kKeys; std::vector> entries = { diff --git a/silkworm/db/datastore/snapshots/schema.cpp b/silkworm/db/datastore/snapshots/schema.cpp index 2b19c2bdce..a5e5e87d01 100644 --- a/silkworm/db/datastore/snapshots/schema.cpp +++ b/silkworm/db/datastore/snapshots/schema.cpp @@ -24,7 +24,7 @@ std::map Schema::RepositoryDef::make_segmen std::map results; for (auto& entry : segment_defs_) { auto tag = entry.first.to_string(); - results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, std::move(tag))); + results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, std::move(tag), kSegmentExtension)); } return results; } diff --git a/silkworm/db/datastore/snapshots/segment/segment_test.cpp b/silkworm/db/datastore/snapshots/segment/segment_test.cpp index 672cdfea01..b0140567bc 100644 --- a/silkworm/db/datastore/snapshots/segment/segment_test.cpp +++ b/silkworm/db/datastore/snapshots/segment/segment_test.cpp @@ -26,7 +26,7 @@ namespace silkworm::snapshots { TEST_CASE("SegmentFile") { TemporaryDirectory tmp_dir; - auto path = SnapshotPath::make(tmp_dir.path(), kSnapshotV1, StepRange{Step{0}, Step{1}}, "headers"); + auto path = SnapshotPath::make(tmp_dir.path(), kSnapshotV1, StepRange{Step{0}, Step{1}}, "headers", ".seg"); std::vector items = { "first", diff --git a/silkworm/db/snapshot_test.cpp b/silkworm/db/snapshot_test.cpp index 7181efa4b2..0dc275dd84 100644 --- a/silkworm/db/snapshot_test.cpp +++ b/silkworm/db/snapshot_test.cpp @@ -50,7 +50,8 @@ class SnapshotPathForTest : public SnapshotPath { tmp_dir, kSnapshotV1, step_range, - "headers"), + "headers", + ".seg"), } {} }; From b6f77001980094225e9f5a619222f4ec5e338d30 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 6 Nov 2024 22:13:42 +0100 Subject: [PATCH 15/15] define SnapshotPath file extensions in schema_config --- cmd/dev/snapshots.cpp | 8 ++++---- silkworm/capi/silkworm.cpp | 10 +++++----- silkworm/capi/silkworm_test.cpp | 6 +++--- silkworm/db/blocks/bodies/body_index.hpp | 4 +++- silkworm/db/blocks/headers/header_index.hpp | 4 +++- silkworm/db/blocks/schema_config.cpp | 3 +++ silkworm/db/blocks/schema_config.hpp | 3 +++ silkworm/db/blocks/transactions/txn_index.hpp | 4 +++- .../db/blocks/transactions/txn_to_block_index.hpp | 2 +- .../datastore/snapshots/common/snapshot_path.cpp | 6 +++--- .../datastore/snapshots/common/snapshot_path.hpp | 11 ++++------- .../snapshots/common/snapshot_path_test.cpp | 5 +++-- silkworm/db/datastore/snapshots/schema.cpp | 4 ++-- silkworm/db/datastore/snapshots/schema.hpp | 15 +++++++++++++++ .../datastore/snapshots/snapshot_repository.cpp | 8 +++++--- .../datastore/snapshots/snapshot_repository.hpp | 14 +++++--------- silkworm/db/snapshot_bundle_factory_impl.cpp | 4 ++-- silkworm/db/snapshot_sync.cpp | 2 +- silkworm/db/snapshot_test.cpp | 14 +++++++------- 19 files changed, 75 insertions(+), 52 deletions(-) diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index 4ce1312376..250e2716b6 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -426,7 +426,7 @@ void open_index(const SnapshotSubcommandSettings& settings) { SILK_INFO << "Open index for snapshot: " << segment_file_path; const auto snapshot_path{snapshots::SnapshotPath::parse(segment_file_path)}; ensure(snapshot_path.has_value(), [&]() { return "open_index: invalid snapshot file " + segment_file_path.filename().string(); }); - const auto index_path{snapshot_path->related_path_ext(kIdxExtension)}; + const auto index_path{snapshot_path->related_path_ext(db::blocks::kIdxExtension)}; SILK_INFO << "Index file: " << index_path.path(); std::chrono::time_point start{std::chrono::steady_clock::now()}; rec_split::RecSplitIndex idx{index_path.path()}; @@ -768,7 +768,7 @@ void lookup_body_in_one(const SnapshotSubcommandSettings& settings, BlockNum blo SegmentFileReader body_segment{*snapshot_path}; body_segment.reopen_segment(); - Index idx_body_number{snapshot_path->related_path_ext(kIdxExtension)}; + Index idx_body_number{snapshot_path->related_path_ext(db::blocks::kIdxExtension)}; idx_body_number.reopen_index(); const auto body = BodyFindByBlockNumQuery{{body_segment, idx_body_number}}.exec(block_number); @@ -873,7 +873,7 @@ void lookup_txn_by_hash_in_one(const SnapshotSubcommandSettings& settings, const txn_segment.reopen_segment(); { - Index idx_txn_hash{snapshot_path->related_path_ext(kIdxExtension)}; + Index idx_txn_hash{snapshot_path->related_path_ext(db::blocks::kIdxExtension)}; idx_txn_hash.reopen_index(); const auto transaction = TransactionFindByHashQuery{{txn_segment, idx_txn_hash}}.exec(hash); @@ -937,7 +937,7 @@ void lookup_txn_by_id_in_one(const SnapshotSubcommandSettings& settings, uint64_ txn_segment.reopen_segment(); { - Index idx_txn_hash{snapshot_path->related_path_ext(kIdxExtension)}; + Index idx_txn_hash{snapshot_path->related_path_ext(db::blocks::kIdxExtension)}; idx_txn_hash.reopen_index(); const auto transaction = TransactionFindByIdQuery{{txn_segment, idx_txn_hash}}.exec(txn_id); diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index bc9d9584d7..8e5ba555f0 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -274,7 +274,7 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc auto index = std::make_shared(snapshots::BodyIndex::make(*snapshot_path, segment_region)); needed_indexes.push_back(index); } else if (name == db::blocks::kTxnSegmentName) { - auto bodies_segment_path = snapshot_path->related_path(db::blocks::kBodySegmentName.to_string(), snapshots::kSegmentExtension); + auto bodies_segment_path = snapshot_path->related_path(db::blocks::kBodySegmentName.to_string(), db::blocks::kSegmentExtension); auto bodies_file = std::find_if(segments, segments + len, [&](SilkwormMemoryMappedFile* file) -> bool { return snapshots::SnapshotPath::parse(file->file_path) == bodies_segment_path; }); @@ -347,7 +347,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn return SILKWORM_INVALID_PATH; } snapshots::SegmentFileReader header_segment{*headers_segment_path, make_region(hs.segment)}; - snapshots::Index idx_header_hash{headers_segment_path->related_path_ext(snapshots::kIdxExtension), make_region(hs.header_hash_index)}; + snapshots::Index idx_header_hash{headers_segment_path->related_path_ext(db::blocks::kIdxExtension), make_region(hs.header_hash_index)}; const SilkwormBodiesSnapshot& bs = snapshot->bodies; if (!bs.segment.file_path || !bs.block_num_index.file_path) { @@ -358,7 +358,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn return SILKWORM_INVALID_PATH; } snapshots::SegmentFileReader body_segment{*bodies_segment_path, make_region(bs.segment)}; - snapshots::Index idx_body_number{bodies_segment_path->related_path_ext(snapshots::kIdxExtension), make_region(bs.block_num_index)}; + snapshots::Index idx_body_number{bodies_segment_path->related_path_ext(db::blocks::kIdxExtension), make_region(bs.block_num_index)}; const SilkwormTransactionsSnapshot& ts = snapshot->transactions; if (!ts.segment.file_path || !ts.tx_hash_index.file_path || !ts.tx_hash_2_block_index.file_path) { @@ -369,8 +369,8 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn return SILKWORM_INVALID_PATH; } snapshots::SegmentFileReader txn_segment{*transactions_segment_path, make_region(ts.segment)}; - snapshots::Index idx_txn_hash{transactions_segment_path->related_path_ext(snapshots::kIdxExtension), make_region(ts.tx_hash_index)}; - snapshots::Index idx_txn_hash_2_block{transactions_segment_path->related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), snapshots::kIdxExtension), make_region(ts.tx_hash_2_block_index)}; + snapshots::Index idx_txn_hash{transactions_segment_path->related_path_ext(db::blocks::kIdxExtension), make_region(ts.tx_hash_index)}; + snapshots::Index idx_txn_hash_2_block{transactions_segment_path->related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), db::blocks::kIdxExtension), make_region(ts.tx_hash_2_block_index)}; snapshots::SnapshotBundleData bundle_data = [&]() { snapshots::SnapshotBundleData data; diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index 274e0986f4..7b14e3510a 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -821,7 +821,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") { REQUIRE_NOTHROW(header_index_builder.build()); snapshots::SegmentFileReader header_segment{header_segment_path}; header_segment.reopen_segment(); - snapshots::Index idx_header_hash{header_segment_path.related_path_ext(snapshots::kIdxExtension)}; + snapshots::Index idx_header_hash{header_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_header_hash.reopen_index(); auto body_index_builder = snapshots::BodyIndex::make(body_segment_path); @@ -829,7 +829,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") { REQUIRE_NOTHROW(body_index_builder.build()); snapshots::SegmentFileReader body_segment{body_segment_path}; body_segment.reopen_segment(); - snapshots::Index idx_body_number{body_segment_path.related_path_ext(snapshots::kIdxExtension)}; + snapshots::Index idx_body_number{body_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_body_number.reopen_index(); auto tx_index_builder = snapshots::TransactionIndex::make(body_segment_path, txn_segment_path); @@ -838,7 +838,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") { tx_index_hash_to_block_builder.build(); snapshots::SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - snapshots::Index idx_txn_hash{txn_segment_path.related_path_ext(snapshots::kIdxExtension)}; + snapshots::Index idx_txn_hash{txn_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_txn_hash.reopen_index(); snapshots::Index idx_txn_hash_2_block{tx_index_hash_to_block_builder.path()}; idx_txn_hash_2_block.reopen_index(); diff --git a/silkworm/db/blocks/bodies/body_index.hpp b/silkworm/db/blocks/bodies/body_index.hpp index b525d8e22b..9d92a62e10 100644 --- a/silkworm/db/blocks/bodies/body_index.hpp +++ b/silkworm/db/blocks/bodies/body_index.hpp @@ -25,6 +25,8 @@ #include #include +#include "../schema_config.hpp" + namespace silkworm::snapshots { class BodyIndex { @@ -43,7 +45,7 @@ class BodyIndex { private: static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) { return { - .index_file = segment_path.related_path_ext(kIdxExtension), + .index_file = segment_path.related_path_ext(db::blocks::kIdxExtension), .key_factory = std::make_unique(), .base_data_id = segment_path.step_range().to_block_num_range().start, }; diff --git a/silkworm/db/blocks/headers/header_index.hpp b/silkworm/db/blocks/headers/header_index.hpp index 59500ca986..88ce4a54d2 100644 --- a/silkworm/db/blocks/headers/header_index.hpp +++ b/silkworm/db/blocks/headers/header_index.hpp @@ -25,6 +25,8 @@ #include #include +#include "../schema_config.hpp" + namespace silkworm::snapshots { class HeaderIndex { @@ -43,7 +45,7 @@ class HeaderIndex { private: static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) { return { - .index_file = segment_path.related_path_ext(kIdxExtension), + .index_file = segment_path.related_path_ext(db::blocks::kIdxExtension), .key_factory = std::make_unique(), .base_data_id = segment_path.step_range().to_block_num_range().start, }; diff --git a/silkworm/db/blocks/schema_config.cpp b/silkworm/db/blocks/schema_config.cpp index 978d41c48e..fb381512eb 100644 --- a/silkworm/db/blocks/schema_config.cpp +++ b/silkworm/db/blocks/schema_config.cpp @@ -23,6 +23,8 @@ namespace silkworm::db::blocks { snapshots::Schema::RepositoryDef make_blocks_repository_schema() { snapshots::Schema::RepositoryDef schema; schema + .segment_file_ext(kSegmentExtension) + .rec_split_index_file_ext(kIdxExtension) .segment(kHeaderSegmentName) .rec_split_index(kIdxHeaderHashName) .segment(kBodySegmentName) @@ -41,6 +43,7 @@ snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_p return snapshots::SnapshotRepository{ std::move(dir_path), open, + make_blocks_repository_schema(), std::make_unique(), make_blocks_bundle_factory(), }; diff --git a/silkworm/db/blocks/schema_config.hpp b/silkworm/db/blocks/schema_config.hpp index db1b3146ca..74fbbce29a 100644 --- a/silkworm/db/blocks/schema_config.hpp +++ b/silkworm/db/blocks/schema_config.hpp @@ -27,6 +27,9 @@ namespace silkworm::db::blocks { inline constexpr datastore::EntityName kBlocksRepositoryName{"Blocks"}; +inline constexpr std::string_view kSegmentExtension{".seg"}; +inline constexpr std::string_view kIdxExtension{".idx"}; + snapshots::Schema::RepositoryDef make_blocks_repository_schema(); std::unique_ptr make_blocks_bundle_factory(); diff --git a/silkworm/db/blocks/transactions/txn_index.hpp b/silkworm/db/blocks/transactions/txn_index.hpp index 562a5f08be..445e74edfe 100644 --- a/silkworm/db/blocks/transactions/txn_index.hpp +++ b/silkworm/db/blocks/transactions/txn_index.hpp @@ -26,6 +26,8 @@ #include #include +#include "../schema_config.hpp" + namespace silkworm::snapshots { struct TransactionKeyFactory : IndexKeyFactory { @@ -66,7 +68,7 @@ class TransactionIndex { private: static IndexDescriptor make_descriptor(const SnapshotPath& segment_path, uint64_t first_tx_id) { return { - .index_file = segment_path.related_path_ext(kIdxExtension), + .index_file = segment_path.related_path_ext(db::blocks::kIdxExtension), .key_factory = std::make_unique(first_tx_id), .base_data_id = first_tx_id, .less_false_positives = true, diff --git a/silkworm/db/blocks/transactions/txn_to_block_index.hpp b/silkworm/db/blocks/transactions/txn_to_block_index.hpp index 20064ddafc..2110b1d730 100644 --- a/silkworm/db/blocks/transactions/txn_to_block_index.hpp +++ b/silkworm/db/blocks/transactions/txn_to_block_index.hpp @@ -96,7 +96,7 @@ class TransactionToBlockIndex { private: static IndexDescriptor make_descriptor(const SnapshotPath& segment_path, BlockNum first_block_num, uint64_t first_tx_id) { return { - .index_file = segment_path.related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), kIdxExtension), + .index_file = segment_path.related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), db::blocks::kIdxExtension), .key_factory = std::make_unique(first_tx_id), .base_data_id = first_block_num, .double_enum_index = false, diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp index fae920559a..031f720cc6 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.cpp @@ -99,7 +99,7 @@ SnapshotPath SnapshotPath::make( uint8_t version, StepRange step_range, std::string tag, - const char* ext) { + std::string_view ext) { const auto filename = SnapshotPath::make_filename(version, step_range, tag, ext); return SnapshotPath{dir / filename, version, step_range, std::move(tag)}; } @@ -108,7 +108,7 @@ fs::path SnapshotPath::make_filename( uint8_t version, StepRange step_range, std::string_view tag, - const char* ext) { + std::string_view ext) { std::string filename = absl::StrFormat( "v%d-%06d-%06d-%s%s", version, @@ -119,7 +119,7 @@ fs::path SnapshotPath::make_filename( return fs::path{filename}; } -SnapshotPath SnapshotPath::related_path(std::string tag, const char* ext) const { +SnapshotPath SnapshotPath::related_path(std::string tag, std::string_view ext) const { return SnapshotPath::make(path_.parent_path(), version_, step_range_, std::move(tag), ext); } diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp index 3d0662d6c6..4dff936ba3 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path.hpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path.hpp @@ -26,9 +26,6 @@ namespace silkworm::snapshots { -inline constexpr const char* kSegmentExtension{".seg"}; -inline constexpr const char* kIdxExtension{".idx"}; - //! The snapshot version 1 aka v1 inline constexpr uint8_t kSnapshotV1{1}; @@ -42,7 +39,7 @@ class SnapshotPath { uint8_t version, StepRange step_range, std::string tag, - const char* ext); + std::string_view ext); std::string filename() const { return path_.filename().string(); } const std::filesystem::path& path() const { return path_; } @@ -52,8 +49,8 @@ class SnapshotPath { const std::string& tag() const { return tag_; } bool exists() const { return std::filesystem::exists(path_); } - SnapshotPath related_path(std::string tag, const char* ext) const; - SnapshotPath related_path_ext(const char* ext) const { + SnapshotPath related_path(std::string tag, std::string_view ext) const; + SnapshotPath related_path_ext(std::string_view ext) const { return related_path(tag_, ext); } @@ -65,7 +62,7 @@ class SnapshotPath { uint8_t version, StepRange step_range, std::string_view tag, - const char* ext); + std::string_view ext); SnapshotPath( std::filesystem::path path, diff --git a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp index 800eca9828..b8265078f4 100644 --- a/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp +++ b/silkworm/db/datastore/snapshots/common/snapshot_path_test.cpp @@ -69,9 +69,10 @@ TEST_CASE("SnapshotPath::parse", "[silkworm][node][snapshot]") { CHECK(snapshot_file->version() == 1); CHECK(snapshot_file->step_range() == StepRange::from_block_num_range(filename_expectation.block_num_range)); CHECK(snapshot_file->tag() == filename_expectation.tag); - const SnapshotPath index_file = snapshot_file->related_path_ext(kIdxExtension); + + const SnapshotPath index_file = snapshot_file->related_path_ext(".idx"); CHECK(index_file.path().stem() == snapshot_file->path().stem()); - CHECK(index_file.path().extension() == kIdxExtension); + CHECK(index_file.path().extension() == ".idx"); CHECK(index_file.version() == 1); CHECK(index_file.step_range() == StepRange::from_block_num_range(filename_expectation.block_num_range)); CHECK(index_file.tag() == filename_expectation.tag); diff --git a/silkworm/db/datastore/snapshots/schema.cpp b/silkworm/db/datastore/snapshots/schema.cpp index a5e5e87d01..99fc39c43a 100644 --- a/silkworm/db/datastore/snapshots/schema.cpp +++ b/silkworm/db/datastore/snapshots/schema.cpp @@ -24,7 +24,7 @@ std::map Schema::RepositoryDef::make_segmen std::map results; for (auto& entry : segment_defs_) { auto tag = entry.first.to_string(); - results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, std::move(tag), kSegmentExtension)); + results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, std::move(tag), segment_file_ext_)); } return results; } @@ -45,7 +45,7 @@ std::map Schema::RepositoryDef::make_rec_sp std::map results; for (auto& entry : rec_split_index_defs_) { auto tag = entry.first.to_string(); - results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, std::move(tag), kIdxExtension)); + results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, std::move(tag), rec_split_index_file_ext_)); } return results; } diff --git a/silkworm/db/datastore/snapshots/schema.hpp b/silkworm/db/datastore/snapshots/schema.hpp index 154c094446..439c9985bf 100644 --- a/silkworm/db/datastore/snapshots/schema.hpp +++ b/silkworm/db/datastore/snapshots/schema.hpp @@ -36,20 +36,35 @@ class Schema { return *this; } + RepositoryDef& segment_file_ext(std::string_view ext) { + segment_file_ext_ = ext; + return *this; + } + RepositoryDef& rec_split_index(datastore::EntityName name) { rec_split_index_defs_.try_emplace(name); return *this; } + RepositoryDef& rec_split_index_file_ext(std::string_view ext) { + rec_split_index_file_ext_ = ext; + return *this; + } + std::map make_segment_paths(const std::filesystem::path& dir_path, StepRange range) const; std::map make_segments(const std::filesystem::path& dir_path, StepRange range) const; std::map make_rec_split_index_paths(const std::filesystem::path& dir_path, StepRange range) const; std::map make_rec_split_indexes(const std::filesystem::path& dir_path, StepRange range) const; std::vector make_all_paths(const std::filesystem::path& dir_path, StepRange range) const; + std::string_view segment_file_ext() const { return segment_file_ext_; } + std::string_view rec_split_index_file_ext() const { return rec_split_index_file_ext_; } + private: std::map segment_defs_; + std::string segment_file_ext_; std::map rec_split_index_defs_; + std::string rec_split_index_file_ext_; }; RepositoryDef& repository(datastore::EntityName name) { diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 48d1d3fe3d..b8ddaaf16f 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -30,9 +30,11 @@ namespace fs = std::filesystem; SnapshotRepository::SnapshotRepository( std::filesystem::path dir_path, bool open, + Schema::RepositoryDef schema, std::unique_ptr step_converter, std::unique_ptr bundle_factory) : dir_path_(std::move(dir_path)), + schema_(std::move(schema)), step_converter_(std::move(step_converter)), bundle_factory_(std::move(bundle_factory)), bundles_(std::make_shared()), @@ -104,7 +106,7 @@ std::pair, std::shared_ptr> Snaps } std::vector> SnapshotRepository::missing_indexes() const { - SnapshotPathList segment_files = get_segment_files(); + SnapshotPathList segment_files = get_files(schema_.segment_file_ext()); auto index_builders = bundle_factory_->index_builders(segment_files); std::erase_if(index_builders, [&](const auto& builder) { @@ -176,7 +178,7 @@ std::vector> SnapshotRepository::bundles_in_rang return bundles; } -SnapshotPathList SnapshotRepository::get_files(const std::string& ext) const { +SnapshotPathList SnapshotRepository::get_files(std::string_view ext) const { ensure(fs::exists(dir_path_), [&]() { return "SnapshotRepository: " + dir_path_.string() + " does not exist"; }); ensure(fs::is_directory(dir_path_), @@ -231,7 +233,7 @@ bool SnapshotRepository::is_stale_index_path(const SnapshotPath& index_path) con SnapshotPathList SnapshotRepository::stale_index_paths() const { SnapshotPathList results; - auto all_files = this->get_idx_files(); + auto all_files = get_files(schema_.rec_split_index_file_ext()); std::copy_if( all_files.begin(), all_files.end(), std::back_inserter(results), diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index d8e4bbf78d..2626b984fa 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -49,6 +49,7 @@ class SnapshotRepository { SnapshotRepository( std::filesystem::path dir_path, bool open, + Schema::RepositoryDef schema, std::unique_ptr step_converter, std::unique_ptr bundle_factory); @@ -113,15 +114,7 @@ class SnapshotRepository { private: Step max_end_step() const; - SnapshotPathList get_segment_files() const { - return get_files(kSegmentExtension); - } - - SnapshotPathList get_idx_files() const { - return get_files(kIdxExtension); - } - - SnapshotPathList get_files(const std::string& ext) const; + SnapshotPathList get_files(std::string_view ext) const; std::vector list_dir_file_ranges() const; bool is_stale_index_path(const SnapshotPath& index_path) const; @@ -130,6 +123,9 @@ class SnapshotRepository { //! Path to the snapshots directory std::filesystem::path dir_path_; + //! Schema + Schema::RepositoryDef schema_; + //! Converts timestamp units to steps std::unique_ptr step_converter_; diff --git a/silkworm/db/snapshot_bundle_factory_impl.cpp b/silkworm/db/snapshot_bundle_factory_impl.cpp index af1e95448f..d3b3a245b8 100644 --- a/silkworm/db/snapshot_bundle_factory_impl.cpp +++ b/silkworm/db/snapshot_bundle_factory_impl.cpp @@ -51,7 +51,7 @@ std::vector> SnapshotBundleFactoryImpl::index_buil if (name == db::blocks::kBodySegmentName) return {std::make_shared(BodyIndex::make(segment_path))}; if (name == db::blocks::kTxnSegmentName) { - auto bodies_segment_path = segment_path.related_path(db::blocks::kBodySegmentName.to_string(), kSegmentExtension); + auto bodies_segment_path = segment_path.related_path(db::blocks::kBodySegmentName.to_string(), db::blocks::kSegmentExtension); if (!bodies_segment_path.exists()) return {}; return { std::make_shared(TransactionIndex::make(bodies_segment_path, segment_path)), @@ -86,7 +86,7 @@ SnapshotPathList SnapshotBundleFactoryImpl::index_dependency_paths(const Snapsho SILKWORM_ASSERT(false); std::abort(); }(); - SnapshotPath snapshot_path = index_path.related_path(segment_name.to_string(), kSegmentExtension); + SnapshotPath snapshot_path = index_path.related_path(segment_name.to_string(), db::blocks::kSegmentExtension); return {snapshot_path}; } diff --git a/silkworm/db/snapshot_sync.cpp b/silkworm/db/snapshot_sync.cpp index b3ed99a775..d654d8958e 100644 --- a/silkworm/db/snapshot_sync.cpp +++ b/silkworm/db/snapshot_sync.cpp @@ -56,7 +56,7 @@ struct PathHasher { static bool snapshot_file_is_fully_merged(std::string_view file_name) { const auto path = SnapshotPath::parse(std::filesystem::path{file_name}); return path.has_value() && - (path->extension() == kSegmentExtension) && + (path->extension() == blocks::kSegmentExtension) && (path->step_range().to_block_num_range().size() >= kMaxMergerSnapshotSize); } diff --git a/silkworm/db/snapshot_test.cpp b/silkworm/db/snapshot_test.cpp index 0dc275dd84..abba627953 100644 --- a/silkworm/db/snapshot_test.cpp +++ b/silkworm/db/snapshot_test.cpp @@ -132,7 +132,7 @@ TEST_CASE("HeaderSnapshot::header_by_number OK", "[silkworm][node][snapshot][ind SegmentFileReader header_segment{header_segment_path}; header_segment.reopen_segment(); - Index idx_header_hash{header_segment_path.related_path_ext(kIdxExtension)}; + Index idx_header_hash{header_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_header_hash.reopen_index(); HeaderFindByBlockNumQuery header_by_number{{header_segment, idx_header_hash}}; @@ -175,7 +175,7 @@ TEST_CASE("BodySnapshot::body_by_number OK", "[silkworm][node][snapshot][index]" SegmentFileReader body_segment{body_segment_path}; body_segment.reopen_segment(); - Index idx_body_number{body_segment_path.related_path_ext(kIdxExtension)}; + Index idx_body_number{body_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_body_number.reopen_index(); BodyFindByBlockNumQuery body_by_number{{body_segment, idx_body_number}}; @@ -204,7 +204,7 @@ TEST_CASE("TransactionSnapshot::txn_by_id OK", "[silkworm][node][snapshot][index SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - Index idx_txn_hash{txn_segment_path.related_path_ext(kIdxExtension)}; + Index idx_txn_hash{txn_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_txn_hash.reopen_index(); TransactionFindByIdQuery txn_by_id{{txn_segment, idx_txn_hash}}; @@ -233,11 +233,11 @@ TEST_CASE("TransactionSnapshot::block_num_by_txn_hash OK", "[silkworm][node][sna SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - Index idx_txn_hash{txn_segment_path.related_path_ext(kIdxExtension)}; + Index idx_txn_hash{txn_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_txn_hash.reopen_index(); TransactionFindByIdQuery txn_by_id{{txn_segment, idx_txn_hash}}; - Index idx_txn_hash_2_block{txn_segment_path.related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), kIdxExtension)}; + Index idx_txn_hash_2_block{txn_segment_path.related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), db::blocks::kIdxExtension)}; idx_txn_hash_2_block.reopen_index(); TransactionBlockNumByTxnHashQuery block_num_by_txn_hash{idx_txn_hash_2_block, TransactionFindByHashQuery{{txn_segment, idx_txn_hash}}}; @@ -275,7 +275,7 @@ TEST_CASE("TransactionSnapshot::txn_range OK", "[silkworm][node][snapshot][index SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - Index idx_txn_hash{txn_segment_path.related_path_ext(kIdxExtension)}; + Index idx_txn_hash{txn_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_txn_hash.reopen_index(); TransactionRangeFromIdQuery query{{txn_segment, idx_txn_hash}}; @@ -307,7 +307,7 @@ TEST_CASE("TransactionSnapshot::txn_rlp_range OK", "[silkworm][node][snapshot][i SegmentFileReader txn_segment{txn_segment_path}; txn_segment.reopen_segment(); - Index idx_txn_hash{txn_segment_path.related_path_ext(kIdxExtension)}; + Index idx_txn_hash{txn_segment_path.related_path_ext(db::blocks::kIdxExtension)}; idx_txn_hash.reopen_index(); TransactionPayloadRlpRangeFromIdQuery query{{txn_segment, idx_txn_hash}};