Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

datastore: EntityName replaces SnapshotType #2491

Merged
merged 15 commits into from
Nov 7, 2024
Merged

datastore: EntityName replaces SnapshotType #2491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
600b7fd
refactor dev/snapshots to use bundle factory
battlmonstr Nov 6, 2024
35a9d08
use EntityName in freezer/merger
battlmonstr Nov 6, 2024
d204e26
use string tag in SnapshotPath test
battlmonstr Nov 6, 2024
1b7cea8
refactor SnapshotRepository::is_stale_index_path
battlmonstr Nov 6, 2024
5c6d624
use EntityName in snapshot_file_recompress
battlmonstr Nov 6, 2024
5d8ecb9
use EntityName in SnapshotBundleFactoryImpl
battlmonstr Nov 6, 2024
3b5022d
use EntityName in silkworm_build_recsplit_indexes
battlmonstr Nov 6, 2024
0d58cc8
replace SnapshotPath::make, related_path
battlmonstr Nov 6, 2024
4c80eaf
remove SnapshotPath::type_
battlmonstr Nov 6, 2024
afcc9b0
use EntityName for find_segment
battlmonstr Nov 6, 2024
384f4b6
dash in kIdxTxnHash2BlockName to eliminate tag conversions
battlmonstr Nov 6, 2024
edcd88f
remove unused magic_enum
battlmonstr Nov 6, 2024
dfbf52c
rename index_file to related_path_ext
battlmonstr Nov 6, 2024
49a30e6
make kSegmentExtension non-optional
battlmonstr Nov 6, 2024
b6f7700
define SnapshotPath file extensions in schema_config
battlmonstr Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 12 additions & 34 deletions cmd/dev/snapshots.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,6 @@ void count_bodies(const SnapshotSubcommandSettings& settings, int repetitions) {
if (settings.segment_file_name) {
const auto snapshot_path{SnapshotPath::parse(std::filesystem::path{*settings.segment_file_name})};
ensure(snapshot_path.has_value(), "count_bodies: invalid snapshot_file path format");
ensure(snapshot_path->type() == SnapshotType::bodies, "count_bodies: snapshot_file must point to body segment");
SegmentFileReader body_segment{*snapshot_path};
body_segment.reopen_segment();
std::tie(num_bodies, num_txns) = count_bodies_in_one(settings, body_segment);
Expand Down Expand Up @@ -390,7 +389,6 @@ void count_headers(const SnapshotSubcommandSettings& settings, int repetitions)
if (settings.segment_file_name) {
const auto snapshot_path{SnapshotPath::parse(std::filesystem::path{*settings.segment_file_name})};
ensure(snapshot_path.has_value(), "count_headers: invalid snapshot_file path format");
ensure(snapshot_path->type() == SnapshotType::headers, "count_headers: snapshot_file must point to header segment");
SegmentFileReader header_segment{*snapshot_path};
header_segment.reopen_segment();
num_headers = count_headers_in_one(settings, header_segment);
Expand All @@ -407,32 +405,12 @@ void create_index(const SnapshotSubcommandSettings& settings, int repetitions) {
ensure(settings.segment_file_name.has_value(), "create_index: --snapshot_file must be specified");
SILK_INFO << "Create index for snapshot: " << *settings.segment_file_name;
std::chrono::time_point start{std::chrono::steady_clock::now()};
auto bundle_factory = db::blocks::make_blocks_bundle_factory();
const auto snapshot_path = SnapshotPath::parse(std::filesystem::path{*settings.segment_file_name});
if (snapshot_path) {
for (int i{0}; i < repetitions; ++i) {
switch (snapshot_path->type()) {
case SnapshotType::headers: {
auto index = HeaderIndex::make(*snapshot_path);
index.build();
break;
}
case SnapshotType::bodies: {
auto index = BodyIndex::make(*snapshot_path);
index.build();
break;
}
case SnapshotType::transactions: {
auto bodies_segment_path = snapshot_path->related_path(SnapshotType::bodies, kSegmentExtension);
auto index = TransactionIndex::make(bodies_segment_path, *snapshot_path);
index.build();

auto index_hash_to_block = TransactionToBlockIndex::make(bodies_segment_path, *snapshot_path);
index_hash_to_block.build();
break;
}
default: {
SILKWORM_ASSERT(false);
}
for (auto& builder : bundle_factory->index_builders(*snapshot_path)) {
builder->build();
}
}
} else {
Expand All @@ -448,7 +426,7 @@ void open_index(const SnapshotSubcommandSettings& settings) {
SILK_INFO << "Open index for snapshot: " << segment_file_path;
const auto snapshot_path{snapshots::SnapshotPath::parse(segment_file_path)};
ensure(snapshot_path.has_value(), [&]() { return "open_index: invalid snapshot file " + segment_file_path.filename().string(); });
const auto index_path{snapshot_path->index_file()};
const auto index_path{snapshot_path->related_path_ext(db::blocks::kIdxExtension)};
SILK_INFO << "Index file: " << index_path.path();
std::chrono::time_point start{std::chrono::steady_clock::now()};
rec_split::RecSplitIndex idx{index_path.path()};
Expand Down Expand Up @@ -722,7 +700,7 @@ void lookup_header_by_hash(const SnapshotSubcommandSettings& settings) {
auto repository = make_repository(settings.settings);
for (const auto& bundle_ptr : repository.view_bundles_reverse()) {
const auto& bundle = *bundle_ptr;
auto segment_and_index = bundle.segment_and_index(SnapshotType::headers);
auto segment_and_index = bundle.segment_and_index(db::blocks::kHeaderSegmentName);
const auto header = HeaderFindByHashQuery{segment_and_index}.exec(*hash);
if (header) {
matching_header = header;
Expand All @@ -749,7 +727,7 @@ void lookup_header_by_number(const SnapshotSubcommandSettings& settings) {
std::chrono::time_point start{std::chrono::steady_clock::now()};

auto repository = make_repository(settings.settings);
const auto [segment_and_index, _] = repository.find_segment(SnapshotType::headers, block_number);
const auto [segment_and_index, _] = repository.find_segment(db::blocks::kHeaderSegmentName, block_number);
if (segment_and_index) {
const auto header = HeaderFindByBlockNumQuery{*segment_and_index}.exec(block_number);
ensure(header.has_value(),
Expand Down Expand Up @@ -790,7 +768,7 @@ void lookup_body_in_one(const SnapshotSubcommandSettings& settings, BlockNum blo
SegmentFileReader body_segment{*snapshot_path};
body_segment.reopen_segment();

Index idx_body_number{snapshot_path->index_file()};
Index idx_body_number{snapshot_path->related_path_ext(db::blocks::kIdxExtension)};
idx_body_number.reopen_index();

const auto body = BodyFindByBlockNumQuery{{body_segment, idx_body_number}}.exec(block_number);
Expand All @@ -810,7 +788,7 @@ void lookup_body_in_all(const SnapshotSubcommandSettings& settings, BlockNum blo
auto repository = make_repository(settings.settings);

std::chrono::time_point start{std::chrono::steady_clock::now()};
const auto [segment_and_index, _] = repository.find_segment(SnapshotType::bodies, block_number);
const auto [segment_and_index, _] = repository.find_segment(db::blocks::kBodySegmentName, block_number);
if (segment_and_index) {
const auto body = BodyFindByBlockNumQuery{*segment_and_index}.exec(block_number);
ensure(body.has_value(),
Expand Down Expand Up @@ -895,7 +873,7 @@ void lookup_txn_by_hash_in_one(const SnapshotSubcommandSettings& settings, const
txn_segment.reopen_segment();

{
Index idx_txn_hash{snapshot_path->index_file()};
Index idx_txn_hash{snapshot_path->related_path_ext(db::blocks::kIdxExtension)};
idx_txn_hash.reopen_index();

const auto transaction = TransactionFindByHashQuery{{txn_segment, idx_txn_hash}}.exec(hash);
Expand All @@ -919,7 +897,7 @@ void lookup_txn_by_hash_in_all(const SnapshotSubcommandSettings& settings, const
std::chrono::time_point start{std::chrono::steady_clock::now()};
for (const auto& bundle_ptr : repository.view_bundles_reverse()) {
const auto& bundle = *bundle_ptr;
auto segment_and_index = bundle.segment_and_index(SnapshotType::transactions);
auto segment_and_index = bundle.segment_and_index(db::blocks::kTxnSegmentName);
const auto transaction = TransactionFindByHashQuery{segment_and_index}.exec(hash);
if (transaction) {
matching_snapshot_path = segment_and_index.segment.path();
Expand Down Expand Up @@ -959,7 +937,7 @@ void lookup_txn_by_id_in_one(const SnapshotSubcommandSettings& settings, uint64_
txn_segment.reopen_segment();

{
Index idx_txn_hash{snapshot_path->index_file()};
Index idx_txn_hash{snapshot_path->related_path_ext(db::blocks::kIdxExtension)};
idx_txn_hash.reopen_index();

const auto transaction = TransactionFindByIdQuery{{txn_segment, idx_txn_hash}}.exec(txn_id);
Expand All @@ -983,7 +961,7 @@ void lookup_txn_by_id_in_all(const SnapshotSubcommandSettings& settings, uint64_
std::chrono::time_point start{std::chrono::steady_clock::now()};
for (const auto& bundle_ptr : repository.view_bundles_reverse()) {
const auto& bundle = *bundle_ptr;
auto segment_and_index = bundle.segment_and_index(SnapshotType::transactions);
auto segment_and_index = bundle.segment_and_index(db::blocks::kTxnSegmentName);
const auto transaction = TransactionFindByIdQuery{segment_and_index}.exec(txn_id);
if (transaction) {
matching_snapshot_path = segment_and_index.segment.path();
Expand Down
34 changes: 14 additions & 20 deletions silkworm/capi/silkworm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,38 +265,32 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc
return SILKWORM_INVALID_PATH;
}

std::shared_ptr<snapshots::IndexBuilder> index;
switch (snapshot_path->type()) {
case snapshots::SnapshotType::headers: {
index = std::make_shared<snapshots::IndexBuilder>(snapshots::HeaderIndex::make(*snapshot_path, segment_region));
datastore::EntityName name{snapshot_path->tag()};
{
if (name == db::blocks::kHeaderSegmentName) {
auto index = std::make_shared<snapshots::IndexBuilder>(snapshots::HeaderIndex::make(*snapshot_path, segment_region));
needed_indexes.push_back(index);
break;
}
case snapshots::SnapshotType::bodies: {
index = std::make_shared<snapshots::IndexBuilder>(snapshots::BodyIndex::make(*snapshot_path, segment_region));
} else if (name == db::blocks::kBodySegmentName) {
auto index = std::make_shared<snapshots::IndexBuilder>(snapshots::BodyIndex::make(*snapshot_path, segment_region));
needed_indexes.push_back(index);
break;
}
case snapshots::SnapshotType::transactions: {
auto bodies_segment_path = snapshot_path->related_path(snapshots::SnapshotType::bodies, snapshots::kSegmentExtension);
} else if (name == db::blocks::kTxnSegmentName) {
auto bodies_segment_path = snapshot_path->related_path(db::blocks::kBodySegmentName.to_string(), db::blocks::kSegmentExtension);
auto bodies_file = std::find_if(segments, segments + len, [&](SilkwormMemoryMappedFile* file) -> bool {
return snapshots::SnapshotPath::parse(file->file_path) == bodies_segment_path;
});

if (bodies_file < segments + len) {
auto bodies_segment_region = make_region(**bodies_file);

index = std::make_shared<snapshots::IndexBuilder>(snapshots::TransactionIndex::make(
auto index = std::make_shared<snapshots::IndexBuilder>(snapshots::TransactionIndex::make(
bodies_segment_path, bodies_segment_region, *snapshot_path, segment_region));
needed_indexes.push_back(index);

index = std::make_shared<snapshots::IndexBuilder>(snapshots::TransactionToBlockIndex::make(
bodies_segment_path, bodies_segment_region, *snapshot_path, segment_region));
needed_indexes.push_back(index);
}
break;
}
default: {
} else {
SILKWORM_ASSERT(false);
}
}
Expand Down Expand Up @@ -353,7 +347,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn
return SILKWORM_INVALID_PATH;
}
snapshots::SegmentFileReader header_segment{*headers_segment_path, make_region(hs.segment)};
snapshots::Index idx_header_hash{headers_segment_path->index_file(), make_region(hs.header_hash_index)};
snapshots::Index idx_header_hash{headers_segment_path->related_path_ext(db::blocks::kIdxExtension), make_region(hs.header_hash_index)};

const SilkwormBodiesSnapshot& bs = snapshot->bodies;
if (!bs.segment.file_path || !bs.block_num_index.file_path) {
Expand All @@ -364,7 +358,7 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn
return SILKWORM_INVALID_PATH;
}
snapshots::SegmentFileReader body_segment{*bodies_segment_path, make_region(bs.segment)};
snapshots::Index idx_body_number{bodies_segment_path->index_file(), make_region(bs.block_num_index)};
snapshots::Index idx_body_number{bodies_segment_path->related_path_ext(db::blocks::kIdxExtension), make_region(bs.block_num_index)};

const SilkwormTransactionsSnapshot& ts = snapshot->transactions;
if (!ts.segment.file_path || !ts.tx_hash_index.file_path || !ts.tx_hash_2_block_index.file_path) {
Expand All @@ -375,8 +369,8 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle handle, SilkwormChainSn
return SILKWORM_INVALID_PATH;
}
snapshots::SegmentFileReader txn_segment{*transactions_segment_path, make_region(ts.segment)};
snapshots::Index idx_txn_hash{transactions_segment_path->related_path(snapshots::SnapshotType::transactions, snapshots::kIdxExtension), make_region(ts.tx_hash_index)};
snapshots::Index idx_txn_hash_2_block{transactions_segment_path->related_path(snapshots::SnapshotType::transactions_to_block, snapshots::kIdxExtension), make_region(ts.tx_hash_2_block_index)};
snapshots::Index idx_txn_hash{transactions_segment_path->related_path_ext(db::blocks::kIdxExtension), make_region(ts.tx_hash_index)};
snapshots::Index idx_txn_hash_2_block{transactions_segment_path->related_path(db::blocks::kIdxTxnHash2BlockName.to_string(), db::blocks::kIdxExtension), make_region(ts.tx_hash_2_block_index)};

snapshots::SnapshotBundleData bundle_data = [&]() {
snapshots::SnapshotBundleData data;
Expand Down
6 changes: 3 additions & 3 deletions silkworm/capi/silkworm_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,15 +821,15 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") {
REQUIRE_NOTHROW(header_index_builder.build());
snapshots::SegmentFileReader header_segment{header_segment_path};
header_segment.reopen_segment();
snapshots::Index idx_header_hash{header_segment_path.index_file()};
snapshots::Index idx_header_hash{header_segment_path.related_path_ext(db::blocks::kIdxExtension)};
idx_header_hash.reopen_index();

auto body_index_builder = snapshots::BodyIndex::make(body_segment_path);
body_index_builder.set_base_data_id(body_segment_file.block_num_range().start);
REQUIRE_NOTHROW(body_index_builder.build());
snapshots::SegmentFileReader body_segment{body_segment_path};
body_segment.reopen_segment();
snapshots::Index idx_body_number{body_segment_path.index_file()};
snapshots::Index idx_body_number{body_segment_path.related_path_ext(db::blocks::kIdxExtension)};
idx_body_number.reopen_index();

auto tx_index_builder = snapshots::TransactionIndex::make(body_segment_path, txn_segment_path);
Expand All @@ -838,7 +838,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") {
tx_index_hash_to_block_builder.build();
snapshots::SegmentFileReader txn_segment{txn_segment_path};
txn_segment.reopen_segment();
snapshots::Index idx_txn_hash{txn_segment_path.index_file()};
snapshots::Index idx_txn_hash{txn_segment_path.related_path_ext(db::blocks::kIdxExtension)};
idx_txn_hash.reopen_index();
snapshots::Index idx_txn_hash_2_block{tx_index_hash_to_block_builder.path()};
idx_txn_hash_2_block.reopen_index();
Expand Down
12 changes: 6 additions & 6 deletions silkworm/db/access_layer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1256,7 +1256,7 @@ bool DataModel::read_block_from_snapshot(BlockNum height, Block& block) const {
std::optional<BlockHeader> DataModel::read_header_from_snapshot(BlockNum height) const {
std::optional<BlockHeader> block_header;
// We know the header snapshot in advance: find it based on target block number
const auto [segment_and_index, _] = repository_.find_segment(SnapshotType::headers, height);
const auto [segment_and_index, _] = repository_.find_segment(blocks::kHeaderSegmentName, height);
if (segment_and_index) {
block_header = HeaderFindByBlockNumQuery{*segment_and_index}.exec(height);
}
Expand All @@ -1268,7 +1268,7 @@ std::optional<BlockHeader> DataModel::read_header_from_snapshot(const Hash& hash
// We don't know the header snapshot in advance: search for block hash in each header snapshot in reverse order
for (const auto& bundle_ptr : repository_.view_bundles_reverse()) {
const auto& bundle = *bundle_ptr;
auto segment_and_index = bundle.segment_and_index(SnapshotType::headers);
auto segment_and_index = bundle.segment_and_index(blocks::kHeaderSegmentName);
block_header = HeaderFindByHashQuery{segment_and_index}.exec(hash);
if (block_header) break;
}
Expand Down Expand Up @@ -1299,7 +1299,7 @@ bool DataModel::read_body_from_snapshot(BlockNum height, BlockBody& body) const

bool DataModel::is_body_in_snapshot(BlockNum height) const {
// We know the body snapshot in advance: find it based on target block number
const auto [segment_and_index, _] = repository_.find_segment(SnapshotType::bodies, height);
const auto [segment_and_index, _] = repository_.find_segment(blocks::kBodySegmentName, height);
if (segment_and_index) {
const auto stored_body = BodyFindByBlockNumQuery{*segment_and_index}.exec(height);
return stored_body.has_value();
Expand All @@ -1313,7 +1313,7 @@ bool DataModel::read_transactions_from_snapshot(BlockNum height, uint64_t base_t
return true;
}

const auto [segment_and_index, _] = repository_.find_segment(SnapshotType::transactions, height);
const auto [segment_and_index, _] = repository_.find_segment(blocks::kTxnSegmentName, height);
if (!segment_and_index) return false;

txs = TransactionRangeFromIdQuery{*segment_and_index}.exec_into_vector(base_txn_id, txn_count);
Expand All @@ -1322,7 +1322,7 @@ bool DataModel::read_transactions_from_snapshot(BlockNum height, uint64_t base_t
}

bool DataModel::read_rlp_transactions_from_snapshot(BlockNum height, std::vector<Bytes>& rlp_txs) const {
const auto [body_segment_and_index, _] = repository_.find_segment(SnapshotType::bodies, height);
const auto [body_segment_and_index, _] = repository_.find_segment(blocks::kBodySegmentName, height);
if (body_segment_and_index) {
auto stored_body = BodyFindByBlockNumQuery{*body_segment_and_index}.exec(height);
if (!stored_body) return false;
Expand All @@ -1333,7 +1333,7 @@ bool DataModel::read_rlp_transactions_from_snapshot(BlockNum height, std::vector

if (txn_count == 0) return true;

const auto [tx_segment_and_index, _2] = repository_.find_segment(SnapshotType::transactions, height);
const auto [tx_segment_and_index, _2] = repository_.find_segment(blocks::kTxnSegmentName, height);
if (!tx_segment_and_index) return false;

rlp_txs = TransactionPayloadRlpRangeFromIdQuery{*tx_segment_and_index}.exec_into_vector(base_txn_id, txn_count);
Expand Down
4 changes: 3 additions & 1 deletion silkworm/db/blocks/bodies/body_index.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
#include <silkworm/db/datastore/snapshots/index_builder.hpp>
#include <silkworm/infra/common/memory_mapped_file.hpp>

#include "../schema_config.hpp"

namespace silkworm::snapshots {

class BodyIndex {
Expand All @@ -43,7 +45,7 @@ class BodyIndex {
private:
static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) {
return {
.index_file = segment_path.index_file(),
.index_file = segment_path.related_path_ext(db::blocks::kIdxExtension),
.key_factory = std::make_unique<KeyFactory>(),
.base_data_id = segment_path.step_range().to_block_num_range().start,
};
Expand Down
3 changes: 2 additions & 1 deletion silkworm/db/blocks/bodies/body_queries.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include <silkworm/db/datastore/snapshots/basic_queries.hpp>
#include <silkworm/db/datastore/snapshots/snapshot_repository.hpp>

#include "../schema_config.hpp"
#include "body_segment.hpp"

namespace silkworm::snapshots {
Expand All @@ -32,7 +33,7 @@ class BodyFindByBlockNumMultiQuery {
: repository_{repository} {}

std::optional<BlockBodyForStorage> exec(BlockNum block_num) {
const auto [segment_and_index, _] = repository_.find_segment(SnapshotType::bodies, block_num);
const auto [segment_and_index, _] = repository_.find_segment(db::blocks::kBodySegmentName, block_num);
if (!segment_and_index) return std::nullopt;
return BodyFindByBlockNumQuery{*segment_and_index}.exec(block_num);
}
Expand Down
Loading
Loading