Skip to content

Commit

Permalink
snapshots: add hash prefix check in header and txn lookup (#1321)
Browse files Browse the repository at this point in the history 
snapshots: add block height check in body lookup
snapshots: add check on block format in snapshot path
snapshots: improvements in snapshot repository
snapshots: improve unit tests
cmd: add lookup_body and lookup_transaction subcommands in snapshots tool + improvements
  • Loading branch information
@canepat
canepat committed Jul 12, 2023
1 parent ba0553d commit a3cfbe5
Show file tree
Hide file tree
Showing 10 changed files with 628 additions and 163 deletions.
263 changes: 247 additions & 16 deletions cmd/dev/snapshots.cpp
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions silkworm/node/snapshot/index_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ TEST_CASE("TransactionIndex::build KO: invalid snapshot", "[silkworm][snapshot][

SECTION("KO: invalid position depth") {
test::SampleBodySnapshotFile invalid_bodies_snapshot{
TemporaryDirectory::get_os_temporary_path(),
"000000000000000e000000000000000000000000000000000000000000000004"
"c100010801c6837004d980c001c6837004d980c001c6837004d980c001c68370" // {c1, 00} <- c1 instead of 01
"04d980c001c6837004d980c001c6837004d980c001c6837004d980c001c68370"
Expand All @@ -86,6 +87,7 @@ TEST_CASE("TransactionIndex::build KO: invalid snapshot", "[silkworm][snapshot][

SECTION("KO: invalid position value") {
test::SampleBodySnapshotFile invalid_bodies_snapshot{
TemporaryDirectory::get_os_temporary_path(),
"000000000000000e000000000000000000000000000000000000000000000004"
"01ff010801c6837004d980c001c6837004d980c001c6837004d980c001c68370" // {01, ff} <- ff instead of 00
"04d980c001c6837004d980c001c6837004d980c001c6837004d980c001c68370"
Expand All @@ -99,6 +101,7 @@ TEST_CASE("TransactionIndex::build KO: invalid snapshot", "[silkworm][snapshot][

SECTION("KO: invalid positions count") {
test::SampleBodySnapshotFile invalid_bodies_snapshot{
TemporaryDirectory::get_os_temporary_path(),
"000000000000000e000000000000000000000000000000000000000000000005" // POSITIONS=5 <- 5 instead of 4
"0100010801c6837004d980c001c6837004d980c001c6837004d980c001c68370"
"04d980c001c6837004d980c001c6837004d980c001c6837004d980c001c68370"
Expand All @@ -112,6 +115,7 @@ TEST_CASE("TransactionIndex::build KO: invalid snapshot", "[silkworm][snapshot][

SECTION("KO: invalid RLP") {
test::SampleBodySnapshotFile invalid_bodies_snapshot{
TemporaryDirectory::get_os_temporary_path(),
"000000000000000e000000000000000000000000000000000000000000000004"
"0100010801c6837004d980c001c6837004d980c001c6837004d980c001c68370"
"04d980c001c6837004d980c001c6837004d980c001c6837004d980c001c78370" // {01, c7837004d980c0} <- c7 instead of c6
Expand Down
4 changes: 4 additions & 0 deletions silkworm/node/snapshot/path.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ std::optional<SnapshotPath> SnapshotPath::parse(fs::path path) {
}

// Expected scaled block format: <dddddd>
if (scaled_from.size() > 6 || scaled_to.size() > 6) {
return std::nullopt;
}

BlockNum scaled_block_from{0};
const auto from_result = std::from_chars(scaled_from.data(), scaled_from.data() + scaled_from.size(), scaled_block_from);
if (from_result.ec == std::errc::invalid_argument) {
Expand Down
81 changes: 52 additions & 29 deletions silkworm/node/snapshot/repository.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,41 @@ namespace silkworm::snapshot {

namespace fs = std::filesystem;

template <ConcreteSnapshot T>
const T* get_segment(const SnapshotsByPath<T>& segments, const SnapshotPath& path) {
if (not segments.contains(path.path())) {
return nullptr;
}
return segments.find(path.path())->second.get();
}

template <ConcreteSnapshot T>
SnapshotRepository::ViewResult view(const SnapshotsByPath<T>& segments, BlockNum number, const SnapshotWalker<T>& walker) {
// Search for target segment in reverse order (from the newest segment to the oldest one)
for (auto it = segments.rbegin(); it != segments.rend(); ++it) {
const auto& snapshot = it->second;
// We're looking for the segment containing the target block number in its block range
if (snapshot->block_from() <= number && number < snapshot->block_to()) {
const bool walk_done = walker(snapshot.get());
return walk_done ? SnapshotRepository::kWalkSuccess : SnapshotRepository::kWalkFailed;
}
}
return SnapshotRepository::kSnapshotNotFound;
}

template <ConcreteSnapshot T>
std::size_t view(const SnapshotsByPath<T>& segments, const SnapshotWalker<T>& walker) {
// Search for target segment in reverse order (from the newest segment to the oldest one)
std::size_t visited_views{0};
bool walk_done{false};
for (auto it = segments.rbegin(); it != segments.rend() && !walk_done; ++it) {
const auto& snapshot = it->second;
walk_done = walker(snapshot.get());
++visited_views;
}
return visited_views;
}

SnapshotRepository::SnapshotRepository(SnapshotSettings settings) : settings_(std::move(settings)) {}

SnapshotRepository::~SnapshotRepository() {
Expand All @@ -37,7 +72,7 @@ SnapshotRepository::~SnapshotRepository() {
void SnapshotRepository::reopen_folder() {
SILK_INFO << "Reopen snapshot repository folder: " << settings_.repository_dir.string();
SnapshotPathList segment_files = get_segment_files();
reopen_list(segment_files, /*.optimistic=*/false);
reopen_list(segment_files);
}

void SnapshotRepository::close() {
Expand Down Expand Up @@ -114,6 +149,18 @@ std::size_t SnapshotRepository::view_tx_segments(const TransactionSnapshotWalker
return view(tx_segments_, walker);
}

const HeaderSnapshot* SnapshotRepository::get_header_segment(const SnapshotPath& path) const {
return get_segment(header_segments_, path);
}

const BodySnapshot* SnapshotRepository::get_body_segment(const SnapshotPath& path) const {
return get_segment(body_segments_, path);
}

const TransactionSnapshot* SnapshotRepository::get_tx_segment(const SnapshotPath& path) const {
return get_segment(tx_segments_, path);
}

const HeaderSnapshot* SnapshotRepository::find_header_segment(BlockNum number) const {
return find_segment(header_segments_, number);
}
Expand Down Expand Up @@ -158,6 +205,10 @@ std::vector<std::shared_ptr<Index>> SnapshotRepository::missing_indexes() const
return missing_index_list;
}

void SnapshotRepository::reopen_file(const SnapshotPath& segment_path, bool optimistic) {
reopen_list(SnapshotPathList{segment_path}, optimistic);
}

void SnapshotRepository::reopen_list(const SnapshotPathList& segment_files, bool optimistic) {
close_segments_not_in_list(segment_files);

Expand Down Expand Up @@ -227,34 +278,6 @@ void SnapshotRepository::close_segments_not_in_list(const SnapshotPathList& /*se
// TODO(canepat): implement
}

template <ConcreteSnapshot T>
SnapshotRepository::ViewResult SnapshotRepository::view(const SnapshotsByPath<T>& segments, BlockNum number,
const SnapshotWalker<T>& walker) {
// Search for target segment in reverse order (from the newest segment to the oldest one)
for (auto it = segments.rbegin(); it != segments.rend(); ++it) {
const auto& snapshot = it->second;
// We're looking for the segment containing the target block number in its block range
if (snapshot->block_from() <= number && number < snapshot->block_to()) {
const bool walk_done = walker(snapshot.get());
return walk_done ? kWalkSuccess : kWalkFailed;
}
}
return kSnapshotNotFound;
}

template <ConcreteSnapshot T>
std::size_t SnapshotRepository::view(const SnapshotsByPath<T>& segments, const SnapshotWalker<T>& walker) {
// Search for target segment in reverse order (from the newest segment to the oldest one)
std::size_t visited_views{0};
bool walk_done{false};
for (auto it = segments.rbegin(); it != segments.rend() && !walk_done; ++it) {
const auto& snapshot = it->second;
walk_done = walker(snapshot.get());
++visited_views;
}
return visited_views;
}

template <ConcreteSnapshot T>
const T* SnapshotRepository::find_segment(const SnapshotsByPath<T>& segments, BlockNum number) const {
if (number > max_block_available()) {
Expand Down
22 changes: 10 additions & 12 deletions silkworm/node/snapshot/repository.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ class SnapshotRepository {

[[nodiscard]] BlockNum max_block_available() const { return std::min(segment_max_block_, idx_max_block_); }

[[nodiscard]] SnapshotPathList get_segment_files() const {
return get_files(kSegmentExtension);
}

void reopen_list(const SnapshotPathList& segment_files, bool optimistic = false);
void reopen_file(const SnapshotPath& segment_path, bool optimistic = false);
void reopen_folder();
void close();

Expand All @@ -84,6 +90,10 @@ class SnapshotRepository {
std::size_t view_body_segments(const BodySnapshotWalker& walker);
std::size_t view_tx_segments(const TransactionSnapshotWalker& walker);

[[nodiscard]] const HeaderSnapshot* get_header_segment(const SnapshotPath& path) const;
[[nodiscard]] const BodySnapshot* get_body_segment(const SnapshotPath& path) const;
[[nodiscard]] const TransactionSnapshot* get_tx_segment(const SnapshotPath& path) const;

[[nodiscard]] const HeaderSnapshot* find_header_segment(BlockNum number) const;
[[nodiscard]] const BodySnapshot* find_body_segment(BlockNum number) const;
[[nodiscard]] const TransactionSnapshot* find_tx_segment(BlockNum number) const;
Expand All @@ -94,30 +104,18 @@ class SnapshotRepository {
[[nodiscard]] BlockNum idx_max_block() const { return idx_max_block_; }

private:
void reopen_list(const SnapshotPathList& segment_files, bool optimistic);

bool reopen_header(const SnapshotPath& seg_file);
bool reopen_body(const SnapshotPath& seg_file);
bool reopen_transaction(const SnapshotPath& seg_file);

void close_segments_not_in_list(const SnapshotPathList& segment_files);

template <ConcreteSnapshot T>
static ViewResult view(const SnapshotsByPath<T>& segments, BlockNum number, const SnapshotWalker<T>& walker);

template <ConcreteSnapshot T>
static std::size_t view(const SnapshotsByPath<T>& segments, const SnapshotWalker<T>& walker);

template <ConcreteSnapshot T>
const T* find_segment(const SnapshotsByPath<T>& segments, BlockNum number) const;

template <ConcreteSnapshot T>
static bool reopen(SnapshotsByPath<T>& segments, const SnapshotPath& seg_file);

[[nodiscard]] SnapshotPathList get_segment_files() const {
return get_files(kSegmentExtension);
}

[[nodiscard]] SnapshotPathList get_idx_files() const {
return get_files(kIdxExtension);
}
Expand Down
75 changes: 71 additions & 4 deletions silkworm/node/snapshot/repository_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@

namespace silkworm::snapshot {

TEST_CASE("SnapshotRepository::SnapshotRepository", "[silkworm][snapshot][snapshot]") {
TEST_CASE("SnapshotRepository::SnapshotRepository", "[silkworm][node][snapshot]") {
test_util::SetLogVerbosityGuard guard{log::Level::kNone};
CHECK_NOTHROW(SnapshotRepository{SnapshotSettings{}});
}

TEST_CASE("SnapshotRepository::reopen_folder", "[silkworm][snapshot][snapshot]") {
TEST_CASE("SnapshotRepository::reopen_folder", "[silkworm][node][snapshot]") {
test_util::SetLogVerbosityGuard guard{log::Level::kNone};

const auto tmp_dir = TemporaryDirectory::get_unique_temporary_path();
Expand All @@ -47,7 +47,7 @@ TEST_CASE("SnapshotRepository::reopen_folder", "[silkworm][snapshot][snapshot]")
CHECK(repository.max_block_available() == 0);
}

TEST_CASE("SnapshotRepository::view", "[silkworm][snapshot][snapshot]") {
TEST_CASE("SnapshotRepository::view", "[silkworm][node][snapshot]") {
test_util::SetLogVerbosityGuard guard{log::Level::kNone};
const auto tmp_dir = TemporaryDirectory::get_unique_temporary_path();
std::filesystem::create_directories(tmp_dir);
Expand Down Expand Up @@ -118,7 +118,7 @@ TEST_CASE("SnapshotRepository::view", "[silkworm][snapshot][snapshot]") {
}
}

TEST_CASE("SnapshotRepository::missing_block_ranges", "[silkworm][snapshot][snapshot]") {
TEST_CASE("SnapshotRepository::missing_block_ranges", "[silkworm][node][snapshot]") {
test_util::SetLogVerbosityGuard guard{log::Level::kNone};
const auto tmp_dir = TemporaryDirectory::get_unique_temporary_path();
std::filesystem::create_directories(tmp_dir);
Expand All @@ -134,4 +134,71 @@ TEST_CASE("SnapshotRepository::missing_block_ranges", "[silkworm][snapshot][snap
BlockNumRange{12'000'000, 14'500'000}});
}

TEST_CASE("SnapshotRepository::find_segment", "[silkworm][node][snapshot]") {
test_util::SetLogVerbosityGuard guard{log::Level::kNone};
const auto tmp_dir = TemporaryDirectory::get_unique_temporary_path();
std::filesystem::create_directories(tmp_dir);
SnapshotSettings settings{tmp_dir};
SnapshotRepository repository{settings};

// These sample snapshot files just contain data for block range [1'500'012, 1'500'013], hence current snapshot
// file name format is not sufficient to support them (see checks commented out below)
test::SampleHeaderSnapshotFile header_snapshot{tmp_dir};
test::SampleBodySnapshotFile body_snapshot{tmp_dir};
test::SampleTransactionSnapshotFile txn_snapshot{tmp_dir};

SECTION("header w/o index") {
CHECK(repository.find_header_segment(1'500'011) == nullptr);
CHECK(repository.find_header_segment(1'500'012) == nullptr);
CHECK(repository.find_header_segment(1'500'013) == nullptr);
CHECK(repository.find_header_segment(1'500'014) == nullptr);
}
SECTION("body w/o index") {
CHECK(repository.find_body_segment(1'500'011) == nullptr);
CHECK(repository.find_body_segment(1'500'012) == nullptr);
CHECK(repository.find_body_segment(1'500'013) == nullptr);
CHECK(repository.find_body_segment(1'500'014) == nullptr);
}
SECTION("tx w/o index") {
CHECK(repository.find_tx_segment(1'500'011) == nullptr);
CHECK(repository.find_tx_segment(1'500'012) == nullptr);
CHECK(repository.find_tx_segment(1'500'013) == nullptr);
CHECK(repository.find_tx_segment(1'500'014) == nullptr);
}

test::SampleHeaderSnapshotPath header_snapshot_path{header_snapshot.path()}; // necessary to tweak the block numbers
HeaderIndex header_index{header_snapshot_path};
REQUIRE_NOTHROW(header_index.build());
test::SampleBodySnapshotPath body_snapshot_path{body_snapshot.path()}; // necessary to tweak the block numbers
BodyIndex body_index{body_snapshot_path};
REQUIRE_NOTHROW(body_index.build());
test::SampleTransactionSnapshotPath txn_snapshot_path{txn_snapshot.path()}; // necessary to tweak the block numbers
TransactionIndex txn_index{txn_snapshot_path};
REQUIRE_NOTHROW(txn_index.build());

REQUIRE_NOTHROW(repository.reopen_folder());

SECTION("header w/ index") {
CHECK(repository.find_header_segment(1'500'011) == nullptr);
// CHECK(repository.find_header_segment(1'500'012) != nullptr); // needs full block number in snapshot file names
// CHECK(repository.find_header_segment(1'500'013) != nullptr); // needs full block number in snapshot file names
CHECK(repository.find_header_segment(1'500'014) == nullptr);
}
SECTION("body w/ index") {
CHECK(repository.find_body_segment(1'500'011) == nullptr);
// CHECK(repository.find_body_segment(1'500'012) != nullptr); // needs full block number in snapshot file names
// CHECK(repository.find_body_segment(1'500'013) != nullptr); // needs full block number in snapshot file names
CHECK(repository.find_body_segment(1'500'014) == nullptr);
}
SECTION("tx w/ index") {
CHECK(repository.find_tx_segment(1'500'011) == nullptr);
// CHECK(repository.find_tx_segment(1'500'012) != nullptr); // needs full block number in snapshot file names
// CHECK(repository.find_tx_segment(1'500'013) != nullptr); // needs full block number in snapshot file names
CHECK(repository.find_tx_segment(1'500'014) == nullptr);
}
SECTION("greater than max_block_available") {
CHECK(repository.find_body_segment(repository.max_block_available() + 1) == nullptr);
}
}

} // namespace silkworm::snapshot
Loading

0 comments on commit a3cfbe5

Please sign in to comment.