Skip to content

Commit

Permalink
node: remove index file when created before snapshot file (#1356)
Browse files Browse the repository at this point in the history
  • Loading branch information
@canepat
canepat authored Jul 22, 2023
1 parent 938ab28 commit 3f87bf7
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 0 deletions.
8 changes: 8 additions & 0 deletions silkworm/node/snapshot/snapshot.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ void HeaderSnapshot::reopen_index() {
idx_header_hash_ = std::make_unique<succinct::RecSplitIndex>(header_index_path.path());
if (idx_header_hash_->last_write_time() < decoder_.last_write_time()) {
// Index has been created before the segment file, needs to be ignored (and rebuilt) as inconsistent
const bool removed = std::filesystem::remove(header_index_path.path());
ensure(removed, "HeaderSnapshot::reopen_index cannot remove index file");
close_index();
}
}
Expand Down Expand Up @@ -283,6 +285,8 @@ void BodySnapshot::reopen_index() {
idx_body_number_ = std::make_unique<succinct::RecSplitIndex>(body_index_path.path());
if (idx_body_number_->last_write_time() < decoder_.last_write_time()) {
// Index has been created before the segment file, needs to be ignored (and rebuilt) as inconsistent
const bool removed = std::filesystem::remove(body_index_path.path());
ensure(removed, "BodySnapshot::reopen_index cannot remove index file");
close_index();
}
}
Expand Down Expand Up @@ -469,6 +473,8 @@ void TransactionSnapshot::reopen_index() {
idx_txn_hash_ = std::make_unique<succinct::RecSplitIndex>(tx_hash_index_path.path());
if (idx_txn_hash_->last_write_time() < decoder_.last_write_time()) {
// Index has been created before the segment file, needs to be ignored (and rebuilt) as inconsistent
const bool removed = std::filesystem::remove(tx_hash_index_path.path());
ensure(removed, "TransactionSnapshot::reopen_index cannot remove tx_hash index file");
close_index();
}
}
Expand All @@ -478,6 +484,8 @@ void TransactionSnapshot::reopen_index() {
idx_txn_hash_2_block_ = std::make_unique<succinct::RecSplitIndex>(tx_hash_2_block_index_path.path());
if (idx_txn_hash_2_block_->last_write_time() < decoder_.last_write_time()) {
// Index has been created before the segment file, needs to be ignored (and rebuilt) as inconsistent
const bool removed = std::filesystem::remove(tx_hash_2_block_index_path.path());
ensure(removed, "TransactionSnapshot::reopen_index cannot remove tx_hash_2_block index file");
close_index();
}
}
Expand Down
75 changes: 75 additions & 0 deletions silkworm/node/snapshot/snapshot_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@

namespace silkworm::snapshot {

using namespace std::chrono_literals;

class Snapshot_ForTest : public Snapshot {
public:
Snapshot_ForTest(std::filesystem::path path, BlockNum block_from, BlockNum block_to)
Expand All @@ -42,6 +44,13 @@ class Snapshot_ForTest : public Snapshot {
void close_index() override {}
};

template <class Rep, class Period>
static auto move_last_write_time(const std::filesystem::path& p, const std::chrono::duration<Rep, Period>& d) {
const auto ftime = std::filesystem::last_write_time(p);
std::filesystem::last_write_time(p, ftime + d);
return std::filesystem::last_write_time(p) - ftime;
}

TEST_CASE("Snapshot::Snapshot", "[silkworm][node][snapshot][snapshot]") {
SECTION("valid") {
std::vector<std::pair<BlockNum, BlockNum>> block_ranges{
Expand Down Expand Up @@ -174,4 +183,70 @@ TEST_CASE("TransactionSnapshot::txn_by_id OK", "[silkworm][node][snapshot][index
}
}

TEST_CASE("HeaderSnapshot::reopen_index regeneration", "[silkworm][node][snapshot][index]") {
test_util::SetLogVerbosityGuard guard{log::Level::kNone};
test::SampleHeaderSnapshotFile sample_header_snapshot{};
test::SampleHeaderSnapshotPath header_snapshot_path{sample_header_snapshot.path()};
HeaderIndex header_index{header_snapshot_path};
REQUIRE_NOTHROW(header_index.build());

HeaderSnapshot header_snapshot{header_snapshot_path.path(), header_snapshot_path.block_from(), header_snapshot_path.block_to()};
header_snapshot.reopen_segment();
header_snapshot.reopen_index();
REQUIRE(std::filesystem::exists(header_snapshot.path().index_file().path()));

// Move 1 hour to the future the last write time for sample header snapshot
const auto last_write_time_diff = move_last_write_time(sample_header_snapshot.path(), 1h);
REQUIRE(last_write_time_diff > std::filesystem::file_time_type::duration::zero());

// Verify that reopening the index removes the index file because it was created in the past
CHECK(std::filesystem::exists(header_snapshot.path().index_file().path()));
header_snapshot.reopen_index();
CHECK(not std::filesystem::exists(header_snapshot.path().index_file().path()));
}

TEST_CASE("BodySnapshot::reopen_index regeneration", "[silkworm][node][snapshot][index]") {
test_util::SetLogVerbosityGuard guard{log::Level::kNone};
test::SampleBodySnapshotFile sample_body_snapshot{};
test::SampleBodySnapshotPath body_snapshot_path{sample_body_snapshot.path()};
BodyIndex body_index{body_snapshot_path};
REQUIRE_NOTHROW(body_index.build());

BodySnapshot body_snapshot{body_snapshot_path.path(), body_snapshot_path.block_from(), body_snapshot_path.block_to()};
body_snapshot.reopen_segment();
body_snapshot.reopen_index();
CHECK(std::filesystem::exists(body_snapshot.path().index_file().path()));

// Move 1 hour to the future the last write time for sample body snapshot
const auto last_write_time_diff = move_last_write_time(sample_body_snapshot.path(), 1h);
REQUIRE(last_write_time_diff > std::filesystem::file_time_type::duration::zero());

// Verify that reopening the index removes the index file if created in the past
CHECK(std::filesystem::exists(body_snapshot.path().index_file().path()));
body_snapshot.reopen_index();
CHECK(not std::filesystem::exists(body_snapshot.path().index_file().path()));
}

TEST_CASE("TransactionSnapshot::reopen_index regeneration", "[silkworm][node][snapshot][index]") {
test_util::SetLogVerbosityGuard guard{log::Level::kNone};
test::SampleTransactionSnapshotFile sample_tx_snapshot{};
test::SampleTransactionSnapshotPath tx_snapshot_path1{sample_tx_snapshot.path()};
TransactionIndex tx_index{tx_snapshot_path1};
REQUIRE_NOTHROW(tx_index.build());

TransactionSnapshot tx_snapshot{tx_snapshot_path1.path(), tx_snapshot_path1.block_from(), tx_snapshot_path1.block_to()};
tx_snapshot.reopen_segment();
tx_snapshot.reopen_index();
CHECK(std::filesystem::exists(tx_snapshot.path().index_file().path()));

// Move 1 hour to the future the last write time for sample tx snapshot
const auto last_write_time_diff = move_last_write_time(sample_tx_snapshot.path(), 1h);
REQUIRE(last_write_time_diff > std::filesystem::file_time_type::duration::zero());

// Verify that reopening the index removes the index file if created in the past
CHECK(std::filesystem::exists(tx_snapshot.path().index_file().path()));
tx_snapshot.reopen_index();
CHECK(not std::filesystem::exists(tx_snapshot.path().index_file().path()));
}

} // namespace silkworm::snapshot

0 comments on commit 3f87bf7

Please sign in to comment.