Skip to content

Commit

Permalink
unit tests for seek
Browse files Browse the repository at this point in the history
  • Loading branch information
@canepat
canepat committed Sep 20, 2024
1 parent 2f01aa7 commit f77791f
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 45 deletions.
5 changes: 1 addition & 4 deletions silkworm/db/snapshots/index/btree_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,10 @@ BTreeIndex::Cursor::Cursor(BTreeIndex* index, ByteView key, ByteView value, Data

std::unique_ptr<BTreeIndex::Cursor> BTreeIndex::seek(ByteView seek_key, DataIterator data_it) {
const auto [found, key, value, data_index] = btree_->seek(seek_key, data_it);
if (!found) {
return nullptr;
}
if (key.compare(seek_key) >= 0) {
return new_cursor(key, value, data_index, data_it);
}
return new_cursor(ByteView{}, ByteView{}, 0, data_it);
return nullptr;
}

std::optional<Bytes> BTreeIndex::get(ByteView key, DataIterator data_it) {
Expand Down
127 changes: 86 additions & 41 deletions silkworm/db/snapshots/index/btree_index_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,8 @@ namespace silkworm::snapshots::index {
using namespace silkworm::test_util;

using KeyAndValue = std::pair<std::string_view, std::string_view>;
using KeysAndValues = std::vector<KeyAndValue>;

static std::filesystem::path sample_kv_file(const std::filesystem::path& tmp_dir_path, const KeysAndValues& kv_pairs) {
static std::filesystem::path sample_kv_file(const std::filesystem::path& tmp_dir_path, std::span<const KeyAndValue> kv_pairs) {
const auto kv_file_path = TemporaryDirectory::get_unique_temporary_path();
seg::Compressor kv_compressor{kv_file_path, tmp_dir_path};
for (const auto& kv_pair : kv_pairs) {
Expand All @@ -56,58 +55,70 @@ static std::filesystem::path sample_bt_index_file(const EliasFanoList32& key_off
return index_file.path();
}

using KvAndBtPaths = std::tuple<std::filesystem::path, std::filesystem::path>;

static KvAndBtPaths sample_3_keys_kv_and_bt_files(const std::filesystem::path& tmp_dir_path) {
// Prepare sample uncompressed KV file containing some key-value pairs
const auto kv_file_path = sample_kv_file(
tmp_dir_path,
std::vector<KeyAndValue>{
KeyAndValue{"0000000000000000000000000000000000000000"sv, "000a0269024e3c8decd159600000"sv},
KeyAndValue{"0000000000000000000000000000000000000004"sv, "0008cf2fa48840ba8add0000"sv},
KeyAndValue{"0000000000000000000000000000000000000008"sv, "0008146c4643c28ed8200000"sv},
});

// Prepare the BT index for such KV file
// Note: key offsets can be computed from KV file layout
// 000000000000000600000000000000000000000000000000000000000000000801000215030F030D
// 01
// 0000000000000000000000000000000000000000 <- 1st key, offset 0
// 03
// 000A0269024E3C8DECD159600000
// 01
// 0000000000000000000000000000000000000004 <- 2nd key, offset 0 + 20 + 1 + 14 + 1
// 07
// 0008CF2FA48840BA8ADD0000
// 01
// 0000000000000000000000000000000000000008 <- 3rd key, offset 0 + 20 + 1 + 14 + 1 + 20 + 1 + 12 + 1
// 07
// 0008146C4643C28ED8200000
EliasFanoList32 encoded_key_offsets{3, 70};
encoded_key_offsets.add_offset(0);
encoded_key_offsets.add_offset(0 + 20 + 1 + 14 + 1);
encoded_key_offsets.add_offset(0 + 20 + 1 + 14 + 1 + 20 + 1 + 12 + 1);
encoded_key_offsets.build();
const auto bt_file_path = sample_bt_index_file(encoded_key_offsets);

return {kv_file_path, bt_file_path};
}

TEST_CASE("BTreeIndex", "[db]") {
TemporaryDirectory tmp_dir;

SECTION("empty") {
// Prepare empty KV and BT index files
const auto kv_file_path = sample_kv_file(tmp_dir.path(), {});
seg::Decompressor kv_decompressor{kv_file_path};
kv_decompressor.open();

TemporaryFile index_file;
index_file.write(Bytes{});

// Open the KV and BT index files
seg::Decompressor kv_decompressor{kv_file_path};
kv_decompressor.open();
BTreeIndex bt_index{kv_decompressor, index_file.path()};
CHECK(bt_index.empty());
}

SECTION("sample KV file") {
// Prepare sample uncompressed KV file containing some key-value pairs
const auto kv_file_path = sample_kv_file(
tmp_dir.path(),
{
{"0000000000000000000000000000000000000000", "000a0269024e3c8decd159600000"},
{"0000000000000000000000000000000000000001", "0008cf2fa48840ba8add0000"},
{"0000000000000000000000000000000000000002", "0008146c4643c28ed8200000"},
});

// Prepare the BT index for such KV file
// Note: key offsets can be computed from KV file layout
// 000000000000000600000000000000000000000000000000000000000000000801000215030F030D
// 01
// 0000000000000000000000000000000000000000 <- 1st key, offset 0
// 03
// 000A0269024E3C8DECD159600000
// 01
// 0000000000000000000000000000000000000001 <- 2nd key, offset 0 + 20 + 1 + 14 + 1
// 07
// 0008CF2FA48840BA8ADD0000
// 01
// 0000000000000000000000000000000000000002 <- 3rd key, offset 0 + 20 + 1 + 14 + 1 + 20 + 1 + 12 + 1
// 07
// 0008146C4643C28ED8200000
EliasFanoList32 encoded_key_offsets{3, 70};
encoded_key_offsets.add_offset(0);
encoded_key_offsets.add_offset(0 + 20 + 1 + 14 + 1);
encoded_key_offsets.add_offset(0 + 20 + 1 + 14 + 1 + 20 + 1 + 12 + 1);
encoded_key_offsets.build();
const auto bt_file_path = sample_bt_index_file(encoded_key_offsets);
// Prepare sample uncompressed KV file containing 3 key-value pairs and its BT index file
const auto [kv_file_path, bt_file_path] = sample_3_keys_kv_and_bt_files(tmp_dir.path());

// Open the KV and BT index files
seg::Decompressor kv_decompressor{kv_file_path};
kv_decompressor.open();
BTreeIndex bt_index{kv_decompressor, bt_file_path};
REQUIRE(bt_index.key_count() == 3);
// Open the KV and BT index files
seg::Decompressor kv_decompressor{kv_file_path};
kv_decompressor.open();
BTreeIndex bt_index{kv_decompressor, bt_file_path};
REQUIRE(!bt_index.empty());
REQUIRE(bt_index.key_count() == 3);

SECTION("BTreeIndex::get") {
// Check that all values retrieved through BT index match
size_t key_count{0};
bool is_key{true};
Expand All @@ -127,6 +138,40 @@ TEST_CASE("BTreeIndex", "[db]") {
}
CHECK(key_count == bt_index.key_count());
}

SECTION("BTreeIndex::seek") {
auto kv_iterator = kv_decompressor.begin();

// Seek using exact keys starting from the first one
auto index_it = bt_index.seek(ByteView{}, kv_iterator);
REQUIRE(index_it);
REQUIRE(index_it->key() == *from_hex("0000000000000000000000000000000000000000"sv));
REQUIRE(index_it->value() == *from_hex("000a0269024e3c8decd159600000"sv));
REQUIRE(index_it->data_index() == 0);
REQUIRE(index_it->next());
REQUIRE(index_it->key() == *from_hex("0000000000000000000000000000000000000004"sv));
REQUIRE(index_it->value() == *from_hex("0008cf2fa48840ba8add0000"sv));
REQUIRE(index_it->data_index() == 1);
REQUIRE(index_it->next());
REQUIRE(index_it->key() == *from_hex("0000000000000000000000000000000000000008"sv));
REQUIRE(index_it->value() == *from_hex("0008146c4643c28ed8200000"sv));
REQUIRE(index_it->data_index() == 2);
REQUIRE(!index_it->next());

// Seek using lower keys than existing ones
index_it = bt_index.seek(*from_hex("0000000000000000000000000000000000000003"sv), kv_iterator);
REQUIRE(index_it->key() == *from_hex("0000000000000000000000000000000000000004"sv));
REQUIRE(index_it->value() == *from_hex("0008cf2fa48840ba8add0000"sv));
REQUIRE(index_it->data_index() == 1);
index_it = bt_index.seek(*from_hex("0000000000000000000000000000000000000007"sv), kv_iterator);
REQUIRE(index_it->key() == *from_hex("0000000000000000000000000000000000000008"sv));
REQUIRE(index_it->value() == *from_hex("0008146c4643c28ed8200000"sv));
REQUIRE(index_it->data_index() == 2);

// Seek beyond the last key
CHECK(!bt_index.seek(*from_hex("0000000000000000000000000000000000000009"), kv_iterator));
CHECK(!bt_index.seek(*from_hex("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), kv_iterator));
}
}

} // namespace silkworm::snapshots::index

0 comments on commit f77791f

Please sign in to comment.