db: add BTree index for kv snapshots

cmd/dev/snapshots.cpp

@@ -43,6 +43,7 @@
 #include <silkworm/db/snapshot_sync.hpp>
 #include <silkworm/db/snapshots/bittorrent/client.hpp>
 #include <silkworm/db/snapshots/bittorrent/web_seed_client.hpp>
+#include <silkworm/db/snapshots/index/btree_index.hpp>
 #include <silkworm/db/snapshots/seg/seg_zip.hpp>
 #include <silkworm/db/snapshots/snapshot_reader.hpp>
 #include <silkworm/db/snapshots/snapshot_repository.hpp>
@@ -97,6 +98,7 @@ enum class SnapshotTool {  // NOLINT(performance-enum-size)
     count_headers,
     create_index,
     open_index,
+    open_btree_index,
     decode_segment,
     download,
     lookup_header,
@@ -218,6 +220,10 @@ void parse_command_line(int argc, char* argv[], CLI::App& app, SnapshotToolboxSe
             ->capture_default_str();
     }
 
+    commands[SnapshotTool::open_btree_index]
+        ->add_option("--file", snapshot_settings.input_file_path, ".kv file to open with associated .bt file")
+        ->required()
+        ->check(CLI::ExistingFile);
     commands[SnapshotTool::recompress]
         ->add_option("--file", snapshot_settings.input_file_path, ".seg file to decompress and compress again")
         ->required()
@@ -382,6 +388,48 @@ void open_index(const SnapshotSubcommandSettings& settings) {
     SILK_INFO << "Open index elapsed: " << duration_as<std::chrono::milliseconds>(elapsed) << " msec";
 }
 
+void open_btree_index(const SnapshotSubcommandSettings& settings) {
+    ensure(!settings.input_file_path.empty(), "open_btree_index: --file must be specified");
+    ensure(settings.input_file_path.extension() == ".kv", "open_btree_index: --file must be .kv file");
+
+    std::filesystem::path bt_index_file_path = settings.input_file_path;
+    bt_index_file_path.replace_extension(".bt");
+    SILK_INFO << "KV file: " << settings.input_file_path.string() << " BT file: " << bt_index_file_path.string();
+    std::chrono::time_point start{std::chrono::steady_clock::now()};
+    seg::Decompressor kv_decompressor{settings.input_file_path};
+    kv_decompressor.open();
+    snapshots::index::BTreeIndex bt_index{kv_decompressor, bt_index_file_path};
+    SILK_INFO << "Starting KV scan and BTreeIndex check, total keys: " << bt_index.key_count();
+    size_t matching_count{0}, key_count{0};
+    bool is_key{true};
+    Bytes key, value;
+    auto kv_iterator = kv_decompressor.begin();
+    while (kv_iterator != kv_decompressor.end()) {
+        if (is_key) {
+            key = *kv_iterator;
+            ++key_count;
+        } else {
+            value = *kv_iterator;
+            const auto v = bt_index.get(key, kv_iterator);
+            SILK_DEBUG << "KV: key=" << to_hex(key) << " value=" << to_hex(value) << " v=" << (v ? to_hex(*v) : "");
+            ensure(v == value,
+                   [&]() { return "open_btree_index: value mismatch for key=" + to_hex(key) + " position=" + std::to_string(key_count); });
+            if (v == value) {
+                ++matching_count;
+            }
+            if (key_count % 10'000'000 == 0) {
+                SILK_INFO << "BTreeIndex check progress: " << key_count << " different: " << (key_count - matching_count);
+            }
+        }
+        ++kv_iterator;
+        is_key = !is_key;
+    }
+    ensure(key_count == bt_index.key_count(), "open_btree_index: total key count does not match");
+    SILK_INFO << "Open btree index matching: " << matching_count << " different: " << (key_count - matching_count);
+    std::chrono::duration elapsed{std::chrono::steady_clock::now() - start};
+    SILK_INFO << "Open btree index elapsed: " << duration_as<std::chrono::milliseconds>(elapsed) << " msec";
+}
+
 static TorrentInfoPtrList download_web_seed(const DownloadSettings& settings) {
     const auto known_config{snapshots::Config::lookup_known_config(settings.chain_id)};
     WebSeedClient web_client{/*url_seeds=*/{settings.url_seed}, known_config.preverified_snapshots()};
@@ -872,6 +920,9 @@ int main(int argc, char* argv[]) {
             case SnapshotTool::open_index:
                 open_index(settings.snapshot_settings);
                 break;
+            case SnapshotTool::open_btree_index:
+                open_btree_index(settings.snapshot_settings);
+                break;
             case SnapshotTool::decode_segment:
                 decode_segment(settings.snapshot_settings, settings.repetitions);
                 break;

silkworm/db/snapshot_decompressor_test.cpp

@@ -258,35 +258,31 @@ TEST_CASE("Decompressor::open invalid files", "[silkworm][node][seg][decompresso
         Decompressor decoder{tmp_file.path()};
         CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("compressed file is too short: 31"));
     }
-    SECTION("cannot build pattern tree: highest_depth reached zero") {
-        TemporaryFile tmp_file;
-        tmp_file.write(*silkworm::from_hex("0x000000000000000C000000000000000400000000000000150309000000000000"));
-        Decompressor decoder{tmp_file.path()};
-        CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("cannot build pattern tree: highest_depth reached zero"));
+    SECTION("invalid pattern_dict_length for compressed file size: 32") {
+        TemporaryFile tmp_file1;
+        tmp_file1.write(*silkworm::from_hex("0x000000000000000C000000000000000400000000000000150309000000000000"));
+        Decompressor decoder1{tmp_file1.path()};
+        CHECK_THROWS_MATCHES(decoder1.open(), std::runtime_error, Message("invalid pattern_dict_length for compressed file size: 32"));
+        TemporaryFile tmp_file2;
+        tmp_file2.write(*silkworm::from_hex("0x0000000000000000000000000000000000000000000000010000000000000000"));
+        Decompressor decoder2{tmp_file2.path()};
+        CHECK_THROWS_MATCHES(decoder2.open(), std::runtime_error, Message("invalid pattern_dict_length for compressed file size: 32"));
     }
-    SECTION("pattern dict is invalid: data skip failed at 22") {
+    SECTION("invalid pattern_dict_length for compressed file size: 34") {
         TemporaryFile tmp_file;
         tmp_file.write(*silkworm::from_hex("0x000000000000000C00000000000000040000000000000016000000000000000003ff"));
         Decompressor decoder{tmp_file.path()};
-        CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("pattern dict is invalid: data skip failed at 11"));
-    }
-    SECTION("pattern dict is invalid: length read failed at 1") {
-        TemporaryFile tmp_file;
-        tmp_file.write(*silkworm::from_hex("0x0000000000000000000000000000000000000000000000010000000000000000"));
-        Decompressor decoder{tmp_file.path()};
-        CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("pattern dict is invalid: length read failed at 1"));
+        CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("invalid pattern_dict_length for compressed file size: 34"));
     }
-    SECTION("cannot build position tree: highest_depth reached zero") {
-        TemporaryFile tmp_file;
-        tmp_file.write(*silkworm::from_hex("0x000000000000000C0000000000000004000000000000000000000000000000160309"));
-        Decompressor decoder{tmp_file.path()};
-        CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("cannot build position tree: highest_depth reached zero"));
-    }
-    SECTION("position dict is invalid: position read failed at 22") {
-        TemporaryFile tmp_file;
-        tmp_file.write(*silkworm::from_hex("0x000000000000000C00000000000000040000000000000000000000000000001603ff"));
-        Decompressor decoder{tmp_file.path()};
-        CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("position dict is invalid: position read failed at 22"));
+    SECTION("invalid position_dict_length for compressed file size: 34") {
+        TemporaryFile tmp_file1;
+        tmp_file1.write(*silkworm::from_hex("0x000000000000000C0000000000000004000000000000000000000000000000160309"));
+        Decompressor decoder1{tmp_file1.path()};
+        CHECK_THROWS_MATCHES(decoder1.open(), std::runtime_error, Message("invalid position_dict_length for compressed file size: 34"));
+        TemporaryFile tmp_file2;
+        tmp_file2.write(*silkworm::from_hex("0x000000000000000C00000000000000040000000000000000000000000000001603ff"));
+        Decompressor decoder2{tmp_file2.path()};
+        CHECK_THROWS_MATCHES(decoder2.open(), std::runtime_error, Message("invalid position_dict_length for compressed file size: 34"));
     }
 }
 

silkworm/db/snapshots/common/bitmask_operators.hpp

@@ -0,0 +1,48 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#pragma once
+
+#include <type_traits>
+
+namespace silkworm::snapshots {
+
+template <typename T>
+    requires(std::is_enum_v<T> and requires(T e) {
+        enable_bitmask_operator_or(e);
+    })
+constexpr auto operator|(const T lhs, const T rhs) {
+    using underlying = std::underlying_type_t<T>;
+    return static_cast<T>(static_cast<underlying>(lhs) | static_cast<underlying>(rhs));
+}
+template <typename T>
+    requires(std::is_enum_v<T> and requires(T e) {
+        enable_bitmask_operator_and(e);
+    })
+constexpr auto operator&(const T lhs, const T rhs) {
+    using underlying = std::underlying_type_t<T>;
+    return static_cast<T>(static_cast<underlying>(lhs) & static_cast<underlying>(rhs));
+}
+template <typename T>
+    requires(std::is_enum_v<T> and requires(T e) {
+        enable_bitmask_operator_not(e);
+    })
+constexpr auto operator~(const T t) {
+    using underlying = std::underlying_type_t<T>;
+    return static_cast<T>(~static_cast<underlying>(t));
+}
+
+}  // namespace silkworm::snapshots

silkworm/db/snapshots/rec_split/encoding/elias_fano.hpp → silkworm/db/snapshots/common/encoding/elias_fano.hpp

@@ -49,6 +49,7 @@
 #include <cstring>
 #include <iostream>
 #include <limits>
+#include <memory>
 #include <span>
 #include <utility>
 #include <vector>
@@ -57,16 +58,21 @@
 #include <silkworm/core/common/base.hpp>
 #include <silkworm/core/common/bytes.hpp>
 #include <silkworm/core/common/endian.hpp>
-#include <silkworm/db/snapshots/rec_split/common/common.hpp>
-#include <silkworm/db/snapshots/rec_split/encoding/sequence.hpp>
 #include <silkworm/infra/common/ensure.hpp>
 #include <silkworm/infra/common/log.hpp>
 
-// EliasFano algo overview https://www.antoniomallia.it/sorted-integers-compression-with-elias-fano-encoding.html
+#include "sequence.hpp"
+#include "util.hpp"
+
+// Elias-Fano encoding is a high bits / low bits representation of a monotonically increasing sequence of N > 0 natural numbers x[i]
+// 0 <= x[0] <= x[1] <= ... <= x[N-2] <= x[N-1] <= U
+// where U > 0 is an upper bound on the last value.
+
+// EliasFano algorithm overview https://www.antoniomallia.it/sorted-integers-compression-with-elias-fano-encoding.html
 // P. Elias. Efficient storage and retrieval by content and address of static files. J. ACM, 21(2):246–260, 1974.
 // Partitioned Elias-Fano Indexes http://groups.di.unipi.it/~ottavian/files/elias_fano_sigir14.pdf
 
-namespace silkworm::snapshots::rec_split::encoding {
+namespace silkworm::snapshots::encoding {
 
 //! Log2Q = Log2(Quantum)
 static constexpr uint64_t kLog2q = 8;
@@ -103,23 +109,37 @@ static void set_bits(std::span<T, Extent> bits, const uint64_t start, const uint
 //! 32-bit Elias-Fano (EF) list that can be used to encode one monotone non-decreasing sequence
 class EliasFanoList32 {
   public:
-    //! Create an empty new 32-bit EF list prepared for specified sequence length and max offset
-    EliasFanoList32(uint64_t sequence_length, uint64_t max_offset)
+    static constexpr std::size_t kCountLength{sizeof(uint64_t)};
+    static constexpr std::size_t kULength{sizeof(uint64_t)};
+
+    //! Create a new 32-bit EF list from the given encoded data (i.e. data plus data header)
+    static std::unique_ptr<EliasFanoList32> from_encoded_data(std::span<uint8_t> encoded_data) {
+        ensure(encoded_data.size() >= kCountLength + kULength, "EliasFanoList32::from_encoded_data data too short");
+        const uint64_t count = endian::load_big_u64(encoded_data.data());
+        const uint64_t u = endian::load_big_u64(encoded_data.subspan(kCountLength).data());
+        const auto remaining_data = encoded_data.subspan(kCountLength + kULength);
+        return std::make_unique<EliasFanoList32>(count, u, remaining_data);
+    }
+
+    //! Create an empty new 32-bit EF list prepared for the given data sequence length and max value
+    //! \param sequence_length the length of the data sequence
+    //! \param max_value the max value in the data sequence
+    EliasFanoList32(uint64_t sequence_length, uint64_t max_value)
         : count_(sequence_length - 1),
-          u_(max_offset + 1),
-          max_offset_(max_offset) {
+          u_(max_value + 1),
+          max_value_(max_value) {
         ensure(sequence_length > 0, "sequence length is zero");
         derive_fields();
     }
 
     //! Create a new 32-bit EF list from an existing data sequence
     //! \param count the number of EF data points
-    //! \param u u
+    //! \param u the strict upper bound on the EF data points, i.e. max value plus one
     //! \param data the existing data sequence (portion exceeding the total words will be ignored)
     EliasFanoList32(uint64_t count, uint64_t u, std::span<uint8_t> data)
         : count_(count),
           u_(u),
-          max_offset_(u - 1) {
+          max_value_(u - 1) {
         const auto total_words = derive_fields();
         SILKWORM_ASSERT(total_words * sizeof(uint64_t) <= data.size());
         data = data.subspan(0, total_words * sizeof(uint64_t));
@@ -130,12 +150,14 @@ class EliasFanoList32 {
 
     [[nodiscard]] std::size_t count() const { return count_; }
 
-    [[nodiscard]] std::size_t max() const { return max_offset_; }
+    [[nodiscard]] std::size_t max() const { return max_value_; }
 
     [[nodiscard]] std::size_t min() const { return get(0); }
 
     [[nodiscard]] const Uint64Sequence& data() const { return data_; }
 
+    [[nodiscard]] std::size_t encoded_data_size() const { return kCountLength + kULength + data_.size() * sizeof(uint64_t); }
+
     [[nodiscard]] uint64_t get(uint64_t i) const {
         uint64_t lower = i * l_;
         std::size_t idx64 = lower / 64;
@@ -256,7 +278,7 @@ class EliasFanoList32 {
     uint64_t count_{0};
     uint64_t u_{0};
     uint64_t l_{0};
-    uint64_t max_offset_{0};
+    uint64_t max_value_{0};
     uint64_t i_{0};
     Uint64Sequence data_;
 };
@@ -554,4 +576,4 @@ class DoubleEliasFanoList16 {
     }
 };
 
-}  // namespace silkworm::snapshots::rec_split::encoding
+}  // namespace silkworm::snapshots::encoding

silkworm/db/snapshots/rec_split/encoding/elias_fano_test.cpp → silkworm/db/snapshots/common/encoding/elias_fano_test.cpp

@@ -26,7 +26,7 @@
 #include <silkworm/core/common/util.hpp>
 #include <silkworm/infra/test_util/log.hpp>
 
-namespace silkworm::snapshots::rec_split::encoding {
+namespace silkworm::snapshots::encoding {
 
 struct EliasFanoList32Test {
     std::vector<uint64_t> offsets;
@@ -168,4 +168,4 @@ TEST_CASE("DoubleEliasFanoList16", "[silkworm][recsplit][elias_fano]") {
                     "0000000000000000010000000000000000000000000000000000000000000000"));
 }
 
-}  // namespace silkworm::snapshots::rec_split::encoding
+}  // namespace silkworm::snapshots::encoding

silkworm/db/snapshots/rec_split/encoding/golomb_rice.hpp → silkworm/db/snapshots/common/encoding/golomb_rice.hpp

@@ -50,11 +50,12 @@
 #include <iostream>
 
 #include <silkworm/core/common/assert.hpp>
-#include <silkworm/db/snapshots/rec_split/common/common.hpp>
-#include <silkworm/db/snapshots/rec_split/encoding/sequence.hpp>
 #include <silkworm/infra/common/log.hpp>
 
-namespace silkworm::snapshots::rec_split::encoding {
+#include "sequence.hpp"
+#include "util.hpp"
+
+namespace silkworm::snapshots::encoding {
 
 //! Storage for Golomb-Rice codes of a RecSplit bucket.
 class GolombRiceVector {
@@ -253,4 +254,4 @@ class GolombRiceVector {
     }
 };
 
-}  // namespace silkworm::snapshots::rec_split::encoding
+}  // namespace silkworm::snapshots::encoding

silkworm/db/snapshots/rec_split/encoding/golomb_rice_test.cpp → silkworm/db/snapshots/common/encoding/golomb_rice_test.cpp

@@ -21,10 +21,9 @@
 #include <catch2/catch_test_macros.hpp>
 
 #include <silkworm/core/common/random_number.hpp>
-#include <silkworm/db/snapshots/rec_split/encoding/sequence.hpp>
 #include <silkworm/infra/test_util/log.hpp>
 
-namespace silkworm::snapshots::rec_split::encoding {
+namespace silkworm::snapshots::encoding {
 
 static const std::size_t kGolombRiceTestNumKeys{128};
 static const std::size_t kGolombRiceTestNumTrees{1'000};
@@ -86,4 +85,4 @@ TEST_CASE("GolombRiceVector", "[silkworm][recsplit][golomb_rice]") {
     }
 }
 
-}  // namespace silkworm::snapshots::rec_split::encoding
+}  // namespace silkworm::snapshots::encoding

silkworm/db/snapshots/rec_split/encoding/sequence.hpp → silkworm/db/snapshots/common/encoding/sequence.hpp

@@ -25,7 +25,7 @@
 #include <silkworm/core/common/endian.hpp>
 #include <silkworm/infra/common/ensure.hpp>
 
-namespace silkworm::snapshots::rec_split::encoding {
+namespace silkworm::snapshots::encoding {
 
 template <UnsignedIntegral T>
 using UnsignedIntegralSequence = std::vector<T>;
@@ -64,4 +64,4 @@ std::istream& operator>>(std::istream& is, UnsignedIntegralSequence<T>& s) {
     return is;
 }
 
-}  // namespace silkworm::snapshots::rec_split::encoding
+}  // namespace silkworm::snapshots::encoding

silkworm/db/snapshots/rec_split/encoding/sequence_test.cpp → silkworm/db/snapshots/common/encoding/sequence_test.cpp

@@ -25,7 +25,7 @@
 #include <silkworm/core/common/endian.hpp>
 #include <silkworm/infra/test_util/log.hpp>
 
-namespace silkworm::snapshots::rec_split::encoding {
+namespace silkworm::snapshots::encoding {
 
 TEST_CASE("Uint64Sequence", "[silkworm][snapshots][recsplit][sequence]") {
     test_util::SetLogVerbosityGuard guard{log::Level::kNone};
@@ -52,4 +52,4 @@ TEST_CASE("Uint64Sequence: size too big", "[silkworm][snapshots][recsplit][seque
     CHECK_THROWS_AS((ss >> input_sequence), std::logic_error);
 }
 
-}  // namespace silkworm::snapshots::rec_split::encoding
+}  // namespace silkworm::snapshots::encoding

silkworm/db/snapshots/rec_split/common/common.hpp → silkworm/db/snapshots/common/encoding/util.hpp

@@ -58,7 +58,7 @@
 
 #include <silkworm/core/common/assert.hpp>
 
-namespace silkworm::snapshots::rec_split {
+namespace silkworm::snapshots::encoding {
 
 using std::memcpy;
 
@@ -206,4 +206,4 @@ inline uint64_t select64(uint64_t x, uint64_t k) {
 #endif
 }
 
-}  // namespace silkworm::snapshots::rec_split
+}  // namespace silkworm::snapshots::encoding