Merge branch 'master' into tf/remove-schnorr

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp

@@ -141,7 +141,7 @@ void check_historic_sibling_path(TreeType& tree,
 void commit_tree(TreeType& tree, bool expected_success = true)
 {
     Signal signal;
-    auto completion = [&](const Response& response) -> void {
+    TreeType::CommitCallback completion = [&](const TypedResponse<CommitResponse>& response) -> void {
         EXPECT_EQ(response.success, expected_success);
         signal.signal_level();
     };
@@ -163,7 +163,7 @@ void rollback_tree(TreeType& tree)
 void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true)
 {
     Signal signal;
-    auto completion = [&](const Response& response) -> void {
+    auto completion = [&](const TypedResponse<RemoveHistoricResponse>& response) -> void {
         EXPECT_EQ(response.success, expected_success);
         signal.signal_level();
     };
@@ -174,7 +174,7 @@ void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expe
 void unwind_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true)
 {
     Signal signal;
-    auto completion = [&](const Response& response) -> void {
+    auto completion = [&](const TypedResponse<UnwindResponse>& response) -> void {
         EXPECT_EQ(response.success, expected_success);
         signal.signal_level();
     };
@@ -455,10 +455,13 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, reports_an_error_if_tree_is_
     }
     add_values(tree, values);
 
+    std::stringstream ss;
+    ss << "Unable to append leaves to tree " << name << " new size: 17 max size: 16";
+
     Signal signal;
     auto add_completion = [&](const TypedResponse<AddDataResponse>& response) {
         EXPECT_EQ(response.success, false);
-        EXPECT_EQ(response.message, "Tree is full");
+        EXPECT_EQ(response.message, ss.str());
         signal.signal_level();
     };
     tree.add_value(VALUES[16], add_completion);
@@ -501,7 +504,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, errors_are_caught_and_handle
 
         // trying to commit that should fail
         Signal signal;
-        auto completion = [&](const Response& response) -> void {
+        auto completion = [&](const TypedResponse<CommitResponse>& response) -> void {
             EXPECT_EQ(response.success, false);
             signal.signal_level();
         };
@@ -1041,7 +1044,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_add_single_whilst_readin
         Signal signal(1 + num_reads);
 
         auto add_completion = [&](const TypedResponse<AddDataResponse>&) {
-            auto commit_completion = [&](const Response&) { signal.signal_decrement(); };
+            auto commit_completion = [&](const TypedResponse<CommitResponse>&) { signal.signal_decrement(); };
             tree.commit(commit_completion);
         };
         tree.add_value(VALUES[0], add_completion);

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp

@@ -64,11 +64,9 @@ inline ThreadPoolPtr make_thread_pool(uint64_t numThreads)
 void inline print_store_data(LMDBTreeStore::SharedPtr db, std::ostream& os)
 {
     LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction();
-    StatsMap stats;
+    TreeDBStats stats;
     db->get_stats(stats, *tx);
 
-    for (const auto& m : stats) {
-        os << m.first << m.second << std::endl;
-    }
+    os << stats;
 }
 } // namespace bb::crypto::merkle_tree

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp

@@ -336,7 +336,7 @@ template <typename TypeOfTree> void check_unfinalised_block_height(TypeOfTree& t
 template <typename TypeOfTree> void commit_tree(TypeOfTree& tree, bool expectedSuccess = true)
 {
     Signal signal;
-    auto completion = [&](const Response& response) -> void {
+    auto completion = [&](const TypedResponse<CommitResponse>& response) -> void {
         EXPECT_EQ(response.success, expectedSuccess);
         signal.signal_level();
     };
@@ -387,7 +387,7 @@ template <typename TypeOfTree>
 void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true)
 {
     Signal signal;
-    auto completion = [&](const Response& response) -> void {
+    auto completion = [&](const TypedResponse<RemoveHistoricResponse>& response) -> void {
         EXPECT_EQ(response.success, expected_success);
         signal.signal_level();
     };
@@ -411,7 +411,7 @@ template <typename TypeOfTree>
 void unwind_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true)
 {
     Signal signal;
-    auto completion = [&](const Response& response) -> void {
+    auto completion = [&](const TypedResponse<UnwindResponse>& response) -> void {
         EXPECT_EQ(response.success, expected_success);
         signal.signal_level();
     };
@@ -505,10 +505,13 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, reports_an_error_if_tree_is_ove
     }
     add_values(tree, values);
 
+    std::stringstream ss;
+    ss << "Unable to insert values into tree " << name << " new size: 17 max size: 16";
+
     Signal signal;
     auto add_completion = [&](const TypedResponse<AddIndexedDataResponse<NullifierLeafValue>>& response) {
         EXPECT_EQ(response.success, false);
-        EXPECT_EQ(response.message, "Tree is full");
+        EXPECT_EQ(response.message, ss.str());
         signal.signal_level();
     };
     tree.add_or_update_value(NullifierLeafValue(VALUES[16]), add_completion);
@@ -939,10 +942,13 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, reports_an_error_if_batch_conta
     }
     values[8] = values[0];
 
+    std::stringstream ss;
+    ss << "Duplicate key not allowed in same batch, key value: " << values[0].value << ", tree: " << name;
+
     Signal signal;
     auto add_completion = [&](const TypedResponse<AddIndexedDataResponse<NullifierLeafValue>>& response) {
         EXPECT_EQ(response.success, false);
-        EXPECT_EQ(response.message, "Duplicate key not allowed in same batch");
+        EXPECT_EQ(response.message, ss.str());
         signal.signal_level();
     };
     tree.add_or_update_values(values, add_completion);
@@ -1205,7 +1211,7 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, can_add_single_whilst_reading)
         Signal signal(1 + num_reads);
 
         auto add_completion = [&](const TypedResponse<AddIndexedDataResponse<NullifierLeafValue>>&) {
-            auto commit_completion = [&](const Response&) { signal.signal_decrement(); };
+            auto commit_completion = [&](const TypedResponse<CommitResponse>&) { signal.signal_decrement(); };
             tree.commit(commit_completion);
         };
         tree.add_or_update_value(VALUES[0], add_completion);

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp

@@ -59,6 +59,8 @@ struct NullifierLeafValue {
     static NullifierLeafValue empty() { return { fr::zero() }; }
 
     static NullifierLeafValue padding(index_t i) { return { i }; }
+
+    static std::string name() { return std::string("NullifierLeafValue"); };
 };
 
 struct PublicDataLeafValue {
@@ -117,6 +119,8 @@ struct PublicDataLeafValue {
     static PublicDataLeafValue empty() { return { fr::zero(), fr::zero() }; }
 
     static PublicDataLeafValue padding(index_t i) { return { i, fr::zero() }; }
+
+    static std::string name() { return std::string("PublicDataLeafValue"); };
 };
 
 template <typename LeafType> struct IndexedLeaf {
@@ -128,7 +132,7 @@ template <typename LeafType> struct IndexedLeaf {
 
     IndexedLeaf() = default;
 
-    IndexedLeaf(const LeafType& val, index_t nextIdx, fr nextVal)
+    IndexedLeaf(const LeafType& val, const index_t& nextIdx, const fr& nextVal)
         : value(val)
         , nextIndex(nextIdx)
         , nextValue(nextVal)
@@ -140,6 +144,8 @@ template <typename LeafType> struct IndexedLeaf {
 
     static bool is_updateable() { return LeafType::is_updateable(); }
 
+    static std::string name() { return LeafType::name(); }
+
     bool operator==(IndexedLeaf<LeafType> const& other) const
     {
         return value == other.value && nextValue == other.nextValue && nextIndex == other.nextIndex;

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.cpp

@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
+#include <endian.h>
 #include <functional>
 #include <vector>
 
@@ -29,7 +30,7 @@ void deserialise_key(void* data, uint8_t& key)
 // 64 bit integers are stored in little endian byte order
 std::vector<uint8_t> serialise_key(uint64_t key)
 {
-    uint64_t le = key;
+    uint64_t le = htole64(key);
     const uint8_t* p = reinterpret_cast<uint8_t*>(&le);
     return std::vector<uint8_t>(p, p + sizeof(key));
 }
@@ -38,10 +39,10 @@ void deserialise_key(void* data, uint64_t& key)
 {
     uint64_t le = 0;
     std::memcpy(&le, data, sizeof(le));
-    key = le;
+    key = le64toh(le);
 }
 
-std::vector<uint8_t> serialise_key(uint256_t key)
+std::vector<uint8_t> serialise_key(const uint256_t& key)
 {
     std::vector<uint8_t> buf(32);
     std::memcpy(buf.data(), key.data, 32);
@@ -58,10 +59,7 @@ int size_cmp(const MDB_val* a, const MDB_val* b)
     if (a->mv_size < b->mv_size) {
         return -1;
     }
-    if (a->mv_size > b->mv_size) {
-        return 1;
-    }
-    return 0;
+    return (a->mv_size > b->mv_size) ? 1 : 0;
 }
 
 std::vector<uint8_t> mdb_val_to_vector(const MDB_val& dbVal)

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp

@@ -19,7 +19,7 @@ int size_cmp(const MDB_val* a, const MDB_val* b);
 
 std::vector<uint8_t> serialise_key(uint8_t key);
 std::vector<uint8_t> serialise_key(uint64_t key);
-std::vector<uint8_t> serialise_key(uint256_t key);
+std::vector<uint8_t> serialise_key(const uint256_t& key);
 
 void deserialise_key(void* data, uint8_t& key);
 void deserialise_key(void* data, uint64_t& key);

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp

@@ -1,6 +1,6 @@
 #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp"
-
 #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp"
+#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp"
 #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp"
 #include <utility>
 

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp

@@ -1,9 +1,10 @@
 #pragma once
 #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp"
-#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp"
 #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp"
 
 namespace bb::crypto::merkle_tree {
+
+class LMDBDatabaseCreationTransaction;
 /**
  * RAII wrapper atound the opening and closing of an LMDB database
  * Contains a reference to its LMDB environment

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp

@@ -29,4 +29,9 @@ void LMDBTransaction::abort()
     call_lmdb_func(mdb_txn_abort, _transaction);
     state = TransactionState::ABORTED;
 }
+
+bool LMDBTransaction::get_value(std::vector<uint8_t>& key, std::vector<uint8_t>& data, const LMDBDatabase& db) const
+{
+    return lmdb_queries::get_value(key, data, db, *this);
+}
 } // namespace bb::crypto::merkle_tree

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp

@@ -1,5 +1,9 @@
 #pragma once
+#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp"
 #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp"
+#include "barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp"
+#include <functional>
+#include <vector>
 
 namespace bb::crypto::merkle_tree {
 
@@ -33,9 +37,68 @@ class LMDBTransaction {
      */
     virtual void abort();
 
+    template <typename T>
+    bool get_value_or_previous(T& key,
+                               std::vector<uint8_t>& data,
+                               const LMDBDatabase& db,
+                               const std::function<bool(const std::vector<uint8_t>&)>& is_valid) const;
+
+    template <typename T> bool get_value_or_previous(T& key, std::vector<uint8_t>& data, const LMDBDatabase& db) const;
+
+    template <typename T> bool get_value(T& key, std::vector<uint8_t>& data, const LMDBDatabase& db) const;
+
+    template <typename T>
+    void get_all_values_greater_or_equal_key(const T& key,
+                                             std::vector<std::vector<uint8_t>>& data,
+                                             const LMDBDatabase& db) const;
+
+    template <typename T>
+    void get_all_values_lesser_or_equal_key(const T& key,
+                                            std::vector<std::vector<uint8_t>>& data,
+                                            const LMDBDatabase& db) const;
+
+    bool get_value(std::vector<uint8_t>& key, std::vector<uint8_t>& data, const LMDBDatabase& db) const;
+
   protected:
     std::shared_ptr<LMDBEnvironment> _environment;
     MDB_txn* _transaction;
     TransactionState state;
 };
+
+template <typename T> bool LMDBTransaction::get_value(T& key, std::vector<uint8_t>& data, const LMDBDatabase& db) const
+{
+    std::vector<uint8_t> keyBuffer = serialise_key(key);
+    return get_value(keyBuffer, data, db);
+}
+
+template <typename T>
+bool LMDBTransaction::get_value_or_previous(T& key, std::vector<uint8_t>& data, const LMDBDatabase& db) const
+{
+    return lmdb_queries::get_value_or_previous(key, data, db, *this);
+}
+
+template <typename T>
+bool LMDBTransaction::get_value_or_previous(T& key,
+                                            std::vector<uint8_t>& data,
+                                            const LMDBDatabase& db,
+                                            const std::function<bool(const std::vector<uint8_t>&)>& is_valid) const
+{
+    return lmdb_queries::get_value_or_previous(key, data, db, is_valid, *this);
+}
+
+template <typename T>
+void LMDBTransaction::get_all_values_greater_or_equal_key(const T& key,
+                                                          std::vector<std::vector<uint8_t>>& data,
+                                                          const LMDBDatabase& db) const
+{
+    lmdb_queries::get_all_values_greater_or_equal_key(key, data, db, *this);
+}
+
+template <typename T>
+void LMDBTransaction::get_all_values_lesser_or_equal_key(const T& key,
+                                                         std::vector<std::vector<uint8_t>>& data,
+                                                         const LMDBDatabase& db) const
+{
+    lmdb_queries::get_all_values_lesser_or_equal_key(key, data, db, *this);
+}
 } // namespace bb::crypto::merkle_tree

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.cpp

@@ -18,20 +18,4 @@ void LMDBTreeReadTransaction::abort()
     LMDBTransaction::abort();
     _environment->release_reader();
 }
-
-bool LMDBTreeReadTransaction::get_value(std::vector<uint8_t>& key,
-                                        std::vector<uint8_t>& data,
-                                        const LMDBDatabase& db) const
-{
-    MDB_val dbKey;
-    dbKey.mv_size = key.size();
-    dbKey.mv_data = (void*)key.data();
-
-    MDB_val dbVal;
-    if (!call_lmdb_func(mdb_get, underlying(), db.underlying(), &dbKey, &dbVal)) {
-        return false;
-    }
-    copy_to_vector(dbVal, data);
-    return true;
-}
 } // namespace bb::crypto::merkle_tree