feat: Replace proof with sibling_hashes

src/Makefile

@@ -257,7 +257,9 @@ PGO_WORKLOAD=\
 	whetstone 2500
 
 # We ignore test-machine-c-api.cpp cause it takes too long.
-LINTER_IGNORE_SOURCES=test-machine-c-api.cpp
+# We ignore uarch-pristine-ram.c because it is generated by xxd.
+# We ignore uarch-pristine-state-hash.cpp because it is generated by compute-uarch-pristine-hash.
+LINTER_IGNORE_SOURCES=test-machine-c-api.cpp uarch-pristine-ram.c uarch-pristine-state-hash.cpp
 LINTER_IGNORE_HEADERS=%.pb.h
 LINTER_SOURCES=$(filter-out $(LINTER_IGNORE_SOURCES),$(strip $(wildcard *.cpp) $(wildcard *.c)))
 LINTER_HEADERS=$(filter-out $(LINTER_IGNORE_HEADERS),$(strip $(wildcard *.hpp) $(wildcard *.h)))
@@ -269,7 +271,9 @@ CLANG_FORMAT=clang-format
 CLANG_FORMAT_UARCH_FILES:=$(wildcard ../uarch/*.cpp)
 CLANG_FORMAT_UARCH_FILES:=$(filter-out %uarch-printf%,$(strip $(CLANG_FORMAT_UARCH_FILES)))
 CLANG_FORMAT_FILES:=$(wildcard *.cpp) $(wildcard *.c) $(wildcard *.h) $(wildcard *.hpp) $(CLANG_FORMAT_UARCH_FILES)
+CLANG_FORMAT_IGNORE_FILES:=uarch-pristine-ram.c uarch-pristine-state-hash.cpp
 CLANG_FORMAT_FILES:=$(filter-out %.pb.h,$(strip $(CLANG_FORMAT_FILES)))
+CLANG_FORMAT_FILES:=$(filter-out $(CLANG_FORMAT_IGNORE_FILES),$(strip $(CLANG_FORMAT_FILES)))
 
 STYLUA=stylua
 STYLUA_FLAGS=--indent-type Spaces --collapse-simple-statement Always
@@ -755,7 +759,6 @@ jsonrpc-discover.cpp: jsonrpc-discover.json
 
 uarch-pristine-state-hash.cpp:  compute-uarch-pristine-hash
 	@echo '// This file is auto-generated and should not be modified' > $@
-	@echo '// clang-format off' >> $@
 	@echo '#include "uarch-pristine-state-hash.h"' >> $@
 	@echo 'namespace cartesi {' >> $@
 	@echo '    const machine_merkle_tree::hash_type uarch_pristine_state_hash{' >> $@

src/access-log.h

@@ -57,8 +57,12 @@ static inline uint64_t get_word_access_data(const access_data &ad) {
 /// NOLINTNEXTLINE(bugprone-exception-escape)
 class access {
 
-    using proof_type = machine_merkle_tree::proof_type;
+    using hasher_type = machine_merkle_tree::hasher_type;
+
+public:
     using hash_type = machine_merkle_tree::hash_type;
+    using sibling_hashes_type = std::vector<hash_type>;
+    using proof_type = machine_merkle_tree::proof_type;
 
 public:
     void set_type(access_type type) {
@@ -158,38 +162,46 @@ class access {
         return m_read_hash;
     }
 
-    /// \brief Sets proof that data read at address was in
-    /// Merkle tree before access.
-    /// \param proof Corresponding Merkle tree proof.
-    void set_proof(const proof_type &proof) {
-        m_proof = proof;
-    }
-    void set_proof(proof_type &&proof) {
-        m_proof = std::move(proof);
+    /// \brief Constructs a proof using this access' data and a given root hash.
+    /// \param root_hash Hash to be used as the root of the proof.
+    /// \return The corresponding proof
+    proof_type make_proof(const hash_type root_hash) const {
+        if (!m_sibling_hashes.has_value()) {
+            throw std::runtime_error("can't make proof if access doesn't have sibling hashes");
+        }
+        // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+        const auto &sibiling_hashes = m_sibling_hashes.value();
+        auto log2_root_size = m_log2_size + sibiling_hashes.size();
+        proof_type proof(log2_root_size, m_log2_size);
+        proof.set_root_hash(root_hash);
+        proof.set_target_address(m_address);
+        proof.set_target_hash(m_read_hash);
+        for (size_t log2_size = m_log2_size; log2_size < log2_root_size; log2_size++) {
+            proof.set_sibling_hash(sibiling_hashes[log2_size - m_log2_size], log2_size);
+        }
+        return proof;
     }
 
-    /// \brief Gets proof that data read at address was in
-    /// Merkle tree before access.
-    /// \returns Proof, if one is available.
-    const std::optional<proof_type> &get_proof(void) const {
-        return m_proof;
+    std::optional<sibling_hashes_type> &get_sibling_hashes() {
+        return m_sibling_hashes;
+    }
+    const std::optional<sibling_hashes_type> &get_sibling_hashes() const {
+        return m_sibling_hashes;
     }
 
-    /// \brief Removes proof that data read at address was in
-    /// Merkle tree before access.
-    void clear_proof(void) {
-        m_proof = std::nullopt;
+    void set_sibling_hashes(const sibling_hashes_type &sibling_hashes) {
+        m_sibling_hashes = sibling_hashes;
     }
 
 private:
-    access_type m_type{0};                     ///< Type of access
-    uint64_t m_address{0};                     ///< Address of access
-    int m_log2_size{0};                        ///< Log2 of size of access
-    std::optional<access_data> m_read{};       ///< Data before access
-    hash_type m_read_hash;                     ///< Hash of data before access
-    std::optional<access_data> m_written{};    ///< Written data
-    std::optional<hash_type> m_written_hash{}; ///< Hash of written data
-    std::optional<proof_type> m_proof{};       ///< Proof of data before access
+    access_type m_type{0};                                 ///< Type of access
+    uint64_t m_address{0};                                 ///< Address of access
+    int m_log2_size{0};                                    ///< Log2 of size of access
+    std::optional<access_data> m_read{};                   ///< Data before access
+    hash_type m_read_hash;                                 ///< Hash of data before access
+    std::optional<access_data> m_written{};                ///< Written data
+    std::optional<hash_type> m_written_hash{};             ///< Hash of written data
+    std::optional<sibling_hashes_type> m_sibling_hashes{}; ///< Hashes of siblings in path from address to root
 };
 
 /// \brief Log of state accesses

src/cartesi-machine-tests.lua

@@ -673,7 +673,6 @@ local function print_machine(test_name, expected_cycles)
  --ram-length=32Mi\
  --ram-image='%s'\
  --no-bootargs\
- --uarch-ram-length=%d\
  --uarch-ram-image=%s\
  --max-mcycle=%d ",
             test_path .. "/" .. test_name,

src/cartesi/util.lua

@@ -232,7 +232,6 @@ function _M.dump_log(log, out)
             j = j + 1
         -- Otherwise, output access
         elseif ai then
-            if ai.proof then indentout(out, indent, "hash %s\n", hexhash8(ai.proof.root_hash)) end
             local read = accessdatastring(ai.read, ai.read_hash, ai.log2_size)
             if ai.type == "read" then
                 indentout(out, indent, "%d: read %s@0x%x(%u): %s\n", i, notes[i] or "", ai.address, ai.address, read)

src/clua-jsonrpc-machine.cpp

@@ -97,9 +97,8 @@ static int jsonrpc_machine_class_verify_uarch_step_state_transition(lua_State *L
     clua_check_cm_hash(L, 3, &target_hash);
     auto &managed_runtime_config = clua_push_to(L,
         clua_managed_cm_ptr<cm_machine_runtime_config>(clua_opt_cm_machine_runtime_config(L, 4, {}, ctxidx)), ctxidx);
-    const bool one_based = lua_toboolean(L, 5);
     TRY_EXECUTE(cm_jsonrpc_verify_uarch_step_state_transition(managed_jsonrpc_mg_mgr.get(), &root_hash,
-        managed_log.get(), &target_hash, managed_runtime_config.get(), one_based, err_msg));
+        managed_log.get(), &target_hash, managed_runtime_config.get(), true, err_msg));
     managed_log.reset();
     managed_runtime_config.reset();
     lua_pop(L, 2);
@@ -122,9 +121,8 @@ static int jsonrpc_machine_class_verify_uarch_reset_state_transition(lua_State *
     clua_check_cm_hash(L, 3, &target_hash);
     auto &managed_runtime_config = clua_push_to(L,
         clua_managed_cm_ptr<cm_machine_runtime_config>(clua_opt_cm_machine_runtime_config(L, 4, {}, ctxidx)), ctxidx);
-    const bool one_based = lua_toboolean(L, 5);
     TRY_EXECUTE(cm_jsonrpc_verify_uarch_reset_state_transition(managed_jsonrpc_mg_mgr.get(), &root_hash,
-        managed_log.get(), &target_hash, managed_runtime_config.get(), one_based, err_msg));
+        managed_log.get(), &target_hash, managed_runtime_config.get(), true, err_msg));
     managed_log.reset();
     managed_runtime_config.reset();
     lua_pop(L, 2);

src/clua-machine-util.cpp

@@ -352,9 +352,9 @@ static void check_sibling_cm_hashes(lua_State *L, int idx, size_t log2_target_si
     }
     sibling_hashes->count = sibling_hashes_count;
     sibling_hashes->entry = new cm_hash[sibling_hashes_count]{};
-    for (; log2_target_size < log2_root_size; ++log2_target_size) {
-        lua_rawgeti(L, idx, static_cast<lua_Integer>(log2_root_size - log2_target_size));
-        auto index = log2_root_size - 1 - log2_target_size;
+    for (size_t log2_size = log2_target_size; log2_size < log2_root_size; ++log2_size) {
+        lua_rawgeti(L, idx, static_cast<lua_Integer>(log2_size - log2_target_size) + 1);
+        auto index = log2_size - log2_target_size;
         clua_check_cm_hash(L, -1, &sibling_hashes->entry[index]);
         lua_pop(L, 1);
     }
@@ -424,8 +424,7 @@ static unsigned char *opt_cm_access_data_field(lua_State *L, int tabidx, const c
 /// \param a Pointer to receive access
 /// \param ctxidx Index (or pseudo-index) of clua context
 static void check_cm_access(lua_State *L, int tabidx, bool proofs, cm_access *a, int ctxidx) {
-    ctxidx = lua_absindex(L, ctxidx);
-    tabidx = lua_absindex(L, tabidx);
+    (void) ctxidx;
     luaL_checktype(L, tabidx, LUA_TTABLE);
     a->type = check_cm_access_type_field(L, tabidx, "type");
     a->address = check_uint_field(L, tabidx, "address");
@@ -435,8 +434,9 @@ static void check_cm_access(lua_State *L, int tabidx, bool proofs, cm_access *a,
             CM_TREE_LOG2_ROOT_SIZE);
     }
     if (proofs) {
-        lua_getfield(L, tabidx, "proof");
-        a->proof = clua_check_cm_merkle_tree_proof(L, -1, ctxidx);
+        lua_getfield(L, tabidx, "sibling_hashes");
+        a->sibling_hashes = new cm_hash_array{};
+        check_sibling_cm_hashes(L, -1, a->log2_size, CM_TREE_LOG2_ROOT_SIZE, a->sibling_hashes);
         lua_pop(L, 1);
     }
 
@@ -651,9 +651,13 @@ void clua_push_cm_access_log(lua_State *L, const cm_access_log *log) {
                 lua_setfield(L, -2, "written");
             }
         }
-        if (log->log_type.proofs && a->proof != nullptr) {
-            clua_push_cm_proof(L, a->proof);
-            lua_setfield(L, -2, "proof");
+        if (log->log_type.proofs && a->sibling_hashes != nullptr) {
+            lua_newtable(L);
+            for (size_t log2_size = a->log2_size; log2_size < CM_TREE_LOG2_ROOT_SIZE; log2_size++) {
+                clua_push_cm_hash(L, &a->sibling_hashes->entry[log2_size - a->log2_size]);
+                lua_rawseti(L, -2, static_cast<lua_Integer>(log2_size - a->log2_size) + 1);
+            }
+            lua_setfield(L, -2, "sibling_hashes");
         }
         lua_rawseti(L, -2, static_cast<lua_Integer>(i) + 1);
     }

src/json-util.cpp

@@ -653,8 +653,15 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, access &access, con
     }
     not_default_constructible<machine_merkle_tree::proof_type> proof;
     ju_get_opt_field(jk, "proof"s, proof, new_path);
-    if (proof.has_value()) {
-        access.set_proof(std::move(proof).value());
+    if (contains(jk, "sibling_hashes")) {
+        access.get_sibling_hashes().emplace();
+        // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+        auto &sibling_hashes = access.get_sibling_hashes().value();
+        ju_get_vector_like_field(jk, "sibling_hashes"s, sibling_hashes, new_path);
+        auto expected_depth = static_cast<size_t>(machine_merkle_tree::get_log2_root_size() - access.get_log2_size());
+        if (sibling_hashes.size() != expected_depth) {
+            throw std::invalid_argument("field \""s + new_path + "sibling_hashes\" has wrong length");
+        }
     }
 }
 
@@ -754,8 +761,9 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constru
     ju_get_vector_like_field(jk, "accesses"s, accesses, new_path);
     if (log_type.value().has_proofs()) {
         for (unsigned i = 0; i < accesses.size(); ++i) {
-            if (!accesses[i].get_proof().has_value()) {
-                throw std::invalid_argument("field \""s + new_path + "accesses/" + to_string(i) + "\" missing proof");
+            if (!accesses[i].get_sibling_hashes().has_value()) {
+                throw std::invalid_argument(
+                    "field \""s + new_path + "accesses/" + to_string(i) + "\" missing sibling hashes");
             }
         }
     }
@@ -1159,9 +1167,15 @@ void to_json(nlohmann::json &j, const access &a) {
             j["written"] = encode_base64(a.get_written().value());
         }
     }
-    if (a.get_proof().has_value()) {
+    if (a.get_sibling_hashes().has_value()) {
         // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
-        j["proof"] = a.get_proof().value();
+        const auto &sibling_hashes = a.get_sibling_hashes().value();
+        auto depth = machine_merkle_tree::get_log2_root_size() - a.get_log2_size();
+        nlohmann::json s = nlohmann::json::array();
+        for (int i = 0; i < depth; i++) {
+            s.push_back(encode_base64(sibling_hashes[i]));
+        }
+        j["sibling_hashes"] = s;
     }
 }
 

src/machine-c-api.cpp

@@ -439,6 +439,25 @@ cartesi::machine_merkle_tree::hash_type convert_from_c(const cm_hash *c_hash) {
     return cpp_hash;
 }
 
+std::vector<cartesi::machine_merkle_tree::hash_type> convert_from_c(const cm_hash_array *c_array) {
+    auto new_array = std::vector<cartesi::machine_merkle_tree::hash_type>(c_array->count);
+    for (size_t i = 0; i < c_array->count; ++i) {
+        new_array[i] = convert_from_c(&c_array->entry[i]);
+    }
+    return new_array;
+}
+
+static cm_hash_array *convert_to_c(const std::vector<cartesi::machine_merkle_tree::hash_type> &cpp_array) {
+    auto *new_array = new cm_hash_array{};
+    new_array->count = cpp_array.size();
+    new_array->entry = new cm_hash[cpp_array.size()];
+    memset(new_array->entry, 0, sizeof(cm_hash) * new_array->count);
+    for (size_t i = 0; i < new_array->count; ++i) {
+        memcpy(&new_array->entry[i], static_cast<const uint8_t *>(cpp_array[i].data()), sizeof(cm_hash));
+    }
+    return new_array;
+}
+
 // ----------------------------------------------
 // Semantic version conversion functions
 // ----------------------------------------------
@@ -458,9 +477,11 @@ cm_semantic_version *convert_to_c(const cartesi::semantic_version &cpp_version)
 // ----------------------------------------------
 
 /// \brief Converts log2_size to index into siblings array
-static int cm_log2_size_to_index(int log2_size, int log2_root_size) {
-    // We know log2_root_size > 0, so log2_root_size-1 >= 0
-    const int index = log2_root_size - 1 - log2_size;
+static int cm_log2_size_to_index(int log2_size, int log2_target_size) {
+    const int index = log2_size - log2_target_size;
+    if (index < 0) {
+        throw std::invalid_argument("log2_size can't be smaller than log2_target_size");
+    }
     return index;
 }
 
@@ -484,8 +505,8 @@ static cm_merkle_tree_proof *convert_to_c(const cartesi::machine_merkle_tree::pr
 
     for (size_t log2_size = new_merkle_tree_proof->log2_target_size; log2_size < new_merkle_tree_proof->log2_root_size;
          ++log2_size) {
-        const int current_index =
-            cm_log2_size_to_index(static_cast<int>(log2_size), static_cast<int>(new_merkle_tree_proof->log2_root_size));
+        const int current_index = cm_log2_size_to_index(static_cast<int>(log2_size),
+            static_cast<int>(new_merkle_tree_proof->log2_target_size));
         const cartesi::machine_merkle_tree::hash_type sibling_hash =
             proof.get_sibling_hash(static_cast<int>(log2_size));
         memcpy(&(new_merkle_tree_proof->sibling_hashes.entry[current_index]),
@@ -495,24 +516,6 @@ static cm_merkle_tree_proof *convert_to_c(const cartesi::machine_merkle_tree::pr
     return new_merkle_tree_proof;
 }
 
-static cartesi::machine_merkle_tree::proof_type convert_from_c(const cm_merkle_tree_proof *c_proof) {
-    cartesi::machine_merkle_tree::proof_type cpp_proof(static_cast<int>(c_proof->log2_root_size),
-        static_cast<int>(c_proof->log2_target_size));
-    cpp_proof.set_target_address(c_proof->target_address);
-
-    cpp_proof.set_root_hash(convert_from_c(&c_proof->root_hash));
-    cpp_proof.set_target_hash(convert_from_c(&c_proof->target_hash));
-
-    for (int log2_size = cpp_proof.get_log2_target_size(); log2_size < cpp_proof.get_log2_root_size(); ++log2_size) {
-        const int current_index = cm_log2_size_to_index(log2_size, cpp_proof.get_log2_root_size());
-        const cartesi::machine_merkle_tree::hash_type cpp_sibling_hash =
-            convert_from_c(&c_proof->sibling_hashes.entry[current_index]);
-        cpp_proof.set_sibling_hash(cpp_sibling_hash, log2_size);
-    }
-
-    return cpp_proof;
-}
-
 // ----------------------------------------------
 // Access log conversion functions
 // ----------------------------------------------
@@ -574,24 +577,23 @@ static cm_access convert_to_c(const cartesi::access &cpp_access) {
         }
     }
 
-    if (cpp_access.get_proof().has_value()) {
+    if (cpp_access.get_sibling_hashes().has_value()) {
         // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
-        new_access.proof = convert_to_c(*cpp_access.get_proof());
+        new_access.sibling_hashes = convert_to_c(*cpp_access.get_sibling_hashes());
     } else {
-        new_access.proof = nullptr;
+        new_access.sibling_hashes = nullptr;
     }
 
     return new_access;
 }
 
-static cartesi::access convert_from_c(const cm_access *c_access) {
+cartesi::access convert_from_c(const cm_access *c_access) {
     cartesi::access cpp_access{};
     cpp_access.set_type(convert_from_c(c_access->type));
     cpp_access.set_log2_size(c_access->log2_size);
     cpp_access.set_address(c_access->address);
-    if (c_access->proof != nullptr) {
-        const cartesi::machine_merkle_tree::proof_type proof = convert_from_c(c_access->proof);
-        cpp_access.set_proof(proof);
+    if (c_access->sibling_hashes != nullptr) {
+        cpp_access.set_sibling_hashes(convert_from_c(c_access->sibling_hashes));
     }
 
     cpp_access.set_read_hash(convert_from_c(&c_access->read_hash));
@@ -612,7 +614,10 @@ static void cm_cleanup_access(cm_access *access) {
     if (access == nullptr) {
         return;
     }
-    cm_delete_merkle_tree_proof(access->proof);
+    if (access->sibling_hashes != nullptr) {
+        delete[] access->sibling_hashes->entry;
+        delete access->sibling_hashes;
+    }
     delete[] access->written_data;
     delete[] access->read_data;
 }