feat: Uarch reset refactoring

.github/workflows/build.yml

@@ -5,10 +5,11 @@ env:
   KERNEL_VERSION: v0.17.0
   LINUX_VERSION: 5.15.63-ctsi-2-v0.17.0
   ROOTFS_VERSION: v0.18.0
-  TEST_VERSION: vshadow-uarch-test
+  TEST_VERSION: vreset-uarch-test
   CARTESI_TESTS_PATH: /usr/share/cartesi-machine/tests
   CARTESI_IMAGES_PATH: /usr/share/cartesi-machine/images
   CARTESI_UARCH_PATH: /usr/share/cartesi-machine/uarch/uarch-ram.bin
+  BUILD_UARCH_IN_DOCKER_LINUX_ENV: no
 jobs:
   build:
     name: Build
@@ -250,7 +251,8 @@ jobs:
       - name: Create uarch json logs to be used to test the Solidity based microarchitecture interpreter
         run: |
           mkdir -p /opt/cartesi/share/logs/uarch-riscv-tests-json-logs
-          docker run --rm -v /opt/cartesi/share/logs:/opt/cartesi/share/logs -v ${{ env.CARTESI_TESTS_PATH }}:${{ env.CARTESI_TESTS_PATH }} -t ${{ github.repository_owner }}/machine-emulator:devel /usr/bin/uarch-riscv-tests --test-path=${{ env.CARTESI_TESTS_PATH }}  --output-dir=/opt/cartesi/share/logs/uarch-riscv-tests-json-logs --proofs --proofs-frequency=1 json-logs
+          docker run --rm -v /opt/cartesi/share/logs:/opt/cartesi/share/logs -v ${{ env.CARTESI_TESTS_PATH }}:${{ env.CARTESI_TESTS_PATH }} -t ${{ github.repository_owner }}/machine-emulator:devel /usr/bin/uarch-riscv-tests --test-path=${{ env.CARTESI_TESTS_PATH }}  --output-dir=/opt/cartesi/share/logs/uarch-riscv-tests-json-logs --proofs --proofs-frequency=1 json-step-logs
+          docker run --rm -v /opt/cartesi/share/logs:/opt/cartesi/share/logs -v ${{ env.CARTESI_TESTS_PATH }}:${{ env.CARTESI_TESTS_PATH }} -t ${{ github.repository_owner }}/machine-emulator:devel /usr/bin/uarch-riscv-tests --test-path=${{ env.CARTESI_TESTS_PATH }}  --output-dir=/opt/cartesi/share/logs/uarch-riscv-tests-json-logs --proofs json-reset-log
 
       - name: Compress uarch json logs
         run: tar -czf uarch-riscv-tests-json-logs.tar.gz -C /opt/cartesi/share/logs/uarch-riscv-tests-json-logs .

.gitignore

@@ -10,6 +10,7 @@ third-party/downloads
 third-party/grpc
 third-party/mongoose-7.9
 src/remote-cartesi-machine-proxy
+src/compute-uarch-pristine-hash
 src/remote-cartesi-machine
 src/jsonrpc-remote-cartesi-machine
 src/merkle-tree-hash

Makefile

@@ -250,7 +250,7 @@ linux-env: check-linux-env
 		-e GROUP=$$(id -g -n) \
 		-e UID=$$(id -u) \
 		-e GID=$$(id -g) \
-		-e UARCH_TOOLCHAIN_AVAILABLE=yes \
+		-e BUILD_UARCH_IN_DOCKER_LINUX_ENV=no \
 		-v `pwd`:/opt/cartesi/machine-emulator \
 		-w /opt/cartesi/machine-emulator \
 		cartesi/linux-env:$(TAG) /bin/bash
@@ -261,7 +261,7 @@ linux-env-exec: check-linux-env
 		-e GROUP=$$(id -g -n) \
 		-e UID=$$(id -u) \
 		-e GID=$$(id -g) \
-		-e UARCH_TOOLCHAIN_AVAILABLE=yes \
+		-e BUILD_UARCH_IN_DOCKER_LINUX_ENV=no \
 		-v `pwd`:/opt/cartesi/machine-emulator \
 		-w /opt/cartesi/machine-emulator \
 		cartesi/linux-env:$(TAG) /bin/bash -c "$(CONTAINER_COMMAND)"

src/.gitignore

@@ -10,3 +10,4 @@ compile_flags.txt
 coverage*
 jsonrpc-discover.cpp
 machine-c-version.h
+uarch-pristine-state-hash.cpp

src/Makefile

@@ -31,11 +31,18 @@ LUA_LIB:=-llua5.4
 LUA_BIN?=lua5.4
 GRPC_DIR:=../lib/grpc-interfaces
 
-# Path to the file containing the uarch ram image to be embedded in the emulator. Override to use a pre-built one.
+# This flag indicates whether your system has the toolchain required to compile the microarchitecture ram image.
+# If "no", the build process will use the toolchain within the linux-env Docker environment.
+# If "yes", the toolchain will run locally.
+BUILD_UARCH_IN_DOCKER_LINUX_ENV ?= yes
+
+# If instead of building the microarchitecture you  wamt to use a pre-built image file, modify this variable to
+# indicate the path of the desired file.
 UARCH_RAM_IMAGE ?= ../uarch/uarch-ram.bin
 
-# Indicates whether the toolchain required to compile the uarch is present. If not, the one within the linux-env Docker environment  will be used.
-UARCH_TOOLCHAIN_AVAILABLE ?= no
+# Allow mongoose to receive large messages
+# The biggest message is reset uarch with large_data: 2x 4MB + small data + overhead
+MG_MAX_RECV_SIZE := 0xc00000
 
 PROTOC:=$(shell which protoc)
 GRPC_CPP_PLUGIN:=$(shell which grpc_cpp_plugin)
@@ -351,16 +358,25 @@ CARTESI_OBJS:= \
 	machine-c-api.o \
 	uarch-machine.o \
 	uarch-step.o \
+	uarch-reset-state.o \
 	uarch-interpret.o \
-	uarch-ram.bin.o
+	uarch-pristine-ram.o \
+	uarch-pristine-state-hash.o
+
+COMPUTE_UARCH_PRISTINE_HASH_OBJS := \
+	compute-uarch-pristine-hash.o \
+  	machine-merkle-tree.o \
+	pristine-merkle-tree.o \
+	uarch-pristine-ram.o
 
 LUACARTESI_OBJS:= \
 	clua-cartesi.o \
 	clua-i-virtual-machine.o \
 	clua-machine.o \
 	clua-htif.o \
 	clua-machine-util.o \
-	clua.o
+	clua.o \
+	uarch-pristine-state-hash.o
 
 CARTESI_PROTOBUF_GEN_OBJS:= \
 	versioning.pb.o \
@@ -537,6 +553,9 @@ compile_flags.txt:
 
 luacartesi: cartesi.so cartesi/grpc.so cartesi/jsonrpc.so
 
+compute-uarch-pristine-hash: $(COMPUTE_UARCH_PRISTINE_HASH_OBJS)
+	$(CXX) $(LDFLAGS) $(CARTESI_EXECUTABLE_LDFLAGS) -o $@ $^ $(HASH_LIBS)
+
 luacartesi-pgo:
 	$(MAKE) --no-print-directory generate
 	./cartesi-machine.lua -- "$(PGO_WORKLOAD)"
@@ -607,8 +626,10 @@ REMOTE_CARTESI_MACHINE_OBJS:= \
 	interpret.o \
 	uarch-machine.o \
 	uarch-step.o \
+	uarch-reset-state.o \
 	uarch-interpret.o \
-	uarch-ram.bin.o
+	uarch-pristine-ram.o \
+	uarch-pristine-state-hash.o
 
 JSONRPC_REMOTE_CARTESI_MACHINE_OBJS:= \
 	slog.o \
@@ -641,8 +662,10 @@ JSONRPC_REMOTE_CARTESI_MACHINE_OBJS:= \
 	interpret.o \
 	uarch-machine.o \
 	uarch-step.o \
+	uarch-reset-state.o \
 	uarch-interpret.o \
-	uarch-ram.bin.o
+	uarch-pristine-ram.o \
+	uarch-pristine-state-hash.o
 
 CYCLE_PERIOD ?= 13
 HASH_DIR = hashes/$(CYCLE_PERIOD)
@@ -716,7 +739,7 @@ grpc-interfaces: $(PROTO_SOURCES)
 remote-cartesi-machine: $(REMOTE_CARTESI_MACHINE_OBJS) $(LIBCARTESI_PROTOBUF)
 	$(CXX) $(LDFLAGS) $(CARTESI_EXECUTABLE_LDFLAGS) -o $@ $(REMOTE_CARTESI_MACHINE_OBJS) $(REMOTE_CARTESI_MACHINE_LIBS)
 
-$(BUILDDIR)/lib/mongoose.o: CFLAGS := $(patsubst %-std=c99,%,$(CFLAGS))
+$(BUILDDIR)/lib/mongoose.o: CFLAGS :=  -D MG_MAX_RECV_SIZE=$(MG_MAX_RECV_SIZE) $(patsubst %-std=c99,%,$(CFLAGS))
 
 jsonrpc-remote-cartesi-machine: $(JSONRPC_REMOTE_CARTESI_MACHINE_OBJS) $(LIBCARTESI_PROTOBUF)
 	$(CXX) $(LDFLAGS) $(CARTESI_EXECUTABLE_LDFLAGS) -o $@ $(JSONRPC_REMOTE_CARTESI_MACHINE_OBJS) $(JSONRPC_REMOTE_CARTESI_MACHINE_LIBS)
@@ -752,6 +775,16 @@ jsonrpc-discover.cpp: jsonrpc-discover.json
 	echo ')json";' >> jsonrpc-discover.cpp
 	echo '} // namespace cartesi' >> jsonrpc-discover.cpp
 
+uarch-pristine-state-hash.cpp:  compute-uarch-pristine-hash
+	@echo '// This file is auto-generated and should not be modified' > $@
+	@echo '// clang-format off' >> $@
+	@echo '#include "uarch-pristine-state-hash.h"' >> $@
+	@echo 'namespace cartesi {' >> $@
+	@echo '    const machine_merkle_tree::hash_type uarch_pristine_state_hash{' >> $@
+	@compute-uarch-pristine-hash  >> $@
+	@echo '    };' >> $@
+	@echo '} // namespace cartesi' >> $@
+
 %.clang-tidy: %.cpp $(PROTO_SOURCES) machine-c-version.h
 	@$(CLANG_TIDY) --header-filter='$(CLANG_TIDY_HEADER_FILTER)' $< -- $(CXXFLAGS) 2>/dev/null
 	@$(CXX) $(CXXFLAGS) $< -MM -MT $@ -MF $@.d > /dev/null 2>&1
@@ -771,21 +804,21 @@ jsonrpc-discover.cpp: jsonrpc-discover.json
 %.o: %.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 
-uarch-ram.bin.o: $(UARCH_RAM_IMAGE)
-	xxd -i -n embedded_uarch_ram $< | $(CXX) $(CFLAGS) -c -x c -o $@ - 
+uarch-pristine-ram.o: $(UARCH_RAM_IMAGE)
+	@xxd -i -n uarch_pristine_ram $< | $(CXX) $(CFLAGS) -c -x c -o $@ - 
 
 ../uarch/uarch-ram.bin:
-	if [ "$(UARCH_TOOLCHAIN_AVAILABLE)" = "yes" ]; then \
-		$(MAKE) -C .. uarch; \
-	else \
+	@if [ "$(BUILD_UARCH_IN_DOCKER_LINUX_ENV)" = "yes" ]; then \
 		$(MAKE) -C .. uarch-with-linux-env; \
+	else \
+		$(MAKE) -C .. uarch; \
 	fi
 
 
 clean: clean-auto-generated clean-coverage clean-profile clean-proto-sources clean-tidy clean-libcartesi clean-executables clean-tests
 
 clean-auto-generated:
-	@rm -f jsonrpc-discover.cpp machine-c-version.h
+	@rm -f jsonrpc-discover.cpp machine-c-version.h uarch-pristine-state-hash.cpp
 
 clean-proto-sources:
 	@rm -f *.pb.cc *.pb.h

src/access-log.h

@@ -58,6 +58,7 @@ static inline uint64_t get_word_access_data(const access_data &ad) {
 class access {
 
     using proof_type = machine_merkle_tree::proof_type;
+    using hash_type = machine_merkle_tree::hash_type;
 
 public:
     void set_type(access_type type) {
@@ -102,10 +103,10 @@ class access {
 
     /// \brief Gets data that can be read at address before access.
     /// \returns Data at address.
-    const access_data &get_read(void) const {
+    const std::optional<access_data> &get_read(void) const {
         return m_read;
     }
-    access_data &get_read(void) {
+    std::optional<access_data> &get_read(void) {
         return m_read;
     }
 
@@ -120,13 +121,43 @@ class access {
 
     /// \brief Gets data that was written at address after access.
     /// \returns Data at address.
-    const access_data &get_written(void) const {
+    const std::optional<access_data> &get_written(void) const {
         return m_written;
     }
-    access_data &get_written(void) {
+    std::optional<access_data> &get_written(void) {
         return m_written;
     }
 
+    /// \brief Sets hash of data that was written at address after access.
+    /// \param hash Hash of new data at address.
+    void set_written_hash(const hash_type &hash) {
+        m_written_hash = hash;
+    }
+
+    /// \brief Gets hash of data that was written at address after access.
+    /// \returns Hash of written data at address.
+    const std::optional<hash_type> &get_written_hash(void) const {
+        return m_written_hash;
+    }
+    std::optional<hash_type> &get_written_hash(void) {
+        return m_written_hash;
+    }
+
+    /// \brief Sets hash of data that can be read at address before access.
+    /// \param hash Hash of data at address.
+    void set_read_hash(const hash_type &hash) {
+        m_read_hash = hash;
+    }
+
+    /// \brief Gets hash of data that can be read at address before access.
+    /// \returns Hash of data at address.
+    const hash_type &get_read_hash(void) const {
+        return m_read_hash;
+    }
+    hash_type &get_read_hash(void) {
+        return m_read_hash;
+    }
+
     /// \brief Sets proof that data read at address was in
     /// Merkle tree before access.
     /// \param proof Corresponding Merkle tree proof.
@@ -151,12 +182,14 @@ class access {
     }
 
 private:
-    access_type m_type{0};               ///< Type of access
-    uint64_t m_address{0};               ///< Address of access
-    int m_log2_size{0};                  ///< Log2 of size of access
-    access_data m_read;                  ///< Data before access
-    access_data m_written;               ///< Data after access (if writing)
-    std::optional<proof_type> m_proof{}; ///< Proof of data before access
+    access_type m_type{0};                     ///< Type of access
+    uint64_t m_address{0};                     ///< Address of access
+    int m_log2_size{0};                        ///< Log2 of size of access
+    std::optional<access_data> m_read{};       ///< Data before access
+    hash_type m_read_hash;                     ///< Hash of data before access
+    std::optional<access_data> m_written{};    ///< Written data
+    std::optional<hash_type> m_written_hash{}; ///< Hash of written data
+    std::optional<proof_type> m_proof{};       ///< Proof of data before access
 };
 
 /// \brief Log of state accesses
@@ -166,11 +199,15 @@ class access_log {
     class type {
         bool m_proofs;      ///< Includes proofs
         bool m_annotations; ///< Includes annotations
+        bool m_large_data;  ///< Includes data bigger than 8 bytes
     public:
         /// \brief Default constructur
         /// \param proofs Include proofs
         /// \param annotations Include annotations (default false)
-        explicit type(bool proofs, bool annotations = false) : m_proofs(proofs), m_annotations(annotations) {
+        explicit type(bool proofs, bool annotations = false, bool large_data = false) :
+            m_proofs(proofs),
+            m_annotations(annotations),
+            m_large_data(large_data) {
             ;
         }
 
@@ -183,6 +220,11 @@ class access_log {
         bool has_annotations(void) const {
             return m_annotations;
         }
+
+        /// \brief Returns whether log includes data bigger than 8 bytes
+        bool has_large_data(void) const {
+            return m_large_data;
+        }
     };
 
 private:

src/cartesi-machine-tests.lua

@@ -355,9 +355,6 @@ where options are:
   --uarch-ram-image=<filename>
     name of file containing microarchitecture RAM image.
 
-  --uarch-ram-length=<number>
-    set microarchitecture RAM length.
-
 and command can be:
 
   run
@@ -559,16 +556,6 @@ local options = {
             return true
         end,
     },
-    {
-        "^%-%-uarch%-ram%-length%=(.+)$",
-        function(n)
-            if not n then return false end
-            uarch = uarch or {}
-            uarch.ram = uarch.ram or {}
-            uarch.ram.length = assert(util.parse_number(n), "invalid microarchitecture RAM length " .. n)
-            return true
-        end,
-    },
     { ".*", function(all) error("unrecognized option " .. all) end },
 }
 
@@ -608,7 +595,7 @@ local function run_machine(machine, ctx, max_mcycle, advance_machine_fn)
 end
 
 local function advance_machine_with_uarch(machine)
-    if machine:run_uarch() == cartesi.UARCH_BREAK_REASON_UARCH_HALTED then machine:reset_uarch_state() end
+    if machine:run_uarch() == cartesi.UARCH_BREAK_REASON_UARCH_HALTED then machine:reset_uarch() end
 end
 
 local function run_machine_with_uarch(machine, ctx, max_mcycle)
@@ -686,12 +673,10 @@ local function print_machine(test_name, expected_cycles)
  --rom-image='%s'\
  --ram-image='%s'\
  --no-rom-bootargs\
- --uarch-ram-length=%d\
  --uarch-ram-image=%s\
  --max-mcycle=%d ",
                 test_path .. "/bootstrap.bin",
                 test_path .. "/" .. test_name,
-                uarch.ram.length,
                 uarch.ram.image_filename,
                 2 * expected_cycles
             )
@@ -852,7 +837,7 @@ local function hash(tests)
                 total_cycles = total_cycles + 1
             end
             if status == cartesi.UARCH_BREAK_REASON_UARCH_HALTED then
-                machine:reset_uarch_state()
+                machine:reset_uarch()
                 if machine:read_iflags_H() then break end
             end
         end
@@ -919,21 +904,21 @@ local function step(tests)
             local final_uarch_cycle = machine:read_uarch_cycle()
             total_uarch_cycles = total_uarch_cycles + (final_uarch_cycle - init_uarch_cycle)
             if machine:read_uarch_halt_flag() then
-                machine:reset_uarch_state()
+                machine:reset_uarch()
                 if machine:read_iflags_H() then break end
             end
             if not periodic_action or total_uarch_cycles == next_action_uarch_cycle then
                 local init_mcycle = machine:read_mcycle()
                 init_uarch_cycle = machine:read_uarch_cycle()
-                local log = machine:step_uarch(log_type)
+                local log = machine:log_uarch_step(log_type)
                 local final_mcycle = machine:read_mcycle()
                 final_uarch_cycle = machine:read_uarch_cycle()
                 if total_logged_steps > 0 then out:write(",\n") end
                 util.dump_json_log(log, init_mcycle, init_uarch_cycle, final_mcycle, final_uarch_cycle, out, 3)
                 total_uarch_cycles = total_uarch_cycles + 1
                 total_logged_steps = total_logged_steps + 1
                 if machine:read_uarch_halt_flag() then
-                    machine:reset_uarch_state()
+                    machine:reset_uarch()
                     if machine:read_iflags_H() then break end
                 end
             end