Machine: gate TLB updates by root register and privilege mode

Ensure that TLBs are only updated when the root register is set, and disable TLB updates while running in Machine mode.

Author: nemecad

src/machine/core.cpp

@@ -2,6 +2,7 @@
 
 #include "common/logging.h"
 #include "execute/alu.h"
+#include "memory/tlb/tlb.h"
 #include "utils.h"
 
 #include <cinttypes>
@@ -158,6 +159,12 @@ bool Core::handle_exception(
             control_state->exception_initiate(
                 state.current_privilege(), CSR::PrivilegeLevel::MACHINE);
             state.set_current_privilege(CSR::PrivilegeLevel::MACHINE);
+            if (auto prog_tlb = dynamic_cast<TLB *>(mem_program)) {
+                prog_tlb->on_privilege_changed(CSR::PrivilegeLevel::MACHINE);
+            }
+            if (auto data_tlb = dynamic_cast<TLB *>(mem_data)) {
+                data_tlb->on_privilege_changed(CSR::PrivilegeLevel::MACHINE);
+            }
             regs->write_pc(control_state->exception_pc_address());
         }
     }
@@ -522,6 +529,12 @@ MemoryState Core::memory(const ExecuteInterstage &dt) {
             CSR::PrivilegeLevel restored
                 = control_state->exception_return(state.current_privilege());
             state.set_current_privilege(restored);
+            if (auto prog_tlb = dynamic_cast<TLB *>(mem_program)) {
+                prog_tlb->on_privilege_changed(restored);
+            }
+            if (auto data_tlb = dynamic_cast<TLB *>(mem_data)) {
+                data_tlb->on_privilege_changed(restored);
+            }
             if (this->xlen == Xlen::_32)
                 computed_next_inst_addr
                     = Address(control_state->read_internal(CSR::Id::MEPC).as_u32());

src/machine/machine.cpp

@@ -72,9 +72,11 @@ Machine::Machine(MachineConfig config, bool load_symtab, bool load_executable)
         cch_program, PROGRAM, machine_config.access_tlb_program(), machine_config.get_vm_enabled());
     tlb_data = new TLB(
         cch_data, DATA, machine_config.access_tlb_data(), machine_config.get_vm_enabled());
-    controlst->write_internal(CSR::Id::SATP, 0);
     tlb_program->on_csr_write(CSR::Id::SATP, 0);
     tlb_data->on_csr_write(CSR::Id::SATP, 0);
+    connect(controlst, &CSR::ControlState::write_signal, tlb_program, &machine::TLB::on_csr_write);
+    connect(controlst, &CSR::ControlState::write_signal, tlb_data, &machine::TLB::on_csr_write);
+    controlst->write_internal(CSR::Id::SATP, 0);
 
     predictor = new BranchPredictor(
         machine_config.get_bp_enabled(), machine_config.get_bp_type(),

src/machine/memory/tlb/tlb.cpp

@@ -3,21 +3,13 @@
 #include "csr/controlstate.h"
 #include "machine.h"
 #include "memory/virtual/page_table_walker.h"
-#include "memory/virtual/sv32.h"
 
 LOG_CATEGORY("machine.TLB");
 
 namespace machine {
 
-static bool is_mmio_region(uint64_t virt) {
-    if (virt >= 0xFFFFC000u && virt <= 0xFFFFC1FFu) return true;
-    if (virt >= 0xFFE00000u && virt <= 0xFFE4AFFFu) return true;
-    if (virt >= 0xFFFD0000u && virt <= 0xFFFDBFFFu) return true;
-    return false;
-}
-
-static Address bypass_mmio(Address vaddr) {
-    return vaddr; // VA == PA for devices
+inline bool is_mode_enabled_in_satp(uint32_t satp_raw) {
+    return (satp_raw & (1u << 31)) != 0;
 }
 
 TLB::TLB(
@@ -31,6 +23,8 @@ TLB::TLB(
     bool memory_access_enable_b)
     : FrontendMemory(memory->simulated_machine_endian)
     , mem(memory)
+    , uncached_start(0xf0000000_addr)
+    , uncached_last(0xfffffffe_addr)
     , type(type_)
     , tlb_config(config)
     , vm_enabled(vm_enabled)
@@ -73,6 +67,18 @@ void TLB::on_csr_write(size_t internal_id, RegisterValue val) {
     update_all_statistics();
 }
 
+void TLB::on_privilege_changed(CSR::PrivilegeLevel new_priv) {
+    if (new_priv == current_priv_) return;
+
+    current_priv_ = new_priv;
+    flush();
+    LOG("TLB: privilege changed -> %d; flushed TLB", static_cast<int>(new_priv));
+}
+
+bool TLB::is_in_uncached_area(Address source) const {
+    return (source >= uncached_start && source <= uncached_last);
+}
+
 void TLB::flush_single(VirtualAddress va, uint16_t asid) {
     uint64_t vpn = va.get_raw() >> 12;
     size_t s = set_index(vpn);
@@ -119,9 +125,11 @@ void TLB::sync() {
 
 Address TLB::translate_virtual_to_physical(Address vaddr) {
     uint64_t virt = vaddr.get_raw();
-    if (is_mmio_region(virt)) { return bypass_mmio(vaddr); }
 
-    if (!vm_enabled) { return vaddr; }
+    if (!vm_enabled || current_priv_ == CSR::PrivilegeLevel::MACHINE
+        || !is_mode_enabled_in_satp(current_satp_raw) || is_in_uncached_area(vaddr)) {
+        return vaddr;
+    }
 
     constexpr unsigned PAGE_SHIFT = 12;
     constexpr uint64_t PAGE_MASK = (1ULL << PAGE_SHIFT) - 1;

src/machine/memory/tlb/tlb.h

@@ -2,6 +2,7 @@
 #define TLB_H
 
 #include "common/logging.h"
+#include "csr/address.h"
 #include "memory/frontend_memory.h"
 #include "memory/virtual/sv32.h"
 #include "memory/virtual/virtual_address.h"
@@ -83,6 +84,9 @@ class TLB : public FrontendMemory {
     void memory_reads_update(uint32_t val);
     void memory_writes_update(uint32_t val);
 
+public slots:
+    void on_privilege_changed(CSR::PrivilegeLevel new_priv);
+
 private:
     struct Entry {
         bool valid = false;
@@ -93,9 +97,12 @@ class TLB : public FrontendMemory {
     };
 
     FrontendMemory *mem;
+    const Address uncached_start;
+    const Address uncached_last;
     TLBType type;
     const TLBConfig tlb_config;
     uint32_t current_satp_raw = 0;
+    CSR::PrivilegeLevel current_priv_ = CSR::PrivilegeLevel::MACHINE;
     const bool vm_enabled;
 
     size_t num_sets_;
@@ -116,6 +123,7 @@ class TLB : public FrontendMemory {
     mutable uint32_t burst_writes = 0;
     mutable uint32_t change_counter = 0;
 
+    bool is_in_uncached_area(Address source) const;
     WriteResult translate_and_write(Address dst, const void *src, size_t sz, WriteOptions opts);
     ReadResult translate_and_read(void *dst, Address src, size_t sz, ReadOptions opts);
     inline size_t set_index(uint64_t vpn) const { return vpn & (num_sets_ - 1); }