Merge branch 'yuji/merklized-not-diffed' (#3293)

* origin/yuji/merklized-not-diffed:
  fix typo
  add comments
  remove is_merklized_storage_key
  fix for tests
  add changelog
  fmt
  add NoDiff subtree
  add diff_key_filter

.changelog/unreleased/features/3293-merklized-not-diffed.md

@@ -0,0 +1,2 @@
+- Enable to write data to be updated in the subspace and Merkle tree, but not to
+  be updated in diffs ([\#3293](https://github.com/anoma/namada/issues/3293))

crates/merkle_tree/Cargo.toml

@@ -17,6 +17,7 @@ migrations = [
     "namada_migrations",
     "linkme",
 ]
+testing = ["namada_core/testing"]
 
 [dependencies]
 namada_core = { path = "../core" }

crates/merkle_tree/src/lib.rs

@@ -48,6 +48,9 @@ use namada_macros::BorshDeserializer;
 use namada_migrations::*;
 use thiserror::Error;
 
+/// Key prefix for the data not stored to diffs
+pub const NO_DIFF_KEY_PREFIX: &str = "no_diff";
+
 /// Trait for reading from a merkle tree that is a sub-tree
 /// of the global merkle tree.
 pub trait SubTreeRead {
@@ -177,6 +180,8 @@ pub enum StoreType {
     PoS,
     /// For the Ethereum bridge Pool transfers
     BridgePool,
+    /// For data not stored to diffs
+    NoDiff,
     /// For the commit only data
     CommitData,
 }
@@ -193,6 +198,8 @@ pub enum Store {
     PoS(SmtStore),
     /// For the Ethereum bridge Pool transfers
     BridgePool(BridgePoolStore),
+    /// For data not stored to diffs
+    NoDiff(SmtStore),
     /// For the commit only data
     CommitData,
 }
@@ -206,6 +213,7 @@ impl Store {
             Self::Ibc(store) => StoreRef::Ibc(store),
             Self::PoS(store) => StoreRef::PoS(store),
             Self::BridgePool(store) => StoreRef::BridgePool(store),
+            Self::NoDiff(store) => StoreRef::NoDiff(store),
             Self::CommitData => StoreRef::CommitData,
         }
     }
@@ -223,6 +231,8 @@ pub enum StoreRef<'a> {
     PoS(&'a SmtStore),
     /// For the Ethereum bridge Pool transfers
     BridgePool(&'a BridgePoolStore),
+    /// For data not stored to diffs
+    NoDiff(&'a SmtStore),
     /// For commit only data
     CommitData,
 }
@@ -236,6 +246,7 @@ impl<'a> StoreRef<'a> {
             Self::Ibc(store) => Store::Ibc(store.to_owned()),
             Self::PoS(store) => Store::PoS(store.to_owned()),
             Self::BridgePool(store) => Store::BridgePool(store.to_owned()),
+            Self::NoDiff(store) => Store::NoDiff(store.to_owned()),
             Self::CommitData => Store::CommitData,
         }
     }
@@ -248,6 +259,7 @@ impl<'a> StoreRef<'a> {
             Self::Ibc(store) => store.serialize_to_vec(),
             Self::PoS(store) => store.serialize_to_vec(),
             Self::BridgePool(store) => store.serialize_to_vec(),
+            Self::NoDiff(store) => store.serialize_to_vec(),
             Self::CommitData => vec![],
         }
     }
@@ -256,24 +268,26 @@ impl<'a> StoreRef<'a> {
 impl StoreType {
     /// Get an iterator for the base tree and subtrees
     pub fn iter() -> std::slice::Iter<'static, Self> {
-        static SUB_TREE_TYPES: [StoreType; 6] = [
+        static SUB_TREE_TYPES: [StoreType; 7] = [
             StoreType::Base,
             StoreType::Account,
             StoreType::PoS,
             StoreType::Ibc,
             StoreType::BridgePool,
+            StoreType::NoDiff,
             StoreType::CommitData,
         ];
         SUB_TREE_TYPES.iter()
     }
 
     /// Get an iterator for subtrees
     pub fn iter_subtrees() -> std::slice::Iter<'static, Self> {
-        static SUB_TREE_TYPES: [StoreType; 5] = [
+        static SUB_TREE_TYPES: [StoreType; 6] = [
             StoreType::Account,
             StoreType::PoS,
             StoreType::Ibc,
             StoreType::BridgePool,
+            StoreType::NoDiff,
             StoreType::CommitData,
         ];
         SUB_TREE_TYPES.iter()
@@ -293,6 +307,14 @@ impl StoreType {
         SUB_TREE_TYPES.iter()
     }
 
+    /// Return true if the subtree should be saved in every block
+    pub fn is_stored_every_block(&self) -> bool {
+        matches!(
+            self,
+            StoreType::Base | StoreType::NoDiff | StoreType::CommitData
+        )
+    }
+
     /// Get the store type and the sub key
     pub fn sub_key(key: &Key) -> Result<(Self, Key)> {
         if key.is_empty() {
@@ -320,6 +342,9 @@ impl StoreType {
                     _ => Ok((StoreType::Account, key.clone())),
                 }
             }
+            Some(DbKeySeg::StringSeg(data)) if data.eq(NO_DIFF_KEY_PREFIX) => {
+                Ok((StoreType::NoDiff, key.clone()))
+            }
             Some(DbKeySeg::StringSeg(data)) if data.eq("commit_data") => {
                 Ok((StoreType::CommitData, key.clone()))
             }
@@ -352,6 +377,7 @@ impl StoreType {
             Self::Ibc => Ok(Store::Ibc(decode(bytes)?)),
             Self::PoS => Ok(Store::PoS(decode(bytes)?)),
             Self::BridgePool => Ok(Store::BridgePool(decode(bytes)?)),
+            Self::NoDiff => Ok(Store::NoDiff(decode(bytes)?)),
             Self::CommitData => Ok(Store::CommitData),
         }
     }
@@ -367,6 +393,7 @@ impl FromStr for StoreType {
             "ibc" => Ok(StoreType::Ibc),
             "pos" => Ok(StoreType::PoS),
             "eth_bridge_pool" => Ok(StoreType::BridgePool),
+            "no_diff" => Ok(StoreType::NoDiff),
             "commit_data" => Ok(StoreType::CommitData),
             _ => Err(Error::StoreType(s.to_string())),
         }
@@ -381,6 +408,7 @@ impl fmt::Display for StoreType {
             StoreType::Ibc => write!(f, "ibc"),
             StoreType::PoS => write!(f, "pos"),
             StoreType::BridgePool => write!(f, "eth_bridge_pool"),
+            StoreType::NoDiff => write!(f, "no_diff"),
             StoreType::CommitData => write!(f, "commit_data"),
         }
     }
@@ -427,6 +455,7 @@ pub struct MerkleTree<H: StorageHasher + Default> {
     ibc: Amt<H>,
     pos: Smt<H>,
     bridge_pool: BridgePoolTree,
+    no_diff: Smt<H>,
     commit_data: CommitDataRoot,
 }
 
@@ -448,6 +477,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
         let pos = Smt::new(stores.pos.0.into(), stores.pos.1);
         let bridge_pool =
             BridgePoolTree::new(stores.bridge_pool.0, stores.bridge_pool.1);
+        let no_diff = Smt::new(stores.no_diff.0.into(), stores.no_diff.1);
         let commit_data = stores.commit.into();
 
         let tree = Self {
@@ -456,6 +486,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
             ibc,
             pos,
             bridge_pool,
+            no_diff,
             commit_data,
         };
 
@@ -468,18 +499,22 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
         let pos_root = tree.base.get(&pos_key.into())?;
         let bp_key = H::hash(StoreType::BridgePool.to_string());
         let bp_root = tree.base.get(&bp_key.into())?;
+        let no_diff_key = H::hash(StoreType::NoDiff.to_string());
+        let no_diff_root = tree.base.get(&no_diff_key.into())?;
         let commit_data_key = H::hash(StoreType::CommitData.to_string());
         let commit_data_root = tree.base.get(&commit_data_key.into())?;
         if tree.base.root().is_zero()
             && tree.account.root().is_zero()
             && tree.ibc.root().is_zero()
             && tree.pos.root().is_zero()
             && tree.bridge_pool.root().is_zero()
+            && tree.no_diff.root().is_zero()
             && tree.commit_data.0.is_zero()
             || (account_root == tree.account.root().into()
                 && ibc_root == tree.ibc.root().into()
                 && pos_root == tree.pos.root().into()
                 && bp_root == tree.bridge_pool.root().into()
+                && no_diff_root == tree.no_diff.root().into()
                 && commit_data_root == tree.commit_data.0)
         {
             Ok(tree)
@@ -498,6 +533,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
         let pos = Smt::new(stores.pos.0.into(), stores.pos.1);
         let bridge_pool =
             BridgePoolTree::new(stores.bridge_pool.0, stores.bridge_pool.1);
+        let no_diff = Smt::new(stores.no_diff.0.into(), stores.no_diff.1);
         let commit_data = stores.commit.into();
 
         Self {
@@ -506,6 +542,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
             ibc,
             pos,
             bridge_pool,
+            no_diff,
             commit_data,
         }
     }
@@ -517,6 +554,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
             StoreType::Ibc => Box::new(&self.ibc),
             StoreType::PoS => Box::new(&self.pos),
             StoreType::BridgePool => Box::new(&self.bridge_pool),
+            StoreType::NoDiff => Box::new(&self.no_diff),
             StoreType::CommitData => Box::new(&self.commit_data),
         }
     }
@@ -531,6 +569,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
             StoreType::Ibc => Box::new(&mut self.ibc),
             StoreType::PoS => Box::new(&mut self.pos),
             StoreType::BridgePool => Box::new(&mut self.bridge_pool),
+            StoreType::NoDiff => Box::new(&mut self.no_diff),
             StoreType::CommitData => Box::new(&mut self.commit_data),
         }
     }
@@ -553,6 +592,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
     }
 
     /// Check if the key exists in the tree
+    #[cfg(any(test, feature = "testing"))]
     pub fn has_key(&self, key: &Key) -> Result<bool> {
         let (store_type, sub_key) = StoreType::sub_key(key)?;
         self.tree(&store_type).subtree_has_key(&sub_key)
@@ -601,6 +641,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
             && self.ibc.validate()
             && self.pos.validate()
             && self.bridge_pool.validate()
+            && self.no_diff.validate()
         {
             let mut reconstructed = Smt::<H>::default();
             reconstructed.update(
@@ -619,6 +660,10 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
                 H::hash(StoreType::BridgePool.to_string()).into(),
                 self.bridge_pool.root().into(),
             )?;
+            reconstructed.update(
+                H::hash(StoreType::NoDiff.to_string()).into(),
+                self.no_diff.root().into(),
+            )?;
             reconstructed.update(
                 H::hash(StoreType::CommitData.to_string()).into(),
                 self.commit_data.0,
@@ -649,6 +694,7 @@ impl<H: StorageHasher + Default> MerkleTree<H> {
                 self.bridge_pool.root().into(),
                 self.bridge_pool.store(),
             ),
+            no_diff: (self.no_diff.root().into(), self.no_diff.store()),
             commit: self.commit_data.0,
         }
     }
@@ -803,6 +849,7 @@ pub struct MerkleTreeStoresRead {
     ibc: (Hash, AmtStore),
     pos: (Hash, SmtStore),
     bridge_pool: (KeccakHash, BridgePoolStore),
+    no_diff: (Hash, SmtStore),
     commit: Hash,
 }
 
@@ -815,6 +862,7 @@ impl MerkleTreeStoresRead {
             StoreType::Ibc => self.ibc.0 = root,
             StoreType::PoS => self.pos.0 = root,
             StoreType::BridgePool => self.bridge_pool.0 = root.into(),
+            StoreType::NoDiff => self.no_diff.0 = root,
             StoreType::CommitData => self.commit = root,
         }
     }
@@ -827,6 +875,7 @@ impl MerkleTreeStoresRead {
             Store::Ibc(store) => self.ibc.1 = store,
             Store::PoS(store) => self.pos.1 = store,
             Store::BridgePool(store) => self.bridge_pool.1 = store,
+            Store::NoDiff(store) => self.no_diff.1 = store,
             Store::CommitData => (),
         }
     }
@@ -839,6 +888,7 @@ impl MerkleTreeStoresRead {
             StoreType::Ibc => StoreRef::Ibc(&self.ibc.1),
             StoreType::PoS => StoreRef::PoS(&self.pos.1),
             StoreType::BridgePool => StoreRef::BridgePool(&self.bridge_pool.1),
+            StoreType::NoDiff => StoreRef::NoDiff(&self.no_diff.1),
             StoreType::CommitData => StoreRef::CommitData,
         }
     }
@@ -851,6 +901,7 @@ impl MerkleTreeStoresRead {
             StoreType::Ibc => self.ibc.0,
             StoreType::PoS => self.pos.0,
             StoreType::BridgePool => Hash(self.bridge_pool.0.0),
+            StoreType::NoDiff => self.no_diff.0,
             StoreType::CommitData => Hash(self.commit.0),
         }
     }
@@ -863,6 +914,7 @@ pub struct MerkleTreeStoresWrite<'a> {
     ibc: (Hash, &'a AmtStore),
     pos: (Hash, &'a SmtStore),
     bridge_pool: (Hash, &'a BridgePoolStore),
+    no_diff: (Hash, &'a SmtStore),
     commit: Hash,
 }
 
@@ -875,6 +927,7 @@ impl<'a> MerkleTreeStoresWrite<'a> {
             StoreType::Ibc => &self.ibc.0,
             StoreType::PoS => &self.pos.0,
             StoreType::BridgePool => &self.bridge_pool.0,
+            StoreType::NoDiff => &self.no_diff.0,
             StoreType::CommitData => &self.commit,
         }
     }
@@ -887,6 +940,7 @@ impl<'a> MerkleTreeStoresWrite<'a> {
             StoreType::Ibc => StoreRef::Ibc(self.ibc.1),
             StoreType::PoS => StoreRef::PoS(self.pos.1),
             StoreType::BridgePool => StoreRef::BridgePool(self.bridge_pool.1),
+            StoreType::NoDiff => StoreRef::NoDiff(self.no_diff.1),
             StoreType::CommitData => StoreRef::CommitData,
         }
     }

crates/node/src/shell/mod.rs

@@ -353,9 +353,9 @@ where
     event_log: EventLog,
 }
 
-/// Merkle tree storage key filter. Return `false` for keys that shouldn't be
-/// merklized.
-pub fn is_merklized_storage_key(key: &namada_sdk::storage::Key) -> bool {
+/// Storage key filter to store the diffs into the storage. Return `false` for
+/// keys whose diffs shouldn't be stored.
+pub fn is_key_diff_storable(key: &namada_sdk::storage::Key) -> bool {
     !(token::storage_key::is_masp_key(key)
         && *key != token::storage_key::masp_convert_anchor_key()
         && *key != token::storage_key::masp_token_map_key()
@@ -441,7 +441,7 @@ where
             chain_id.clone(),
             native_token,
             config.shell.storage_read_past_height_limit,
-            is_merklized_storage_key,
+            is_key_diff_storable,
         );
         let vp_wasm_cache_dir =
             base_dir.join(chain_id.as_str()).join("vp_wasm_cache");