Merge branch 'bat+brent/native-updatetype' (#2870)

* bat+brent/native-updatetype:
  changelog: add #2870
  fmt
  Update the migration example to perform migration on the conversion state
  [fix]: Added the column family to the query-db cli command
  specify the column family
  Added more type guardrails
  add conversion state token map to migration
  [fix]: Made migrating a single command
  [fix]: Fixed log lines from migrations and refactored the dsl implementation to be DRYer
  clippy and fmt
  Cleanupt up the foreign type macros. Tidied up some logging. Added a migration step to remove old keys
  fmt
  working
  fix iteration
  test state migration first commit

.changelog/unreleased/features/2870-native-updatetype.md

@@ -0,0 +1,2 @@
+- Implements state migration functionality.
+  ([\#2870](https://github.com/anoma/namada/pull/2870))

crates/apps/src/bin/namada-node/cli.rs

@@ -3,7 +3,7 @@
 use eyre::{Context, Result};
 use namada::core::time::{DateTimeUtc, Utc};
 use namada_apps::cli::{self, cmds};
-use namada_apps::config::ValidatorLocalConfig;
+use namada_apps::config::{Action, ActionAtHeight, ValidatorLocalConfig};
 use namada_apps::node::ledger;
 #[cfg(not(feature = "migrations"))]
 use namada_sdk::display_line;
@@ -48,13 +48,31 @@ pub fn main() -> Result<()> {
                          \"migrations\" feature."
                     )
                 }
-                let chain_ctx = ctx.take_chain_or_exit();
+                let mut chain_ctx = ctx.take_chain_or_exit();
                 #[cfg(feature = "migrations")]
                 ledger::update_db_keys(
-                    chain_ctx.config.ledger,
+                    chain_ctx.config.ledger.clone(),
                     args.updates,
                     args.dry_run,
                 );
+                if !args.dry_run {
+                    let wasm_dir = chain_ctx.wasm_dir();
+                    chain_ctx.config.ledger.shell.action_at_height =
+                        Some(ActionAtHeight {
+                            height: args.last_height + 2,
+                            action: Action::Halt,
+                        });
+                    std::env::set_var(
+                        "NAMADA_INITIAL_HEIGHT",
+                        args.last_height.to_string(),
+                    );
+                    // don't stop on panics
+                    let handle = std::thread::spawn(|| {
+                        ledger::run(chain_ctx.config.ledger, wasm_dir)
+                    });
+                    _ = handle.join();
+                    std::env::remove_var("NAMADA_INITIAL_HEIGHT");
+                }
             }
             cmds::Ledger::QueryDB(cmds::LedgerQueryDB(args)) => {
                 #[cfg(not(feature = "migrations"))]
@@ -66,7 +84,12 @@ pub fn main() -> Result<()> {
                 }
                 let chain_ctx = ctx.take_chain_or_exit();
                 #[cfg(feature = "migrations")]
-                ledger::query_db(chain_ctx.config.ledger, &args.key_hash_pairs);
+                ledger::query_db(
+                    chain_ctx.config.ledger,
+                    &args.key,
+                    &args.hash,
+                    &args.cf,
+                );
             }
         },
         cmds::NamadaNode::Config(sub) => match sub {

crates/apps/src/lib/cli.rs

@@ -3107,6 +3107,11 @@ pub mod args {
         arg_opt("success-sleep");
     pub const DATA_PATH_OPT: ArgOpt<PathBuf> = arg_opt("data-path");
     pub const DATA_PATH: Arg<PathBuf> = arg("data-path");
+    pub const DB_KEY: Arg<String> = arg("db-key");
+    pub const DB_COLUMN_FAMILY: ArgDefault<String> = arg_default(
+        "db-column-family",
+        DefaultFn(|| storage::SUBSPACE_CF.to_string()),
+    );
     pub const DECRYPT: ArgFlag = flag("decrypt");
     pub const DESCRIPTION_OPT: ArgOpt<String> = arg_opt("description");
     pub const DISPOSABLE_SIGNING_KEY: ArgFlag = flag("disposable-gas-payer");
@@ -3163,6 +3168,7 @@ pub mod args {
     pub const GENESIS_VALIDATOR_ADDRESS: Arg<EstablishedAddress> =
         arg("validator");
     pub const HALT_ACTION: ArgFlag = flag("halt");
+    pub const HASH: Arg<String> = arg("hash");
     pub const HASH_LIST: Arg<String> = arg("hash-list");
     pub const HD_DERIVATION_PATH: ArgDefault<String> =
         arg_default("hd-path", DefaultFn(|| "default".to_string()));
@@ -3177,8 +3183,7 @@ pub mod args {
          scheme is not supplied, it is assumed to be TCP.";
     pub const CONFIG_RPC_LEDGER_ADDRESS: ArgDefaultFromCtx<ConfigRpcAddress> =
         arg_default_from_ctx("node", DefaultFn(|| "".to_string()));
-    pub const KEY_HASH_LIST: ArgMulti<String, GlobPlus> =
-        arg_multi("key-hash-list");
+
     pub const LEDGER_ADDRESS: ArgDefault<Url> =
         arg("node").default(DefaultFn(|| {
             let raw = "http://127.0.0.1:26657";
@@ -3462,13 +3467,19 @@ pub mod args {
     pub struct LedgerUpdateDb {
         pub updates: PathBuf,
         pub dry_run: bool,
+        pub last_height: BlockHeight,
     }
 
     impl Args for LedgerUpdateDb {
         fn parse(matches: &ArgMatches) -> Self {
             let updates = PATH.parse(matches);
             let dry_run = DRY_RUN_TX.parse(matches);
-            Self { updates, dry_run }
+            let last_height = BLOCK_HEIGHT.parse(matches);
+            Self {
+                updates,
+                dry_run,
+                last_height,
+            }
         }
 
         fn def(app: App) -> App {
@@ -3479,43 +3490,46 @@ pub mod args {
                 "If set, applies the updates but does not persist them. Using \
                  for testing and debugging.",
             ))
+            .arg(
+                BLOCK_HEIGHT
+                    .def()
+                    .help("The height at which the hard fork is happening."),
+            )
         }
     }
 
     #[derive(Clone, Debug)]
     pub struct LedgerQueryDb {
-        pub key_hash_pairs: Vec<(storage::Key, [u8; 32])>,
+        pub key: storage::Key,
+        pub hash: [u8; 32],
+        pub cf: storage::DbColFam,
     }
 
     impl Args for LedgerQueryDb {
         fn parse(matches: &ArgMatches) -> Self {
-            let pairs = KEY_HASH_LIST
-                .parse(matches)
-                .into_iter()
-                .map(|pair| {
-                    let (hash, key) = pair
-                        .split_once(':')
-                        .expect("Key hash pairs must be colon separated.");
-                    let hash: [u8; 32] = HEXUPPER
-                        .decode(hash.to_uppercase().as_bytes())
-                        .unwrap()
-                        .try_into()
-                        .unwrap();
-                    let key = storage::Key::parse(key).unwrap();
-                    (key, hash)
-                })
-                .collect();
-
-            Self {
-                key_hash_pairs: pairs,
-            }
+            let key = storage::Key::parse(DB_KEY.parse(matches)).unwrap();
+            let hex_hash = HASH.parse(matches);
+            let hash: [u8; 32] = HEXUPPER
+                .decode(hex_hash.to_uppercase().as_bytes())
+                .unwrap()
+                .try_into()
+                .unwrap();
+            let cf =
+                storage::DbColFam::from_str(&DB_COLUMN_FAMILY.parse(matches))
+                    .unwrap();
+            Self { key, hash, cf }
         }
 
         fn def(app: App) -> App {
-            app.arg(KEY_HASH_LIST.def().help(
-                "A comma separated list of entries of the form \
-                 <hex_type_hash>:<key>.",
-            ))
+            app.arg(DB_KEY.def().help("A database key to query"))
+                .arg(HASH.def().help(
+                    "The hex encoded type hash of the value contained under \
+                     the provided key.",
+                ))
+                .arg(DB_COLUMN_FAMILY.def().help(
+                    "The column family under which the key is kept. Defaults \
+                     to the subspace column family if none is provided.",
+                ))
         }
     }
 

crates/apps/src/lib/node/ledger/mod.rs

@@ -13,12 +13,14 @@ use std::str::FromStr;
 use std::thread;
 
 use byte_unit::Byte;
+use data_encoding::HEXUPPER;
 use futures::future::TryFutureExt;
 use namada::core::storage::{BlockHeight, Key};
 use namada::core::time::DateTimeUtc;
 use namada::eth_bridge::ethers::providers::{Http, Provider};
 use namada::governance::storage::keys as governance_storage;
 use namada::state::DB;
+use namada::storage::DbColFam;
 use namada::tendermint::abci::request::CheckTxKind;
 use namada_sdk::state::StateRead;
 use once_cell::unsync::Lazy;
@@ -231,30 +233,38 @@ pub fn dump_db(
 }
 
 #[cfg(feature = "migrations")]
-pub fn query_db(config: config::Ledger, keys: &[(Key, [u8; 32])]) {
+pub fn query_db(
+    config: config::Ledger,
+    key: &Key,
+    type_hash: &[u8; 32],
+    cf: &DbColFam,
+) {
+    use namada_sdk::migrations::DBUpdateVisitor;
     let chain_id = config.chain_id;
     let db_path = config.shell.db_dir(&chain_id);
 
     let db = storage::PersistentDB::open(db_path, None);
-    for (key, type_hash) in keys {
-        let bytes = db.read_subspace_val(key).unwrap();
-        if let Some(bytes) = bytes {
-            let deserializer = namada_migrations::get_deserializer(type_hash)
-                .unwrap_or_else(|| {
-                    panic!(
-                        "Could not find a deserializer for the type provided \
-                         with key <{}>",
-                        key
-                    )
-                });
-            let value = deserializer(bytes).unwrap_or_else(|| {
-                panic!("Unable to deserialize the value under key <{}>", key)
-            });
-            tracing::info!("Key <{}>: {}", key, value);
-        } else {
-            tracing::info!("Key <{}> is not present in storage.", key);
-        }
-    }
+    let db_visitor = storage::RocksDBUpdateVisitor::new(&db);
+    let bytes = db_visitor.read(key, cf).unwrap();
+
+    let deserializer = namada_migrations::get_deserializer(type_hash)
+        .unwrap_or_else(|| {
+            panic!(
+                "Could not find a deserializer for the type provided with key \
+                 <{}>",
+                key
+            )
+        });
+    let hex_bytes = HEXUPPER.encode(&bytes);
+    let value = deserializer(bytes).unwrap_or_else(|| {
+        panic!("Unable to deserialize the value under key <{}>", key)
+    });
+    tracing::info!(
+        "Key <{}>: {}\nThe value in bytes is {}",
+        key,
+        value,
+        hex_bytes
+    );
 }
 
 /// Change the funds of an account in-place. Use with
@@ -287,7 +297,7 @@ pub fn update_db_keys(config: config::Ledger, updates: PathBuf, dry_run: bool) {
             e => {
                 tracing::error!(
                     "Attempt to write to key/pattern <{}> failed.",
-                    change.key()
+                    change.pattern()
                 );
                 e.unwrap();
             }