Storage migration (#1057)

* Brainstorm

* Basic idea

* Add encrypted db

* Seems to work

* Define types in migration, refactor

* where in stronghold is the data?

* Make stronghold snapshot migration work

* Add migration to bindings, add get_chrysalis_data()

* Save chrysalis data from snapshot in db

* Address some TODOs

* Address todos

* Add chrysalis-backup-work-factor-0.stronghold

* Fix wasm

* Don't error on ClientDataNotPresent

* Clippy

* Address review comments

* Remove extra function

* Update bindings/nodejs/CHANGELOG.md

Co-authored-by: Thibault Martinez <[email protected]>

* Remove `from`

* Clippy

* Changelog

* Update sdk/src/wallet/storage/constants.rs

* Move function to storage

* Address review comments

* Address review comments

* Review suggestions

* Address review comments

* Add db_encryption_key to Rust, export migrateDbChrysalisToStardust and add nodejs example

* Single import, lint

* Address review comments

---------

Co-authored-by: Thibault Martinez <[email protected]>
Co-authored-by: Thibault Martinez <[email protected]>

bindings/core/src/method/wallet.rs

@@ -52,6 +52,11 @@ pub enum WalletMethod {
     /// Read accounts.
     /// Expected response: [`Accounts`](crate::Response::Accounts)
     GetAccounts,
+    /// Get historic chrysalis data.
+    /// Expected response: [`ChrysalisData`](crate::Response::ChrysalisData)
+    #[cfg(feature = "storage")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "storage")))]
+    GetChrysalisData,
     /// Consume an account method.
     /// Returns [`Response`](crate::Response)
     #[serde(rename_all = "camelCase")]

bindings/core/src/method_handler/wallet.rs

@@ -63,6 +63,7 @@ pub(crate) async fn call_wallet_method_internal(wallet: &Wallet, method: WalletM
             }
             Response::Accounts(account_dtos)
         }
+        WalletMethod::GetChrysalisData => Response::ChrysalisData(wallet.get_chrysalis_data().await?),
         WalletMethod::CallAccountMethod { account_id, method } => {
             let account = wallet.get_account(account_id).await?;
             call_account_method_internal(&account, method).await?

bindings/core/src/response.rs

@@ -1,6 +1,7 @@
 // Copyright 2023 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+use std::collections::HashMap;
 #[cfg(not(target_family = "wasm"))]
 use std::collections::HashSet;
 
@@ -46,7 +47,6 @@ use serde::Serialize;
 use {
     iota_sdk::types::api::plugins::participation::types::{ParticipationEventId, ParticipationEventStatus},
     iota_sdk::wallet::account::{AccountParticipationOverview, ParticipationEventWithNodes},
-    std::collections::HashMap,
 };
 
 use crate::{error::Error, OmittedDebug};
@@ -308,6 +308,9 @@ pub enum Response {
     /// - [`AddressesWithUnspentOutputs`](crate::method::AccountMethod::AddressesWithUnspentOutputs)
     AddressesWithUnspentOutputs(Vec<AddressWithUnspentOutputs>),
     /// Response for:
+    /// - [`GetChrysalisData`](crate::method::WalletMethod::GetChrysalisData)
+    ChrysalisData(Option<HashMap<String, String>>),
+    /// Response for:
     /// - [`MinimumRequiredStorageDeposit`](crate::method::ClientMethod::MinimumRequiredStorageDeposit)
     /// - [`ComputeStorageDeposit`](crate::method::UtilsMethod::ComputeStorageDeposit)
     MinimumRequiredStorageDeposit(String),

bindings/nodejs/CHANGELOG.md

@@ -27,6 +27,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## 1.0.6 - 2023-08-25
 
+### Added
+
+- `migrateDbChrysalisToStardust` function;
+- `Wallet::getChrysalisData` method;
+
 ### Fixed
 
 - `Account::prepareBurn()` return type;

bindings/nodejs/examples/wallet/migrate-db-chrysalis-to-stardust.ts

@@ -0,0 +1,53 @@
+// Copyright 2023 IOTA Stiftung
+// SPDX-License-Identifier: Apache-2.0
+
+import { WalletOptions, Wallet, migrateDbChrysalisToStardust } from '@iota/sdk';
+require('dotenv').config({ path: '.env' });
+
+// Run with command:
+// yarn run-example wallet/migrate-db-chrysalis-to-stardust.ts
+
+const walletDbPath = './chrysalis-db';
+
+async function run() {
+    const { initLogger } = require('@iota/sdk');
+    initLogger({
+        name: './wallet.log',
+        levelFilter: 'debug',
+        targetExclusions: ['h2', 'hyper', 'rustls'],
+    });
+    if (!process.env.NODE_URL) {
+        throw new Error('.env NODE_URL is undefined, see .env.example');
+    }
+    if (!process.env.STRONGHOLD_PASSWORD) {
+        throw new Error(
+            '.env STRONGHOLD_PASSWORD is undefined, see .env.example',
+        );
+    }
+
+    migrateDbChrysalisToStardust(walletDbPath, 'password');
+
+    const walletOptions: WalletOptions = {
+        storagePath: walletDbPath,
+        clientOptions: {
+            nodes: [process.env.NODE_URL],
+        },
+        secretManager: {
+            stronghold: {
+                snapshotPath: walletDbPath + 'wallet.stronghold',
+                password: process.env.STRONGHOLD_PASSWORD,
+            },
+        },
+    };
+    console.log(walletOptions);
+    const wallet = new Wallet(walletOptions);
+
+    // Accounts migrated from the Chrysalis db
+    const accounts = await wallet.getAccounts();
+    console.log(accounts);
+
+    const historicChrysalisData = await wallet.getChrysalisData();
+    console.log(historicChrysalisData);
+}
+
+run().then(() => process.exit());

bindings/nodejs/lib/bindings.ts

@@ -27,6 +27,7 @@ const {
     getClientFromWallet,
     getSecretManagerFromWallet,
     migrateStrongholdSnapshotV2ToV3,
+    migrateDbChrysalisToStardust,
 } = addon;
 
 const callClientMethodAsync = (
@@ -116,4 +117,5 @@ export {
     getSecretManagerFromWallet,
     listenMqtt,
     migrateStrongholdSnapshotV2ToV3,
+    migrateDbChrysalisToStardust,
 };

bindings/nodejs/lib/types/wallet/bridge/index.ts

@@ -64,6 +64,7 @@ import type {
     __GetAccountMethod__,
     __GetAccountIndexesMethod__,
     __GetAccountsMethod__,
+    __GetChrysalisDataMethod__,
     __GetLedgerNanoStatusMethod__,
     __GenerateEd25519AddressMethod__,
     __IsStrongholdPasswordAvailableMethod__,
@@ -153,6 +154,7 @@ export type __Method__ =
     | __GetAccountMethod__
     | __GetAccountIndexesMethod__
     | __GetAccountsMethod__
+    | __GetChrysalisDataMethod__
     | __GetLedgerNanoStatusMethod__
     | __GenerateEd25519AddressMethod__
     | __IsStrongholdPasswordAvailableMethod__

bindings/nodejs/lib/types/wallet/bridge/wallet.ts

@@ -55,6 +55,10 @@ export type __GetAccountMethod__ = {
     data: { accountId: AccountId };
 };
 
+export type __GetChrysalisDataMethod__ = {
+    name: 'getChrysalisData';
+};
+
 export type __GetLedgerNanoStatusMethod__ = {
     name: 'getLedgerNanoStatus';
 };

bindings/nodejs/lib/wallet/index.ts

@@ -5,3 +5,4 @@ export * from './account';
 export * from './wallet';
 export * from './wallet-method-handler';
 export * from '../types/wallet';
+export { migrateDbChrysalisToStardust } from '../bindings';

bindings/nodejs/lib/wallet/wallet.ts

@@ -148,6 +148,17 @@ export class Wallet {
         return this.methodHandler.getClient();
     }
 
+    /**
+     * Get chrysalis data.
+     */
+    async getChrysalisData(): Promise<Record<string, string>> {
+        const response = await this.methodHandler.callMethod({
+            name: 'getChrysalisData',
+        });
+
+        return JSON.parse(response).payload;
+    }
+
     /**
      * Get secret manager.
      */

bindings/nodejs/src/lib.rs

@@ -65,6 +65,7 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> {
     cx.export_function("getClientFromWallet", wallet::get_client)?;
     cx.export_function("getSecretManagerFromWallet", wallet::get_secret_manager)?;
     cx.export_function("listenWallet", wallet::listen_wallet)?;
+    cx.export_function("migrateDbChrysalisToStardust", wallet::migrate_db_chrysalis_to_stardust)?;
 
     Ok(())
 }

bindings/nodejs/src/wallet.rs

@@ -7,6 +7,7 @@ use iota_sdk_bindings_core::{
     call_wallet_method as rust_call_wallet_method,
     iota_sdk::wallet::{
         events::types::{Event, WalletEventType},
+        migration::migrate_db_chrysalis_to_stardust as rust_migrate_db_chrysalis_to_stardust,
         Wallet,
     },
     Response, Result, WalletMethod, WalletOptions,
@@ -232,3 +233,27 @@ pub fn get_secret_manager(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     Ok(promise)
 }
+
+pub fn migrate_db_chrysalis_to_stardust(mut cx: FunctionContext) -> JsResult<JsPromise> {
+    let storage_path = cx.argument::<JsString>(0)?.value(&mut cx);
+    let password = cx
+        .argument_opt(1)
+        .map(|opt| opt.downcast_or_throw::<JsString, _>(&mut cx))
+        .transpose()?
+        .map(|opt| opt.value(&mut cx))
+        .map(Into::into);
+
+    let channel = cx.channel();
+    let (deferred, promise) = cx.promise();
+    crate::RUNTIME.spawn(async move {
+        if let Err(err) = rust_migrate_db_chrysalis_to_stardust(storage_path, password, None).await {
+            deferred.settle_with(&channel, move |mut cx| {
+                cx.error(serde_json::to_string(&Response::Error(err.into())).expect("json to string error"))
+            });
+        } else {
+            deferred.settle_with(&channel, move |mut cx| Ok(cx.boxed(())));
+        }
+    });
+
+    Ok(promise)
+}

bindings/wasm/lib/bindings.ts

@@ -9,7 +9,7 @@ import { __UtilsMethods__ } from './utils';
 // Import needs to be in a single line, otherwise it breaks
 // prettier-ignore
 // @ts-ignore: path is set to match runtime transpiled js path when bundled.
-import { initLogger, createClient, destroyClient, createSecretManager, createWallet, callClientMethodAsync, callSecretManagerMethodAsync, callUtilsMethodRust, callWalletMethodAsync, destroyWallet, listenWalletAsync, getClientFromWallet, getSecretManagerFromWallet, listenMqtt, migrateStrongholdSnapshotV2ToV3 } from '../wasm/iota_sdk_wasm';
+import { initLogger, createClient, destroyClient, createSecretManager, createWallet, callClientMethodAsync, callSecretManagerMethodAsync, callUtilsMethodRust, callWalletMethodAsync, destroyWallet, listenWalletAsync, getClientFromWallet, getSecretManagerFromWallet, listenMqtt, migrateStrongholdSnapshotV2ToV3, migrateDbChrysalisToStardust } from '../wasm/iota_sdk_wasm';
 
 const callUtilsMethod = (method: __UtilsMethods__): any => {
     const response = JSON.parse(callUtilsMethodRust(JSON.stringify(method)));
@@ -36,4 +36,5 @@ export {
     getSecretManagerFromWallet,
     listenMqtt,
     migrateStrongholdSnapshotV2ToV3,
+    migrateDbChrysalisToStardust,
 };

bindings/wasm/src/wallet.rs

@@ -141,3 +141,14 @@ pub async fn listen_wallet(
 
     Ok(JsValue::UNDEFINED)
 }
+
+/// Rocksdb chrysalis migration is not supported for WebAssembly bindings.
+///
+/// Throws an error if called, only included for compatibility
+/// with the Node.js bindings TypeScript definitions.
+#[wasm_bindgen(js_name = migrateDbChrysalisToStardust)]
+pub fn migrate_db_chrysalis_to_stardust(_storage_path: String, _password: Option<String>) -> Result<(), JsValue> {
+    let js_error = js_sys::Error::new("Rocksdb chrysalis migration is not supported for WebAssembly");
+
+    Err(JsValue::from(js_error))
+}

sdk/CHANGELOG.md

@@ -21,6 +21,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## 1.0.3 - 2023-MM-DD
 
+### Added
+
+- `migrate_db_chrysalis_to_stardust()` function;
+- `Wallet::get_chrysalis_data()` method;
+
 ### Fixed
 
 - `Clients` returning the default protocol parameters when multiple `Client` instances are used;

sdk/src/client/stronghold/common.rs

@@ -25,7 +25,7 @@ pub(super) const DERIVE_OUTPUT_RECORD_PATH: &[u8] = b"iota-wallet-derived";
 /// The client path for the seed.
 ///
 /// The value has been hard-coded historically.
-pub(super) const PRIVATE_DATA_CLIENT_PATH: &[u8] = b"iota_seed";
+pub(crate) const PRIVATE_DATA_CLIENT_PATH: &[u8] = b"iota_seed";
 
 /// The path for the user-data encryption key for the Stronghold store.
 pub(super) const USERDATA_STORE_KEY_RECORD_PATH: &[u8] = b"userdata-store-key";

sdk/src/client/stronghold/mod.rs

@@ -67,7 +67,7 @@ use tokio::{
 };
 use zeroize::Zeroizing;
 
-use self::common::PRIVATE_DATA_CLIENT_PATH;
+pub(crate) use self::common::PRIVATE_DATA_CLIENT_PATH;
 pub use self::error::Error;
 use super::{storage::StorageAdapter, utils::Password};
 

sdk/src/wallet/core/operations/storage.rs

@@ -13,8 +13,8 @@ mod storage_stub {
         },
         wallet::{
             core::builder::dto::WalletBuilderDto,
-            storage::constants::{SECRET_MANAGER_KEY, WALLET_INDEXATION_KEY},
-            WalletBuilder,
+            storage::constants::{CHRYSALIS_STORAGE_KEY, SECRET_MANAGER_KEY, WALLET_INDEXATION_KEY},
+            Wallet, WalletBuilder,
         },
     };
 
@@ -84,6 +84,14 @@ mod storage_stub {
             Ok(res.map(Into::into))
         }
     }
+
+    impl Wallet {
+        pub async fn get_chrysalis_data(
+            &self,
+        ) -> crate::wallet::Result<Option<std::collections::HashMap<String, String>>> {
+            self.storage_manager.read().await.get(CHRYSALIS_STORAGE_KEY).await
+        }
+    }
 }
 #[cfg(not(feature = "storage"))]
 mod storage_stub {

sdk/src/wallet/core/operations/stronghold_backup/mod.rs

@@ -13,10 +13,11 @@ use crate::wallet::WalletBuilder;
 use crate::{
     client::{
         secret::{stronghold::StrongholdSecretManager, SecretManager, SecretManagerConfig, SecretManagerDto},
+        storage::StorageAdapter,
         utils::Password,
     },
     types::block::address::Hrp,
-    wallet::{Account, Wallet},
+    wallet::{storage::constants::CHRYSALIS_STORAGE_KEY, Account, Wallet},
 };
 
 impl Wallet {
@@ -102,7 +103,7 @@ impl Wallet {
             .password(stronghold_password.clone())
             .build(backup_path.clone())?;
 
-        let (read_client_options, read_coin_type, read_secret_manager, read_accounts) =
+        let (read_client_options, read_coin_type, read_secret_manager, read_accounts, chrysalis_data) =
             read_data_from_stronghold_snapshot::<SecretManager>(&new_stronghold).await?;
 
         // If the coin type is not matching the current one, then the addresses in the accounts will also not be
@@ -204,6 +205,13 @@ impl Wallet {
             for account in accounts.iter() {
                 account.save(None).await?;
             }
+            if let Some(chrysalis_data) = chrysalis_data {
+                self.storage_manager
+                    .read()
+                    .await
+                    .set(CHRYSALIS_STORAGE_KEY, &chrysalis_data)
+                    .await?;
+            }
         }
 
         Ok(())
@@ -272,7 +280,7 @@ impl Wallet<StrongholdSecretManager> {
             .password(stronghold_password.clone())
             .build(backup_path.clone())?;
 
-        let (read_client_options, read_coin_type, read_secret_manager, read_accounts) =
+        let (read_client_options, read_coin_type, read_secret_manager, read_accounts, chrysalis_data) =
             read_data_from_stronghold_snapshot::<StrongholdSecretManager>(&new_stronghold).await?;
 
         // If the coin type is not matching the current one, then the addresses in the accounts will also not be
@@ -367,6 +375,13 @@ impl Wallet<StrongholdSecretManager> {
             for account in accounts.iter() {
                 account.save(None).await?;
             }
+            if let Some(chrysalis_data) = chrysalis_data {
+                self.storage_manager
+                    .read()
+                    .await
+                    .set(CHRYSALIS_STORAGE_KEY, &chrysalis_data)
+                    .await?;
+            }
         }
 
         Ok(())