feat: add sync and full_scan methods on esplora client

bdk-android/lib/src/androidTest/kotlin/org/bitcoindevkit/LiveTxBuilderTest.kt

@@ -26,7 +26,8 @@ class LiveTxBuilderTest {
         val descriptor = Descriptor("wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/0h/0/*)", Network.TESTNET)
         val wallet = Wallet(descriptor, null, persistenceFilePath, Network.TESTNET)
         val esploraClient = EsploraClient("https://esplora.testnet.kuutamo.cloud/")
-        val update = esploraClient.fullScan(wallet, 10uL, 1uL)
+        val fullScanRequest: FullScanRequest = wallet.startFullScan()
+        val update = esploraClient.fullScan(fullScanRequest, 10uL, 1uL)
         wallet.applyUpdate(update)
         println("Balance: ${wallet.getBalance().total}")
 
@@ -48,7 +49,8 @@ class LiveTxBuilderTest {
         val changeDescriptor = Descriptor("wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/0h/1/*)", Network.TESTNET)
         val wallet = Wallet(externalDescriptor, changeDescriptor, persistenceFilePath, Network.TESTNET)
         val esploraClient = EsploraClient("https://esplora.testnet.kuutamo.cloud/")
-        val update = esploraClient.fullScan(wallet, 10uL, 1uL)
+        val fullScanRequest: FullScanRequest = wallet.startFullScan()
+        val update = esploraClient.fullScan(fullScanRequest, 10uL, 1uL)
         wallet.applyUpdate(update)
         println("Balance: ${wallet.getBalance().total}")
 

bdk-android/lib/src/androidTest/kotlin/org/bitcoindevkit/LiveWalletTest.kt

@@ -26,7 +26,8 @@ class LiveWalletTest {
         val descriptor: Descriptor = Descriptor("wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/0h/0/*)", Network.TESTNET)
         val wallet: Wallet = Wallet(descriptor, null, persistenceFilePath, Network.TESTNET)
         val esploraClient: EsploraClient = EsploraClient("https://esplora.testnet.kuutamo.cloud/")
-        val update = esploraClient.fullScan(wallet, 10uL, 1uL)
+        val fullScanRequest: FullScanRequest = wallet.startFullScan()
+        val update = esploraClient.fullScan(fullScanRequest, 10uL, 1uL)
         wallet.applyUpdate(update)
         println("Balance: ${wallet.getBalance().total}")
         val balance: Balance = wallet.getBalance()
@@ -49,14 +50,15 @@ class LiveWalletTest {
         val descriptor = Descriptor("wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/0h/0/*)", Network.TESTNET)
         val wallet = Wallet(descriptor, null, persistenceFilePath, Network.TESTNET)
         val esploraClient = EsploraClient("https://esplora.testnet.kuutamo.cloud/")
-        val update = esploraClient.fullScan(wallet, 10uL, 1uL)
+        val fullScanRequest: FullScanRequest = wallet.startFullScan()
+        val update = esploraClient.fullScan(fullScanRequest, 10uL, 1uL)
 
         wallet.applyUpdate(update)
         println("Balance: ${wallet.getBalance().total}")
-        println("New address: ${wallet.getAddress(AddressIndex.New).address}")
+        println("New address: ${wallet.revealNextAddress(KeychainKind.EXTERNAL).address}")
 
         assert(wallet.getBalance().total > 0uL) {
-            "Wallet balance must be greater than 0! Please send funds to ${wallet.getAddress(AddressIndex.New).address} and try again."
+            "Wallet balance must be greater than 0! Please send funds to ${wallet.revealNextAddress(KeychainKind.EXTERNAL).address} and try again."
         }
 
         val recipient: Address = Address("tb1qrnfslnrve9uncz9pzpvf83k3ukz22ljgees989", Network.TESTNET)

bdk-android/lib/src/androidTest/kotlin/org/bitcoindevkit/OfflineWalletTest.kt

@@ -44,7 +44,7 @@ class OfflineWalletTest {
             persistenceFilePath,
             Network.TESTNET
         )
-        val addressInfo: AddressInfo = wallet.getAddress(AddressIndex.New)
+        val addressInfo: AddressInfo = wallet.revealNextAddress(KeychainKind.EXTERNAL)
 
         assertTrue(addressInfo.address.isValidForNetwork(Network.TESTNET), "Address is not valid for testnet network")
         assertTrue(addressInfo.address.isValidForNetwork(Network.SIGNET), "Address is not valid for signet network")

bdk-ffi/src/bdk.udl

@@ -238,6 +238,10 @@ dictionary CanonicalTx {
   ChainPosition chain_position;
 };
 
+interface FullScanRequest {};
+
+interface SyncRequest {};
+
 // ------------------------------------------------------------------------
 // bdk crate - wallet module
 // ------------------------------------------------------------------------
@@ -283,6 +287,10 @@ interface Wallet {
   sequence<LocalOutput> list_unspent();
 
   sequence<LocalOutput> list_output();
+
+  FullScanRequest start_full_scan();
+
+  SyncRequest start_sync_with_revealed_spks();
 };
 
 interface Update {};
@@ -429,7 +437,10 @@ interface EsploraClient {
   constructor(string url);
 
   [Throws=EsploraError]
-  Update full_scan(Wallet wallet, u64 stop_gap, u64 parallel_requests);
+  Update full_scan(FullScanRequest full_scan_request, u64 stop_gap, u64 parallel_requests);
+
+  [Throws=EsploraError]
+  Update sync(SyncRequest sync_request, u64 parallel_requests);
 
   [Throws=EsploraError]
   void broadcast([ByRef] Transaction transaction);

bdk-ffi/src/error.rs

@@ -1,28 +1,23 @@
 use crate::bitcoin::OutPoint;
 
+use bdk::bitcoin::address::ParseError;
+use bdk::bitcoin::bip32::Error as BdkBip32Error;
 use bdk::bitcoin::psbt::PsbtParseError as BdkPsbtParseError;
 use bdk::bitcoin::Network;
+use bdk::chain;
 use bdk::chain::tx_graph::CalculateFeeError as BdkCalculateFeeError;
 use bdk::descriptor::DescriptorError as BdkDescriptorError;
+use bdk::keys::bip39::Error as BdkBip39Error;
+use bdk::miniscript::descriptor::DescriptorKeyParseError as BdkDescriptorKeyParseError;
 use bdk::wallet::error::BuildFeeBumpError;
+use bdk::wallet::error::CreateTxError as BdkCreateTxError;
 use bdk::wallet::signer::SignerError as BdkSignerError;
 use bdk::wallet::tx_builder::AddUtxoError;
 use bdk::wallet::NewOrLoadError;
 use bdk_esplora::esplora_client::{Error as BdkEsploraError, Error};
 use bdk_file_store::FileError as BdkFileError;
-use bdk_file_store::IterError;
 use bitcoin_internals::hex::display::DisplayHex;
 
-use crate::error::bip32::Error as BdkBip32Error;
-use bdk::bitcoin::address::ParseError;
-use bdk::keys::bip39::Error as BdkBip39Error;
-use bdk::miniscript::descriptor::DescriptorKeyParseError as BdkDescriptorKeyParseError;
-
-use bdk::bitcoin::bip32;
-
-use bdk::chain;
-
-use bdk::wallet::error::CreateTxError as BdkCreateTxError;
 use std::convert::TryInto;
 
 // ------------------------------------------------------------------------
@@ -406,6 +401,7 @@ pub enum TxidParseError {
     InvalidTxid { txid: String },
 }
 
+// This error combines the Rust bdk::wallet::NewOrLoadError and bdk_file_store::FileError
 #[derive(Debug, thiserror::Error)]
 pub enum WalletCreationError {
     #[error("io error trying to read file: {error_message}")]
@@ -417,17 +413,14 @@ pub enum WalletCreationError {
     #[error("error with descriptor")]
     Descriptor,
 
-    #[error("failed to write to persistence")]
-    Write,
-
-    #[error("failed to load from persistence")]
-    Load,
+    #[error("failed to either write to or load from persistence, {error_message}")]
+    Persist { error_message: String },
 
     #[error("wallet is not initialized, persistence backend is empty")]
     NotInitialized,
 
-    #[error("loaded genesis hash does not match the expected one")]
-    LoadedGenesisDoesNotMatch,
+    #[error("loaded genesis hash '{got}' does not match the expected one '{expected}'")]
+    LoadedGenesisDoesNotMatch { expected: String, got: String },
 
     #[error("loaded network type is not {expected}, got {got:?}")]
     LoadedNetworkDoesNotMatch {
@@ -557,8 +550,8 @@ impl From<chain::local_chain::CannotConnectError> for CannotConnectError {
     }
 }
 
-impl From<BdkCreateTxError<std::io::Error>> for CreateTxError {
-    fn from(error: BdkCreateTxError<std::io::Error>) -> Self {
+impl From<BdkCreateTxError> for CreateTxError {
+    fn from(error: BdkCreateTxError) -> Self {
         match error {
             BdkCreateTxError::Descriptor(e) => CreateTxError::Descriptor {
                 error_message: e.to_string(),
@@ -748,6 +741,44 @@ impl From<BdkEsploraError> for EsploraError {
     }
 }
 
+impl From<Box<BdkEsploraError>> for EsploraError {
+    fn from(error: Box<BdkEsploraError>) -> Self {
+        match *error {
+            BdkEsploraError::Minreq(e) => EsploraError::Minreq {
+                error_message: e.to_string(),
+            },
+            BdkEsploraError::HttpResponse { status, message } => EsploraError::HttpResponse {
+                status,
+                error_message: message,
+            },
+            BdkEsploraError::Parsing(e) => EsploraError::Parsing {
+                error_message: e.to_string(),
+            },
+            Error::StatusCode(e) => EsploraError::StatusCode {
+                error_message: e.to_string(),
+            },
+            BdkEsploraError::BitcoinEncoding(e) => EsploraError::BitcoinEncoding {
+                error_message: e.to_string(),
+            },
+            BdkEsploraError::HexToArray(e) => EsploraError::HexToArray {
+                error_message: e.to_string(),
+            },
+            BdkEsploraError::HexToBytes(e) => EsploraError::HexToBytes {
+                error_message: e.to_string(),
+            },
+            BdkEsploraError::TransactionNotFound(_) => EsploraError::TransactionNotFound,
+            BdkEsploraError::HeaderHeightNotFound(height) => {
+                EsploraError::HeaderHeightNotFound { height }
+            }
+            BdkEsploraError::HeaderHashNotFound(_) => EsploraError::HeaderHashNotFound,
+            Error::InvalidHttpHeaderName(name) => EsploraError::InvalidHttpHeaderName { name },
+            BdkEsploraError::InvalidHttpHeaderValue(value) => {
+                EsploraError::InvalidHttpHeaderValue { value }
+            }
+        }
+    }
+}
+
 impl From<bdk::bitcoin::psbt::ExtractTxError> for ExtractTxError {
     fn from(error: bdk::bitcoin::psbt::ExtractTxError) -> Self {
         match error {
@@ -855,15 +886,19 @@ impl From<BdkFileError> for WalletCreationError {
     }
 }
 
-impl From<NewOrLoadError<std::io::Error, IterError>> for WalletCreationError {
-    fn from(error: NewOrLoadError<std::io::Error, IterError>) -> Self {
+impl From<NewOrLoadError> for WalletCreationError {
+    fn from(error: NewOrLoadError) -> Self {
         match error {
             NewOrLoadError::Descriptor(_) => WalletCreationError::Descriptor,
-            NewOrLoadError::Write(_) => WalletCreationError::Write,
-            NewOrLoadError::Load(_) => WalletCreationError::Load,
+            NewOrLoadError::Persist(e) => WalletCreationError::Persist {
+                error_message: e.to_string(),
+            },
             NewOrLoadError::NotInitialized => WalletCreationError::NotInitialized,
-            NewOrLoadError::LoadedGenesisDoesNotMatch { .. } => {
-                WalletCreationError::LoadedGenesisDoesNotMatch
+            NewOrLoadError::LoadedGenesisDoesNotMatch { expected, got } => {
+                WalletCreationError::LoadedGenesisDoesNotMatch {
+                    expected: expected.to_string(),
+                    got: format!("{:?}", got),
+                }
             }
             NewOrLoadError::LoadedNetworkDoesNotMatch { expected, got } => {
                 WalletCreationError::LoadedNetworkDoesNotMatch { expected, got }
@@ -1544,20 +1579,21 @@ mod test {
                 "error with descriptor".to_string(),
             ),
             (
-                WalletCreationError::Write,
-                "failed to write to persistence".to_string(),
-            ),
-            (
-                WalletCreationError::Load,
-                "failed to load from persistence".to_string(),
+                WalletCreationError::Persist {
+                    error_message: "persistence error".to_string(),
+                },
+                "failed to either write to or load from persistence, persistence error".to_string(),
             ),
             (
                 WalletCreationError::NotInitialized,
                 "wallet is not initialized, persistence backend is empty".to_string(),
             ),
             (
-                WalletCreationError::LoadedGenesisDoesNotMatch,
-                "loaded genesis hash does not match the expected one".to_string(),
+                WalletCreationError::LoadedGenesisDoesNotMatch {
+                    expected: "abc".to_string(),
+                    got: "def".to_string(),
+                },
+                "loaded genesis hash 'def' does not match the expected one 'abc'".to_string(),
             ),
             (
                 WalletCreationError::LoadedNetworkDoesNotMatch {

bdk-ffi/src/esplora.rs

@@ -1,9 +1,15 @@
+use crate::bitcoin::Transaction;
 use crate::error::EsploraError;
-use crate::wallet::{Update, Wallet};
+use crate::types::{FullScanRequest, SyncRequest};
+use crate::wallet::Update;
+use std::collections::BTreeMap;
 
-use crate::bitcoin::Transaction;
 use bdk::bitcoin::Transaction as BdkTransaction;
-use bdk::wallet::Update as BdkUpdate;
+use bdk::chain::spk_client::FullScanRequest as BdkFullScanRequest;
+use bdk::chain::spk_client::FullScanResult as BdkFullScanResult;
+use bdk::chain::spk_client::SyncRequest as BdkSyncRequest;
+use bdk::chain::spk_client::SyncResult as BdkSyncResult;
+use bdk::KeychainKind;
 use bdk_esplora::esplora_client::{BlockingClient, Builder};
 use bdk_esplora::EsploraExt;
 
@@ -17,40 +23,48 @@ impl EsploraClient {
         Self(client)
     }
 
-    // This is a temporary solution for scanning. The long-term solution involves not passing
-    // the wallet to the client at all.
     pub fn full_scan(
         &self,
-        wallet: Arc<Wallet>,
+        request: Arc<FullScanRequest>,
         stop_gap: u64,
         parallel_requests: u64,
     ) -> Result<Arc<Update>, EsploraError> {
-        let wallet = wallet.get_wallet();
-
-        let previous_tip = wallet.latest_checkpoint();
-        let keychain_spks = wallet.all_unbounded_spk_iters().into_iter().collect();
-
-        let (update_graph, last_active_indices) = self
-            .0
-            .full_scan(keychain_spks, stop_gap as usize, parallel_requests as usize)
-            .map_err(|e| EsploraError::from(*e))?;
-
-        let missing_heights = update_graph.missing_heights(wallet.local_chain());
-        let chain_update = self
-            .0
-            .update_local_chain(previous_tip, missing_heights)
-            .map_err(|e| EsploraError::from(*e))?;
-
-        let update = BdkUpdate {
-            last_active_indices,
-            graph: update_graph,
-            chain: Some(chain_update),
+        // using option and take is not ideal but the only way to take full ownership of the request
+        // TODO: if the option is None should throw error like "already consumed request" or "invalid request"
+        let request: BdkFullScanRequest<KeychainKind> = request.0.lock().unwrap().take().unwrap();
+
+        let result: BdkFullScanResult<KeychainKind> =
+            self.0
+                .full_scan(request, stop_gap as usize, parallel_requests as usize)?;
+
+        let update = bdk::wallet::Update {
+            last_active_indices: result.last_active_indices,
+            graph: result.graph_update,
+            chain: Some(result.chain_update),
         };
 
         Ok(Arc::new(Update(update)))
     }
 
-    // pub fn sync();
+    pub fn sync(
+        &self,
+        request: Arc<SyncRequest>,
+        parallel_requests: u64,
+    ) -> Result<Arc<Update>, EsploraError> {
+        // using option and take is not ideal but the only way to take full ownership of the request
+        // TODO: if the option is None should throw error like "already consumed request" or "invalid request"
+        let request: BdkSyncRequest = request.0.lock().unwrap().take().unwrap();
+
+        let result: BdkSyncResult = self.0.sync(request, parallel_requests as usize)?;
+
+        let update = bdk::wallet::Update {
+            last_active_indices: BTreeMap::default(),
+            graph: result.graph_update,
+            chain: Some(result.chain_update),
+        };
+
+        Ok(Arc::new(Update(update)))
+    }
 
     pub fn broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
         let bdk_transaction: BdkTransaction = transaction.into();

bdk-ffi/src/lib.rs

@@ -40,8 +40,10 @@ use crate::types::AddressInfo;
 use crate::types::Balance;
 use crate::types::CanonicalTx;
 use crate::types::ChainPosition;
+use crate::types::FullScanRequest;
 use crate::types::LocalOutput;
 use crate::types::ScriptAmount;
+use crate::types::SyncRequest;
 use crate::wallet::BumpFeeTxBuilder;
 use crate::wallet::SentAndReceivedValues;
 use crate::wallet::TxBuilder;

bdk-ffi/src/types.rs

@@ -1,14 +1,15 @@
 use crate::bitcoin::{Address, OutPoint, Script, Transaction, TxOut};
 
+use bdk::chain::spk_client::FullScanRequest as BdkFullScanRequest;
+use bdk::chain::spk_client::SyncRequest as BdkSyncRequest;
 use bdk::chain::tx_graph::CanonicalTx as BdkCanonicalTx;
 use bdk::chain::{ChainPosition as BdkChainPosition, ConfirmationTimeHeightAnchor};
-use bdk::wallet::AddressIndex as BdkAddressIndex;
 use bdk::wallet::AddressInfo as BdkAddressInfo;
 use bdk::wallet::Balance as BdkBalance;
 use bdk::KeychainKind;
 use bdk::LocalOutput as BdkLocalOutput;
 
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum ChainPosition {
@@ -108,3 +109,7 @@ impl From<BdkLocalOutput> for LocalOutput {
         }
     }
 }
+
+pub struct FullScanRequest(pub(crate) Mutex<Option<BdkFullScanRequest<KeychainKind>>>);
+
+pub struct SyncRequest(pub(crate) Mutex<Option<BdkSyncRequest>>);