Use dockerized utxo coin daemon in integration docker_tests crate. (j…

…l777#539)

* WIP. Added first integration test using native UTXO daemon in Docker.

* Use Github fork of testcontainers instead of local copy.

* Stop using hardcoded path for coin config in test.

* Create coin data dir on host if not exist.

* Enable logger in test. Run only test using Docker.

* Fix CI config.

* Fix CI config again.

* Remove comment.

* Use OS specific Zcash params path.

* Remove unused import.

* Wait until conf file is copied from container. Run all tests.

* Try custom test runner.

* Rm the containers at the end of test despite success state.
Also clean up before container is started.

* Move test_search_for_swap_tx_spend_native_was_spent from utxo to docker.

* Prevent the UTXO coins to be concurrently initialized in tests.

Cargo.toml

@@ -15,7 +15,7 @@ default-run = "mm2"
 
 [features]
 native = [
-  "hyper", "tokio-core", "dirs",
+  "hyper", "tokio-core", "dirs", "docker_tests",
   "common/native", "peers/native", "coins/native", # "portfolio/native"
 ]
 
@@ -60,6 +60,7 @@ crc = "1.8"
 crc32fast = { version = "1.2", features = ["std", "nightly"] }
 crossbeam = "0.6"
 dirs = { version = "1", optional = true }
+docker_tests = { path = "mm2src/docker_tests", optional = true }
 enum-primitive-derive = "0.1"
 fomat-macros = "0.2"
 futures01 = { version = "0.1", package = "futures" }
@@ -104,7 +105,7 @@ mocktopus = "0.7.0"
 
 [workspace]
 members = [
-  "mm2src/coins"
+  "mm2src/coins",
 ]
 
 # The backtrace disables build.define("HAVE_DL_ITERATE_PHDR", "1"); for android which results in "unknown" function

mm2src/coins/eth.rs

@@ -704,6 +704,7 @@ impl MarketCoinOps for EthCoin {
         tx: &[u8],
         confirmations: u64,
         wait_until: u64,
+        check_every: u64,
     ) -> Result<(), String> {
         let unsigned: UnverifiedTransaction = try_s!(rlp::decode(tx));
         let tx = try_s!(SignedEthTx::new(unsigned));
@@ -718,7 +719,7 @@ impl MarketCoinOps for EthCoin {
                 Ok(r) => r,
                 Err(e) => {
                     log!("Error " [e] " getting the " (self.ticker()) " transaction " [tx.tx_hash()] ", retrying in 15 seconds");
-                    thread::sleep(Duration::from_secs(15));
+                    thread::sleep(Duration::from_secs(check_every));
                     continue;
                 }
             };
@@ -732,7 +733,7 @@ impl MarketCoinOps for EthCoin {
                         Ok(b) => b,
                         Err(e) => {
                             log!("Error " [e] " getting the " (self.ticker()) " block number retrying in 15 seconds");
-                            thread::sleep(Duration::from_secs(15));
+                            thread::sleep(Duration::from_secs(check_every));
                             continue;
                         }
                     };
@@ -741,7 +742,7 @@ impl MarketCoinOps for EthCoin {
                     }
                 }
             }
-            thread::sleep(Duration::from_secs(15));
+            thread::sleep(Duration::from_secs(check_every));
         }
     }
 

mm2src/coins/lp_coins.rs

@@ -218,6 +218,7 @@ pub trait MarketCoinOps {
         tx: &[u8],
         confirmations: u64,
         wait_until: u64,
+        check_every: u64,
     ) -> Result<(), String>;
 
     fn wait_for_tx_spend(&self, transaction: &[u8], wait_until: u64, from_block: u64) -> Result<TransactionEnum, String>;

mm2src/coins/test_coin.rs

@@ -36,6 +36,7 @@ impl MarketCoinOps for TestCoin {
         tx: &[u8],
         confirmations: u64,
         wait_until: u64,
+        check_every: u64,
     ) -> Result<(), String> {
         unimplemented!()
     }

mm2src/coins/utxo.rs

@@ -64,12 +64,12 @@ use std::time::Duration;
 pub use chain::Transaction as UtxoTx;
 
 use self::rpc_clients::{electrum_script_hash, ElectrumClient, ElectrumClientImpl, EstimateFeeMethod, NativeClient, UtxoRpcClientEnum, UnspentInfo };
-use super::{CoinsContext, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, MmCoinEnum, SwapOps, TradeInfo,
+use super::{CoinsContext, FoundSwapTxSpend, HistorySyncState, MarketCoinOps, MmCoin, SwapOps, TradeInfo,
             Transaction, TransactionEnum, TransactionFut, TransactionDetails, WithdrawFee, WithdrawRequest};
 use crate::utxo::rpc_clients::{NativeClientImpl, UtxoRpcClientOps, ElectrumRpcRequest};
 
 #[cfg(test)]
-mod utxo_tests;
+pub mod utxo_tests;
 
 #[cfg(windows)]
 #[cfg(feature = "native")]
@@ -284,6 +284,14 @@ impl UtxoCoinImpl {
             None => Ok(None),
         }
     }
+
+    pub fn my_public_key(&self) -> &Public {
+        self.key_pair.public()
+    }
+
+    pub fn rpc_client(&self) -> &UtxoRpcClientEnum {
+        &self.rpc_client
+    }
 }
 
 fn payment_script(
@@ -1252,12 +1260,14 @@ impl MarketCoinOps for UtxoCoin {
         tx: &[u8],
         confirmations: u64,
         wait_until: u64,
+        check_every: u64,
     ) -> Result<(), String> {
         let tx: UtxoTx = try_s!(deserialize(tx).map_err(|e| ERRL!("{:?}", e)));
         self.rpc_client.wait_for_confirmations(
             &tx,
             confirmations as u32,
             wait_until,
+            check_every,
         )
     }
 
@@ -1299,7 +1309,7 @@ impl MarketCoinOps for UtxoCoin {
 }
 
 async fn withdraw_impl(coin: UtxoCoin, req: WithdrawRequest) -> Result<TransactionDetails, String> {
-    let to: Address = try_s!(Address::from_str(&req.to));
+    let to = try_s!(Address::from_str(&req.to));
     if to.prefix != coin.pub_addr_prefix || to.t_addr_prefix != coin.pub_t_addr_prefix {
         return ERR!("Address {} has invalid format, it must start with {}", to, &coin.my_address()[..1]);
     }
@@ -1644,6 +1654,61 @@ impl MmCoin for UtxoCoin {
     }
 }
 
+#[cfg(feature = "native")]
+// https://github.com/KomodoPlatform/komodo/blob/master/zcutil/fetch-params.sh#L5
+// https://github.com/KomodoPlatform/komodo/blob/master/zcutil/fetch-params.bat#L4
+pub fn zcash_params_path() -> PathBuf {
+    if cfg! (windows) {
+        // >= Vista: c:\Users\$username\AppData\Roaming
+        get_special_folder_path().join("ZcashParams")
+    } else if cfg! (target_os = "macos") {
+        unwrap!(home_dir()).join("Library").join("Application Support").join("ZcashParams")
+    } else {
+        unwrap!(home_dir()).join(".zcash-params")
+    }
+}
+
+#[cfg(feature = "native")]
+pub fn coin_daemon_data_dir(name: &str, is_asset_chain: bool) -> PathBuf {
+    // komodo/util.cpp/GetDefaultDataDir
+    let mut data_dir = match dirs::home_dir() {
+        Some (hd) => hd,
+        None => Path::new ("/") .to_path_buf()
+    };
+
+    if cfg! (windows) {
+        // >= Vista: c:\Users\$username\AppData\Roaming
+        data_dir = get_special_folder_path();
+        if is_asset_chain {
+            data_dir.push ("Komodo");
+        } else {
+            data_dir.push (first_char_to_upper(name));
+        }
+    } else if cfg! (target_os = "macos") {
+        data_dir.push ("Library");
+        data_dir.push ("Application Support");
+        if is_asset_chain {
+            data_dir.push ("Komodo");
+        } else {
+            data_dir.push (first_char_to_upper(name));
+        }
+    } else {
+        if is_asset_chain {
+            data_dir.push (".komodo");
+        } else {
+            data_dir.push (format!(".{}", name));
+        }
+    }
+
+    if is_asset_chain {data_dir.push (name)};
+    data_dir
+}
+
+#[cfg(not(feature = "native"))]
+pub fn coin_daemon_data_dir(name: &str, is_asset_chain: bool) -> PathBuf {
+    unimplemented!()
+}
+
 #[cfg(feature = "native")]
 /// Returns a path to the native coin wallet configuration.
 /// (This path is used in to read the wallet credentials).
@@ -1660,37 +1725,7 @@ fn confpath (coins_en: &Json) -> Result<PathBuf, String> {
             }
         };
 
-        // komodo/util.cpp/GetDefaultDataDir
-        let mut data_dir = match dirs::home_dir() {
-            Some (hd) => hd,
-            None => Path::new ("/") .to_path_buf()
-        };
-
-        if cfg! (windows) {
-            // >= Vista: c:\Users\$username\AppData\Roaming
-            data_dir = get_special_folder_path();
-            if is_asset_chain {
-                data_dir.push ("Komodo");
-            } else {
-                data_dir.push (first_char_to_upper(name));
-            }
-        } else if cfg! (target_os = "macos") {
-            data_dir.push ("Library");
-            data_dir.push ("Application Support");
-            if is_asset_chain {
-                data_dir.push ("Komodo");
-            } else {
-                data_dir.push (first_char_to_upper(name));
-            }
-        } else {
-            if is_asset_chain {
-                data_dir.push (".komodo");
-            } else {
-                data_dir.push (format!(".{}", name));
-            }
-        }
-
-        if is_asset_chain {data_dir.push (name)}
+        let data_dir = coin_daemon_data_dir(name, is_asset_chain);
 
         let confname = format! ("{}.conf", name);
 
@@ -1741,7 +1776,7 @@ pub fn utxo_coin_from_conf_and_request(
     conf: &Json,
     req: &Json,
     priv_key: &[u8],
-) -> Result<MmCoinEnum, String> {
+) -> Result<UtxoCoin, String> {
     let checksum_type = if ticker == "GRS" {
         ChecksumType::DGROESTL512
     } else if ticker == "SMART" {
@@ -1904,7 +1939,7 @@ pub fn utxo_coin_from_conf_and_request(
         history_sync_state: Mutex::new(initial_history_state),
         required_confirmations: conf["required_confirmations"].as_u64().unwrap_or(1).into(),
     };
-    Ok(UtxoCoin(Arc::new(coin)).into())
+    Ok(UtxoCoin(Arc::new(coin)))
 }
 
 /// Function calculating KMD interest

mm2src/coins/utxo/rpc_clients.rs

@@ -111,7 +111,7 @@ pub trait UtxoRpcClientOps: Debug + 'static {
 
     // TODO This operation is synchronous because it's currently simpler to do it this way.
     // Might consider refactoring when async/await is released.
-    fn wait_for_confirmations(&self, tx: &UtxoTx, confirmations: u32, wait_until: u64) -> Result<(), String> {
+    fn wait_for_confirmations(&self, tx: &UtxoTx, confirmations: u32, wait_until: u64, check_every: u64) -> Result<(), String> {
         loop {
             if now_ms() / 1000 > wait_until {
                 return ERR!("Waited too long until {} for transaction {:?} to be confirmed {} times", wait_until, tx, confirmations);
@@ -128,7 +128,7 @@ pub trait UtxoRpcClientOps: Debug + 'static {
                 Err(e) => log!("Error " [e] " getting the transaction " [tx.hash().reversed()] ", retrying in 10 seconds"),
             }
 
-            thread::sleep(Duration::from_secs(10));
+            thread::sleep(Duration::from_secs(check_every));
         }
     }
 
@@ -488,6 +488,11 @@ impl NativeClientImpl {
     fn get_block_hash(&self, block_number: u64) -> RpcRes<H256Json> {
         rpc_func!(self, "getblockhash", block_number)
     }
+
+    /// https://bitcoin.org/en/developer-reference#sendtoaddress
+    pub fn send_to_address(&self, addr: &str, amount: &BigDecimal) -> RpcRes<H256Json> {
+        rpc_func!(self, "sendtoaddress", addr, amount)
+    }
 }
 
 #[derive(Debug, Deserialize)]

mm2src/coins/utxo/utxo_tests.rs

@@ -328,60 +328,6 @@ fn test_search_for_swap_tx_spend_electrum_was_refunded() {
     assert_eq!(FoundSwapTxSpend::Refunded(spend_tx), found);
 }
 
-#[test]
-#[ignore]
-// ignored, will work only when ETOMIC daemon is running locally and has following addresses imported
-// with rescan: R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW, bVrUo5BSwzghCobtJs9Su1JxHyBGAhFVzc
-fn test_search_for_swap_tx_spend_native_was_spent() {
-    let conf = json!({"asset":"ETOMIC"});
-    let req = json!({"method":"enable"});
-    let priv_key = unwrap!(hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f"));
-    let coin = unwrap!(utxo_coin_from_conf_and_request("ETOMIC", &conf, &req, &priv_key));
-
-    // raw tx bytes of https://etomic.explorer.dexstats.info/tx/c514b3163d66636ebc3574817cb5853d5ab39886183de71ffedf5c5768570a6b
-    let payment_tx_bytes = unwrap!(hex::decode("0400008085202f89013ac014d4926c8b435f7a5c58f38975d14f1aba597b1eef2dfdc093457678eb83010000006a47304402204ddb9b10237a1267a02426d923528213ad1e0b62d45be7d9629e2909f099d90c02205eecadecf6fd09cb8465170eb878c5d54e563f067b64e23c418da0f6519ca354012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff02809698000000000017a914bbd726b74f27b476d5d932e903b5893fd4e8bd2187acdaaa87010000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac2771515d000000000000000000000000000000"));
-
-    // raw tx bytes of https://etomic.explorer.dexstats.info/tx/e72be40bab15f3914e70507e863e26b8ccfaa75a9861d6fe706b39cab1272617
-    let spend_tx_bytes = unwrap!(hex::decode("0400008085202f89016b0a5768575cdffe1fe73d188698b35a3d85b57c817435bc6e63663d16b314c500000000d8483045022100d3cf75d26d977c0358e46c5db3753aa332ba12130b36b24b541cb90416f4606102201d805c5bdfc4d630cb78adb63239911c97a09c41125af37be219e876610f15f201209ac4a742e81a47dc26a3ddf83de84783fdb49c2322c4a3fdafc0613bf3335c40004c6b6304218f515db1752102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ac6782012088a914954f5a3f3b5de4410e5e1a82949410de95a4b6ba882102631dcf1d4b1b693aa8c2751afc68e4794b1e5996566cfc701a663f8b7bbbe640ac68ffffffff0198929800000000001976a91464ae8510aac9546d5e7704e31ce177451386455588ac3963515d000000000000000000000000000000"));
-    let spend_tx = TransactionEnum::UtxoTx(unwrap!(deserialize(spend_tx_bytes.as_slice())));
-
-    let found = unwrap!(unwrap!(coin.search_for_swap_tx_spend_my(
-        1565626145,
-        &unwrap!(hex::decode("02631dcf1d4b1b693aa8c2751afc68e4794b1e5996566cfc701a663f8b7bbbe640")),
-        &unwrap!(hex::decode("954f5a3f3b5de4410e5e1a82949410de95a4b6ba")),
-        &payment_tx_bytes,
-        165597
-    )));
-    assert_eq!(FoundSwapTxSpend::Spent(spend_tx), found);
-}
-
-#[test]
-#[ignore]
-// ignored, will work only when ETOMIC daemon is running locally and has following addresses imported
-// with rescan: R9o9xTocqr6CeEDGDH6mEYpwLoMz6jNjMW, bQGZYDnEvifgehVKxEzFt8ygNSnLV6Dqiz
-fn test_search_for_swap_tx_spend_native_was_refunded() {
-    let conf = json!({"asset":"ETOMIC"});
-    let req = json!({"method":"enable"});
-    let priv_key = unwrap!(hex::decode("809465b17d0a4ddb3e4c69e8f23c2cabad868f51f8bed5c765ad1d6516c3306f"));
-    let coin = unwrap!(utxo_coin_from_conf_and_request("ETOMIC", &conf, &req, &priv_key));
-
-    // raw tx bytes of https://etomic.explorer.dexstats.info/tx/c9a47cc6e80a98355cd4e69d436eae6783cbee5991756caa6e64a0743442fa96
-    let payment_tx_bytes = unwrap!(hex::decode("0400008085202f8901887e809b10738b1625b7f47fd5d2201f32e8a4c6c0aaefc3b9ab6c07dc6a5925010000006a47304402203966f49ba8acc9fcc0e53e7b917ca5599ce6054a0c2d22752c57a3dc1b0fc83502206fde12c869da20a21cedd5bbc4bcd12977d25ff4b00e0999de5ac4254668e891012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff02809698000000000017a9147e9456f37fa53cf9053e192ea4951d2c8b58647c8784631684010000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac0fb3525d000000000000000000000000000000"));
-
-    // raw tx bytes of https://etomic.explorer.dexstats.info/tx/a5f11bdb657ee834a4c410e2001beccce0374bfa3f662bd890fd3d01b0b3d101
-    let spend_tx_bytes = unwrap!(hex::decode("0400008085202f890196fa423474a0646eaa6c759159eecb8367ae6e439de6d45c35980ae8c67ca4c900000000c4483045022100969c3b2c1ab630b67a6ee74316c9356e38872276a070d126da1d731503bb6e3e02204398d8c23cb59c2caddc33ce9c9716c54b11574126a3c3350a17043f3751696d01514c786304cf93525db1752102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ac6782012088a92102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3882102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ac68feffffff0198929800000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac01a5525d000000000000000000000000000000"));
-    let spend_tx = TransactionEnum::UtxoTx(unwrap!(deserialize(spend_tx_bytes.as_slice())));
-
-    let found = unwrap!(unwrap!(coin.search_for_swap_tx_spend_my(
-        1565692879,
-        &unwrap!(hex::decode("02031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3")),
-        &unwrap!(hex::decode("02031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3")),
-        &payment_tx_bytes,
-        166864
-    )));
-    assert_eq!(FoundSwapTxSpend::Refunded(spend_tx), found);
-}
-
 #[test]
 fn test_withdraw_impl_set_fixed_fee() {
     NativeClient::list_unspent_ordered.mock_safe(|_,_| {

mm2src/docker_tests/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "docker_tests"
+version = "0.1.0"
+authors = ["Artem Pikulin <[email protected]>"]
+edition = "2018"
+
+[lib]
+path = "src/docker_tests.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+[dev-dependencies]
+common = { path = "../common" }
+coins = { path = "../coins", features = ["native"] }
+fomat-macros = "0.2"
+futures01 = { version = "0.1", package = "futures" }
+gstuff = { version = "0.6", features = ["nightly"] }
+hex = "0.3.2"
+lazy_static = "1.3.0"
+libsecp256k1 = "0.2.2"
+rand = { version = "0.4" }
+serde_json = "1.0"
+testcontainers = { git = "https://github.com/artemii235/testcontainers-rs.git" }
+unwrap = "1.2"