Mm2.1 new order keep alive and refactor (jl777#742)

* WIP. * WIP. Compilation fails. * Remove some unused fields from ordermatching structs. Code compiles. * WIP. Compilation fails. * WIP. Playing with Patricia tries. * WIP major refactoring. Commented out the code that fails to compile. Many ordermatching tests fail as of now. * WIP. Code does not compile. * WIP. Playing with tries. * WIP. Test that sync works properly when new orders are created. * WIP. Fixing alice_can_see_the_active_order_after_connection. * WIP. Almost all tests pass. * Save history diffs when order is removed. * Do not remove own pubkey state by timeout. * Ensure that kick started maker order is added to orderbook. * Remove some unused fns. * Do not add root hash of pairs we are not subscribed to. * Prevent cycles in TrieDiffHistory. * Refactor. Remove unwraps wherever possible. * Fixes for review.
DeckerSU · Nov 18, 2020 · 48de827 · 48de827
1 parent e8fa43d
commit 48de827
Show file tree

Hide file tree

Showing 15 changed files with 2,279 additions and 809 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -69,6 +69,7 @@ async-trait = "0.1"
 atomic = "0.4"
 bigdecimal = { version = "0.1", features = ["serde"] }
 bitcrypto = { git = "https://github.com/artemii235/parity-bitcoin.git" }
+blake2 = "0.9.1"
 bytes = "0.4"
 chain = { git = "https://github.com/artemii235/parity-bitcoin.git" }
 coins = { path = "mm2src/coins" }
@@ -88,7 +89,10 @@ futures-cpupool = "0.1"
 futures-timer = "0.1"
 futures = { version = "0.3.1", package = "futures", features = ["compat", "async-await"] }
 gstuff = { version = "0.6", features = ["nightly"] }
+hash256-std-hasher = "0.15.2"
+hash-db = "0.15.2"
 hex = "0.3.2"
+hex-literal = "0.3.1"
 http = "0.2"
 hyper = { version = "0.13", optional = true }
 hyper-rustls = { version = "0.21", optional = true }
@@ -106,6 +110,7 @@ parking_lot = { version = "0.11", features = ["nightly"] }
 # portfolio = { path = "mm2src/portfolio" }
 primitives = { git = "https://github.com/artemii235/parity-bitcoin.git" }
 rand = { version = "0.7", features = ["std", "small_rng"] }
+rmp-serde = "0.14.3"
 # TODO: Reduce the size of regex by disabling the features we don't use.
 # cf. https://github.com/rust-lang/regex/issues/583
 regex = "1"
@@ -116,12 +121,15 @@ serde_json = { version = "1.0", features = ["preserve_order"] }
 serde_derive = "1.0"
 serialization = { git = "https://github.com/artemii235/parity-bitcoin.git" }
 serialization_derive = { git = "https://github.com/artemii235/parity-bitcoin.git" }
+sp-trie = "2.0.0"
 
 # Pin `term` to 0.5.1 because `dirs` is not portable, cf.
 # https://github.com/Stebalien/term/commit/84cfdb51775b327fedf21784749d862fdffa10b4#diff-80398c5faae3c069e4e6aa2ed11b28c0
 term = "=0.5.1"
 
 tokio = { version = "0.2.22", features = ["io-util", "rt-threaded", "stream", "tcp"] }
+trie-db = "0.22.1"
+trie-root = "0.16.0"
 unwrap = "1.2"
 uuid = { version = "0.7", features = ["serde", "v4"] }
 wasm-timer = "0.2.4"

diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs
@@ -1280,6 +1280,9 @@ fn test_unavailable_electrum_proto_version() {
 }
 
 #[test]
+#[ignore]
+// The test provided to dimxy to recreate "stuck mempool" problem of komodod on RICK chain.
+// Leaving this test here for a while because it might be still useful
 fn test_spam_rick() {
     let conf = json!({"coin":"RICK","asset":"RICK","fname":"RICK (TESTCOIN)","rpcport":25435,"txversion":4,"overwintered":1,"mm2":1,"required_confirmations":1,"avg_blocktime":1,"protocol":{"type":"UTXO"}});
     let req = json!({

diff --git a/mm2src/common/mm_ctx.rs b/mm2src/common/mm_ctx.rs
@@ -213,28 +213,6 @@ impl MmCtx {
         }
     }
 
-    /// Sends the P2P message to a processing thread
-    #[cfg(feature = "native")]
-    pub fn broadcast_p2p_msg(&self, _topic: String, _msg: Vec<u8>) { unimplemented!() }
-
-    #[cfg(not(feature = "native"))]
-    pub fn broadcast_p2p_msg(&self, msg: &str) {
-        use crate::executor::spawn;
-        use crate::{helperᶜ, BroadcastP2pMessageArgs};
-
-        let args = BroadcastP2pMessageArgs {
-            ctx: self.ffi_handle.copy_or(0),
-            msg: msg.into(),
-        };
-        let args = unwrap!(bencode(&args));
-        spawn(async move {
-            let rc = helperᶜ("broadcast_p2p_msg", args).await;
-            if let Err(err) = rc {
-                log!("!broadcast_p2p_msg: "(err))
-            }
-        });
-    }
-
     /// Get a reference to the secp256k1 key pair.
     /// Panics if the key pair is not available.
     pub fn secp256k1_key_pair(&self) -> &KeyPair {

diff --git a/mm2src/common/mm_number.rs b/mm2src/common/mm_number.rs
@@ -9,7 +9,7 @@ use std::str::FromStr;
 
 pub use num_bigint::{BigInt, Sign};
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Serialize)]
 pub struct MmNumber(BigRational);
 
 /// Rational number representation de/serializable in human readable form
@@ -45,7 +45,7 @@ impl Into<BigRational> for Fraction {
 }
 
 impl From<BigDecimal> for Fraction {
-    fn from(dec: BigDecimal) -> Fraction { from_dec_to_ratio(dec).into() }
+    fn from(dec: BigDecimal) -> Fraction { from_dec_to_ratio(&dec).into() }
 }
 
 impl<'de> Deserialize<'de> for Fraction {
@@ -75,7 +75,7 @@ pub fn from_ratio_to_dec(r: &BigRational) -> BigDecimal {
     BigDecimal::from(r.numer().clone()) / BigDecimal::from(r.denom().clone())
 }
 
-pub fn from_dec_to_ratio(d: BigDecimal) -> BigRational {
+pub fn from_dec_to_ratio(d: &BigDecimal) -> BigRational {
     let (num, scale) = d.as_bigint_and_exponent();
     let ten = BigInt::from(10);
     if scale >= 0 {
@@ -99,7 +99,7 @@ impl<'de> Deserialize<'de> for MmNumber {
         let raw: Box<RawValue> = Deserialize::deserialize(deserializer)?;
 
         match BigDecimal::from_str(&raw.get().trim_matches('"')) {
-            Ok(dec) => return Ok(MmNumber(from_dec_to_ratio(dec))),
+            Ok(dec) => return Ok(MmNumber(from_dec_to_ratio(&dec))),
             Err(_) => (),
         };
 
@@ -125,7 +125,7 @@ impl std::fmt::Display for MmNumber {
 }
 
 impl From<BigDecimal> for MmNumber {
-    fn from(n: BigDecimal) -> MmNumber { from_dec_to_ratio(n).into() }
+    fn from(n: BigDecimal) -> MmNumber { from_dec_to_ratio(&n).into() }
 }
 
 impl From<BigRational> for MmNumber {
@@ -208,33 +208,14 @@ impl Div for &MmNumber {
     }
 }
 
-impl PartialOrd<MmNumber> for MmNumber {
-    fn partial_cmp(&self, rhs: &MmNumber) -> Option<std::cmp::Ordering> {
-        let lhs = from_ratio_to_dec(&self.0);
-        let rhs = from_ratio_to_dec(&rhs.0);
-        Some(lhs.cmp(&rhs))
-    }
-}
-
 impl PartialOrd<BigDecimal> for MmNumber {
     fn partial_cmp(&self, other: &BigDecimal) -> Option<std::cmp::Ordering> {
-        Some(from_ratio_to_dec(&self.0).cmp(other))
-    }
-}
-
-impl PartialEq for MmNumber {
-    fn eq(&self, rhs: &MmNumber) -> bool {
-        let lhs = from_ratio_to_dec(&self.0);
-        let rhs = from_ratio_to_dec(&rhs.0);
-        lhs == rhs
+        Some(self.0.cmp(&from_dec_to_ratio(other)))
     }
 }
 
 impl PartialEq<BigDecimal> for MmNumber {
-    fn eq(&self, rhs: &BigDecimal) -> bool {
-        let dec = from_ratio_to_dec(&self.0);
-        &dec == rhs
-    }
+    fn eq(&self, rhs: &BigDecimal) -> bool { self.0 == from_dec_to_ratio(rhs) }
 }
 
 impl Default for MmNumber {
@@ -282,17 +263,17 @@ mod tests {
     #[test]
     fn test_from_dec_to_ratio() {
         let number: BigDecimal = "11.00000000000000000000000000000000000000".parse().unwrap();
-        let rational = from_dec_to_ratio(number);
+        let rational = from_dec_to_ratio(&number);
         assert_eq!(*rational.numer(), 11.into());
         assert_eq!(*rational.denom(), 1.into());
 
         let number: BigDecimal = "0.00000001".parse().unwrap();
-        let rational = from_dec_to_ratio(number);
+        let rational = from_dec_to_ratio(&number);
         assert_eq!(*rational.numer(), 1.into());
         assert_eq!(*rational.denom(), 100000000.into());
 
         let number: BigDecimal = 1.into();
-        let rational = from_dec_to_ratio(number);
+        let rational = from_dec_to_ratio(&number);
         assert_eq!(*rational.numer(), 1.into());
         assert_eq!(*rational.denom(), 1.into());
     }
@@ -312,7 +293,7 @@ mod tests {
 
         for num in vals {
             let decimal: BigDecimal = BigDecimal::from_str(num).unwrap();
-            let expected: MmNumber = from_dec_to_ratio(decimal).into();
+            let expected: MmNumber = from_dec_to_ratio(&decimal).into();
             let actual: MmNumber = json::from_str(&num).unwrap();
             assert_eq!(expected, actual);
         }

diff --git a/mm2src/docker_tests.rs b/mm2src/docker_tests.rs
@@ -634,6 +634,15 @@ mod docker_tests {
         log!([block_on(enable_native(&mm_alice, "MYCOIN", vec![]))]);
         log!([block_on(enable_native(&mm_alice, "MYCOIN1", vec![]))]);
 
+        log!("Get MYCOIN/MYCOIN1 orderbook on Alice side to trigger subscription");
+        let rc = unwrap!(block_on(mm_alice.rpc(json! ({
+            "userpass": mm_alice.userpass,
+            "method": "orderbook",
+            "base": "MYCOIN",
+            "rel": "MYCOIN1",
+        }))));
+        assert!(rc.0.is_success(), "!orderbook: {}", rc.1);
+
         let rc = unwrap!(block_on(mm_bob.rpc(json! ({
             "userpass": mm_bob.userpass,
             "method": "setprice",
@@ -768,6 +777,15 @@ mod docker_tests {
         log!([block_on(enable_native(&mm_alice, "MYCOIN", vec![]))]);
         log!([block_on(enable_native(&mm_alice, "MYCOIN1", vec![]))]);
 
+        log!("Get MYCOIN/MYCOIN1 orderbook on Alice side to trigger subscription");
+        let rc = unwrap!(block_on(mm_alice.rpc(json! ({
+            "userpass": mm_alice.userpass,
+            "method": "orderbook",
+            "base": "MYCOIN",
+            "rel": "MYCOIN1",
+        }))));
+        assert!(rc.0.is_success(), "!orderbook: {}", rc.1);
+
         let rc = unwrap!(block_on(mm_bob.rpc(json! ({
             "userpass": mm_bob.userpass,
             "method": "setprice",
@@ -792,6 +810,7 @@ mod docker_tests {
         let asks = bob_orderbook["asks"].as_array().unwrap();
         assert_eq!(asks.len(), 1, "Bob MYCOIN/MYCOIN1 orderbook must have exactly 1 ask");
 
+        thread::sleep(Duration::from_secs(2));
         log!("Get MYCOIN/MYCOIN1 orderbook on Alice side");
         let rc = unwrap!(block_on(mm_alice.rpc(json! ({
             "userpass": mm_alice.userpass,
@@ -913,6 +932,16 @@ mod docker_tests {
         log!([block_on(enable_native(&mm_bob, "MYCOIN1", vec![]))]);
         log!([block_on(enable_native(&mm_alice, "MYCOIN", vec![]))]);
         log!([block_on(enable_native(&mm_alice, "MYCOIN1", vec![]))]);
+        // TODO remove this request when orderbook request is reimplemented using tries
+        log!("Get MYCOIN/MYCOIN1 orderbook on Alice side to trigger subscription");
+        let rc = unwrap!(block_on(mm_alice.rpc(json! ({
+            "userpass": mm_alice.userpass,
+            "method": "orderbook",
+            "base": "MYCOIN",
+            "rel": "MYCOIN1",
+        }))));
+        assert!(rc.0.is_success(), "!orderbook: {}", rc.1);
+
         let rc = unwrap!(block_on(mm_bob.rpc(json! ({
             "userpass": mm_bob.userpass,
             "method": "setprice",
@@ -1532,4 +1561,69 @@ mod docker_tests {
             })));
         }
     }
+
+    #[test]
+    fn test_maker_order_should_kick_start_and_appear_in_orderbook_on_restart() {
+        let (_ctx, _, bob_priv_key) = generate_coin_with_random_privkey("MYCOIN", 1000);
+        let (_ctx, _, alice_priv_key) = generate_coin_with_random_privkey("MYCOIN1", 2000);
+        let coins = json! ([
+            {"coin":"MYCOIN","asset":"MYCOIN","txversion":4,"overwintered":1,"txfee":1000,"protocol":{"type":"UTXO"}},
+            {"coin":"MYCOIN1","asset":"MYCOIN1","txversion":4,"overwintered":1,"txfee":1000,"protocol":{"type":"UTXO"}},
+        ]);
+        let mut bob_conf = json! ({
+            "gui": "nogui",
+            "netid": 9000,
+            "dht": "on",  // Enable DHT without delay.
+            "passphrase": format!("0x{}", hex::encode(bob_priv_key)),
+            "coins": coins,
+            "rpc_password": "pass",
+            "i_am_seed": true,
+        });
+        let mut mm_bob = unwrap!(MarketMakerIt::start(bob_conf.clone(), "pass".to_string(), None,));
+        let (_bob_dump_log, _bob_dump_dashboard) = mm_dump(&mm_bob.log_path);
+        unwrap!(block_on(
+            mm_bob.wait_for_log(22., |log| log.contains(">>>>>>>>> DEX stats "))
+        ));
+
+        log!([block_on(enable_native(&mm_bob, "MYCOIN", vec![]))]);
+        log!([block_on(enable_native(&mm_bob, "MYCOIN1", vec![]))]);
+        let rc = unwrap!(block_on(mm_bob.rpc(json! ({
+            "userpass": mm_bob.userpass,
+            "method": "setprice",
+            "base": "MYCOIN",
+            "rel": "MYCOIN1",
+            "price": 1,
+            "max": true,
+        }))));
+        assert!(rc.0.is_success(), "!setprice: {}", rc.1);
+
+        // mm_bob using same DB dir that should kick start the order
+        bob_conf["dbdir"] = mm_bob.folder.join("DB").to_str().unwrap().into();
+        bob_conf["log"] = mm_bob.folder.join("mm2_dup.log").to_str().unwrap().into();
+        unwrap!(block_on(mm_bob.stop()));
+
+        let mut mm_bob_dup = unwrap!(MarketMakerIt::start(bob_conf, "pass".to_string(), None,));
+        let (_bob_dup_dump_log, _bob_dup_dump_dashboard) = mm_dump(&mm_bob_dup.log_path);
+        unwrap!(block_on(
+            mm_bob_dup.wait_for_log(22., |log| log.contains(">>>>>>>>> DEX stats "))
+        ));
+        log!([block_on(enable_native(&mm_bob_dup, "MYCOIN", vec![]))]);
+        log!([block_on(enable_native(&mm_bob_dup, "MYCOIN1", vec![]))]);
+
+        thread::sleep(Duration::from_secs(2));
+
+        log!("Get RICK/MORTY orderbook on Bob side");
+        let rc = unwrap!(block_on(mm_bob_dup.rpc(json! ({
+            "userpass": mm_bob_dup.userpass,
+            "method": "orderbook",
+            "base": "MYCOIN",
+            "rel": "MYCOIN1",
+        }))));
+        assert!(rc.0.is_success(), "!orderbook: {}", rc.1);
+
+        let bob_orderbook: Json = unwrap!(json::from_str(&rc.1));
+        log!("Bob orderbook "[bob_orderbook]);
+        let asks = bob_orderbook["asks"].as_array().unwrap();
+        assert_eq!(asks.len(), 1, "Bob MYCOIN/MYCOIN1 orderbook must have exactly 1 asks");
+    }
 }
diff --git a/mm2src/lp_native_dex.rs b/mm2src/lp_native_dex.rs
@@ -41,29 +41,11 @@ use crate::common::mm_ctx::{MmArc, MmCtx};
 use crate::common::privkey::key_pair_from_seed;
 use crate::common::{slurp_url, MM_DATETIME, MM_VERSION};
 use crate::mm2::lp_network::{p2p_event_process_loop, P2PContext};
-use crate::mm2::lp_ordermatch::{broadcast_maker_keep_alives_loop, lp_ordermatch_loop, migrate_saved_orders,
-                                orders_kick_start, BalanceUpdateOrdermatchHandler};
+use crate::mm2::lp_ordermatch::{broadcast_maker_orders_keep_alive_loop, lp_ordermatch_loop, orders_kick_start,
+                                BalanceUpdateOrdermatchHandler};
 use crate::mm2::lp_swap::{running_swaps_num, swap_kick_starts};
 use crate::mm2::rpc::spawn_rpc;
 
-// TODO: Use MM2-nightly seed nodes.
-//       MM1 nodes no longer compatible due to the UTXO reforms in particular.
-//       We might also diverge in how we handle the p2p communication in the future.
-
-/// Aka `default_LPnodes`. Initial nodes of the peer-to-peer network.
-#[allow(dead_code)]
-const P2P_SEED_NODES: [&str; 5] = [
-    "5.9.253.195",
-    "173.212.225.176",
-    "136.243.45.140",
-    "23.254.202.142",
-    "45.32.19.196",
-];
-
-/// Default seed nodes for netid 9999 that is used for MM2 testing
-#[allow(dead_code)]
-const P2P_SEED_NODES_9999: [&str; 3] = ["195.201.116.176", "46.4.87.18", "46.4.78.11"];
-
 pub fn lp_ports(netid: u16) -> Result<(u16, u16, u16), String> {
     const LP_RPCPORT: u16 = 7783;
     let max_netid = (65535 - 40 - LP_RPCPORT) / 4;
@@ -252,10 +234,7 @@ fn migrate_db(ctx: &MmArc) -> Result<(), String> {
 }
 
 #[cfg(feature = "native")]
-fn migration_1(ctx: &MmArc) -> Result<(), String> {
-    try_s!(migrate_saved_orders(ctx));
-    Ok(())
-}
+fn migration_1(_ctx: &MmArc) -> Result<(), String> { Ok(()) }
 
 /// Resets the context (most of which resides currently in `lp::G` but eventually would move into `MmCtx`).
 /// Restarts the peer connections.
@@ -514,7 +493,7 @@ pub async fn lp_init(mypubport: u16, ctx: MmArc) -> Result<(), String> {
     spawn(lp_ordermatch_loop(ctxʹ));
 
     let ctxʹ = ctx.clone();
-    spawn(broadcast_maker_keep_alives_loop(ctxʹ));
+    spawn(broadcast_maker_orders_keep_alive_loop(ctxʹ));
 
     #[cfg(not(feature = "native"))]
     {