diff --git a/Cargo.lock b/Cargo.lock index c987e890e0..706dc2b91c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,15 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "addr2line" version = "0.12.2" @@ -108,6 +118,15 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "anyhow" version = "1.0.31" @@ -385,17 +404,35 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitvec" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41262f11d771fd4a61aa3ce019fca363b4b6c282fca9da2a31186d3965a47a5c" +dependencies = [ + "either", + "radium", +] + [[package]] name = "blake2" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ce5b6108f8e154604bd4eb76a2f726066c3464d5a552a4229262a18c9bb471" +checksum = "10a5720225ef5daecf08657f23791354e1685a8c91a4c60c7f3d3b2892f978f4" dependencies = [ - "byte-tools 0.3.1", - "byteorder 1.3.4", "crypto-mac 0.8.0", "digest 0.9.0", - "opaque-debug 0.2.3", + "opaque-debug 0.3.0", +] + +[[package]] +name = "blake2-rfc" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" +dependencies = [ + "arrayvec 0.4.12", + "constant_time_eq", ] [[package]] @@ -512,6 +549,12 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +[[package]] +name = "byte-slice-cast" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" + [[package]] name = "byte-tools" version = "0.2.0" @@ -669,7 +712,7 @@ version = "2.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" dependencies = [ - "ansi_term", + "ansi_term 0.11.0", "atty", "bitflags", "strsim", @@ -1109,6 +1152,17 @@ dependencies = [ "syn 0.11.11", ] +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + [[package]] name = "digest" version = "0.6.2" @@ -1174,6 +1228,33 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dyn-clonable" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e9232f0e607a262ceb9bd5141a3dfb3e4db6994b31989bbfd845878cba59fd4" +dependencies = [ + "dyn-clonable-impl", + "dyn-clone", +] + +[[package]] +name = "dyn-clonable-impl" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558e40ea573c374cf53507fd240b7ee2f5477df7cfebdb97323ec61c719399c5" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + +[[package]] +name = "dyn-clone" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55796afa1b20c2945ca8eabfc421839f2b766619209f1ede813cf2484f31804" + [[package]] name = "ed25519" version = "1.0.1" @@ -1246,6 +1327,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "environmental" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6576a1755ddffd988788025e75bce9e74b018f7cc226198fe931d077911c6d7e" + [[package]] name = "error-chain" version = "0.12.2" @@ -1267,7 +1354,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "tiny-keccak", + "tiny-keccak 1.4.4", ] [[package]] @@ -1278,9 +1365,9 @@ checksum = "a6294da962646baa738414e8e718d1a1f0360a51d92de89ccbf91870418f5360" dependencies = [ "crunchy 0.1.6", "ethereum-types-serialize", - "fixed-hash", + "fixed-hash 0.2.5", "serde", - "tiny-keccak", + "tiny-keccak 1.4.4", ] [[package]] @@ -1305,9 +1392,9 @@ dependencies = [ "crunchy 0.1.6", "ethbloom", "ethereum-types-serialize", - "fixed-hash", + "fixed-hash 0.2.5", "serde", - "uint", + "uint 0.4.1", ] [[package]] @@ -1334,7 +1421,29 @@ dependencies = [ "rustc-hex 1.0.0", "serde", "serde_derive", - "tiny-keccak", + "tiny-keccak 1.4.4", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", + "synstructure", ] [[package]] @@ -1381,6 +1490,18 @@ dependencies = [ "rustc-hex 2.1.0", ] +[[package]] +name = "fixed-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11498d382790b7a8f2fd211780bec78619bba81cdad3a283997c0c41f836759c" +dependencies = [ + "byteorder 1.3.4", + "rand 0.7.3", + "rustc-hex 2.1.0", + "static_assertions", +] + [[package]] name = "fixedbitset" version = "0.2.0" @@ -1590,6 +1711,19 @@ version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" +[[package]] +name = "generator" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc" +dependencies = [ + "cc", + "libc", + "log 0.4.11", + "rustc_version", + "winapi 0.3.8", +] + [[package]] name = "generic-array" version = "0.8.3" @@ -1733,6 +1867,21 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "hash-db" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" + +[[package]] +name = "hash256-std-hasher" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" +dependencies = [ + "crunchy 0.2.2", +] + [[package]] name = "hashbrown" version = "0.6.3" @@ -1807,6 +1956,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" +[[package]] +name = "hex-literal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5af1f635ef1bc545d78392b136bfe1c9809e029023c84a3638a864a10b8819c8" + [[package]] name = "hmac" version = "0.4.2" @@ -2008,6 +2163,35 @@ dependencies = [ "typenum", ] +[[package]] +name = "impl-codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-serde" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47ca4d2b6931707a55fce5cf66aff80e2178c8b63bbb4ecb5695cbc870ddf6f" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + [[package]] name = "indexmap" version = "1.4.0" @@ -2087,7 +2271,7 @@ version = "0.1.2" source = "git+https://github.com/artemii235/parity-common#986aba04f5e11e855d08a41c40bf03b4ba2862b0" dependencies = [ "ethereum-types", - "tiny-keccak", + "tiny-keccak 1.4.4", ] [[package]] @@ -2513,6 +2697,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "loom" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", +] + [[package]] name = "lru" version = "0.4.3" @@ -2531,6 +2728,15 @@ dependencies = [ "hashbrown 0.8.2", ] +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.8" @@ -2563,6 +2769,35 @@ dependencies = [ "autocfg 1.0.0", ] +[[package]] +name = "memory-db" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f36ddb0b2cdc25d38babba472108798e3477f02be5165f038c5e393e50c57a" +dependencies = [ + "hash-db", + "hashbrown 0.8.2", + "parity-util-mem", +] + +[[package]] +name = "memory_units" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" + +[[package]] +name = "merlin" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6feca46f4fa3443a01769d768727f10c10a20fdb65e52dc16a81f0c8269bb78" +dependencies = [ + "byteorder 1.3.4", + "keccak", + "rand_core 0.5.1", + "zeroize", +] + [[package]] name = "metrics" version = "0.12.1" @@ -2694,6 +2929,7 @@ dependencies = [ "atomic 0.4.6", "bigdecimal", "bitcrypto", + "blake2", "bytes 0.4.12", "chain", "coins", @@ -2711,7 +2947,10 @@ dependencies = [ "futures-cpupool", "futures-timer 0.1.1", "gstuff", + "hash-db", + "hash256-std-hasher", "hex 0.3.2", + "hex-literal", "http 0.2.1", "hyper", "hyper-rustls 0.21.0", @@ -2730,6 +2969,7 @@ dependencies = [ "rand 0.4.6", "rand 0.7.3", "regex", + "rmp-serde", "rpc", "script", "serde", @@ -2738,9 +2978,12 @@ dependencies = [ "serde_json", "serialization", "serialization_derive", + "sp-trie", "term 0.5.1", "testcontainers", "tokio", + "trie-db", + "trie-root", "unwrap", "uuid", "wasm-bindgen", @@ -2950,6 +3193,9 @@ name = "once_cell" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +dependencies = [ + "parking_lot 0.10.2", +] [[package]] name = "opaque-debug" @@ -2995,6 +3241,63 @@ dependencies = [ "url 2.1.1", ] +[[package]] +name = "parity-scale-codec" +version = "1.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c740e5fbcb6847058b40ac7e5574766c6388f585e184d769910fe0d3a2ca861" +dependencies = [ + "arrayvec 0.5.1", + "bitvec", + "byte-slice-cast", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "198db82bb1c18fc00176004462dd809b2a6d851669550aa17af6dacd21ae0c14" +dependencies = [ + "proc-macro-crate", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + +[[package]] +name = "parity-util-mem" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297ff91fa36aec49ce183484b102f6b75b46776822bd81525bfc4cc9b0dd0f5c" +dependencies = [ + "cfg-if", + "hashbrown 0.8.2", + "impl-trait-for-tuples", + "parity-util-mem-derive", + "parking_lot 0.10.2", + "primitive-types", + "winapi 0.3.8", +] + +[[package]] +name = "parity-util-mem-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" +dependencies = [ + "proc-macro2 1.0.18", + "syn 1.0.33", + "synstructure", +] + +[[package]] +name = "parity-wasm" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" + [[package]] name = "parking" version = "1.0.2" @@ -3022,6 +3325,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "parking_lot" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.7.2", +] + [[package]] name = "parking_lot" version = "0.11.0" @@ -3061,6 +3374,20 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "parking_lot_core" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" +dependencies = [ + "cfg-if", + "cloudabi 0.0.3", + "libc", + "redox_syscall", + "smallvec 1.4.0", + "winapi 0.3.8", +] + [[package]] name = "parking_lot_core" version = "0.8.0" @@ -3076,6 +3403,16 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "pbkdf2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" +dependencies = [ + "byteorder 1.3.4", + "crypto-mac 0.7.0", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -3167,6 +3504,18 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" +[[package]] +name = "primitive-types" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55c21c64d0eaa4d7ed885d959ef2d62d9e488c27c0e02d9aa5ce6c877b7d5f8" +dependencies = [ + "fixed-hash 0.6.1", + "impl-codec", + "impl-serde", + "uint 0.8.5", +] + [[package]] name = "primitives" version = "0.1.0" @@ -3178,10 +3527,19 @@ dependencies = [ ] [[package]] -name = "proc-macro-hack" -version = "0.5.16" +name = "proc-macro-crate" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" [[package]] name = "proc-macro-nested" @@ -3321,6 +3679,12 @@ dependencies = [ "proc-macro2 1.0.18", ] +[[package]] +name = "radium" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def50a86306165861203e7f84ecffbbdfdea79f0e51039b33de1e952358c47ac" + [[package]] name = "rand" version = "0.3.23" @@ -3552,6 +3916,26 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "ref-cast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17626b2f4bcf35b84bf379072a66e28cfe5c3c6ae58b38e4914bb8891dabece" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c523ccaed8ac4b0288948849a350b37d3035827413c458b6a40ddb614bb4f72" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + [[package]] name = "regex" version = "1.3.9" @@ -3564,6 +3948,16 @@ dependencies = [ "thread_local", ] +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder 1.3.4", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.18" @@ -3683,6 +4077,12 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hex" version = "1.0.0" @@ -3781,6 +4181,24 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "schnorrkel" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "021b403afe70d81eea68f6ea12f6b3c9588e5d536a94c3bf80f15e7faa267862" +dependencies = [ + "arrayref", + "arrayvec 0.5.1", + "curve25519-dalek 2.1.0", + "getrandom", + "merlin", + "rand 0.7.3", + "rand_core 0.5.1", + "sha2 0.8.2", + "subtle 2.2.3", + "zeroize", +] + [[package]] name = "scoped-tls" version = "1.0.0" @@ -3840,6 +4258,15 @@ dependencies = [ "cc", ] +[[package]] +name = "secrecy" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9182278ed645df3477a9c27bfee0621c621aa16f6972635f7f795dae3d81070f" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "0.4.4" @@ -4054,6 +4481,16 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "sharded-slab" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127" +dependencies = [ + "lazy_static", + "loom", +] + [[package]] name = "signature" version = "1.2.2" @@ -4169,6 +4606,165 @@ dependencies = [ "sha-1", ] +[[package]] +name = "sp-core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e92ac5c674ee2cd9219d084301b4cbb82b28a94a0f3087bf4bea0ef3067ebb5c" +dependencies = [ + "base58", + "blake2-rfc", + "byteorder 1.3.4", + "derive_more", + "dyn-clonable", + "ed25519-dalek", + "futures 0.3.5", + "hash-db", + "hash256-std-hasher", + "hex 0.4.2", + "impl-serde", + "lazy_static", + "libsecp256k1 0.3.5", + "log 0.4.11", + "merlin", + "num-traits 0.2.12", + "parity-scale-codec", + "parity-util-mem", + "parking_lot 0.10.2", + "primitive-types", + "rand 0.7.3", + "regex", + "schnorrkel", + "secrecy", + "serde", + "sha2 0.8.2", + "sp-debug-derive", + "sp-externalities", + "sp-runtime-interface", + "sp-std", + "sp-storage", + "substrate-bip39", + "tiny-bip39", + "tiny-keccak 2.0.2", + "twox-hash", + "wasmi", + "zeroize", +] + +[[package]] +name = "sp-debug-derive" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a3750b084e0f4677f6e834a974f30b1ba97fc2fe00185c9d03611a2228446dc" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + +[[package]] +name = "sp-externalities" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d87fcd0e0fc5e025459cfe769803488d4894e36d0f8cef80b5239d2e7ef6580" +dependencies = [ + "environmental", + "parity-scale-codec", + "sp-std", + "sp-storage", +] + +[[package]] +name = "sp-runtime-interface" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7e363c480cc8c9019b84f85d10c0b56a184079d5d840d2d1d55087ad835dc6" +dependencies = [ + "parity-scale-codec", + "primitive-types", + "sp-externalities", + "sp-runtime-interface-proc-macro", + "sp-std", + "sp-storage", + "sp-tracing", + "sp-wasm-interface", + "static_assertions", +] + +[[package]] +name = "sp-runtime-interface-proc-macro" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cf56a38544293e54dbe0aa7b6aed1e046bfc704b6fc3de7255897dca98ccb1" +dependencies = [ + "Inflector", + "proc-macro-crate", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + +[[package]] +name = "sp-std" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa2d6e166cead2d3b1d3d8fe0e787d076b7d0296b1760a0d7d340846d0ba42c5" + +[[package]] +name = "sp-storage" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4625e6f8f40995939560f48f89028f658b7929657c68d01c571c81ab5619ff" +dependencies = [ + "impl-serde", + "parity-scale-codec", + "ref-cast", + "serde", + "sp-debug-derive", + "sp-std", +] + +[[package]] +name = "sp-tracing" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9a5c42c5450991ca3a28c190e75122f5ccedbcb024953e7c357e7aa2afd8534" +dependencies = [ + "log 0.4.11", + "parity-scale-codec", + "sp-std", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "sp-trie" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3aae57c8ae81ba978503137a8c625d2963eb425dd90dec0d96b4ed18d8bfd55" +dependencies = [ + "hash-db", + "memory-db", + "parity-scale-codec", + "sp-core", + "sp-std", + "trie-db", + "trie-root", +] + +[[package]] +name = "sp-wasm-interface" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1c28225e8b7ec7e260f8b46443f8731abda206334cb75c740d2407693f38167" +dependencies = [ + "impl-trait-for-tuples", + "parity-scale-codec", + "sp-std", + "wasmi", +] + [[package]] name = "spin" version = "0.5.2" @@ -4202,6 +4798,19 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +[[package]] +name = "substrate-bip39" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bed6646a0159b9935b5d045611560eeef842b78d7adc3ba36f5ca325a13a0236" +dependencies = [ + "hmac 0.7.1", + "pbkdf2", + "schnorrkel", + "sha2 0.8.2", + "zeroize", +] + [[package]] name = "subtle" version = "1.0.0" @@ -4489,6 +5098,22 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "tiny-bip39" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0165e045cc2ae1660270ca65e1676dbaab60feb0f91b10f7d0665e9b47e31f2" +dependencies = [ + "failure", + "hmac 0.7.1", + "once_cell", + "pbkdf2", + "rand 0.7.3", + "rustc-hash", + "sha2 0.8.2", + "unicode-normalization", +] + [[package]] name = "tiny-keccak" version = "1.4.4" @@ -4498,6 +5123,15 @@ dependencies = [ "crunchy 0.2.2", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy 0.2.2", +] + [[package]] name = "tinyvec" version = "0.3.3" @@ -4580,6 +5214,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cf45bb0bef80604d001caaec0d09da99611b3c0fd39d3080468875cdb65645" +dependencies = [ + "serde", +] + [[package]] name = "tower-service" version = "0.3.0" @@ -4595,9 +5238,21 @@ dependencies = [ "cfg-if", "log 0.4.11", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + [[package]] name = "tracing-core" version = "0.1.17" @@ -4607,12 +5262,88 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-log" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9" +dependencies = [ + "lazy_static", + "log 0.4.11", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401" +dependencies = [ + "ansi_term 0.12.1", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec 1.4.0", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "trie-db" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e55f7ace33d6237e14137e386f4e1672e2a5c6bbc97fef9f438581a143971f0" +dependencies = [ + "hash-db", + "hashbrown 0.8.2", + "log 0.4.11", + "rustc-hex 2.1.0", + "smallvec 1.4.0", +] + +[[package]] +name = "trie-root" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" +dependencies = [ + "hash-db", +] + [[package]] name = "try-lock" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" +[[package]] +name = "twox-hash" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04f8ab788026715fa63b31960869617cba39117e520eb415b0139543e325ab59" +dependencies = [ + "cfg-if", + "rand 0.7.3", + "static_assertions", +] + [[package]] name = "typenum" version = "1.12.0" @@ -4630,6 +5361,18 @@ dependencies = [ "rustc-hex 2.1.0", ] +[[package]] +name = "uint" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9db035e67dfaf7edd9aebfe8676afcd63eed53c8a4044fed514c8cccf1835177" +dependencies = [ + "byteorder 1.3.4", + "crunchy 0.2.2", + "rustc-hex 2.1.0", + "static_assertions", +] + [[package]] name = "unexpected" version = "0.1.0" @@ -4925,6 +5668,29 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmi" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf617d864d25af3587aa745529f7aaa541066c876d57e050c0d0c85c61c92aff" +dependencies = [ + "libc", + "memory_units", + "num-rational", + "num-traits 0.2.12", + "parity-wasm", + "wasmi-validation", +] + +[[package]] +name = "wasmi-validation" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" +dependencies = [ + "parity-wasm", +] + [[package]] name = "web-sys" version = "0.3.40" diff --git a/Cargo.toml b/Cargo.toml index bf90152a40..7e77d591bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ async-trait = "0.1" atomic = "0.4" bigdecimal = { version = "0.1", features = ["serde"] } bitcrypto = { git = "https://github.com/artemii235/parity-bitcoin.git" } +blake2 = "0.9.1" bytes = "0.4" chain = { git = "https://github.com/artemii235/parity-bitcoin.git" } coins = { path = "mm2src/coins" } @@ -88,7 +89,10 @@ futures-cpupool = "0.1" futures-timer = "0.1" futures = { version = "0.3.1", package = "futures", features = ["compat", "async-await"] } gstuff = { version = "0.6", features = ["nightly"] } +hash256-std-hasher = "0.15.2" +hash-db = "0.15.2" hex = "0.3.2" +hex-literal = "0.3.1" http = "0.2" hyper = { version = "0.13", optional = true } hyper-rustls = { version = "0.21", optional = true } @@ -106,6 +110,7 @@ parking_lot = { version = "0.11", features = ["nightly"] } # portfolio = { path = "mm2src/portfolio" } primitives = { git = "https://github.com/artemii235/parity-bitcoin.git" } rand = { version = "0.7", features = ["std", "small_rng"] } +rmp-serde = "0.14.3" # TODO: Reduce the size of regex by disabling the features we don't use. # cf. https://github.com/rust-lang/regex/issues/583 regex = "1" @@ -116,12 +121,15 @@ serde_json = { version = "1.0", features = ["preserve_order"] } serde_derive = "1.0" serialization = { git = "https://github.com/artemii235/parity-bitcoin.git" } serialization_derive = { git = "https://github.com/artemii235/parity-bitcoin.git" } +sp-trie = "2.0.0" # Pin `term` to 0.5.1 because `dirs` is not portable, cf. # https://github.com/Stebalien/term/commit/84cfdb51775b327fedf21784749d862fdffa10b4#diff-80398c5faae3c069e4e6aa2ed11b28c0 term = "=0.5.1" tokio = { version = "0.2.22", features = ["io-util", "rt-threaded", "stream", "tcp"] } +trie-db = "0.22.1" +trie-root = "0.16.0" unwrap = "1.2" uuid = { version = "0.7", features = ["serde", "v4"] } wasm-timer = "0.2.4" diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 4df1cd0b27..b885801e84 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -1280,6 +1280,9 @@ fn test_unavailable_electrum_proto_version() { } #[test] +#[ignore] +// The test provided to dimxy to recreate "stuck mempool" problem of komodod on RICK chain. +// Leaving this test here for a while because it might be still useful fn test_spam_rick() { let conf = json!({"coin":"RICK","asset":"RICK","fname":"RICK (TESTCOIN)","rpcport":25435,"txversion":4,"overwintered":1,"mm2":1,"required_confirmations":1,"avg_blocktime":1,"protocol":{"type":"UTXO"}}); let req = json!({ diff --git a/mm2src/common/mm_ctx.rs b/mm2src/common/mm_ctx.rs index 2fbe2936bf..7dbe4c0abb 100644 --- a/mm2src/common/mm_ctx.rs +++ b/mm2src/common/mm_ctx.rs @@ -213,28 +213,6 @@ impl MmCtx { } } - /// Sends the P2P message to a processing thread - #[cfg(feature = "native")] - pub fn broadcast_p2p_msg(&self, _topic: String, _msg: Vec) { unimplemented!() } - - #[cfg(not(feature = "native"))] - pub fn broadcast_p2p_msg(&self, msg: &str) { - use crate::executor::spawn; - use crate::{helperᶜ, BroadcastP2pMessageArgs}; - - let args = BroadcastP2pMessageArgs { - ctx: self.ffi_handle.copy_or(0), - msg: msg.into(), - }; - let args = unwrap!(bencode(&args)); - spawn(async move { - let rc = helperᶜ("broadcast_p2p_msg", args).await; - if let Err(err) = rc { - log!("!broadcast_p2p_msg: "(err)) - } - }); - } - /// Get a reference to the secp256k1 key pair. /// Panics if the key pair is not available. pub fn secp256k1_key_pair(&self) -> &KeyPair { diff --git a/mm2src/common/mm_number.rs b/mm2src/common/mm_number.rs index dfdd07a27c..1037a62e74 100644 --- a/mm2src/common/mm_number.rs +++ b/mm2src/common/mm_number.rs @@ -9,7 +9,7 @@ use std::str::FromStr; pub use num_bigint::{BigInt, Sign}; -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Serialize)] pub struct MmNumber(BigRational); /// Rational number representation de/serializable in human readable form @@ -45,7 +45,7 @@ impl Into for Fraction { } impl From for Fraction { - fn from(dec: BigDecimal) -> Fraction { from_dec_to_ratio(dec).into() } + fn from(dec: BigDecimal) -> Fraction { from_dec_to_ratio(&dec).into() } } impl<'de> Deserialize<'de> for Fraction { @@ -75,7 +75,7 @@ pub fn from_ratio_to_dec(r: &BigRational) -> BigDecimal { BigDecimal::from(r.numer().clone()) / BigDecimal::from(r.denom().clone()) } -pub fn from_dec_to_ratio(d: BigDecimal) -> BigRational { +pub fn from_dec_to_ratio(d: &BigDecimal) -> BigRational { let (num, scale) = d.as_bigint_and_exponent(); let ten = BigInt::from(10); if scale >= 0 { @@ -99,7 +99,7 @@ impl<'de> Deserialize<'de> for MmNumber { let raw: Box = Deserialize::deserialize(deserializer)?; match BigDecimal::from_str(&raw.get().trim_matches('"')) { - Ok(dec) => return Ok(MmNumber(from_dec_to_ratio(dec))), + Ok(dec) => return Ok(MmNumber(from_dec_to_ratio(&dec))), Err(_) => (), }; @@ -125,7 +125,7 @@ impl std::fmt::Display for MmNumber { } impl From for MmNumber { - fn from(n: BigDecimal) -> MmNumber { from_dec_to_ratio(n).into() } + fn from(n: BigDecimal) -> MmNumber { from_dec_to_ratio(&n).into() } } impl From for MmNumber { @@ -208,33 +208,14 @@ impl Div for &MmNumber { } } -impl PartialOrd for MmNumber { - fn partial_cmp(&self, rhs: &MmNumber) -> Option { - let lhs = from_ratio_to_dec(&self.0); - let rhs = from_ratio_to_dec(&rhs.0); - Some(lhs.cmp(&rhs)) - } -} - impl PartialOrd for MmNumber { fn partial_cmp(&self, other: &BigDecimal) -> Option { - Some(from_ratio_to_dec(&self.0).cmp(other)) - } -} - -impl PartialEq for MmNumber { - fn eq(&self, rhs: &MmNumber) -> bool { - let lhs = from_ratio_to_dec(&self.0); - let rhs = from_ratio_to_dec(&rhs.0); - lhs == rhs + Some(self.0.cmp(&from_dec_to_ratio(other))) } } impl PartialEq for MmNumber { - fn eq(&self, rhs: &BigDecimal) -> bool { - let dec = from_ratio_to_dec(&self.0); - &dec == rhs - } + fn eq(&self, rhs: &BigDecimal) -> bool { self.0 == from_dec_to_ratio(rhs) } } impl Default for MmNumber { @@ -282,17 +263,17 @@ mod tests { #[test] fn test_from_dec_to_ratio() { let number: BigDecimal = "11.00000000000000000000000000000000000000".parse().unwrap(); - let rational = from_dec_to_ratio(number); + let rational = from_dec_to_ratio(&number); assert_eq!(*rational.numer(), 11.into()); assert_eq!(*rational.denom(), 1.into()); let number: BigDecimal = "0.00000001".parse().unwrap(); - let rational = from_dec_to_ratio(number); + let rational = from_dec_to_ratio(&number); assert_eq!(*rational.numer(), 1.into()); assert_eq!(*rational.denom(), 100000000.into()); let number: BigDecimal = 1.into(); - let rational = from_dec_to_ratio(number); + let rational = from_dec_to_ratio(&number); assert_eq!(*rational.numer(), 1.into()); assert_eq!(*rational.denom(), 1.into()); } @@ -312,7 +293,7 @@ mod tests { for num in vals { let decimal: BigDecimal = BigDecimal::from_str(num).unwrap(); - let expected: MmNumber = from_dec_to_ratio(decimal).into(); + let expected: MmNumber = from_dec_to_ratio(&decimal).into(); let actual: MmNumber = json::from_str(&num).unwrap(); assert_eq!(expected, actual); } diff --git a/mm2src/docker_tests.rs b/mm2src/docker_tests.rs index 151b0beafc..0fbd422aae 100644 --- a/mm2src/docker_tests.rs +++ b/mm2src/docker_tests.rs @@ -634,6 +634,15 @@ mod docker_tests { log!([block_on(enable_native(&mm_alice, "MYCOIN", vec![]))]); log!([block_on(enable_native(&mm_alice, "MYCOIN1", vec![]))]); + log!("Get MYCOIN/MYCOIN1 orderbook on Alice side to trigger subscription"); + let rc = unwrap!(block_on(mm_alice.rpc(json! ({ + "userpass": mm_alice.userpass, + "method": "orderbook", + "base": "MYCOIN", + "rel": "MYCOIN1", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + let rc = unwrap!(block_on(mm_bob.rpc(json! ({ "userpass": mm_bob.userpass, "method": "setprice", @@ -768,6 +777,15 @@ mod docker_tests { log!([block_on(enable_native(&mm_alice, "MYCOIN", vec![]))]); log!([block_on(enable_native(&mm_alice, "MYCOIN1", vec![]))]); + log!("Get MYCOIN/MYCOIN1 orderbook on Alice side to trigger subscription"); + let rc = unwrap!(block_on(mm_alice.rpc(json! ({ + "userpass": mm_alice.userpass, + "method": "orderbook", + "base": "MYCOIN", + "rel": "MYCOIN1", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + let rc = unwrap!(block_on(mm_bob.rpc(json! ({ "userpass": mm_bob.userpass, "method": "setprice", @@ -792,6 +810,7 @@ mod docker_tests { let asks = bob_orderbook["asks"].as_array().unwrap(); assert_eq!(asks.len(), 1, "Bob MYCOIN/MYCOIN1 orderbook must have exactly 1 ask"); + thread::sleep(Duration::from_secs(2)); log!("Get MYCOIN/MYCOIN1 orderbook on Alice side"); let rc = unwrap!(block_on(mm_alice.rpc(json! ({ "userpass": mm_alice.userpass, @@ -913,6 +932,16 @@ mod docker_tests { log!([block_on(enable_native(&mm_bob, "MYCOIN1", vec![]))]); log!([block_on(enable_native(&mm_alice, "MYCOIN", vec![]))]); log!([block_on(enable_native(&mm_alice, "MYCOIN1", vec![]))]); + // TODO remove this request when orderbook request is reimplemented using tries + log!("Get MYCOIN/MYCOIN1 orderbook on Alice side to trigger subscription"); + let rc = unwrap!(block_on(mm_alice.rpc(json! ({ + "userpass": mm_alice.userpass, + "method": "orderbook", + "base": "MYCOIN", + "rel": "MYCOIN1", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + let rc = unwrap!(block_on(mm_bob.rpc(json! ({ "userpass": mm_bob.userpass, "method": "setprice", @@ -1532,4 +1561,69 @@ mod docker_tests { }))); } } + + #[test] + fn test_maker_order_should_kick_start_and_appear_in_orderbook_on_restart() { + let (_ctx, _, bob_priv_key) = generate_coin_with_random_privkey("MYCOIN", 1000); + let (_ctx, _, alice_priv_key) = generate_coin_with_random_privkey("MYCOIN1", 2000); + let coins = json! ([ + {"coin":"MYCOIN","asset":"MYCOIN","txversion":4,"overwintered":1,"txfee":1000,"protocol":{"type":"UTXO"}}, + {"coin":"MYCOIN1","asset":"MYCOIN1","txversion":4,"overwintered":1,"txfee":1000,"protocol":{"type":"UTXO"}}, + ]); + let mut bob_conf = json! ({ + "gui": "nogui", + "netid": 9000, + "dht": "on", // Enable DHT without delay. + "passphrase": format!("0x{}", hex::encode(bob_priv_key)), + "coins": coins, + "rpc_password": "pass", + "i_am_seed": true, + }); + let mut mm_bob = unwrap!(MarketMakerIt::start(bob_conf.clone(), "pass".to_string(), None,)); + let (_bob_dump_log, _bob_dump_dashboard) = mm_dump(&mm_bob.log_path); + unwrap!(block_on( + mm_bob.wait_for_log(22., |log| log.contains(">>>>>>>>> DEX stats ")) + )); + + log!([block_on(enable_native(&mm_bob, "MYCOIN", vec![]))]); + log!([block_on(enable_native(&mm_bob, "MYCOIN1", vec![]))]); + let rc = unwrap!(block_on(mm_bob.rpc(json! ({ + "userpass": mm_bob.userpass, + "method": "setprice", + "base": "MYCOIN", + "rel": "MYCOIN1", + "price": 1, + "max": true, + })))); + assert!(rc.0.is_success(), "!setprice: {}", rc.1); + + // mm_bob using same DB dir that should kick start the order + bob_conf["dbdir"] = mm_bob.folder.join("DB").to_str().unwrap().into(); + bob_conf["log"] = mm_bob.folder.join("mm2_dup.log").to_str().unwrap().into(); + unwrap!(block_on(mm_bob.stop())); + + let mut mm_bob_dup = unwrap!(MarketMakerIt::start(bob_conf, "pass".to_string(), None,)); + let (_bob_dup_dump_log, _bob_dup_dump_dashboard) = mm_dump(&mm_bob_dup.log_path); + unwrap!(block_on( + mm_bob_dup.wait_for_log(22., |log| log.contains(">>>>>>>>> DEX stats ")) + )); + log!([block_on(enable_native(&mm_bob_dup, "MYCOIN", vec![]))]); + log!([block_on(enable_native(&mm_bob_dup, "MYCOIN1", vec![]))]); + + thread::sleep(Duration::from_secs(2)); + + log!("Get RICK/MORTY orderbook on Bob side"); + let rc = unwrap!(block_on(mm_bob_dup.rpc(json! ({ + "userpass": mm_bob_dup.userpass, + "method": "orderbook", + "base": "MYCOIN", + "rel": "MYCOIN1", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + + let bob_orderbook: Json = unwrap!(json::from_str(&rc.1)); + log!("Bob orderbook "[bob_orderbook]); + let asks = bob_orderbook["asks"].as_array().unwrap(); + assert_eq!(asks.len(), 1, "Bob MYCOIN/MYCOIN1 orderbook must have exactly 1 asks"); + } } diff --git a/mm2src/lp_native_dex.rs b/mm2src/lp_native_dex.rs index 7f6188b65b..fb0b82dbc8 100644 --- a/mm2src/lp_native_dex.rs +++ b/mm2src/lp_native_dex.rs @@ -41,29 +41,11 @@ use crate::common::mm_ctx::{MmArc, MmCtx}; use crate::common::privkey::key_pair_from_seed; use crate::common::{slurp_url, MM_DATETIME, MM_VERSION}; use crate::mm2::lp_network::{p2p_event_process_loop, P2PContext}; -use crate::mm2::lp_ordermatch::{broadcast_maker_keep_alives_loop, lp_ordermatch_loop, migrate_saved_orders, - orders_kick_start, BalanceUpdateOrdermatchHandler}; +use crate::mm2::lp_ordermatch::{broadcast_maker_orders_keep_alive_loop, lp_ordermatch_loop, orders_kick_start, + BalanceUpdateOrdermatchHandler}; use crate::mm2::lp_swap::{running_swaps_num, swap_kick_starts}; use crate::mm2::rpc::spawn_rpc; -// TODO: Use MM2-nightly seed nodes. -// MM1 nodes no longer compatible due to the UTXO reforms in particular. -// We might also diverge in how we handle the p2p communication in the future. - -/// Aka `default_LPnodes`. Initial nodes of the peer-to-peer network. -#[allow(dead_code)] -const P2P_SEED_NODES: [&str; 5] = [ - "5.9.253.195", - "173.212.225.176", - "136.243.45.140", - "23.254.202.142", - "45.32.19.196", -]; - -/// Default seed nodes for netid 9999 that is used for MM2 testing -#[allow(dead_code)] -const P2P_SEED_NODES_9999: [&str; 3] = ["195.201.116.176", "46.4.87.18", "46.4.78.11"]; - pub fn lp_ports(netid: u16) -> Result<(u16, u16, u16), String> { const LP_RPCPORT: u16 = 7783; let max_netid = (65535 - 40 - LP_RPCPORT) / 4; @@ -252,10 +234,7 @@ fn migrate_db(ctx: &MmArc) -> Result<(), String> { } #[cfg(feature = "native")] -fn migration_1(ctx: &MmArc) -> Result<(), String> { - try_s!(migrate_saved_orders(ctx)); - Ok(()) -} +fn migration_1(_ctx: &MmArc) -> Result<(), String> { Ok(()) } /// Resets the context (most of which resides currently in `lp::G` but eventually would move into `MmCtx`). /// Restarts the peer connections. @@ -514,7 +493,7 @@ pub async fn lp_init(mypubport: u16, ctx: MmArc) -> Result<(), String> { spawn(lp_ordermatch_loop(ctxʹ)); let ctxʹ = ctx.clone(); - spawn(broadcast_maker_keep_alives_loop(ctxʹ)); + spawn(broadcast_maker_orders_keep_alive_loop(ctxʹ)); #[cfg(not(feature = "native"))] { diff --git a/mm2src/lp_network.rs b/mm2src/lp_network.rs index c81f6cda20..99d6ad30be 100644 --- a/mm2src/lp_network.rs +++ b/mm2src/lp_network.rs @@ -99,12 +99,14 @@ async fn process_p2p_message( i_am_relay: bool, ) { let mut to_propagate = false; + let mut orderbook_pairs = vec![]; + for topic in message.topics { let mut split = topic.as_str().split(TOPIC_SEPARATOR); match split.next() { Some(lp_ordermatch::ORDERBOOK_PREFIX) => { - if lp_ordermatch::process_msg(ctx.clone(), topic.as_str(), peer_id.to_string(), &message.data).await { - to_propagate = true; + if let Some(pair) = split.next() { + orderbook_pairs.push(pair.to_string()); } }, Some(lp_swap::SWAP_PREFIX) => { @@ -114,6 +116,14 @@ async fn process_p2p_message( None | Some(_) => (), } } + + if !orderbook_pairs.is_empty() { + let process_fut = lp_ordermatch::process_msg(ctx.clone(), orderbook_pairs, peer_id.to_string(), &message.data); + if process_fut.await { + to_propagate = true; + } + } + if to_propagate && i_am_relay { propagate_message(&ctx, message_id, peer_id); } @@ -143,10 +153,10 @@ async fn process_p2p_request( } #[cfg(feature = "native")] -pub fn broadcast_p2p_msg(ctx: &MmArc, topic: String, msg: Vec) { +pub fn broadcast_p2p_msg(ctx: &MmArc, topics: Vec, msg: Vec) { let ctx = ctx.clone(); spawn(async move { - let cmd = AdexBehaviourCmd::PublishMsg { topic, msg }; + let cmd = AdexBehaviourCmd::PublishMsg { topics, msg }; let p2p_ctx = P2PContext::fetch_from_mm_arc(&ctx); if let Err(e) = p2p_ctx.cmd_tx.lock().await.try_send(cmd) { log!("broadcast_p2p_msg cmd_tx.send error "[e]); diff --git a/mm2src/lp_ordermatch.rs b/mm2src/lp_ordermatch.rs index c584498bac..22d96e4f13 100644 --- a/mm2src/lp_ordermatch.rs +++ b/mm2src/lp_ordermatch.rs @@ -22,15 +22,19 @@ use async_trait::async_trait; use bigdecimal::BigDecimal; +use blake2::digest::{Update, VariableOutput}; +use blake2::VarBlake2b; use coins::utxo::{compressed_pub_key_from_priv_raw, ChecksumType}; use coins::{lp_coinfindᵃ, BalanceTradeFeeUpdatedHandler, MmCoinEnum, TradeFee}; use common::executor::{spawn, Timer}; use common::mm_ctx::{from_ctx, MmArc, MmWeak}; -use common::mm_number::{from_dec_to_ratio, Fraction, MmNumber}; +use common::mm_number::{Fraction, MmNumber}; use common::{bits256, block_on, json_dir_entries, new_uuid, now_ms, remove_file, write}; use either::Either; use futures::{compat::Future01CompatExt, lock::Mutex as AsyncMutex, StreamExt}; use gstuff::slurp; +use hash256_std_hasher::Hash256StdHasher; +use hash_db::{Hasher, EMPTY_PREFIX}; use http::Response; use mm2_libp2p::{decode_signed, encode_and_sign, encode_message, pub_sub_topic, TopicPrefix, TOPIC_SEPARATOR}; #[cfg(test)] use mocktopus::macros::*; @@ -38,12 +42,16 @@ use num_rational::BigRational; use num_traits::identities::Zero; use rpc::v1::types::H256 as H256Json; use serde_json::{self as json, Value as Json}; -use std::collections::hash_map::{Entry, HashMap}; +use sp_trie::{delta_trie_root, DBValue, HashDBT, MemoryDB, Trie, TrieConfiguration, TrieDB, TrieDBMut, TrieHash, + TrieMut}; +use std::collections::hash_map::{Entry, HashMap, RawEntryMut}; use std::collections::{BTreeSet, HashSet}; +use std::convert::TryInto; use std::fmt; use std::fs::DirEntry; use std::path::PathBuf; use std::sync::Arc; +use trie_db::NodeCodec as NodeCodecT; use uuid::Uuid; use crate::mm2::{lp_network::{broadcast_p2p_msg, request_one_peer, request_relays, subscribe_to_topic, P2PRequest, @@ -72,79 +80,132 @@ const ORDERBOOK_REQUESTING_TIMEOUT: u64 = MIN_ORDER_KEEP_ALIVE_INTERVAL * 2; const INACTIVE_ORDER_TIMEOUT: u64 = 240; const MIN_TRADING_VOL: &str = "0.00777"; -impl From<(new_protocol::MakerOrderCreated, Vec, String, String)> for PricePingRequest { - fn from(tuple: (new_protocol::MakerOrderCreated, Vec, String, String)) -> PricePingRequest { - let (order, initial_message, pubsecp, peer_id) = tuple; - let price_mm = MmNumber::from(order.price); - let max_vol_mm = MmNumber::from(order.max_volume); - let min_vol_mm = MmNumber::from(order.min_volume); +/// Alphabetically ordered orderbook pair +type AlbOrderedOrderbookPair = String; - PricePingRequest { - method: "".to_string(), - pubkey: "".to_string(), +impl From<(new_protocol::MakerOrderCreated, String)> for OrderbookItem { + fn from(tuple: (new_protocol::MakerOrderCreated, String)) -> OrderbookItem { + let (order, pubkey) = tuple; + + OrderbookItem { + pubkey, base: order.base, rel: order.rel, - price: price_mm.to_decimal(), - price_rat: Some(price_mm), - price64: "".to_string(), - timestamp: now_ms() / 1000, - pubsecp, - sig: "".to_string(), - balance: max_vol_mm.to_decimal(), - balance_rat: Some(max_vol_mm), - min_volume: min_vol_mm, - uuid: Some(order.uuid.into()), - peer_id, - initial_message, - update_messages: Vec::new(), + price: order.price, + max_volume: order.max_volume, + min_volume: order.min_volume, + uuid: order.uuid.into(), + created_at: order.created_at, } } } -async fn process_order_keep_alive( +async fn process_orders_keep_alive( ctx: MmArc, propagated_from_peer: String, from_pubkey: String, - keep_alive: new_protocol::MakerOrderKeepAlive, + topics: Vec, + keep_alive: new_protocol::PubkeyKeepAlive, ) -> bool { - let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); - let uuid = keep_alive.uuid.into(); - if let Some(order) = ordermatch_ctx + let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).expect("from_ctx failed"); + let to_request = ordermatch_ctx .orderbook .lock() .await - .find_order_by_uuid_and_pubkey(&uuid, &from_pubkey) - { - order.timestamp = keep_alive.timestamp; - return true; - } - - if let Some(mut order) = ordermatch_ctx.inactive_orders.lock().await.remove(&uuid) { - order.timestamp = keep_alive.timestamp; - ordermatch_ctx - .orderbook - .lock() - .await - .insert_or_update_order(uuid, order); - return true; - } - - log!("Couldn't find an order " [uuid] ", try request it from peers"); - match request_order(ctx, uuid, propagated_from_peer, &from_pubkey).await { - Ok(Some(order)) => { - ordermatch_ctx - .orderbook - .lock() - .await - .insert_or_update_order(uuid, order); - return true; - }, - Ok(None) => log!("None of peers responded to the GetOrder request"), - Err(e) => log!("Error on GetOrder request: "(e)), + .process_keep_alive(&from_pubkey, topics, keep_alive); + + let req = match to_request { + Some(req) => req, + // The message was processed, simply forward it + None => return true, }; - log!("Skip the order "[uuid]); - false + let resp = + request_one_peer::(ctx.clone(), P2PRequest::Ordermatch(req), propagated_from_peer) + .await; + + let response = match resp { + Ok(Some(resp)) => resp, + _ => return false, + }; + + let mut orderbook = ordermatch_ctx.orderbook.lock().await; + let all_orders_root = pubkey_state_mut(&mut orderbook.pubkeys_state, &from_pubkey).all_orders_trie_root; + let new_orders_root = match response.all_orders_diff { + DeltaOrFullTrie::Delta(delta) => { + let delta = delta.into_iter().map(|(pair, hash)| (pair.into_bytes(), hash)); + delta_trie_root::(&mut orderbook.memory_db, all_orders_root, delta) + .expect("All orders trie should always exist") + }, + DeltaOrFullTrie::FullTrie(values) => { + orderbook.memory_db.remove_and_purge(&all_orders_root, EMPTY_PREFIX); + + let mut new_root = H64::default(); + let values: Vec<_> = values + .into_iter() + .map(|(pair, hash)| (pair.into_bytes(), hash.to_vec())) + .collect(); + if let Err(e) = populate_trie::(&mut orderbook.memory_db, &mut new_root, &values) { + log!("Error " (e) " on trie population with values " [values]); + return false; + } + + new_root + }, + }; + pubkey_state_mut(&mut orderbook.pubkeys_state, &from_pubkey).all_orders_trie_root = new_orders_root; + + for (pair, diff) in response.pair_orders_diff { + let old_root = orderbook + .pubkeys_state + .get(&from_pubkey) + .expect("Pubkey state always exists at this point") + .order_pairs_trie_roots + .get(&pair) + .map(|val| *val) + .unwrap_or(H64::default()); + let new_root = match diff { + DeltaOrFullTrie::Delta(delta) => { + let delta = delta.into_iter().map(|(uuid, order)| { + ( + *uuid.as_bytes(), + order.map(|o| rmp_serde::to_vec(&o).expect("Serialization failed")), + ) + }); + delta_trie_root::(&mut orderbook.memory_db, old_root, delta) + .expect("Pair trie should always exist") + }, + DeltaOrFullTrie::FullTrie(values) => { + orderbook.memory_db.remove_and_purge(&old_root, EMPTY_PREFIX); + + let mut new_root = H64::default(); + let trie_values: Vec<_> = values + .iter() + .map(|(uuid, order)| { + ( + uuid.as_bytes().to_vec(), + rmp_serde::to_vec(&order).expect("Serialization failed"), + ) + }) + .collect(); + if let Err(e) = populate_trie::(&mut orderbook.memory_db, &mut new_root, &trie_values) { + log!("Error " (e) " on trie population with values " [trie_values]); + return false; + } + for value in values { + orderbook.insert_or_update_order(value.1); + } + new_root + }, + }; + orderbook + .pubkeys_state + .get_mut(&from_pubkey) + .expect("Pubkey state always exists at this point") + .order_pairs_trie_roots + .insert(pair, new_root); + } + true } async fn process_maker_order_updated( @@ -154,82 +215,19 @@ async fn process_maker_order_updated( updated_msg: new_protocol::MakerOrderUpdated, serialized: Vec, ) -> bool { - let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); + let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).expect("from_ctx failed"); let uuid = updated_msg.uuid(); - if let Some(order) = ordermatch_ctx - .orderbook - .lock() - .await - .find_order_by_uuid_and_pubkey(&uuid, &from_pubkey) - { - order.apply_updated(&updated_msg, serialized); - return true; - } - - if let Some(mut order) = ordermatch_ctx.inactive_orders.lock().await.remove(&uuid) { - order.apply_updated(&updated_msg, serialized); - ordermatch_ctx - .orderbook - .lock() - .await - .insert_or_update_order(uuid, order); - return true; - } - - log!("Couldn't find an order " [uuid] ", try request it from peers"); - match request_order(ctx, uuid, propagated_from_peer, &from_pubkey).await { - Ok(Some(order)) => { - ordermatch_ctx - .orderbook - .lock() - .await - .insert_or_update_order(uuid, order); - return true; + let mut orderbook = ordermatch_ctx.orderbook.lock().await; + match orderbook.find_order_by_uuid_and_pubkey(&uuid, &from_pubkey) { + Some(mut order) => { + order.apply_updated(&updated_msg); + orderbook.insert_or_update_order_update_trie(order); + true }, - Ok(None) => log!("None of peers responded to the GetOrder request"), - Err(e) => log!("Error on GetOrder request: "(e)), - }; - log!("Skip the order "[uuid]); - false -} - -async fn request_order( - ctx: MmArc, - uuid: Uuid, - propagated_from_peer: String, - from_pubkey: &str, -) -> Result, String> { - let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); - if ordermatch_ctx - .order_requests_tracker - .lock() - .await - .limit_reached(&propagated_from_peer) - { - return ERR!("Reached requests per second limit to peer {}", propagated_from_peer); - } - - ordermatch_ctx - .order_requests_tracker - .lock() - .await - .peer_requested(&propagated_from_peer); - - let get_order = OrdermatchRequest::GetOrder { - uuid, - from_pubkey: from_pubkey.to_string(), - }; - let req = P2PRequest::Ordermatch(get_order); - match try_s!(request_one_peer::(ctx, req, propagated_from_peer).await) { - Some(order) => { - let order = try_s!(PricePingRequest::from_initial_msg( - order.initial_message, - order.update_messages, - order.from_peer, - )); - Ok(Some(order)) + None => { + log!("Couldn't find an order " [uuid] ", ignoring, it will be synced upon pubkey keep alive"); + false }, - None => Ok(None), } } @@ -246,19 +244,20 @@ async fn request_and_fill_orderbook( asks_num: Option, bids_num: Option, ) -> Result<(), String> { - // The function converts the given Vec to Iter. + /* + // The function converts the given Vec to Iter. fn process_initial_messages( initial_msgs: Vec, - ) -> impl Iterator { + ) -> impl Iterator { initial_msgs.into_iter().filter_map( |new_protocol::OrderInitialMessage { initial_message, from_peer, update_messages, - }| match PricePingRequest::from_initial_msg(initial_message, update_messages, from_peer) { + }| match OrderbookItem::from_initial_msg(initial_message, update_messages, from_peer) { Ok(order) => Some(order), Err(e) => { - log!("Error on parse PricePingRequest from initial message: "[e]); + log!("Error on parse OrderbookItem from initial message: "[e]); None }, }, @@ -305,48 +304,36 @@ async fn request_and_fill_orderbook( let mut orderbook = ordermatch_ctx.orderbook.lock().await; for ask in asks { - orderbook.insert_or_update_order(ask.uuid.clone().unwrap(), ask); + orderbook.insert_or_update_order(ask); } for bid in bids { - orderbook.insert_or_update_order(bid.uuid.clone().unwrap(), bid); + orderbook.insert_or_update_order(bid); } orderbook .topics_subscribed_to .insert(orderbook_topic(base, rel), OrderbookRequestingState::Requested); - Ok(()) -} - -/// Processes keep alive message of our own node, returns whether operation was successful (order exists) -async fn process_my_order_keep_alive(ctx: &MmArc, keep_alive: &new_protocol::MakerOrderKeepAlive) -> bool { - let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); - let mut orderbook = ordermatch_ctx.orderbook.lock().await; - - let uuid = keep_alive.uuid.into(); - if let Some(mut order) = orderbook.find_order_by_uuid(&uuid) { - order.timestamp = keep_alive.timestamp; - return true; - } - false + */ + Ok(()) } /// Insert or update an order `req`. /// Note this function locks the [`OrdermatchContext::orderbook`] async mutex. -async fn insert_or_update_order(ctx: &MmArc, req: PricePingRequest, uuid: Uuid) { - let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); +async fn insert_or_update_order(ctx: &MmArc, item: OrderbookItem) { + let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).expect("from_ctx failed"); let mut orderbook = ordermatch_ctx.orderbook.lock().await; - orderbook.insert_or_update_order(uuid, req) + orderbook.insert_or_update_order_update_trie(item) } async fn delete_order(ctx: &MmArc, pubkey: &str, uuid: Uuid) { - let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); + let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).expect("from_ctx failed"); let mut inactive = ordermatch_ctx.inactive_orders.lock().await; match inactive.get(&uuid) { // don't remove the order if the pubkey is not equal - Some(order) if order.pubsecp != pubkey => (), + Some(order) if order.pubkey != pubkey => (), Some(_) => { inactive.remove(&uuid); }, @@ -356,38 +343,31 @@ async fn delete_order(ctx: &MmArc, pubkey: &str, uuid: Uuid) { let mut orderbook = ordermatch_ctx.orderbook.lock().await; match orderbook.order_set.get(&uuid) { // don't remove the order if the pubkey is not equal - Some(order) if order.pubsecp != pubkey => (), + Some(order) if order.pubkey != pubkey => (), Some(_) => { - orderbook.remove_order(uuid); + orderbook.remove_order_trie_update(uuid); }, None => (), } } async fn delete_my_order(ctx: &MmArc, uuid: Uuid) { - let ordermatch_ctx: Arc = OrdermatchContext::from_ctx(&ctx).unwrap(); + let ordermatch_ctx: Arc = OrdermatchContext::from_ctx(&ctx).expect("from_ctx failed"); let mut orderbook = ordermatch_ctx.orderbook.lock().await; - orderbook.remove_order(uuid); + orderbook.remove_order_trie_update(uuid); } /// Attempts to decode a message and process it returning whether the message is valid and worth rebroadcasting -pub async fn process_msg(ctx: MmArc, _initial_topic: &str, from_peer: String, msg: &[u8]) -> bool { +pub async fn process_msg(ctx: MmArc, topics: Vec, from_peer: String, msg: &[u8]) -> bool { match decode_signed::(msg) { Ok((message, _sig, pubkey)) => match message { new_protocol::OrdermatchMessage::MakerOrderCreated(created_msg) => { - let req: PricePingRequest = ( - created_msg, - msg.to_vec(), - hex::encode(pubkey.to_bytes().as_slice()), - from_peer, - ) - .into(); - let uuid = req.uuid.unwrap(); - insert_or_update_order(&ctx, req, uuid).await; + let order: OrderbookItem = (created_msg, hex::encode(pubkey.to_bytes().as_slice())).into(); + insert_or_update_order(&ctx, order).await; true }, - new_protocol::OrdermatchMessage::MakerOrderKeepAlive(keep_alive) => { - process_order_keep_alive(ctx, from_peer, pubkey.to_hex(), keep_alive).await + new_protocol::OrdermatchMessage::PubkeyKeepAlive(keep_alive) => { + process_orders_keep_alive(ctx, from_peer, pubkey.to_hex(), topics, keep_alive).await }, new_protocol::OrdermatchMessage::TakerRequest(taker_request) => { let msg = TakerRequest::from_new_proto_and_pubkey(taker_request, pubkey.unprefixed().into()); @@ -422,11 +402,8 @@ pub async fn process_msg(ctx: MmArc, _initial_topic: &str, from_peer: String, ms } } -#[derive(Eq, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum OrdermatchRequest { - /// Get an order using uuid and the order maker's pubkey. - /// Actual we expect to receive [`OrderInitialMessage`] that will be parsed into [`OrdermatchMessage::MakerOrderCreated`]. - GetOrder { uuid: Uuid, from_pubkey: String }, /// Get an orderbook for the given pair. GetOrderbook { base: String, @@ -436,31 +413,80 @@ pub enum OrdermatchRequest { /// Get the given number of best bids if the `bids_num` is some, else get all of the bids. bids_num: Option, }, + /// Sync specific pubkey orderbook state if our known Patricia trie state doesn't match the latest keep alive message + SyncPubkeyOrderbookState { + pubkey: String, + /// Latest known orders trie root by our node + current_orders_trie_root: H64, + /// Expected orders trie root + expected_orders_trie_root: H64, + /// Request using this condition + pairs_trie_roots: HashMap, + }, +} + +#[derive(Debug)] +struct TryFromBytesError(String); + +impl From for TryFromBytesError { + fn from(string: String) -> Self { TryFromBytesError(string) } +} + +trait TryFromBytes { + fn try_from_bytes(bytes: Vec) -> Result + where + Self: Sized; +} + +impl TryFromBytes for String { + fn try_from_bytes(bytes: Vec) -> Result { + String::from_utf8(bytes).map_err(|e| ERRL!("{}", e).into()) + } +} + +impl TryFromBytes for OrderbookItem { + fn try_from_bytes(bytes: Vec) -> Result { + rmp_serde::from_read(bytes.as_slice()).map_err(|e| ERRL!("{}", e).into()) + } +} + +impl TryFromBytes for H64 { + fn try_from_bytes(bytes: Vec) -> Result { + bytes.try_into().map_err(|e| ERRL!("{:?}", e).into()) + } +} + +impl TryFromBytes for Uuid { + fn try_from_bytes(bytes: Vec) -> Result { + Uuid::from_slice(&bytes).map_err(|e| ERRL!("{}", e).into()) + } } pub async fn process_peer_request(ctx: MmArc, request: OrdermatchRequest) -> Result>, String> { println!("Got ordermatch request {:?}", request); match request { - OrdermatchRequest::GetOrder { uuid, from_pubkey } => process_get_order_request(ctx, uuid, from_pubkey).await, OrdermatchRequest::GetOrderbook { base, rel, asks_num, bids_num, } => process_get_orderbook_request(ctx, base, rel, asks_num, bids_num).await, - } -} - -async fn process_get_order_request(ctx: MmArc, uuid: Uuid, from_pubkey: String) -> Result>, String> { - let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); - let mut orderbook = ordermatch_ctx.orderbook.lock().await; - match orderbook.find_order_by_uuid_and_pubkey(&uuid, &from_pubkey) { - Some(order) => { - let response: new_protocol::OrderInitialMessage = order.clone().into(); - let encoded = try_s!(encode_message(&response)); - Ok(Some(encoded)) + OrdermatchRequest::SyncPubkeyOrderbookState { + pubkey, + current_orders_trie_root, + expected_orders_trie_root, + pairs_trie_roots, + } => { + let response = process_sync_pubkey_orderbook_state( + ctx, + pubkey, + current_orders_trie_root, + expected_orders_trie_root, + pairs_trie_roots, + ) + .await; + response.map(|res| res.map(|r| encode_message(&r).expect("Serialization failed"))) }, - None => Ok(None), } } @@ -471,6 +497,7 @@ async fn process_get_orderbook_request( asks_num: Option, bids_num: Option, ) -> Result>, String> { + /* enum PriceOrdering { LowestToHighest, HighestToLowest, @@ -524,9 +551,126 @@ async fn process_get_orderbook_request( let response = new_protocol::Orderbook { asks, bids }; let encoded = try_s!(encode_message(&response)); Ok(Some(encoded)) + + */ + Ok(None) } -fn alb_ordered_pair(base: &str, rel: &str) -> String { +#[derive(Debug, Deserialize, Serialize)] +enum DeltaOrFullTrie { + Delta(HashMap>), + FullTrie(Vec<(Key, Value)>), +} + +#[derive(Debug)] +enum TrieDiffHistoryError { + TrieDbError(Box>), + TryFromBytesError(TryFromBytesError), +} + +impl std::fmt::Display for TrieDiffHistoryError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "({:?})", self) } +} + +impl From for TrieDiffHistoryError { + fn from(error: TryFromBytesError) -> TrieDiffHistoryError { TrieDiffHistoryError::TryFromBytesError(error) } +} + +impl From>> for TrieDiffHistoryError { + fn from(error: Box>) -> TrieDiffHistoryError { + TrieDiffHistoryError::TrieDbError(error) + } +} + +impl DeltaOrFullTrie { + fn from_history( + history: &TrieDiffHistory, + from_hash: H64, + trie_from_hash: H64, + db: &MemoryDB, + ) -> Result, TrieDiffHistoryError> { + match history.get(&from_hash) { + Some(delta) => { + let mut current_delta = delta; + let mut total_delta = HashMap::new(); + for (key, new_value) in &delta.delta { + total_delta.insert(key.clone(), new_value.clone()); + } + while let Some(cur) = history.get(¤t_delta.next_root) { + current_delta = cur; + for (key, new_value) in ¤t_delta.delta { + total_delta.insert(key.clone(), new_value.clone()); + } + } + Ok(DeltaOrFullTrie::Delta(total_delta)) + }, + None => { + let trie = TrieDB::::new(db, &trie_from_hash)?; + let trie: Result, TrieDiffHistoryError> = trie + .iter()? + .map(|key_value| { + let (key, value) = key_value?; + Ok((TryFromBytes::try_from_bytes(key)?, TryFromBytes::try_from_bytes(value)?)) + }) + .collect(); + Ok(DeltaOrFullTrie::FullTrie(trie?)) + }, + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +struct SyncPubkeyOrderbookStateRes { + all_orders_diff: DeltaOrFullTrie, + pair_orders_diff: HashMap>, +} + +async fn process_sync_pubkey_orderbook_state( + ctx: MmArc, + pubkey: String, + current_orders_trie_root: H64, + expected_orders_trie_root: H64, + pairs_trie_roots: HashMap, +) -> Result, String> { + let ordermatch_ctx = unwrap!(OrdermatchContext::from_ctx(&ctx)); + let orderbook = ordermatch_ctx.orderbook.lock().await; + let pubkey_state = match orderbook.pubkeys_state.get(&pubkey) { + Some(s) => s, + None => return Ok(None), + }; + + let all_orders_diff = try_s!(DeltaOrFullTrie::from_history( + &pubkey_state.orders_trie_state_history, + current_orders_trie_root, + pubkey_state.all_orders_trie_root, + &orderbook.memory_db, + )); + + let pair_orders_diff: Result<_, _> = pairs_trie_roots + .into_iter() + .map(|(pair, root)| { + let pair_root = try_s!(pubkey_state + .order_pairs_trie_roots + .get(&pair) + .ok_or(format!("No pair trie root for {}", pair))); + let delta = try_s!(DeltaOrFullTrie::from_history( + &pubkey_state.order_pairs_trie_state_history, + root, + *pair_root, + &orderbook.memory_db, + )); + Ok((pair, delta)) + }) + .collect(); + let pair_orders_diff = try_s!(pair_orders_diff); + let result = SyncPubkeyOrderbookStateRes { + all_orders_diff, + pair_orders_diff, + }; + Ok(Some(result)) +} + +fn alb_ordered_pair(base: &str, rel: &str) -> AlbOrderedOrderbookPair { let (first, second) = if base < rel { (base, rel) } else { (rel, base) }; let mut res = first.to_owned(); res.push(':'); @@ -534,7 +678,11 @@ fn alb_ordered_pair(base: &str, rel: &str) -> String { res } -fn orderbook_topic(base: &str, rel: &str) -> String { pub_sub_topic(ORDERBOOK_PREFIX, &alb_ordered_pair(base, rel)) } +fn orderbook_topic_from_base_rel(base: &str, rel: &str) -> String { + pub_sub_topic(ORDERBOOK_PREFIX, &alb_ordered_pair(base, rel)) +} + +fn orderbook_topic_from_ordered_pair(pair: &str) -> String { pub_sub_topic(ORDERBOOK_PREFIX, pair) } #[test] fn test_alb_ordered_pair() { @@ -574,45 +722,44 @@ fn test_parse_orderbook_pair_from_topic() { } async fn maker_order_created_p2p_notify(ctx: MmArc, order: &MakerOrder) { - let topic = orderbook_topic(&order.base, &order.rel); + let topic = orderbook_topic_from_base_rel(&order.base, &order.rel); let message = new_protocol::MakerOrderCreated { uuid: order.uuid.into(), base: order.base.clone(), rel: order.rel.clone(), price: order.price.to_ratio(), - max_volume: order.max_base_vol.to_ratio(), + max_volume: order.available_amount().to_ratio(), min_volume: order.min_base_vol.to_ratio(), conf_settings: order.conf_settings.unwrap(), + created_at: now_ms() / 1000, }; let key_pair = ctx.secp256k1_key_pair.or(&&|| panic!()); let to_broadcast = new_protocol::OrdermatchMessage::MakerOrderCreated(message.clone()); let encoded_msg = encode_and_sign(&to_broadcast, &*key_pair.private().secret).unwrap(); - let peer = ctx.peer_id.or(&&|| panic!()).clone(); - let price_ping_req: PricePingRequest = - (message, encoded_msg.clone(), hex::encode(&**key_pair.public()), peer).into(); - let uuid = price_ping_req.uuid.unwrap(); - insert_or_update_order(&ctx, price_ping_req, uuid).await; - broadcast_p2p_msg(&ctx, topic, encoded_msg); + let order: OrderbookItem = (message, hex::encode(&**key_pair.public())).into(); + insert_or_update_order(&ctx, order).await; + broadcast_p2p_msg(&ctx, vec![topic], encoded_msg); } async fn process_my_maker_order_updated(ctx: &MmArc, message: &new_protocol::MakerOrderUpdated, serialized: Vec) { - let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); + let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).expect("from_ctx failed"); let mut orderbook = ordermatch_ctx.orderbook.lock().await; let uuid = message.uuid(); - if let Some(order) = orderbook.find_order_by_uuid(&uuid) { - order.apply_updated(message, serialized); + if let Some(mut order) = orderbook.find_order_by_uuid(&uuid) { + order.apply_updated(message); + orderbook.insert_or_update_order_update_trie(order); } } async fn maker_order_updated_p2p_notify(ctx: MmArc, base: &str, rel: &str, message: new_protocol::MakerOrderUpdated) { let msg: new_protocol::OrdermatchMessage = message.clone().into(); - let topic = orderbook_topic(base, rel); + let topic = orderbook_topic_from_base_rel(base, rel); let key_pair = ctx.secp256k1_key_pair.or(&&|| panic!()); let encoded_msg = encode_and_sign(&msg, &*key_pair.private().secret).unwrap(); process_my_maker_order_updated(&ctx, &message, encoded_msg.clone()).await; - broadcast_p2p_msg(&ctx, topic, encoded_msg); + broadcast_p2p_msg(&ctx, vec![topic], encoded_msg); } async fn maker_order_cancelled_p2p_notify(ctx: MmArc, order: &MakerOrder) { @@ -621,7 +768,11 @@ async fn maker_order_cancelled_p2p_notify(ctx: MmArc, order: &MakerOrder) { }); delete_my_order(&ctx, order.uuid).await; println!("maker_order_cancelled_p2p_notify called, message {:?}", message); - broadcast_ordermatch_message(&ctx, orderbook_topic(&order.base, &order.rel), message); + broadcast_ordermatch_message( + &ctx, + vec![orderbook_topic_from_base_rel(&order.base, &order.rel)], + message, + ); } pub struct BalanceUpdateOrdermatchHandler { @@ -696,13 +847,10 @@ impl OrderConfirmationsSettings { pub struct TakerRequest { base: String, rel: String, - base_amount: BigDecimal, - base_amount_rat: Option, - rel_amount: BigDecimal, - rel_amount_rat: Option, + base_amount: MmNumber, + rel_amount: MmNumber, action: TakerAction, uuid: Uuid, - method: String, sender_pubkey: H256Json, dest_pub_key: H256Json, #[serde(default)] @@ -712,19 +860,16 @@ pub struct TakerRequest { impl TakerRequest { fn from_new_proto_and_pubkey(message: new_protocol::TakerRequest, sender_pubkey: H256Json) -> Self { - let base_amount_mm = MmNumber::from(message.base_amount); - let rel_amount_mm = MmNumber::from(message.rel_amount); + let base_amount = MmNumber::from(message.base_amount); + let rel_amount = MmNumber::from(message.rel_amount); TakerRequest { base: message.base, rel: message.rel, - base_amount: base_amount_mm.to_decimal(), - base_amount_rat: Some(base_amount_mm.into()), - rel_amount: rel_amount_mm.to_decimal(), - rel_amount_rat: Some(rel_amount_mm.into()), + base_amount, + rel_amount, action: message.action, uuid: message.uuid.into(), - method: "".to_string(), sender_pubkey, dest_pub_key: Default::default(), match_by: message.match_by.into(), @@ -750,8 +895,8 @@ impl TakerRequest { impl Into for TakerRequest { fn into(self) -> new_protocol::OrdermatchMessage { new_protocol::OrdermatchMessage::TakerRequest(new_protocol::TakerRequest { - base_amount: self.get_base_amount().into(), - rel_amount: self.get_rel_amount().into(), + base_amount: self.get_base_amount().to_ratio(), + rel_amount: self.get_rel_amount().to_ratio(), base: self.base, rel: self.rel, action: self.action, @@ -763,19 +908,9 @@ impl Into for TakerRequest { } impl TakerRequest { - fn get_base_amount(&self) -> MmNumber { - match &self.base_amount_rat { - Some(r) => r.clone().into(), - None => self.base_amount.clone().into(), - } - } + fn get_base_amount(&self) -> &MmNumber { &self.base_amount } - fn get_rel_amount(&self) -> MmNumber { - match &self.rel_amount_rat { - Some(r) => r.clone().into(), - None => self.rel_amount.clone().into(), - } - } + fn get_rel_amount(&self) -> &MmNumber { &self.rel_amount } } struct TakerRequestBuilder { @@ -889,7 +1024,7 @@ impl TakerRequestBuilder { /// Validate fields and build fn build(self) -> Result { - let min_vol = MmNumber::from(MIN_TRADING_VOL.parse::().unwrap()); + let min_vol = MmNumber::from(MIN_TRADING_VOL); if self.base.is_empty() { return Err(TakerRequestBuildError::BaseCoinEmpty); @@ -928,13 +1063,10 @@ impl TakerRequestBuilder { Ok(TakerRequest { base: self.base, rel: self.rel, - base_amount: self.base_amount.to_decimal(), - base_amount_rat: Some(self.base_amount.into()), - rel_amount: self.rel_amount.to_decimal(), - rel_amount_rat: Some(self.rel_amount.into()), + base_amount: self.base_amount, + rel_amount: self.rel_amount, action: self.action, uuid: new_uuid(), - method: "request".to_string(), sender_pubkey: self.sender_pubkey, dest_pub_key: Default::default(), match_by: self.match_by, @@ -948,13 +1080,10 @@ impl TakerRequestBuilder { TakerRequest { base: self.base, rel: self.rel, - base_amount: self.base_amount.to_decimal(), - base_amount_rat: Some(self.base_amount.into()), - rel_amount: self.rel_amount.to_decimal(), - rel_amount_rat: Some(self.rel_amount.into()), + base_amount: self.base_amount, + rel_amount: self.rel_amount, action: self.action, uuid: new_uuid(), - method: "request".to_string(), sender_pubkey: self.sender_pubkey, dest_pub_key: Default::default(), match_by: self.match_by, @@ -1021,10 +1150,10 @@ impl TakerOrder { }, } - let my_base_amount: MmNumber = self.request.get_base_amount(); - let my_rel_amount: MmNumber = self.request.get_rel_amount(); - let other_base_amount: MmNumber = reserved.get_base_amount(); - let other_rel_amount: MmNumber = reserved.get_rel_amount(); + let my_base_amount = self.request.get_base_amount(); + let my_rel_amount = self.request.get_rel_amount(); + let other_base_amount = reserved.get_base_amount(); + let other_rel_amount = reserved.get_rel_amount(); match self.request.action { TakerAction::Buy => { @@ -1274,7 +1403,7 @@ impl MakerOrder { fn available_amount(&self) -> MmNumber { let reserved: MmNumber = self.matches.iter().fold( MmNumber::from(BigRational::from_integer(0.into())), - |reserved, (_, order_match)| reserved + order_match.reserved.get_base_amount(), + |reserved, (_, order_match)| &reserved + order_match.reserved.get_base_amount(), ); &self.max_base_vol - &reserved } @@ -1292,38 +1421,38 @@ impl MakerOrder { } fn match_with_request(&self, taker: &TakerRequest) -> OrderMatchResult { - let taker_base_amount: MmNumber = taker.get_base_amount(); - let taker_rel_amount: MmNumber = taker.get_rel_amount(); + let taker_base_amount = taker.get_base_amount(); + let taker_rel_amount = taker.get_rel_amount(); let zero = MmNumber::from(0); - if taker_base_amount <= zero || taker_rel_amount <= zero { + if taker_base_amount <= &zero || taker_rel_amount <= &zero { return OrderMatchResult::NotMatched; } match taker.action { TakerAction::Buy => { - let taker_price = &taker_rel_amount / &taker_base_amount; + let taker_price = taker_rel_amount / taker_base_amount; if self.base == taker.base && self.rel == taker.rel - && taker_base_amount <= self.available_amount() - && taker_base_amount >= self.min_base_vol + && taker_base_amount <= &self.available_amount() + && taker_base_amount >= &self.min_base_vol && taker_price >= self.price { - OrderMatchResult::Matched((taker_base_amount.clone(), &taker_base_amount * &self.price)) + OrderMatchResult::Matched((taker_base_amount.clone(), taker_base_amount * &self.price)) } else { OrderMatchResult::NotMatched } }, TakerAction::Sell => { - let taker_price = &taker_base_amount / &taker_rel_amount; + let taker_price = taker_base_amount / taker_rel_amount; if self.base == taker.rel && self.rel == taker.base - && taker_rel_amount <= self.available_amount() - && taker_rel_amount >= self.min_base_vol + && taker_rel_amount <= &self.available_amount() + && taker_rel_amount >= &self.min_base_vol && taker_price >= self.price { - OrderMatchResult::Matched((&taker_base_amount / &self.price, taker_base_amount)) + OrderMatchResult::Matched((taker_base_amount / &self.price, taker_base_amount.clone())) } else { OrderMatchResult::NotMatched } @@ -1337,7 +1466,7 @@ impl Into for TakerOrder { match self.request.action { TakerAction::Sell => MakerOrder { price: (self.request.get_rel_amount() / self.request.get_base_amount()), - max_base_vol: self.request.get_base_amount(), + max_base_vol: self.request.get_base_amount().clone(), min_base_vol: 0.into(), created_at: now_ms(), base: self.request.base, @@ -1350,7 +1479,7 @@ impl Into for TakerOrder { // The "buy" taker order is recreated with reversed pair as Maker order is always considered as "sell" TakerAction::Buy => MakerOrder { price: (self.request.get_base_amount() / self.request.get_rel_amount()), - max_base_vol: self.request.get_rel_amount(), + max_base_vol: self.request.get_rel_amount().clone(), min_base_vol: 0.into(), created_at: now_ms(), base: self.request.rel, @@ -1368,7 +1497,6 @@ impl Into for TakerOrder { pub struct TakerConnect { taker_order_uuid: Uuid, maker_order_uuid: Uuid, - method: String, sender_pubkey: H256Json, dest_pub_key: H256Json, } @@ -1378,7 +1506,6 @@ impl From for TakerConnect { TakerConnect { taker_order_uuid: message.taker_order_uuid.into(), maker_order_uuid: message.maker_order_uuid.into(), - method: "".to_string(), sender_pubkey: Default::default(), dest_pub_key: Default::default(), } @@ -1399,49 +1526,33 @@ impl Into for TakerConnect { pub struct MakerReserved { base: String, rel: String, - base_amount: BigDecimal, - base_amount_rat: Option, - rel_amount: BigDecimal, - rel_amount_rat: Option, + base_amount: MmNumber, + rel_amount: MmNumber, taker_order_uuid: Uuid, maker_order_uuid: Uuid, - method: String, sender_pubkey: H256Json, dest_pub_key: H256Json, conf_settings: Option, } impl MakerReserved { - fn get_base_amount(&self) -> MmNumber { - match &self.base_amount_rat { - Some(r) => r.clone().into(), - None => self.base_amount.clone().into(), - } - } + fn get_base_amount(&self) -> &MmNumber { &self.base_amount } - fn get_rel_amount(&self) -> MmNumber { - match &self.rel_amount_rat { - Some(r) => r.clone().into(), - None => self.rel_amount.clone().into(), - } - } + fn get_rel_amount(&self) -> &MmNumber { &self.rel_amount } } impl MakerReserved { fn from_new_proto_and_pubkey(message: new_protocol::MakerReserved, sender_pubkey: H256Json) -> Self { - let base_amount_mm = MmNumber::from(message.base_amount); - let rel_amount_mm = MmNumber::from(message.rel_amount); + let base_amount = MmNumber::from(message.base_amount); + let rel_amount = MmNumber::from(message.rel_amount); MakerReserved { base: message.base, rel: message.rel, - base_amount: base_amount_mm.to_decimal(), - rel_amount: rel_amount_mm.to_decimal(), - base_amount_rat: Some(base_amount_mm.into()), - rel_amount_rat: Some(rel_amount_mm.into()), + base_amount, + rel_amount, taker_order_uuid: message.taker_order_uuid.into(), maker_order_uuid: message.maker_order_uuid.into(), - method: "".to_string(), sender_pubkey, dest_pub_key: Default::default(), conf_settings: Some(message.conf_settings), @@ -1452,8 +1563,8 @@ impl MakerReserved { impl Into for MakerReserved { fn into(self) -> new_protocol::OrdermatchMessage { new_protocol::OrdermatchMessage::MakerReserved(new_protocol::MakerReserved { - base_amount: self.get_base_amount().into(), - rel_amount: self.get_rel_amount().into(), + base_amount: self.get_base_amount().to_ratio(), + rel_amount: self.get_rel_amount().to_ratio(), base: self.base, rel: self.rel, taker_order_uuid: self.taker_order_uuid.into(), @@ -1493,49 +1604,47 @@ impl Into for MakerConnected { } } -pub async fn broadcast_maker_keep_alives_loop(ctx: MmArc) { - let interval = MIN_ORDER_KEEP_ALIVE_INTERVAL as f64; +pub async fn broadcast_maker_orders_keep_alive_loop(ctx: MmArc) { + let my_pubsecp = hex::encode(&**ctx.secp256k1_key_pair().public()); while !ctx.is_stopping() { - let ordermatch_ctx: Arc = OrdermatchContext::from_ctx(&ctx).unwrap(); - let to_keep_alive: Vec<_> = ordermatch_ctx - .my_maker_orders - .lock() - .await - .iter() - .map(|(uuid, order)| (*uuid, orderbook_topic(&order.base, &order.rel))) - .collect(); - if to_keep_alive.is_empty() { - Timer::sleep(interval).await; - } else { - let to_sleep = interval / to_keep_alive.len() as f64; - for (uuid, topic) in to_keep_alive { - Timer::sleep(to_sleep).await; - let msg = new_protocol::MakerOrderKeepAlive { - uuid: uuid.into(), - timestamp: now_ms() / 1000, - }; - if process_my_order_keep_alive(&ctx, &msg).await { - broadcast_ordermatch_message(&ctx, topic, msg.into()); - } else { - if let Some(order) = ordermatch_ctx.my_maker_orders.lock().await.get(&uuid) { - maker_order_created_p2p_notify(ctx.clone(), order).await; - } - } + Timer::sleep(MIN_ORDER_KEEP_ALIVE_INTERVAL as f64).await; + let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).expect("from_ctx failed"); + if let Some(state) = ordermatch_ctx.orderbook.lock().await.pubkeys_state.get(&my_pubsecp) { + if state.all_orders_trie_root == H64::default() + || state.all_orders_trie_root == hashed_null_node::() + { + continue; } - } + + let message = new_protocol::PubkeyKeepAlive { + orders_trie_root: state.all_orders_trie_root, + timestamp: now_ms() / 1000, + }; + + let topics: HashSet<_> = state + .orders_uuids + .iter() + .map(|(_, pair)| orderbook_topic_from_ordered_pair(pair)) + .collect(); + broadcast_ordermatch_message(&ctx, topics, message.into()); + }; } } -fn broadcast_ordermatch_message(ctx: &MmArc, topic: String, msg: new_protocol::OrdermatchMessage) { +fn broadcast_ordermatch_message( + ctx: &MmArc, + topics: impl IntoIterator, + msg: new_protocol::OrdermatchMessage, +) { let key_pair = ctx.secp256k1_key_pair.or(&&|| panic!()); let encoded_msg = encode_and_sign(&msg, &*key_pair.private().secret).unwrap(); - broadcast_p2p_msg(ctx, topic, encoded_msg); + broadcast_p2p_msg(ctx, topics.into_iter().collect(), encoded_msg); } -/// The order is ordered by [`PricePingRequest::price`] and [`PricePingRequest::uuid`]. +/// The order is ordered by [`OrderbookItem::price`] and [`OrderbookItem::uuid`]. #[derive(Eq, Ord, PartialEq, PartialOrd)] struct OrderedByPriceOrder { - price: BigDecimal, + price: MmNumber, uuid: Uuid, } @@ -1547,54 +1656,259 @@ enum OrderbookRequestingState { NotRequested { subscribed_at: u64 }, } +type H64 = [u8; 8]; + +#[derive(Debug, Eq, PartialEq)] +struct TrieDiff { + delta: Vec<(Key, Option)>, + next_root: H64, +} + +#[derive(Debug, Eq, PartialEq)] +struct TrieDiffHistory { + inner: HashMap>, +} + +impl Default for TrieDiffHistory { + fn default() -> Self { + TrieDiffHistory { + inner: Default::default(), + } + } +} + +impl TrieDiffHistory { + fn insert_new_diff(&mut self, insert_at: H64, diff: TrieDiff) { + if insert_at == diff.next_root { + // do nothing to avoid cycles in diff history + return; + } + + match self.inner.remove(&diff.next_root) { + Some(mut diff) => { + // we reached a state that was already reached previously + // history can be cleaned up to this state hash + while let Some(next_diff) = self.inner.remove(&diff.next_root) { + diff = next_diff; + } + }, + None => { + self.inner.insert(insert_at, diff); + }, + }; + } + + fn contains_key(&self, key: &H64) -> bool { self.inner.contains_key(key) } + + fn get(&self, key: &H64) -> Option<&TrieDiff> { self.inner.get(key) } +} + +#[derive(Default)] +struct OrderbookPubkeyState { + /// Timestamp of the latest keep alive message received + last_keep_alive: u64, + /// Current Patricia Trie root hash of the pubkey orderbooks tries root hashes + all_orders_trie_root: H64, + /// The map storing historical data about orders trie changes, maps the root hash to the items delta with resulting root hash + /// Used to get diffs of orders_trie between specific root hashes + orders_trie_state_history: TrieDiffHistory, + /// The map storing historical data about specific pair subtrie changes + /// Used to get diffs of orders of pair between specific root hashes + order_pairs_trie_state_history: TrieDiffHistory, + /// The known UUIDs owned by pubkey with alphabetically ordered pair to ease the lookup during pubkey orderbook requests + orders_uuids: HashSet<(Uuid, AlbOrderedOrderbookPair)>, + order_pairs_trie_roots: HashMap, +} + +fn get_trie_mut<'a>( + mem_db: &'a mut MemoryDB, + root: &'a mut H64, +) -> Result, String> { + if *root == H64::default() { + Ok(TrieDBMut::new(mem_db, root)) + } else { + TrieDBMut::from_existing(mem_db, root).map_err(|e| ERRL!("{}", e)) + } +} + +fn pubkey_state_mut<'a>( + state: &'a mut HashMap, + from_pubkey: &str, +) -> &'a mut OrderbookPubkeyState { + match state.raw_entry_mut().from_key(from_pubkey) { + RawEntryMut::Occupied(e) => e.into_mut(), + RawEntryMut::Vacant(e) => { + let mut state: OrderbookPubkeyState = Default::default(); + state.last_keep_alive = now_ms() / 1000; + e.insert(from_pubkey.to_string(), state).1 + }, + } +} + +fn order_pair_root_mut<'a>(state: &'a mut HashMap, pair: &str) -> &'a mut H64 { + match state.raw_entry_mut().from_key(pair) { + RawEntryMut::Occupied(e) => e.into_mut(), + RawEntryMut::Vacant(e) => e.insert(pair.to_string(), Default::default()).1, + } +} + +fn populate_pubkey_trie<'db, T: TrieConfiguration>( + db: &'db mut dyn HashDBT, + root: &'db mut TrieHash, + v: &[(Vec, H64)], +) -> Result, String> { + let mut t = TrieDBMut::::new(db, root); + for (key, val) in v { + try_s!(t.insert(key, val)); + } + Ok(t) +} + +fn populate_trie<'db, T: TrieConfiguration>( + db: &'db mut dyn HashDBT, + root: &'db mut TrieHash, + v: &[(Vec, Vec)], +) -> Result, String> { + let mut t = TrieDBMut::::new(db, root); + for (key, val) in v { + try_s!(t.insert(key, val)); + } + Ok(t) +} + #[derive(Default)] struct Orderbook { /// A map from (base, rel). ordered: HashMap<(String, String), BTreeSet>, /// A map from (base, rel). unordered: HashMap<(String, String), HashSet>, - order_set: HashMap, + order_set: HashMap, + /// a map of orderbook states of known maker pubkeys + pubkeys_state: HashMap, topics_subscribed_to: HashMap, + /// MemoryDB instance to store Patricia Tries data + memory_db: MemoryDB, } +fn hashed_null_node() -> TrieHash { ::hashed_null_node() } + impl Orderbook { - fn find_order_by_uuid_and_pubkey(&mut self, uuid: &Uuid, from_pubkey: &str) -> Option<&mut PricePingRequest> { - self.order_set.get_mut(uuid).and_then(|order| { - if order.pubsecp == from_pubkey { - Some(order) + fn find_order_by_uuid_and_pubkey(&self, uuid: &Uuid, from_pubkey: &str) -> Option { + self.order_set.get(uuid).and_then(|order| { + if order.pubkey == from_pubkey { + Some(order.clone()) } else { None } }) } - fn find_order_by_uuid(&mut self, uuid: &Uuid) -> Option<&mut PricePingRequest> { self.order_set.get_mut(uuid) } + fn find_order_by_uuid(&self, uuid: &Uuid) -> Option { + self.order_set.get(uuid).map(|order| order.clone()) + } + + fn insert_or_update_order_update_trie(&mut self, order: OrderbookItem) { + let zero = BigRational::from_integer(0.into()); + if order.max_volume <= zero || order.price <= zero || order.min_volume < zero { + self.remove_order_trie_update(order.uuid); + return; + } // else insert the order + + self.insert_or_update_order(order.clone()); + + let pubkey_state = pubkey_state_mut(&mut self.pubkeys_state, &order.pubkey); + + let alb_ordered = alb_ordered_pair(&order.base, &order.rel); + let pair_root = order_pair_root_mut(&mut pubkey_state.order_pairs_trie_roots, &alb_ordered); + let prev_root = *pair_root; + + pubkey_state.orders_uuids.insert((order.uuid, alb_ordered.clone())); + + let mut pair_trie = match get_trie_mut(&mut self.memory_db, pair_root) { + Ok(trie) => trie, + Err(e) => { + log!("Error getting "(e)" trie with root "[prev_root]); + return; + }, + }; + let order_bytes = rmp_serde::to_vec(&order).expect("Serialization should never fail"); + if let Err(e) = pair_trie.insert(order.uuid.as_bytes(), &order_bytes) { + log!("Error " (e) " on insertion to trie. Key " (order.uuid) ", value " [order_bytes]); + return; + }; + drop(pair_trie); + + if prev_root != H64::default() { + pubkey_state + .order_pairs_trie_state_history + .insert_new_diff(prev_root, TrieDiff { + delta: vec![(order.uuid, Some(order.clone()))], + next_root: *pair_root, + }); + } + + let old_root = pubkey_state.all_orders_trie_root; + let delta = vec![(alb_ordered.clone(), Some(pair_root.clone()))]; + + if old_root == H64::default() { + let to_populate = vec![(alb_ordered.into_bytes(), pair_root.clone())]; + if let Err(e) = populate_pubkey_trie::( + &mut self.memory_db, + &mut pubkey_state.all_orders_trie_root, + &to_populate, + ) { + log!("Failed to populate trie: "(e) ", to_populate: "[to_populate]); + return; + }; + } else { + let delta_bytes = vec![(alb_ordered.into_bytes(), Some(pair_root.clone()))]; + pubkey_state.all_orders_trie_root = + match delta_trie_root::(&mut self.memory_db, old_root, delta_bytes) { + Ok(root) => root, + Err(e) => { + log!("Failed to get existing trie for "[old_root]", error: "(e)); + return; + }, + }; + } + + if old_root != H64::default() { + pubkey_state + .orders_trie_state_history + .insert_new_diff(old_root, TrieDiff { + delta, + next_root: pubkey_state.all_orders_trie_root, + }); + } + } - fn insert_or_update_order(&mut self, uuid: Uuid, req: PricePingRequest) { - if req.balance <= 0.into() || req.price <= 0.into() { - self.remove_order(uuid); + fn insert_or_update_order(&mut self, order: OrderbookItem) { + log!("Inserting order "[order]); + let zero = BigRational::from_integer(0.into()); + if order.max_volume <= zero || order.price <= zero || order.min_volume < zero { + self.remove_order_trie_update(order.uuid); return; } // else insert the order - let base_rel = (req.base.clone(), req.rel.clone()); + let base_rel = (order.base.clone(), order.rel.clone()); self.ordered .entry(base_rel.clone()) .or_insert_with(BTreeSet::new) .insert(OrderedByPriceOrder { - price: req.price.clone(), - uuid, + price: order.price.clone().into(), + uuid: order.uuid, }); self.unordered .entry(base_rel) .or_insert_with(HashSet::new) - .insert(uuid.clone()); + .insert(order.uuid); - self.order_set.insert(uuid, req); + self.order_set.insert(order.uuid, order); } - fn remove_order(&mut self, uuid: Uuid) -> Option { + fn remove_order(&mut self, uuid: Uuid) -> Option { let order = match self.order_set.remove(&uuid) { Some(order) => order, None => return None, @@ -1603,7 +1917,7 @@ impl Orderbook { // create an `order_to_delete` that allows to find and remove an element from `self.ordered` by hash let order_to_delete = OrderedByPriceOrder { - price: order.price.clone(), + price: order.price.clone().into(), uuid, }; @@ -1620,10 +1934,122 @@ impl Orderbook { if orders.is_empty() { self.unordered.remove(&base_rel); } + }; + Some(order) + } + + fn remove_order_trie_update(&mut self, uuid: Uuid) -> Option { + let order = match self.order_set.remove(&uuid) { + Some(order) => order, + None => return None, + }; + let base_rel = (order.base.clone(), order.rel.clone()); + + // create an `order_to_delete` that allows to find and remove an element from `self.ordered` by hash + let order_to_delete = OrderedByPriceOrder { + price: order.price.clone().into(), + uuid, + }; + + if let Some(orders) = self.ordered.get_mut(&base_rel) { + orders.remove(&order_to_delete); + if orders.is_empty() { + self.ordered.remove(&base_rel); + } } + if let Some(orders) = self.unordered.get_mut(&base_rel) { + // use the same uuid to remove an order + orders.remove(&order_to_delete.uuid); + if orders.is_empty() { + self.unordered.remove(&base_rel); + } + } + + let alb_ordered = alb_ordered_pair(&order.base, &order.rel); + let pubkey_state = pubkey_state_mut(&mut self.pubkeys_state, &order.pubkey); + let pair_state = order_pair_root_mut(&mut pubkey_state.order_pairs_trie_roots, &alb_ordered); + let old_state = *pair_state; + *pair_state = match delta_trie_root::(&mut self.memory_db, *pair_state, vec![( + *order.uuid.as_bytes(), + None::>, + )]) { + Ok(root) => root, + Err(_) => { + log!("Failed to get existing trie with root "[pair_state]); + return Some(order); + }, + }; + + pubkey_state + .order_pairs_trie_state_history + .insert_new_diff(old_state, TrieDiff { + delta: vec![(uuid, None)], + next_root: *pair_state, + }); + + let all_orders_delta = if *pair_state == hashed_null_node::() { + vec![(alb_ordered, None)] + } else { + vec![(alb_ordered, Some(*pair_state))] + }; + + let old_state = pubkey_state.all_orders_trie_root; + pubkey_state.all_orders_trie_root = match delta_trie_root::( + &mut self.memory_db, + pubkey_state.all_orders_trie_root, + all_orders_delta + .clone() + .into_iter() + .map(|(pair, value)| (pair.into_bytes(), value)), + ) { + Ok(root) => root, + Err(_) => { + log!("Failed to get existing trie with root "[pubkey_state.all_orders_trie_root]); + return Some(order); + }, + }; + + pubkey_state + .orders_trie_state_history + .insert_new_diff(old_state, TrieDiff { + delta: all_orders_delta, + next_root: pubkey_state.all_orders_trie_root, + }); Some(order) } + + fn is_subscribed_to(&self, topic: &str) -> bool { self.topics_subscribed_to.contains_key(topic) } + + fn process_keep_alive( + &mut self, + from_pubkey: &str, + pairs: Vec, + message: new_protocol::PubkeyKeepAlive, + ) -> Option { + let pubkey_state = pubkey_state_mut(&mut self.pubkeys_state, from_pubkey); + for pair in pairs { + if !pubkey_state.order_pairs_trie_roots.contains_key(&pair) + && self + .topics_subscribed_to + .contains_key(&orderbook_topic_from_ordered_pair(&pair)) + { + pubkey_state.order_pairs_trie_roots.insert(pair, H64::default()); + } + } + + if pubkey_state.all_orders_trie_root == message.orders_trie_root { + pubkey_state.last_keep_alive = message.timestamp; + None + } else { + Some(OrdermatchRequest::SyncPubkeyOrderbookState { + pubkey: from_pubkey.into(), + current_orders_trie_root: pubkey_state.all_orders_trie_root, + expected_orders_trie_root: message.orders_trie_root, + pairs_trie_roots: pubkey_state.order_pairs_trie_roots.clone(), + }) + } + } } #[derive(Default)] @@ -1633,7 +2059,7 @@ struct OrdermatchContext { pub my_cancelled_orders: AsyncMutex>, pub orderbook: AsyncMutex, pub order_requests_tracker: AsyncMutex, - pub inactive_orders: AsyncMutex>, + pub inactive_orders: AsyncMutex>, } #[cfg_attr(test, mockable)] @@ -1682,8 +2108,8 @@ fn lp_connect_start_bob(ctx: MmArc, maker_match: MakerMatch, maker_order: MakerO }; let mut alice = bits256::default(); alice.bytes = maker_match.request.sender_pubkey.0; - let maker_amount = maker_match.reserved.get_base_amount().into(); - let taker_amount = maker_match.reserved.get_rel_amount().into(); + let maker_amount = maker_match.reserved.get_base_amount().to_decimal(); + let taker_amount = maker_match.reserved.get_rel_amount().to_decimal(); let privkey = &ctx.secp256k1_key_pair().private().secret; let my_persistent_pub = unwrap!(compressed_pub_key_from_priv_raw(&privkey[..], ChecksumType::DSHA256)); let uuid = maker_match.request.uuid; @@ -1754,8 +2180,8 @@ fn lp_connected_alice(ctx: MmArc, taker_request: TakerRequest, taker_match: Take let privkey = &ctx.secp256k1_key_pair().private().secret; let my_persistent_pub = unwrap!(compressed_pub_key_from_priv_raw(&privkey[..], ChecksumType::DSHA256)); - let maker_amount = taker_match.reserved.get_base_amount().into(); - let taker_amount = taker_match.reserved.get_rel_amount().into(); + let maker_amount = taker_match.reserved.get_base_amount().to_decimal(); + let taker_amount = taker_match.reserved.get_rel_amount().to_decimal(); let uuid = taker_match.reserved.taker_order_uuid; let my_conf_settings = @@ -1795,6 +2221,7 @@ fn lp_connected_alice(ctx: MmArc, taker_request: TakerRequest, taker_match: Take } pub async fn lp_ordermatch_loop(ctx: MmArc) { + let my_pubsecp = hex::encode(&**ctx.secp256k1_key_pair().public()); loop { if ctx.is_stopping() { break; @@ -1855,38 +2282,45 @@ pub async fn lp_ordermatch_loop(ctx: MmArc) { } { - // remove "timed out" orders from inactive_orders - // ones they are inactive for 240 seconds or more - let mut inactive = ordermatch_ctx.inactive_orders.lock().await; - - let current = now_ms() / 1000; - inactive.retain(|_, order| current < order.timestamp + INACTIVE_ORDER_TIMEOUT); - - // remove "timed out" orders from orderbook - // ones that didn't receive an update for 30 seconds or more - // store them in inactive orders temporary in order not to request them from relays in case we start - // receiving keep alive again + // remove "timed out" pubkeys states with their orders from orderbook let mut orderbook = ordermatch_ctx.orderbook.lock().await; - - let inactive_uuids: Vec = orderbook - .order_set - .iter() - .filter_map(|(uuid, order)| { - if order.timestamp + MAKER_ORDER_TIMEOUT < current { - Some(*uuid) - } else { - None + let mut uuids_to_remove = vec![]; + let mut keys_to_remove = vec![]; + orderbook.pubkeys_state.retain(|pubkey, state| { + let to_retain = pubkey == &my_pubsecp || state.last_keep_alive + MAKER_ORDER_TIMEOUT > now_ms() / 1000; + if !to_retain { + for (uuid, _) in &state.orders_uuids { + uuids_to_remove.push(*uuid); } - }) - .collect(); - - for uuid in inactive_uuids { - let order = orderbook.remove_order(uuid.clone()).unwrap(); - inactive.insert(uuid, order); + keys_to_remove.push(state.all_orders_trie_root); + for (_, root) in &state.order_pairs_trie_roots { + keys_to_remove.push(*root); + } + } + to_retain + }); + for uuid in uuids_to_remove { + orderbook.remove_order(uuid); } + for key in keys_to_remove { + orderbook.memory_db.remove_and_purge(&key, EMPTY_PREFIX); + } mm_gauge!(ctx.metrics, "orderbook.len", orderbook.order_set.len() as i64); - mm_gauge!(ctx.metrics, "inactive_orders.len", inactive.len() as i64); + // mm_gauge!(ctx.metrics, "inactive_orders.len", inactive.len() as i64); + } + + { + let my_maker_orders = ordermatch_ctx.my_maker_orders.lock().await; + for (uuid, order) in my_maker_orders.iter() { + if !ordermatch_ctx.orderbook.lock().await.order_set.contains_key(uuid) { + if let Ok(Some(_)) = lp_coinfindᵃ(&ctx, &order.base).await { + if let Ok(Some(_)) = lp_coinfindᵃ(&ctx, &order.rel).await { + maker_order_created_p2p_notify(ctx.clone(), order).await; + } + } + } + } } Timer::sleep(0.777).await; @@ -1919,12 +2353,11 @@ async fn process_maker_reserved(ctx: MmArc, reserved_msg: MakerReserved) { let connect = TakerConnect { sender_pubkey: H256Json::from(our_public_id.bytes), dest_pub_key: reserved_msg.sender_pubkey.clone(), - method: "connect".into(), taker_order_uuid: reserved_msg.taker_order_uuid, maker_order_uuid: reserved_msg.maker_order_uuid, }; - let topic = orderbook_topic(&my_order.request.base, &my_order.request.rel); - broadcast_ordermatch_message(&ctx, topic, connect.clone().into()); + let topic = orderbook_topic_from_base_rel(&my_order.request.base, &my_order.request.rel); + broadcast_ordermatch_message(&ctx, vec![topic], connect.clone().into()); let taker_match = TakerMatch { reserved: reserved_msg, connect, @@ -2009,12 +2442,9 @@ async fn process_taker_request(ctx: MmArc, taker_request: TakerRequest) { dest_pub_key: taker_request.sender_pubkey.clone(), sender_pubkey: our_public_id, base: order.base.clone(), - base_amount: base_amount.clone().into(), - base_amount_rat: Some(base_amount.into()), - rel_amount: rel_amount.clone().into(), - rel_amount_rat: Some(rel_amount.into()), + base_amount: base_amount.clone(), + rel_amount: rel_amount.clone(), rel: order.rel.clone(), - method: "reserved".into(), taker_order_uuid: taker_request.uuid, maker_order_uuid: *uuid, conf_settings: order.conf_settings.or_else(|| { @@ -2026,9 +2456,9 @@ async fn process_taker_request(ctx: MmArc, taker_request: TakerRequest) { }) }), }; - let topic = orderbook_topic(&order.base, &order.rel); + let topic = orderbook_topic_from_base_rel(&order.base, &order.rel); log!({"Request matched sending reserved {:?}", reserved}); - broadcast_ordermatch_message(&ctx, topic, reserved.clone().into()); + broadcast_ordermatch_message(&ctx, vec![topic], reserved.clone().into()); let maker_match = MakerMatch { request: taker_request, reserved, @@ -2080,8 +2510,8 @@ async fn process_taker_connect(ctx: MmArc, sender_pubkey: H256Json, connect_msg: maker_order_uuid: connect_msg.maker_order_uuid, method: "connected".into(), }; - let topic = orderbook_topic(&my_order.base, &my_order.rel); - broadcast_ordermatch_message(&ctx, topic, connected.clone().into()); + let topic = orderbook_topic_from_base_rel(&my_order.base, &my_order.rel); + broadcast_ordermatch_message(&ctx, vec![topic], connected.clone().into()); order_match.connect = Some(connect_msg); order_match.connected = Some(connected); my_order.started_swaps.push(order_match.request.uuid); @@ -2221,7 +2651,11 @@ pub async fn lp_auto_buy( .with_conf_settings(conf_settings) .with_sender_pubkey(H256Json::from(our_public_id.bytes)); let request = try_s!(request_builder.build()); - broadcast_ordermatch_message(&ctx, orderbook_topic(&input.base, &input.rel), request.clone().into()); + broadcast_ordermatch_message( + &ctx, + vec![orderbook_topic_from_base_rel(&input.base, &input.rel)], + request.clone().into(), + ); let result = json!({ "result": request }).to_string(); let order = TakerOrder { created_at: now_ms(), @@ -2236,86 +2670,64 @@ pub async fn lp_auto_buy( } #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -struct PricePingRequest { - method: String, +struct OrderbookItem { pubkey: String, base: String, rel: String, - price: BigDecimal, - price_rat: Option, - price64: String, - timestamp: u64, - pubsecp: String, - sig: String, - // TODO rename, it's called "balance", but it's actual meaning is max available volume to trade - #[serde(rename = "bal")] - balance: BigDecimal, - balance_rat: Option, - min_volume: MmNumber, - uuid: Option, - peer_id: String, - initial_message: Vec, - update_messages: Vec>, -} - -impl PricePingRequest { - fn from_initial_msg( - initial_message: Vec, - update_messages: Vec>, - from_peer: String, - ) -> Result { + price: BigRational, + max_volume: BigRational, + min_volume: BigRational, + uuid: Uuid, + created_at: u64, +} + +/// Concrete implementation of Hasher using Blake2b 64-bit hashes +#[derive(Debug)] +pub struct Blake2Hasher64; + +impl Hasher for Blake2Hasher64 { + type Out = [u8; 8]; + type StdHasher = Hash256StdHasher; + const LENGTH: usize = 8; + + fn hash(x: &[u8]) -> Self::Out { + let mut hasher = VarBlake2b::new(8).expect("8 is valid VarBlake2b output_size"); + hasher.update(x); + let mut res: [u8; 8] = Default::default(); + hasher.finalize_variable(|hash| res.copy_from_slice(hash)); + res + } +} + +type Layout = sp_trie::Layout; + +impl OrderbookItem { + fn from_initial_msg(initial_message: Vec, from_peer: String) -> Result { let (message, _sig, init_pubkey) = try_s!(decode_signed::(&initial_message)); let order = match message { new_protocol::OrdermatchMessage::MakerOrderCreated(order) => order, msg => return ERR!("Expected MakerOrderCreated, found {:?}", msg), }; - let mut req: PricePingRequest = ( - order, - initial_message, - hex::encode(init_pubkey.to_bytes().as_slice()), - from_peer, - ) - .into(); - - for update in update_messages { - let (message, _sig, pubkey) = try_s!(decode_signed::(&update)); - if pubkey != init_pubkey { - return ERR!("Init pubkey not equal to 1 of update messages pubkeys"); - } - - let update_message = match message { - new_protocol::OrdermatchMessage::MakerOrderUpdated(update_message) => update_message, - msg => return ERR!("Expected MakerOrderUpdated, found {:?}", msg), - }; - req.apply_updated(&update_message, update); - } + let req: OrderbookItem = (order, hex::encode(init_pubkey.to_bytes().as_slice())).into(); Ok(req) } - fn apply_updated(&mut self, msg: &new_protocol::MakerOrderUpdated, serialized: Vec) { - self.timestamp = now_ms() / 1000; - + fn apply_updated(&mut self, msg: &new_protocol::MakerOrderUpdated) { if let Some(new_price) = msg.new_price() { - self.price = new_price.to_decimal(); - self.price_rat = Some(new_price.clone()); + self.price = new_price.into(); } if let Some(new_max_volume) = msg.new_max_volume() { - self.balance = new_max_volume.to_decimal(); - self.balance_rat = Some(new_max_volume.clone()); + self.max_volume = new_max_volume.into(); } if let Some(new_min_volume) = msg.new_min_volume() { - self.min_volume = new_min_volume.clone(); + self.min_volume = new_min_volume.into(); } - - self.update_messages.push(serialized); } } -fn one() -> u8 { 1 } - fn get_true() -> bool { true } fn min_volume() -> MmNumber { MmNumber::from(MIN_TRADING_VOL) } @@ -2327,9 +2739,6 @@ struct SetPriceReq { price: MmNumber, #[serde(default)] max: bool, - #[allow(dead_code)] - #[serde(default = "one")] - broadcast: u8, #[serde(default)] volume: MmNumber, #[serde(default = "min_volume")] @@ -2487,7 +2896,7 @@ pub async fn order_status(ctx: MmArc, req: Json) -> Result>, St "order": MakerOrderForRpc::from(order), }); return Response::builder() - .body(json::to_vec(&res).unwrap()) + .body(json::to_vec(&res).expect("Serialization failed")) .map_err(|e| ERRL!("{}", e)); } @@ -2498,7 +2907,7 @@ pub async fn order_status(ctx: MmArc, req: Json) -> Result>, St "order": TakerOrderForRpc::from(order), }); return Response::builder() - .body(json::to_vec(&res).unwrap()) + .body(json::to_vec(&res).expect("Serialization failed")) .map_err(|e| ERRL!("{}", e)); } @@ -2507,7 +2916,7 @@ pub async fn order_status(ctx: MmArc, req: Json) -> Result>, St }); Response::builder() .status(404) - .body(json::to_vec(&res).unwrap()) + .body(json::to_vec(&res).expect("Serialization failed")) .map_err(|e| ERRL!("{}", e)) } @@ -2532,7 +2941,7 @@ pub async fn cancel_order(ctx: MmArc, req: Json) -> Result>, St "result": "success" }); return Response::builder() - .body(json::to_vec(&res).unwrap()) + .body(json::to_vec(&res).expect("Serialization failed")) .map_err(|e| ERRL!("{}", e)); }, // look for taker order with provided uuid @@ -2551,7 +2960,7 @@ pub async fn cancel_order(ctx: MmArc, req: Json) -> Result>, St "result": "success" }); return Response::builder() - .body(json::to_vec(&res).unwrap()) + .body(json::to_vec(&res).expect("Serialization failed")) .map_err(|e| ERRL!("{}", e)); }, // error is returned @@ -2563,7 +2972,7 @@ pub async fn cancel_order(ctx: MmArc, req: Json) -> Result>, St }); Response::builder() .status(404) - .body(json::to_vec(&res).unwrap()) + .body(json::to_vec(&res).expect("Serialization failed")) .map_err(|e| ERRL!("{}", e)) } @@ -2620,7 +3029,7 @@ pub async fn my_orders(ctx: MmArc) -> Result>, String> { } }); Response::builder() - .body(json::to_vec(&res).unwrap()) + .body(json::to_vec(&res).expect("Serialization failed")) .map_err(|e| ERRL!("{}", e)) } @@ -2804,7 +3213,7 @@ pub async fn cancel_all_orders(ctx: MmArc, req: Json) -> Result } }); Response::builder() - .body(json::to_vec(&res).unwrap()) + .body(json::to_vec(&res).expect("Serialization failed")) .map_err(|e| ERRL!("{}", e)) } @@ -2825,7 +3234,7 @@ async fn subscribe_to_orderbook_topic( const BIDS_NUMBER: Option = Some(20); let current_timestamp = now_ms() / 1000; - let topic = orderbook_topic(base, rel); + let topic = orderbook_topic_from_base_rel(base, rel); let is_orderbook_filled = { let ordermatch_ctx = try_s!(OrdermatchContext::from_ctx(ctx)); let mut orderbook = ordermatch_ctx.orderbook.lock().await; @@ -2922,7 +3331,8 @@ pub async fn orderbook(ctx: MmArc, req: Json) -> Result>, Strin let rel_coin = try_s!(rel_coin.ok_or("Rel coin is not found or inactive")); let base_coin = try_s!(lp_coinfindᵃ(&ctx, &req.base).await); let base_coin: MmCoinEnum = try_s!(base_coin.ok_or("Base coin is not found or inactive")); - let request_orderbook = true; + // TODO change this to `true` when orderbook request is reimplemented using tries + let request_orderbook = false; try_s!(subscribe_to_orderbook_topic(&ctx, &req.base, &req.rel, request_orderbook).await); let ordermatch_ctx: Arc = try_s!(OrdermatchContext::from_ctx(&ctx)); let orderbook = ordermatch_ctx.orderbook.lock().await; @@ -2936,39 +3346,27 @@ pub async fn orderbook(ctx: MmArc, req: Json) -> Result>, Strin "Orderbook::unordered contains {:?} uuid that is not in Orderbook::order_set", uuid ))?; + let price_mm: MmNumber = ask.price.clone().into(); + let max_vol_mm: MmNumber = ask.max_volume.clone().into(); + let min_vol_mm: MmNumber = ask.min_volume.clone().into(); + orderbook_entries.push(OrderbookEntry { coin: req.base.clone(), - address: try_s!(base_coin.address_from_pubkey_str(&ask.pubsecp)), - price: ask.price.clone(), - price_rat: ask - .price_rat - .as_ref() - .map(|p| p.to_ratio()) - .unwrap_or_else(|| from_dec_to_ratio(ask.price.clone())), - price_fraction: ask - .price_rat - .as_ref() - .map(|p| p.to_fraction()) - .unwrap_or_else(|| ask.price.clone().into()), - max_volume: ask.balance.clone(), - max_volume_rat: ask - .balance_rat - .as_ref() - .map(|p| p.to_ratio()) - .unwrap_or_else(|| from_dec_to_ratio(ask.balance.clone())), - max_volume_fraction: ask - .balance_rat - .as_ref() - .map(|p| p.to_fraction()) - .unwrap_or_else(|| ask.balance.clone().into()), - min_volume: ask.min_volume.to_decimal(), - min_volume_rat: ask.min_volume.to_ratio(), - min_volume_fraction: ask.min_volume.to_fraction(), + address: try_s!(base_coin.address_from_pubkey_str(&ask.pubkey)), + price: price_mm.to_decimal(), + price_rat: price_mm.to_ratio(), + price_fraction: price_mm.to_fraction(), + max_volume: max_vol_mm.to_decimal(), + max_volume_rat: max_vol_mm.to_ratio(), + max_volume_fraction: max_vol_mm.to_fraction(), + min_volume: min_vol_mm.to_decimal(), + min_volume_rat: min_vol_mm.to_ratio(), + min_volume_fraction: min_vol_mm.to_fraction(), pubkey: ask.pubkey.clone(), - age: (now_ms() as i64 / 1000) - ask.timestamp as i64, + age: (now_ms() as i64 / 1000), zcredits: 0, uuid: *uuid, - is_mine: my_pubsecp == ask.pubsecp, + is_mine: my_pubsecp == ask.pubkey, }) } orderbook_entries @@ -2984,38 +3382,28 @@ pub async fn orderbook(ctx: MmArc, req: Json) -> Result>, Strin "Orderbook::unordered contains {:?} uuid that is not in Orderbook::order_set", uuid ))?; - let price_mm = MmNumber::from(1i32) - / bid - .price_rat - .clone() - .unwrap_or_else(|| from_dec_to_ratio(bid.price.clone()).into()); + let price_mm = &MmNumber::from(1i32) / &bid.price.clone().into(); + let max_vol_mm: MmNumber = bid.max_volume.clone().into(); + let min_vol_mm: MmNumber = bid.min_volume.clone().into(); orderbook_entries.push(OrderbookEntry { coin: req.rel.clone(), - address: try_s!(rel_coin.address_from_pubkey_str(&bid.pubsecp)), + address: try_s!(rel_coin.address_from_pubkey_str(&bid.pubkey)), // NB: 1/x can not be represented as a decimal and introduces a rounding error // cf. https://github.com/KomodoPlatform/atomicDEX-API/issues/495#issuecomment-516365682 - price: BigDecimal::from(1) / &bid.price, + price: price_mm.to_decimal(), price_rat: price_mm.to_ratio(), price_fraction: price_mm.to_fraction(), - max_volume: bid.balance.clone(), - max_volume_rat: bid - .balance_rat - .as_ref() - .map(|p| p.to_ratio()) - .unwrap_or_else(|| from_dec_to_ratio(bid.balance.clone())), - max_volume_fraction: bid - .balance_rat - .as_ref() - .map(|p| p.to_fraction()) - .unwrap_or_else(|| from_dec_to_ratio(bid.balance.clone()).into()), - min_volume: bid.min_volume.to_decimal(), - min_volume_rat: bid.min_volume.to_ratio(), - min_volume_fraction: bid.min_volume.to_fraction(), + max_volume: max_vol_mm.to_decimal(), + max_volume_rat: max_vol_mm.to_ratio(), + max_volume_fraction: max_vol_mm.to_fraction(), + min_volume: min_vol_mm.to_decimal(), + min_volume_rat: min_vol_mm.to_ratio(), + min_volume_fraction: min_vol_mm.to_fraction(), pubkey: bid.pubkey.clone(), - age: (now_ms() as i64 / 1000) - bid.timestamp as i64, + age: (now_ms() as i64 / 1000), zcredits: 0, uuid: *uuid, - is_mine: my_pubsecp == bid.pubsecp, + is_mine: my_pubsecp == bid.pubkey, }) } orderbook_entries @@ -3038,22 +3426,6 @@ pub async fn orderbook(ctx: MmArc, req: Json) -> Result>, Strin Ok(try_s!(Response::builder().body(responseʲ))) } -pub fn migrate_saved_orders(ctx: &MmArc) -> Result<(), String> { - let taker_entries: Vec = try_s!(json_dir_entries(&my_taker_orders_dir(&ctx))); - taker_entries.iter().for_each(|entry| { - if let Ok(mut order) = json::from_slice::(&slurp(&entry.path())) { - if order.request.base_amount_rat.is_none() { - order.request.base_amount_rat = Some(from_dec_to_ratio(order.request.base_amount.clone())); - } - if order.request.rel_amount_rat.is_none() { - order.request.rel_amount_rat = Some(from_dec_to_ratio(order.request.rel_amount.clone())); - } - save_my_taker_order(ctx, &order) - } - }); - Ok(()) -} - fn choose_maker_confs_and_notas( maker_confs: Option, taker_req: &TakerRequest, diff --git a/mm2src/lp_ordermatch/new_protocol.rs b/mm2src/lp_ordermatch/new_protocol.rs index ca7ed5ff60..b596d22354 100644 --- a/mm2src/lp_ordermatch/new_protocol.rs +++ b/mm2src/lp_ordermatch/new_protocol.rs @@ -1,9 +1,9 @@ -use super::{MatchBy as SuperMatchBy, PricePingRequest, TakerAction}; +use super::{MatchBy as SuperMatchBy, OrderbookItem, TakerAction}; use crate::mm2::lp_ordermatch::OrderConfirmationsSettings; use common::mm_number::MmNumber; use compact_uuid::CompactUuid; use num_rational::BigRational; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use uuid::Uuid; #[derive(Debug, Deserialize, Serialize)] @@ -11,7 +11,7 @@ use uuid::Uuid; pub enum OrdermatchMessage { MakerOrderCreated(MakerOrderCreated), MakerOrderUpdated(MakerOrderUpdated), - MakerOrderKeepAlive(MakerOrderKeepAlive), + PubkeyKeepAlive(PubkeyKeepAlive), MakerOrderCancelled(MakerOrderCancelled), TakerRequest(TakerRequest), MakerReserved(MakerReserved), @@ -19,33 +19,8 @@ pub enum OrdermatchMessage { MakerConnected(MakerConnected), } -/// Get an order using uuid and the order maker's pubkey. -/// Actual we expect to receive [`OrdermatchMessage::MakerOrderCreated`] that will be parsed into [`PricePingRequest`]. -#[derive(Debug, Deserialize, Serialize)] -pub struct OrderInitialMessage { - pub initial_message: Vec, - pub update_messages: Vec>, - pub from_peer: String, -} - -impl From for OrderInitialMessage { - fn from(order: PricePingRequest) -> Self { - OrderInitialMessage { - initial_message: order.initial_message, - update_messages: order.update_messages, - from_peer: order.peer_id, - } - } -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct Orderbook { - pub asks: Vec, - pub bids: Vec, -} - -impl From for OrdermatchMessage { - fn from(keep_alive: MakerOrderKeepAlive) -> Self { OrdermatchMessage::MakerOrderKeepAlive(keep_alive) } +impl From for OrdermatchMessage { + fn from(keep_alive: PubkeyKeepAlive) -> Self { OrdermatchMessage::PubkeyKeepAlive(keep_alive) } } impl From for OrdermatchMessage { @@ -141,12 +116,13 @@ pub struct MakerOrderCreated { pub price: BigRational, pub max_volume: BigRational, pub min_volume: BigRational, + pub created_at: u64, pub conf_settings: OrderConfirmationsSettings, } #[derive(Debug, Deserialize, Serialize)] -pub struct MakerOrderKeepAlive { - pub uuid: CompactUuid, +pub struct PubkeyKeepAlive { + pub orders_trie_root: [u8; 8], pub timestamp: u64, } diff --git a/mm2src/lp_swap.rs b/mm2src/lp_swap.rs index 2f1041ef9b..b9f629a47a 100644 --- a/mm2src/lp_swap.rs +++ b/mm2src/lp_swap.rs @@ -138,7 +138,7 @@ pub fn broadcast_swap_message_every(ctx: MmArc, topic: String, msg: SwapMsg, int pub fn broadcast_swap_message(ctx: &MmArc, topic: String, msg: SwapMsg) { let key_pair = ctx.secp256k1_key_pair.or(&&|| panic!()); let encoded_msg = encode_and_sign(&msg, &*key_pair.private().secret).unwrap(); - broadcast_p2p_msg(ctx, topic, encoded_msg); + broadcast_p2p_msg(ctx, vec![topic], encoded_msg); } pub fn process_msg(ctx: MmArc, topic: &str, msg: &[u8]) { @@ -792,7 +792,7 @@ fn broadcast_my_swap_status(uuid: &Uuid, ctx: &MmArc) -> Result<(), String> { data: status, }; let msg = json::to_vec(&status).expect("Swap status ser should never fail"); - broadcast_p2p_msg(ctx, swap_topic(uuid), msg); + broadcast_p2p_msg(ctx, vec![swap_topic(uuid)], msg); Ok(()) } diff --git a/mm2src/lp_swap/taker_swap.rs b/mm2src/lp_swap/taker_swap.rs index 027555390b..ab5de7b972 100644 --- a/mm2src/lp_swap/taker_swap.rs +++ b/mm2src/lp_swap/taker_swap.rs @@ -225,7 +225,7 @@ impl RunTakerSwapInput { /// Starts the taker swap and drives it to completion (until None next command received). /// Panics in case of command or event apply fails, not sure yet how to handle such situations /// because it's usually means that swap is in invalid state which is possible only if there's developer error -/// Every produced event is saved to local DB. Swap status is broadcasted to P2P network after completion. +/// Every produced event is saved to local DB. Swap status is broadcast to P2P network after completion. pub async fn run_taker_swap(swap: RunTakerSwapInput, ctx: MmArc) { let uuid = swap.uuid().to_owned(); let lock_path = my_swaps_dir(&ctx).join(fomat!((uuid) ".lock")); diff --git a/mm2src/mm2_libp2p/src/atomicdex_behaviour.rs b/mm2src/mm2_libp2p/src/atomicdex_behaviour.rs index bc87a157d9..46a5d828e7 100644 --- a/mm2src/mm2_libp2p/src/atomicdex_behaviour.rs +++ b/mm2src/mm2_libp2p/src/atomicdex_behaviour.rs @@ -106,7 +106,7 @@ pub enum AdexBehaviourCmd { topic: String, }, PublishMsg { - topic: String, + topics: Vec, msg: Vec, }, /// Request relays sequential until a response is received. @@ -259,8 +259,9 @@ impl AtomicDexBehaviour { let topic = Topic::new(topic); self.gossipsub.subscribe(topic); }, - AdexBehaviourCmd::PublishMsg { topic, msg } => { - self.gossipsub.publish(&Topic::new(topic), msg); + AdexBehaviourCmd::PublishMsg { topics, msg } => { + self.gossipsub + .publish_many(topics.into_iter().map(|topic| Topic::new(topic)), msg); }, AdexBehaviourCmd::RequestAnyRelay { req, response_tx } => { let relays = self.gossipsub.get_relay_mesh(); diff --git a/mm2src/mm2_tests.rs b/mm2src/mm2_tests.rs index 8160186ef7..80a4644cf6 100644 --- a/mm2src/mm2_tests.rs +++ b/mm2src/mm2_tests.rs @@ -303,6 +303,7 @@ fn alice_can_see_the_active_order_after_connection() { })))); assert!(rc.0.is_success(), "!setprice: {}", rc.1); + thread::sleep(Duration::from_secs(40)); log!("Get RICK/MORTY orderbook on Eve side"); let rc = unwrap!(block_on(mm_eve.rpc(json! ({ "userpass": mm_eve.userpass, @@ -356,6 +357,15 @@ fn alice_can_see_the_active_order_after_connection() { // Enable coins on Alice side. Print the replies in case we need the "address". log!({ "enable_coins (alice): {:?}", block_on(enable_coins_eth_electrum(&mm_alice, vec!["https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b"])) }); + log!("Get RICK/MORTY orderbook on Alice side to trigger subscription"); + let rc = unwrap!(block_on(mm_alice.rpc(json! ({ + "userpass": mm_alice.userpass, + "method": "orderbook", + "base": "RICK", + "rel": "MORTY", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + thread::sleep(Duration::from_secs(40)); log!("Get RICK/MORTY orderbook on Alice side"); let rc = unwrap!(block_on(mm_alice.rpc(json! ({ "userpass": mm_alice.userpass, @@ -1607,8 +1617,16 @@ fn test_cancel_order() { // Enable coins on Alice side. Print the replies in case we need the "address". log! ({"enable_coins (alice): {:?}", block_on (enable_coins_eth_electrum (&mm_alice, vec!["https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b"]))}); - log!("Give Alice 15 seconds to import the order…"); - thread::sleep(Duration::from_secs(15)); + log!("Get RICK/MORTY orderbook on Alice side to trigger subscription"); + let rc = unwrap!(block_on(mm_alice.rpc(json! ({ + "userpass": mm_alice.userpass, + "method": "orderbook", + "base": "RICK", + "rel": "MORTY", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + log!("Give Alice 40 seconds to import the order…"); + thread::sleep(Duration::from_secs(40)); log!("Get RICK/MORTY orderbook on Alice side"); let rc = unwrap!(block_on(mm_alice.rpc(json! ({ @@ -1951,6 +1969,15 @@ fn test_order_should_not_be_displayed_when_node_is_down() { ]))] ); + log!("Get RICK/MORTY orderbook on Alice side to trigger subscription"); + let rc = unwrap!(block_on(mm_alice.rpc(json! ({ + "userpass": mm_alice.userpass, + "method": "orderbook", + "base": "RICK", + "rel": "MORTY", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + // issue sell request on Bob side by setting base/rel price log!("Issue bob sell request"); let rc = unwrap!(block_on(mm_bob.rpc(json! ({ @@ -1963,7 +1990,7 @@ fn test_order_should_not_be_displayed_when_node_is_down() { })))); assert!(rc.0.is_success(), "!setprice: {}", rc.1); - thread::sleep(Duration::from_secs(12)); + thread::sleep(Duration::from_secs(2)); log!("Get RICK/MORTY orderbook on Alice side"); let rc = unwrap!(block_on(mm_alice.rpc(json! ({ @@ -1980,7 +2007,7 @@ fn test_order_should_not_be_displayed_when_node_is_down() { assert_eq!(asks.len(), 1, "Alice RICK/MORTY orderbook must have exactly 1 ask"); unwrap!(block_on(mm_bob.stop())); - thread::sleep(Duration::from_secs(65)); + thread::sleep(Duration::from_secs(95)); let rc = unwrap!(block_on(mm_alice.rpc(json! ({ "userpass": mm_alice.userpass, @@ -1998,6 +2025,86 @@ fn test_order_should_not_be_displayed_when_node_is_down() { unwrap!(block_on(mm_alice.stop())); } +#[test] +#[cfg(feature = "native")] +fn test_own_orders_should_not_be_removed_from_orderbook() { + let coins = json!([ + {"coin":"RICK","asset":"RICK","protocol":{"type":"UTXO"}}, + {"coin":"MORTY","asset":"MORTY","protocol":{"type":"UTXO"}}, + ]); + + // start bob and immediately place the order + let mut mm_bob = unwrap!(MarketMakerIt::start( + json! ({ + "gui": "nogui", + "netid": 9998, + "dht": "on", // Enable DHT without delay. + "myipaddr": env::var ("BOB_TRADE_IP") .ok(), + "rpcip": env::var ("BOB_TRADE_IP") .ok(), + "canbind": env::var ("BOB_TRADE_PORT") .ok().map (|s| unwrap! (s.parse::())), + "passphrase": "bob passphrase", + "coins": coins, + "i_am_seed": true, + "rpc_password": "pass", + }), + "pass".into(), + match var("LOCAL_THREAD_MM") { + Ok(ref e) if e == "bob" => Some(local_start()), + _ => None, + } + )); + let (_bob_dump_log, _bob_dump_dashboard) = mm_dump(&mm_bob.log_path); + log!({"Bob log path: {}", mm_bob.log_path.display()}); + unwrap!(block_on( + mm_bob.wait_for_log(22., |log| log.contains(">>>>>>>>> DEX stats ")) + )); + + log!( + "Bob enable RICK "[block_on(enable_electrum(&mm_bob, "RICK", vec![ + "electrum3.cipig.net:10017", + "electrum2.cipig.net:10017", + "electrum1.cipig.net:10017", + ]))] + ); + + log!( + "Bob enable MORTY "[block_on(enable_electrum(&mm_bob, "MORTY", vec![ + "electrum3.cipig.net:10018", + "electrum2.cipig.net:10018", + "electrum1.cipig.net:10018", + ]))] + ); + + // issue sell request on Bob side by setting base/rel price + log!("Issue bob sell request"); + let rc = unwrap!(block_on(mm_bob.rpc(json! ({ + "userpass": mm_bob.userpass, + "method": "setprice", + "base": "RICK", + "rel": "MORTY", + "price": 0.9, + "volume": "0.9", + })))); + assert!(rc.0.is_success(), "!setprice: {}", rc.1); + + thread::sleep(Duration::from_secs(95)); + + let rc = unwrap!(block_on(mm_bob.rpc(json! ({ + "userpass": mm_bob.userpass, + "method": "orderbook", + "base": "RICK", + "rel": "MORTY", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + + let bob_orderbook: Json = unwrap!(json::from_str(&rc.1)); + log!("Bob orderbook "[bob_orderbook]); + let asks = unwrap!(bob_orderbook["asks"].as_array()); + assert_eq!(asks.len(), 1, "Bob RICK/MORTY orderbook must have exactly 1 ask"); + + unwrap!(block_on(mm_bob.stop())); +} + #[test] #[cfg(feature = "native")] // https://github.com/KomodoPlatform/atomicDEX-API/issues/511 @@ -2531,6 +2638,7 @@ fn setprice_min_volume_should_be_displayed_in_orderbook() { })))); assert!(rc.0.is_success(), "!setprice: {}", rc.1); + thread::sleep(Duration::from_secs(2)); log!("Get ETH/JST orderbook on Bob side"); let rc = unwrap!(block_on(mm_bob.rpc(json! ({ "userpass": mm_bob.userpass, @@ -2902,8 +3010,17 @@ fn set_price_with_cancel_previous_should_broadcast_cancelled_message() { // Enable coins on Alice side. Print the replies in case we need the "address". log! ({"enable_coins (alice): {:?}", block_on (enable_coins_eth_electrum (&mm_alice, vec!["https://ropsten.infura.io/v3/c01c1b4cf66642528547624e1d6d9d6b"]))}); - log!("Give Alice 15 seconds to import the order…"); - thread::sleep(Duration::from_secs(15)); + log!("Get RICK/MORTY orderbook on Alice side to trigger subscription"); + let rc = unwrap!(block_on(mm_alice.rpc(json! ({ + "userpass": mm_alice.userpass, + "method": "orderbook", + "base": "RICK", + "rel": "MORTY", + })))); + assert!(rc.0.is_success(), "!orderbook: {}", rc.1); + + log!("Give Alice 40 seconds to import the order…"); + thread::sleep(Duration::from_secs(40)); log!("Get RICK/MORTY orderbook on Alice side"); let rc = unwrap!(block_on(mm_alice.rpc(json! ({ diff --git a/mm2src/ordermatch_tests.rs b/mm2src/ordermatch_tests.rs index d969f06bce..4d4cfc4c19 100644 --- a/mm2src/ordermatch_tests.rs +++ b/mm2src/ordermatch_tests.rs @@ -1,5 +1,6 @@ use super::*; use crate::mm2::lp_network::P2PContext; +use crate::mm2::lp_ordermatch::new_protocol::PubkeyKeepAlive; use coins::{MmCoin, TestCoin}; use common::{executor::spawn, mm_ctx::{MmArc, MmCtx, MmCtxBuilder}, @@ -8,8 +9,9 @@ use futures::{channel::mpsc, lock::Mutex as AsyncMutex, StreamExt}; use mm2_libp2p::atomicdex_behaviour::{AdexBehaviourCmd, AdexResponse}; use mm2_libp2p::{decode_message, PeerId}; use mocktopus::mocking::*; -use rand::Rng; +use rand::{seq::SliceRandom, thread_rng, Rng}; use std::collections::HashSet; +use std::iter::{self, FromIterator}; #[test] fn test_match_maker_order_and_taker_request() { @@ -30,13 +32,10 @@ fn test_match_maker_order_and_taker_request() { base: "BASE".into(), rel: "REL".into(), uuid: Uuid::new_v4(), - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 20.into(), - rel_amount_rat: Some(BigRational::from_integer(20.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -63,13 +62,10 @@ fn test_match_maker_order_and_taker_request() { base: "BASE".into(), rel: "REL".into(), uuid: Uuid::new_v4(), - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 20.into(), - rel_amount_rat: Some(BigRational::from_integer(20.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -96,13 +92,10 @@ fn test_match_maker_order_and_taker_request() { base: "BASE".into(), rel: "REL".into(), uuid: Uuid::new_v4(), - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 2.into(), - rel_amount_rat: Some(BigRational::from_integer(2.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -129,13 +122,10 @@ fn test_match_maker_order_and_taker_request() { base: "REL".into(), rel: "BASE".into(), uuid: Uuid::new_v4(), - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 5.into(), - base_amount_rat: Some(BigRational::from_integer(5.into())), rel_amount: 10.into(), - rel_amount_rat: Some(BigRational::from_integer(10.into())), action: TakerAction::Sell, match_by: MatchBy::Any, conf_settings: None, @@ -162,13 +152,10 @@ fn test_match_maker_order_and_taker_request() { base: "REL".into(), rel: "BASE".into(), uuid: Uuid::new_v4(), - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 10.into(), - rel_amount_rat: Some(BigRational::from_integer(10.into())), action: TakerAction::Sell, match_by: MatchBy::Any, conf_settings: None, @@ -195,13 +182,10 @@ fn test_match_maker_order_and_taker_request() { base: "REL".into(), rel: "BASE".into(), uuid: Uuid::new_v4(), - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), - rel_amount: "0.9".parse().unwrap(), - rel_amount_rat: Some(BigRational::new(9.into(), 10.into())), + rel_amount: "0.9".into(), action: TakerAction::Sell, match_by: MatchBy::Any, conf_settings: None, @@ -260,24 +244,18 @@ fn test_maker_order_available_amount() { base: "BASE".into(), rel: "REL".into(), base_amount: 5.into(), - base_amount_rat: None, rel_amount: 5.into(), - rel_amount_rat: None, sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), - method: "request".into(), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, }, reserved: MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 5.into(), - base_amount_rat: Some(BigRational::from_integer(5.into())), rel_amount: 5.into(), - rel_amount_rat: Some(BigRational::from_integer(5.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -294,24 +272,18 @@ fn test_maker_order_available_amount() { base: "BASE".into(), rel: "REL".into(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 1.into(), - rel_amount_rat: Some(BigRational::from_integer(1.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), - method: "request".into(), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, }, reserved: MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 1.into(), - rel_amount_rat: Some(BigRational::from_integer(1.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -336,13 +308,10 @@ fn test_taker_match_reserved() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 10.into(), - rel_amount_rat: Some(BigRational::from_integer(10.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -356,13 +325,10 @@ fn test_taker_match_reserved() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 10.into(), - rel_amount_rat: Some(BigRational::from_integer(10.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -376,13 +342,10 @@ fn test_taker_match_reserved() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 10.into(), - rel_amount_rat: Some(BigRational::from_integer(10.into())), action: TakerAction::Sell, match_by: MatchBy::Any, conf_settings: None, @@ -396,13 +359,10 @@ fn test_taker_match_reserved() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "REL".into(), rel: "BASE".into(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 10.into(), - rel_amount_rat: Some(BigRational::from_integer(10.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -416,13 +376,10 @@ fn test_taker_match_reserved() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), - rel_amount: "0.9".parse().unwrap(), - rel_amount_rat: Some(BigRational::new(9.into(), 10.into())), + rel_amount: "0.9".into(), action: TakerAction::Sell, match_by: MatchBy::Any, conf_settings: None, @@ -436,13 +393,10 @@ fn test_taker_match_reserved() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "REL".into(), rel: "BASE".into(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 1.into(), - rel_amount_rat: Some(BigRational::from_integer(1.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -456,13 +410,10 @@ fn test_taker_match_reserved() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), - rel_amount: "0.9".parse().unwrap(), - rel_amount_rat: Some(BigRational::new(9.into(), 10.into())), + rel_amount: "0.9".into(), action: TakerAction::Sell, match_by: MatchBy::Any, conf_settings: None, @@ -476,13 +427,10 @@ fn test_taker_match_reserved() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "REL".into(), rel: "BASE".into(), - base_amount: "0.8".parse().unwrap(), - base_amount_rat: Some(BigRational::new(8.into(), 10.into())), + base_amount: "0.8".into(), rel_amount: 1.into(), - rel_amount_rat: Some(BigRational::from_integer(1.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -496,13 +444,10 @@ fn test_taker_match_reserved() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 2.into(), - rel_amount_rat: Some(BigRational::from_integer(2.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -516,13 +461,10 @@ fn test_taker_match_reserved() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 1.into(), - rel_amount_rat: Some(BigRational::from_integer(1.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -536,13 +478,10 @@ fn test_taker_match_reserved() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: None, rel_amount: 2.into(), - rel_amount_rat: None, action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -556,13 +495,10 @@ fn test_taker_match_reserved() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 1.into(), - rel_amount_rat: Some(BigRational::from_integer(1.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -576,13 +512,10 @@ fn test_taker_match_reserved() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 2.into(), - rel_amount_rat: Some(BigRational::from_integer(2.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -596,13 +529,10 @@ fn test_taker_match_reserved() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 1.into(), - base_amount_rat: None, rel_amount: 1.into(), - rel_amount_rat: None, sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -616,13 +546,10 @@ fn test_taker_match_reserved() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 2.into(), - rel_amount_rat: Some(BigRational::from_integer(2.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -636,13 +563,10 @@ fn test_taker_match_reserved() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 3.into(), - rel_amount_rat: Some(BigRational::from_integer(3.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -659,14 +583,10 @@ fn test_taker_match_reserved() { rel: "MORTY".into(), base_amount: "0.3333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333" - .parse() - .unwrap(), - base_amount_rat: Some(BigRational::new(1.into(), 3.into())), + .into(), rel_amount: 1.into(), - rel_amount_rat: Some(BigRational::from_integer(1.into())), action: TakerAction::Buy, uuid, - method: "request".into(), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), match_by: MatchBy::Any, @@ -679,13 +599,10 @@ fn test_taker_match_reserved() { let reserved = MakerReserved { base: "RICK".into(), rel: "MORTY".into(), - base_amount: "0.3333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333".parse().unwrap(), - base_amount_rat: None, - rel_amount: "0.777777776666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666588888889".parse().unwrap(), - rel_amount_rat: None, + base_amount: "0.3333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333".into(), + rel_amount: "0.777777776666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666588888889".into(), taker_order_uuid: uuid, maker_order_uuid: uuid, - method: "reserved".into(), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), conf_settings: None, @@ -700,13 +617,10 @@ fn test_taker_order_cancellable() { base: "BASE".into(), rel: "REL".into(), uuid: Uuid::new_v4(), - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 2.into(), - rel_amount_rat: Some(BigRational::from_integer(2.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -725,13 +639,10 @@ fn test_taker_order_cancellable() { base: "BASE".into(), rel: "REL".into(), uuid: Uuid::new_v4(), - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 2.into(), - rel_amount_rat: Some(BigRational::from_integer(2.into())), action: TakerAction::Buy, match_by: MatchBy::Any, conf_settings: None, @@ -747,13 +658,10 @@ fn test_taker_order_cancellable() { order.matches.insert(Uuid::new_v4(), TakerMatch { last_updated: now_ms(), reserved: MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 1.into(), - base_amount_rat: Some(BigRational::from_integer(1.into())), rel_amount: 3.into(), - rel_amount_rat: Some(BigRational::from_integer(3.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -761,7 +669,6 @@ fn test_taker_order_cancellable() { conf_settings: None, }, connect: TakerConnect { - method: "connect".into(), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -827,11 +734,8 @@ fn prepare_for_cancel_by(ctx: &MmArc) -> mpsc::Receiver { uuid: Uuid::from_bytes([3; 16]), action: TakerAction::Buy, base_amount: 0.into(), - base_amount_rat: Some(BigRational::from_integer(0.into())), rel_amount: 0.into(), - rel_amount_rat: Some(BigRational::from_integer(0.into())), dest_pub_key: H256Json::default(), - method: "request".into(), sender_pubkey: H256Json::default(), match_by: MatchBy::Any, conf_settings: None, @@ -910,13 +814,10 @@ fn test_taker_order_match_by() { base: "BASE".into(), rel: "REL".into(), uuid, - method: "request".into(), dest_pub_key: H256Json::default(), sender_pubkey: H256Json::default(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 10.into(), - rel_amount_rat: Some(BigRational::from_integer(10.into())), action: TakerAction::Buy, match_by: MatchBy::Orders(not_matching_uuids), conf_settings: None, @@ -930,13 +831,10 @@ fn test_taker_order_match_by() { }; let reserved = MakerReserved { - method: "reserved".into(), base: "BASE".into(), rel: "REL".into(), base_amount: 10.into(), - base_amount_rat: Some(BigRational::from_integer(10.into())), rel_amount: 10.into(), - rel_amount_rat: Some(BigRational::from_integer(10.into())), sender_pubkey: H256Json::default(), dest_pub_key: H256Json::default(), maker_order_uuid: Uuid::new_v4(), @@ -1381,14 +1279,7 @@ fn make_ctx_for_tests() -> (MmArc, String, [u8; 32]) { (ctx, pubkey, secret) } -fn make_random_orders( - pubkey: String, - secret: &[u8; 32], - peer_id: String, - base: String, - rel: String, - n: usize, -) -> Vec { +fn make_random_orders(pubkey: String, secret: &[u8; 32], base: String, rel: String, n: usize) -> Vec { let mut rng = rand::thread_rng(); let mut orders = Vec::with_capacity(n); for _i in 0..n { @@ -1401,6 +1292,7 @@ fn make_random_orders( max_volume: BigRational::from_integer(1.into()), min_volume: BigRational::from_integer(0.into()), conf_settings: OrderConfirmationsSettings::default(), + created_at: now_ms() / 1000, }; // create an initial_message and encode it with the secret @@ -1410,7 +1302,7 @@ fn make_random_orders( ) .unwrap(); - orders.push((order, initial_message, pubkey.clone(), peer_id.clone()).into()); + orders.push((order, pubkey.clone()).into()); } orders @@ -1427,6 +1319,7 @@ fn p2p_context_mock() -> (mpsc::Sender, mpsc::Receiver(&encoded).unwrap(); assert!(orderbook.bids.is_empty()); - let asks: Vec = orderbook + let asks: Vec = orderbook .asks .into_iter() - .map(|order| PricePingRequest::from_initial_msg(order.initial_message, Vec::new(), order.from_peer).unwrap()) + .map(|order| OrderbookItem::from_initial_msg(order.initial_message, order.from_peer).unwrap()) .collect(); assert_eq!(asks, vec![price_ping_request2]); @@ -1519,10 +1412,10 @@ fn test_process_get_orderbook_request() { let orderbook = decode_message::(&encoded).unwrap(); assert!(orderbook.asks.is_empty()); - let bids: Vec = orderbook + let bids: Vec = orderbook .bids .into_iter() - .map(|order| PricePingRequest::from_initial_msg(order.initial_message, Vec::new(), order.from_peer).unwrap()) + .map(|order| OrderbookItem::from_initial_msg(order.initial_message, order.from_peer).unwrap()) .collect(); assert_eq!(bids, vec![price_ping_request1]); } @@ -1628,7 +1521,7 @@ fn test_request_and_fill_orderbook() { // check if the best asks and bids are in the orderbook let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); let orderbook = block_on(ordermatch_ctx.orderbook.lock()); - let asks: Vec = orderbook + let asks: Vec = orderbook .ordered .get(&("RICK".into(), "MORTY".into())) .unwrap() @@ -1642,7 +1535,7 @@ fn test_request_and_fill_orderbook() { .clone() }) .collect(); - let bids: Vec = orderbook + let bids: Vec = orderbook .ordered .get(&("MORTY".into(), "RICK".into())) .unwrap() @@ -1690,8 +1583,8 @@ fn test_process_order_keep_alive_requested_from_peer() { ) .unwrap(); - let expected_request = P2PRequest::Ordermatch(OrdermatchRequest::GetOrder { - uuid: uuid.clone(), + let expected_request = P2PRequest::Ordermatch(OrdermatchRequest::GetOrders { + pairs: vec![("RICK".into(), "MORTY".into())], from_pubkey: pubkey.clone(), }); let from_peer = peer.clone(); @@ -1709,11 +1602,11 @@ fn test_process_order_keep_alive_requested_from_peer() { assert_eq!(actual, expected_request); // create a response with the initial_message and random from_peer - let response = new_protocol::OrderInitialMessage { + let response = vec![new_protocol::OrderInitialMessage { initial_message, from_peer: from_peer.clone(), update_messages: Vec::new(), - }; + }]; let response = AdexResponse::Ok { response: encode_message(&response).unwrap(), @@ -1721,13 +1614,13 @@ fn test_process_order_keep_alive_requested_from_peer() { response_tx.send(vec![(PeerId::random(), response)]).unwrap(); }); - let keep_alive = new_protocol::MakerOrderKeepAlive { - uuid: uuid.clone().into(), + let keep_alive = new_protocol::MakerOrdersKeepAlive { timestamp: now_ms(), + num_orders: HashMap::from_iter(iter::once((("RICK".into(), "MORTY".into()), 1))), }; - // process_order_keep_alive() should return true because an order should be requested from a peer. - assert!(block_on(process_order_keep_alive( + // process_order_keep_alive() should return true because an order was successfully requested from a peer. + assert!(block_on(process_orders_keep_alive( ctx, peer.clone(), pubkey.clone(), @@ -1737,10 +1630,8 @@ fn test_process_order_keep_alive_requested_from_peer() { let mut orderbook = block_on(ordermatch_ctx.orderbook.lock()); // try to find the order within OrdermatchContext::orderbook and check if this order equals to the expected let actual = orderbook.find_order_by_uuid_and_pubkey(&uuid, &pubkey).unwrap(); - let expected: PricePingRequest = (order, initial_order_message, pubkey, peer).into(); + let expected: OrderbookItem = (order, pubkey).into(); - // the expected.timestamp may be greater than actual.timestamp because of two now_ms() calls - actual.timestamp = expected.timestamp; assert_eq!(actual, &expected); } @@ -1752,7 +1643,6 @@ fn test_process_get_order_request() { OrdermatchContext::from_ctx.mock_safe(move |_| MockResult::Return(Ok(ordermatch_ctx_clone.clone()))); let mut orderbook = block_on(ordermatch_ctx.orderbook.lock()); - let peer = PeerId::random().to_string(); let order = new_protocol::MakerOrderCreated { uuid: Uuid::new_v4().into(), @@ -1769,23 +1659,22 @@ fn test_process_get_order_request() { &secret, ) .unwrap(); - let price_ping_request: PricePingRequest = (order, initial_message, pubkey.clone(), peer.clone()).into(); - orderbook.insert_or_update_order(price_ping_request.uuid.unwrap(), price_ping_request.clone()); + let price_ping_request: OrderbookItem = (order, pubkey.clone()).into(); + orderbook.insert_or_update_order(price_ping_request.clone()); // avoid dead lock on orderbook as process_get_orderbook_request also acquires it drop(orderbook); let encoded = block_on(process_get_order_request( ctx.clone(), - price_ping_request.uuid.unwrap(), + price_ping_request.uuid, pubkey.clone(), )) .unwrap() .unwrap(); let order = decode_message::(&encoded).unwrap(); - let actual_price_ping_request = - PricePingRequest::from_initial_msg(order.initial_message, order.update_messages, order.from_peer).unwrap(); + let actual_price_ping_request = OrderbookItem::from_initial_msg(order.initial_message, order.from_peer).unwrap(); assert_eq!(actual_price_ping_request, price_ping_request); } @@ -1917,7 +1806,7 @@ fn test_subscribe_to_ordermatch_topic_subscribed_filled() { let expected = Some(OrderbookRequestingState::NotRequested { subscribed_at }); assert_eq!(actual, expected); } - +*/ #[test] fn test_taker_request_can_match_with_maker_pubkey() { let maker_pubkey = H256Json::default(); @@ -1959,3 +1848,299 @@ fn test_taker_request_can_match_with_uuid() { request.match_by = MatchBy::Orders(HashSet::new()); assert!(!request.can_match_with_uuid(&uuid)); } + +#[test] +fn test_orderbook_insert_or_update_order() { + let (_, pubkey, secret) = make_ctx_for_tests(); + let mut orderbook = Orderbook::default(); + let order = make_random_orders(pubkey.clone(), &secret, "C1".into(), "C2".into(), 1).remove(0); + orderbook.insert_or_update_order_update_trie(order.clone()); +} + +fn all_orders_trie_root_by_pub(ctx: &MmArc, pubkey: &str) -> H64 { + let ordermatch_ctx = OrdermatchContext::from_ctx(ctx).unwrap(); + let orderbook = block_on(ordermatch_ctx.orderbook.lock()); + orderbook.pubkeys_state.get(pubkey).unwrap().all_orders_trie_root +} + +fn pair_trie_root_by_pub(ctx: &MmArc, pubkey: &str, pair: &str) -> H64 { + let ordermatch_ctx = OrdermatchContext::from_ctx(ctx).unwrap(); + let orderbook = block_on(ordermatch_ctx.orderbook.lock()); + *orderbook + .pubkeys_state + .get(pubkey) + .unwrap() + .order_pairs_trie_roots + .get(pair) + .unwrap() +} + +fn clone_orderbook_memory_db(ctx: &MmArc) -> MemoryDB { + let ordermatch_ctx = OrdermatchContext::from_ctx(ctx).unwrap(); + let orderbook = block_on(ordermatch_ctx.orderbook.lock()); + orderbook.memory_db.clone() +} + +fn remove_order(ctx: &MmArc, uuid: Uuid) { + let ordermatch_ctx = OrdermatchContext::from_ctx(ctx).unwrap(); + let mut orderbook = block_on(ordermatch_ctx.orderbook.lock()); + orderbook.remove_order_trie_update(uuid); +} + +#[test] +fn test_process_sync_pubkey_orderbook_state_after_new_orders_added() { + let (ctx, pubkey, secret) = make_ctx_for_tests(); + let orders = make_random_orders(pubkey.clone(), &secret, "C1".into(), "C2".into(), 100); + + for order in orders { + block_on(insert_or_update_order(&ctx, order)); + } + + let alb_ordered_pair = alb_ordered_pair("C1", "C2"); + let current_root_hash = all_orders_trie_root_by_pub(&ctx, &pubkey); + let pair_trie_root = pair_trie_root_by_pub(&ctx, &pubkey, &alb_ordered_pair); + + let prev_pairs_state = HashMap::from_iter(iter::once((alb_ordered_pair.clone(), pair_trie_root))); + + let mut old_mem_db = clone_orderbook_memory_db(&ctx); + + let new_orders = make_random_orders(pubkey.clone(), &secret, "C1".into(), "C2".into(), 100); + for order in new_orders { + block_on(insert_or_update_order(&ctx, order.clone())); + } + + let expected_root_hash = all_orders_trie_root_by_pub(&ctx, &pubkey); + let mut result = block_on(process_sync_pubkey_orderbook_state( + ctx.clone(), + pubkey.clone(), + current_root_hash, + expected_root_hash, + prev_pairs_state, + )) + .unwrap() + .unwrap(); + + // check all orders trie root first + let delta = match result.all_orders_diff { + DeltaOrFullTrie::Delta(delta) => delta, + DeltaOrFullTrie::FullTrie(_) => panic!("Must be DeltaOrFullTrie::Delta"), + }; + + let actual_root_hash = delta_trie_root::( + &mut old_mem_db, + current_root_hash, + delta.into_iter().map(|(pair, hash)| (pair.into_bytes(), hash)), + ) + .unwrap(); + assert_eq!(expected_root_hash, actual_root_hash); + + // check pair trie root + let expected_root_hash = pair_trie_root_by_pub(&ctx, &pubkey, &alb_ordered_pair); + + let delta = match result.pair_orders_diff.remove(&alb_ordered_pair).unwrap() { + DeltaOrFullTrie::Delta(delta) => delta, + DeltaOrFullTrie::FullTrie(_) => panic!("Must be DeltaOrFullTrie::Delta"), + }; + + let actual_root_hash = delta_trie_root::( + &mut old_mem_db, + pair_trie_root, + delta + .into_iter() + .map(|(uuid, order)| (*uuid.as_bytes(), order.map(|o| encode_message(&o).unwrap()))), + ) + .unwrap(); + assert_eq!(expected_root_hash, actual_root_hash); +} + +#[test] +fn test_diff_should_not_be_written_if_hash_not_changed_on_insert() { + let (ctx, pubkey, secret) = make_ctx_for_tests(); + let orders = make_random_orders(pubkey.clone(), &secret, "C1".into(), "C2".into(), 100); + + for order in orders.clone() { + block_on(insert_or_update_order(&ctx, order)); + } + + let alb_ordered_pair = alb_ordered_pair("C1", "C2"); + let current_root_hash = all_orders_trie_root_by_pub(&ctx, &pubkey); + let pair_trie_root = pair_trie_root_by_pub(&ctx, &pubkey, &alb_ordered_pair); + for order in orders.clone() { + block_on(insert_or_update_order(&ctx, order)); + } + + let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); + let orderbook = block_on(ordermatch_ctx.orderbook.lock()); + let pubkey_state = orderbook.pubkeys_state.get(&pubkey).unwrap(); + assert!(!pubkey_state + .order_pairs_trie_state_history + .contains_key(¤t_root_hash)); + assert!(!pubkey_state + .order_pairs_trie_state_history + .contains_key(&pair_trie_root)); +} + +#[test] +fn test_process_sync_pubkey_orderbook_state_after_orders_removed() { + let (ctx, pubkey, secret) = make_ctx_for_tests(); + let orders = make_random_orders(pubkey.clone(), &secret, "C1".into(), "C2".into(), 100); + + for order in orders.clone() { + block_on(insert_or_update_order(&ctx, order)); + } + + let alb_ordered_pair = alb_ordered_pair("C1", "C2"); + let current_root_hash = all_orders_trie_root_by_pub(&ctx, &pubkey); + let pair_trie_root = pair_trie_root_by_pub(&ctx, &pubkey, &alb_ordered_pair); + + let prev_pairs_state = HashMap::from_iter(iter::once((alb_ordered_pair.clone(), pair_trie_root))); + + let mut old_mem_db = clone_orderbook_memory_db(&ctx); + + // pick 10 orders at random and remove them + let mut rng = thread_rng(); + let to_remove = orders.choose_multiple(&mut rng, 10); + for order in to_remove { + remove_order(&ctx, order.uuid); + } + + let expected_root_hash = all_orders_trie_root_by_pub(&ctx, &pubkey); + let mut result = block_on(process_sync_pubkey_orderbook_state( + ctx.clone(), + pubkey.clone(), + current_root_hash, + expected_root_hash, + prev_pairs_state, + )) + .unwrap() + .unwrap(); + + // check all orders trie root first + let delta = match result.all_orders_diff { + DeltaOrFullTrie::Delta(delta) => delta, + DeltaOrFullTrie::FullTrie(_) => panic!("Must be DeltaOrFullTrie::Delta"), + }; + + let actual_root_hash = delta_trie_root::( + &mut old_mem_db, + current_root_hash, + delta.into_iter().map(|(pair, hash)| (pair.into_bytes(), hash)), + ) + .unwrap(); + assert_eq!(expected_root_hash, actual_root_hash); + + // check pair trie root + let expected_root_hash = pair_trie_root_by_pub(&ctx, &pubkey, &alb_ordered_pair); + + let delta = match result.pair_orders_diff.remove(&alb_ordered_pair).unwrap() { + DeltaOrFullTrie::Delta(delta) => delta, + DeltaOrFullTrie::FullTrie(_) => panic!("Must be DeltaOrFullTrie::Delta"), + }; + + let actual_root_hash = delta_trie_root::( + &mut old_mem_db, + pair_trie_root, + delta + .into_iter() + .map(|(uuid, order)| (*uuid.as_bytes(), order.map(|o| encode_message(&o).unwrap()))), + ) + .unwrap(); + assert_eq!(expected_root_hash, actual_root_hash); +} + +#[test] +fn test_diff_should_not_be_written_if_hash_not_changed_on_remove() { + let (ctx, pubkey, secret) = make_ctx_for_tests(); + let orders = make_random_orders(pubkey.clone(), &secret, "C1".into(), "C2".into(), 100); + + for order in orders.clone() { + block_on(insert_or_update_order(&ctx, order)); + } + + let to_remove: Vec<_> = orders + .choose_multiple(&mut thread_rng(), 10) + .map(|order| order.uuid) + .collect(); + for uuid in &to_remove { + remove_order(&ctx, *uuid); + } + for uuid in &to_remove { + remove_order(&ctx, *uuid); + } + + let alb_ordered_pair = alb_ordered_pair("C1", "C2"); + let current_root_hash = all_orders_trie_root_by_pub(&ctx, &pubkey); + let pair_trie_root = pair_trie_root_by_pub(&ctx, &pubkey, &alb_ordered_pair); + + let ordermatch_ctx = OrdermatchContext::from_ctx(&ctx).unwrap(); + let orderbook = block_on(ordermatch_ctx.orderbook.lock()); + let pubkey_state = orderbook.pubkeys_state.get(&pubkey).unwrap(); + assert!(!pubkey_state + .order_pairs_trie_state_history + .contains_key(¤t_root_hash)); + assert!(!pubkey_state + .order_pairs_trie_state_history + .contains_key(&pair_trie_root)); +} + +#[test] +fn test_orderbook_pubkey_sync_request() { + let mut orderbook = Orderbook::default(); + orderbook.topics_subscribed_to.insert( + orderbook_topic_from_base_rel("C1", "C2"), + OrderbookRequestingState::Requested, + ); + let pairs = vec!["C1:C2".into(), "C2:C3".into()]; + let pubkey = "pubkey"; + let message = PubkeyKeepAlive { + orders_trie_root: [1; 8], + timestamp: now_ms() / 1000, + }; + + let request = orderbook.process_keep_alive(pubkey, pairs, message).unwrap(); + match request { + OrdermatchRequest::SyncPubkeyOrderbookState { pairs_trie_roots, .. } => { + assert!(pairs_trie_roots.contains_key("C1:C2")); + assert!(!pairs_trie_roots.contains_key("C2:C3")); + }, + _ => panic!("Invalid request {:?}", request), + } +} + +#[test] +fn test_trie_diff_avoid_cycle_on_insertion() { + let mut history = TrieDiffHistory::::default(); + history.insert_new_diff([1; 8], TrieDiff { + delta: vec![], + next_root: [2; 8], + }); + + history.insert_new_diff([2; 8], TrieDiff { + delta: vec![], + next_root: [3; 8], + }); + + history.insert_new_diff([3; 8], TrieDiff { + delta: vec![], + next_root: [4; 8], + }); + + history.insert_new_diff([4; 8], TrieDiff { + delta: vec![], + next_root: [5; 8], + }); + + history.insert_new_diff([5; 8], TrieDiff { + delta: vec![], + next_root: [2; 8], + }); + + let expected = TrieDiffHistory { + inner: HashMap::from_iter(iter::once(([1; 8], TrieDiff { + delta: vec![], + next_root: [2; 8], + }))), + }; + + assert_eq!(expected, history); +}