Add an example test and finnish the framework

.gitignore

@@ -4,4 +4,5 @@ Cargo.lock
 perf.*
 *.svg
 config.toml
-/builds
+/builds
+/data

Cargo.toml

@@ -5,7 +5,8 @@ edition = "2021"
 
 [dependencies]
 rustreexo = { git = "https://www.github.com/mit-dci/rustreexo" }
-btcd-rpc = { git = "https://github.com/Davidson-Souza/rust-btcd-rpc", features = ["utreexod"], branch = "use-reqwest"}
+# btcd-rpc = { git = "https://github.com/Davidson-Souza/rust-btcd-rpc", features = ["utreexod"], branch = "use-reqwest"}
+btcd-rpc = {  path = "../utreexod-cli/client/", features = ["utreexod"] }
 clap = {version = "4.0.29", features = ["derive"]}
 sha2 = "^0.10.6"
 async-std = "1.12.0"

tests/basic/restart.py

@@ -0,0 +1,23 @@
+import subprocess
+import time
+import os
+import filecmp
+from test_framework.test_framework import TestFramework
+
+
+class TestRestart(TestFramework):
+    def run_test(self):
+        """
+            Tests if we don't corrupt our data dir between restarts. This would have caught,
+            the error fixed in #9
+        """
+        print("Running restart test")
+        base_testdir = "data/TestRestart/"
+        self.run_node(base_testdir + "1/")
+        time.sleep(10)
+        self.stop_node(0)
+        self.run_node(base_testdir + "2/")
+        time.sleep(10)
+        self.stop_node(0)
+        assert (filecmp.dircmp(base_testdir + "2/", base_testdir + "1/"))
+        print("restart test passed!")

tests/data/rpc/block_index.json

@@ -0,0 +1,12 @@
+[
+    "",
+    "00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53",
+    "00000032bb881de703dcc968e8258080c7ed4a2933e3a35888fa0b2f75f36029",
+    "000000e8daac2a2e973ecaab46dc948181c638adecf5ae0fd5d3e13aa14364b0",
+    "00000194763f1233e40afd5f6eb2e64abd2e5e158a4139198a224f8a7503d47e",
+    "000003776c984edd753b876f0e70d08450f92dd76a768df03d2c42d1880aea5b",
+    "000001b62ff822d0b1c68e231670eba2cf8d494075f9616447194b370db6f3da",
+    "000000b6f018c5b53240616c7868eab7e1338de4200ec25b65a5e1c2f3c57c45",
+    "000002213230ed813e42076f753f666244db421b3e36a3a3ba31384feb61eba2",
+    "000001e163d0a8e294c3406a713ba4363f422f1322a1d6ab6de4e54b51e868a8"
+]

tests/data/rpc/blocks.json

@@ -1,5 +1,5 @@
-[
-    {
+{
+    "00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53": {
         "0": "00000020f61eee3b63a380a477a063af32b2bbc97c9ff9f01f2c4225e973988108000000f575c83235984e7dc4afc1f30944c170462e84437ab6f2d52e16878a79e4678bd1914d5fae77031eccf4070001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025151feffffff0200f2052a010000001600149243f727dd5343293eb83174324019ec16c2630f0000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa2490047304402205e423a8754336ca99dbe16509b877ef1bf98d008836c725005b3c787c41ebe46022047246e4467ad7cc7f1ad98662afcaf14c115e0095a227c7b05c5182591c23e7e01000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53",
@@ -22,7 +22,7 @@
             "nextblockhash": "00000032bb881de703dcc968e8258080c7ed4a2933e3a35888fa0b2f75f36029"
         }
     },
-    {
+    "00000032bb881de703dcc968e8258080c7ed4a2933e3a35888fa0b2f75f36029": {
         "0": "00000020533b53ded9bff4adc94101d32400a144c54edc5ed492a3b26c63b2d686000000b38fef50592017cfafbcab88eb3d9cf50b2c801711cad8299495d26df5e54812e7914d5fae77031ecfdd0b0001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025251feffffff0200f2052a01000000160014fd09839740f0e0b4fc6d5e2527e4022aa9b89dfa0000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa24900473044022031d64a1692cdad1fc0ced69838169fe19ae01be524d831b95fcf5ea4e6541c3c02204f9dea0801df8b4d0cd0857c62ab35c6c25cc47c930630dc7fe723531daa3e9b01000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "00000032bb881de703dcc968e8258080c7ed4a2933e3a35888fa0b2f75f36029",
@@ -45,7 +45,7 @@
             "nextblockhash": "000000e8daac2a2e973ecaab46dc948181c638adecf5ae0fd5d3e13aa14364b0"
         }
     },
-    {
+    "000000e8daac2a2e973ecaab46dc948181c638adecf5ae0fd5d3e13aa14364b0": {
         "0": "000000202960f3752f0bfa8858a3e333294aedc7808025e868c9dc03e71d88bb320000007765fcd3d5b4966beb338bba2675dc2cf2ad28d4ad1d83bdb6f286e7e27ac1f807924d5fae77031e81d60b0001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025351feffffff0200f2052a010000001600141e5fb426042692ae0e87c070e78c39307a5661c20000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa2490047304402205de93694763a42954865bcf1540cb82958bc62d0ec4eee02070fb7937cd037f4022067f333753bce47b10bc25eb6e1f311482e994c862a7e0b2d41ab1c8679fd1b1101000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "000000e8daac2a2e973ecaab46dc948181c638adecf5ae0fd5d3e13aa14364b0",
@@ -68,7 +68,7 @@
             "nextblockhash": "00000194763f1233e40afd5f6eb2e64abd2e5e158a4139198a224f8a7503d47e"
         }
     },
-    {
+    "00000194763f1233e40afd5f6eb2e64abd2e5e158a4139198a224f8a7503d47e": {
         "0": "00000020b06443a13ae1d3d50faef5ecad38c6818194dc46abca3e972e2aacdae800000069a5829097e80fee00ac49a56ea9f82d741a6af84d32b3bc455cf31871e2a8ac27924d5fae77031e9c91050001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025451feffffff0200f2052a0100000016001430db2f8225dcf7751361ab38735de08190318cb70000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa2490047304402200936f5f9872f6df5dd242026ad52241a68423f7f682e79169a8d85a374eab9b802202cd2979c48b321b3453e65e8f92460db3fca93cbea8539b450c959f4fbe630c601000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "00000194763f1233e40afd5f6eb2e64abd2e5e158a4139198a224f8a7503d47e",
@@ -91,7 +91,7 @@
             "nextblockhash": "000003776c984edd753b876f0e70d08450f92dd76a768df03d2c42d1880aea5b"
         }
     },
-    {
+    "000003776c984edd753b876f0e70d08450f92dd76a768df03d2c42d1880aea5b": {
         "0": "000000207ed403758a4f228a1939418a155e2ebd4ae6b26e5ffd0ae433123f7694010000542e80b609c5bc58af5bdf492e26d4f60cd43a3966c2e063c50444c29b3757a636924d5fae77031ee8601d0001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025551feffffff0200f2052a01000000160014edc207e014df34fa3885dff97d1129d356e1186a0000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa24900473044022021a3656609f85a66a2c5672ed9322c2158d57251040d2716ed202a1fe14f0c12022057d68bc6611f7a9424a7e00bbf3e27e6ae6b096f60bac624a094bc97a59aa1ff01000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "000003776c984edd753b876f0e70d08450f92dd76a768df03d2c42d1880aea5b",
@@ -114,7 +114,7 @@
             "nextblockhash": "000001b62ff822d0b1c68e231670eba2cf8d494075f9616447194b370db6f3da"
         }
     },
-    {
+    "000001b62ff822d0b1c68e231670eba2cf8d494075f9616447194b370db6f3da": {
         "0": "000000205bea0a88d1422c3df08d766ad72df95084d0700e6f873b75dd4e986c7703000002b57516d33ed60c2bdd9f93d6d5614083324c837e68e5ba6e04287a7285633585924d5fae77031ed171960001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025651feffffff0200f2052a010000001600143ae612599cf96f2442ce572633e0251116eaa52f0000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa24900473044022059a7c54de76bfdbb1dd44c78ea2dbd2bb4e97f4abad38965f41e76433e56423c022054bf17f04fe17415c0141f60eebd2b839200f574d8ad8d55a0917b92b0eb913401000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "000001b62ff822d0b1c68e231670eba2cf8d494075f9616447194b370db6f3da",
@@ -137,7 +137,7 @@
             "nextblockhash": "000000b6f018c5b53240616c7868eab7e1338de4200ec25b65a5e1c2f3c57c45"
         }
     },
-    {
+    "000000b6f018c5b53240616c7868eab7e1338de4200ec25b65a5e1c2f3c57c45": {
         "0": "00000020daf3b60d374b19476461f97540498dcfa2eb7016238ec6b1d022f82fb60100007a7ae65b53cb988c2ec92d2384996713821d5645ffe61c9acea60da75cd5edfa1a944d5fae77031e9dbb050001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025751feffffff0200f2052a01000000160014ef2dceae02e35f8137de76768ae3345d99ca68860000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa2490047304402202b3f946d6447f9bf17d00f3696cede7ee70b785495e5498274ee682a493befd5022045fc0bcf9332243168b5d35507175f9f374a8eba2336873885d12aada67ea5f601000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "000000b6f018c5b53240616c7868eab7e1338de4200ec25b65a5e1c2f3c57c45",
@@ -160,7 +160,7 @@
             "nextblockhash": "000002213230ed813e42076f753f666244db421b3e36a3a3ba31384feb61eba2"
         }
     },
-    {
+    "000002213230ed813e42076f753f666244db421b3e36a3a3ba31384feb61eba2": {
         "0": "00000020457cc5f3c2e1a5655bc20e20e48d33e1b7ea68786c614032b5c518f0b6000000541f36942d82c6e7248275ff15c8933487fbe1819c67a9ecc0f4b70bb7e6cf672a944d5fae77031e8f39860001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025851feffffff0200f2052a0100000016001472a27906947c06d034b38ba2fa13c6391a4832790000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa2490047304402202d62805ce60cbd60591f97f949b5ea5bd7e2307bcde343e6ea8394da92758e72022053a25370b0aa20da100189b7899a8f8675a0fdc60e38ece6b8a4f98edd94569e01000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "000002213230ed813e42076f753f666244db421b3e36a3a3ba31384feb61eba2",
@@ -183,7 +183,7 @@
             "nextblockhash": "000001e163d0a8e294c3406a713ba4363f422f1322a1d6ab6de4e54b51e868a8"
         }
     },
-    {
+    "000001e163d0a8e294c3406a713ba4363f422f1322a1d6ab6de4e54b51e868a8": {
         "0": "00000020a2eb61eb4f3831baa3a3363e1b42db4462663f756f07423e81ed30322102000077224de7dea0f8d0ec22b1d2e2e255f0a987b96fe7200e1a2e6373f48a2f5b7894954d5fae77031e36867e0001010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025951feffffff0200f2052a01000000160014aa0ad9f26801258382e0734dceec03a4a75f60240000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa2490047304402206fa0d59990eed369bd7375767c9a6c9369fae209152b8674e520da270605528c0220749eed3b12dbe3f583f505d21803e4aef59c8e24c5831951eafa4f15a8f92c4e01000120000000000000000000000000000000000000000000000000000000000000000000000000",
         "1": {
             "hash": "000001e163d0a8e294c3406a713ba4363f422f1322a1d6ab6de4e54b51e868a8",
@@ -206,4 +206,4 @@
             "nextblockhash": "000001418e9af8310ef3644093dbc4a8ad9edb0c9ac80397f3479ad1729d965b"
         }
     }
-]
+}

tests/data/rpc/proofs.json

@@ -1,83 +1,83 @@
-[
-    {
-        "proof": {
+{
+    "00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     },
-    {
-        "proof": {
+    "00000032bb881de703dcc968e8258080c7ed4a2933e3a35888fa0b2f75f36029": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     },
-    {
-        "proof": {
+    "000000e8daac2a2e973ecaab46dc948181c638adecf5ae0fd5d3e13aa14364b0": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     },
-    {
-        "proof": {
+    "00000194763f1233e40afd5f6eb2e64abd2e5e158a4139198a224f8a7503d47e": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     },
-    {
-        "proof": {
+    "000003776c984edd753b876f0e70d08450f92dd76a768df03d2c42d1880aea5b": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     },
-    {
-        "proof": {
+    "000001b62ff822d0b1c68e231670eba2cf8d494075f9616447194b370db6f3da": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     },
-    {
-        "proof": {
+    "000000b6f018c5b53240616c7868eab7e1338de4200ec25b65a5e1c2f3c57c45": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     },
-    {
-        "proof": {
+    "000002213230ed813e42076f753f666244db421b3e36a3a3ba31384feb61eba2": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     },
-    {
-        "proof": {
+    "000001e163d0a8e294c3406a713ba4363f422f1322a1d6ab6de4e54b51e868a8": {
+        "1": {
             "proofhashes": [],
             "rememberindexes": [],
             "targethashes": [],
             "targetpreimages": [],
             "prooftargets": []
         }
     }
-]
+}

tests/example_test.py

@@ -0,0 +1,35 @@
+"""
+    This is an example of how tests should look lke, see the class bellow for more info
+"""
+import time
+import os
+
+from test_framework.test_framework import TestFramework
+from test_framework.electrum_client import ElectrumClient
+from test_framework.mock_rpc import MockUtreexod
+
+# Tests should be a child class from TestFramework
+
+
+class ExampleTest(TestFramework):
+    # All tests should override the run_test method
+    def run_test(self):
+        # This creates a dummy rpc listening on port 8080
+        self.run_rpc()
+        # Wait until the rpc is ready
+        time.sleep(1)
+        # Start a new node (this crate's binary)
+        node1 = self.run_node("./data/test1", "signet")
+        # Wait the node to start
+        time.sleep(10)
+        # Create an instance of the Electrum Client, a small implementation of the electrum
+        # protocol, to test our own electrum implementation
+        electrum = ElectrumClient("localhost", 50001)
+        print(electrum.get_version())
+
+        # .... cleanup ....
+        os.rmdir("./data")
+        # Stop any rpc server that is running
+        self.stop_rpc()
+        # Stop the node that is running
+        self.stop_node(0)

tests/test_framework/electrum_client.py

@@ -23,7 +23,6 @@ def _request(self, method, params):
             if c == b'\n':
                 break
             buff += c.decode("utf-8")
-
         return str.encode(buff)
 
 
@@ -177,4 +176,4 @@ def get_version(self) -> str:
             Identify the client to the server and negotiate the protocol version.
             Only the first server.version() message is accepted.
         """
-        return self.con._request("server.version", [])
+        return self.con._request("server.version", ["test-client", "1.2"])

tests/test_framework/mock_rpc.py

@@ -11,14 +11,19 @@ def __init__(self):
         self.server.register_function(self.getblock)
         self.server.register_function(self.stop)
         self.server.register_function(self.getutreexoproof)
-        self.server.register_function(self.get_best_block)
-        self.server.register_function(self.get_block_hash)
+        self.server.register_function(self.getbestblock)
+        self.server.register_function(self.getblockhash)
+        self.server.register_function(self.getheaders)
 
+    def run(self):
+        print("rpc running")
         self.server.serve_forever()
 
     def get(self, what: str, key: str | int, verbosity=bool | int) -> dict:
-        fp = open("tests/data/rpc/{}.json" % key)
+        fp = open(f"tests/data/rpc/{what}.json")
         data = json.load(fp)
+        if verbosity == None:
+            return data[key]
 
         if verbosity:
             return data[key]["1"]
@@ -30,13 +35,21 @@ def getblock(self, height: int, verbose: int = 0):
     def stop(self):
         return self.server.shutdown()
 
-    def get_best_block(self):
-        fp = open("tests/data/rpc/blocks.json")
+    def getbestblock(self):
+        fp = open("tests/data/rpc/block_index.json")
         data = json.load(fp)
-        return {"height": len(data), "hash": data[-1]["1"]["hash"]}
+        return {"height": len(data) - 1, "hash": data[-1]}
 
-    def get_block_hash(self, height: str) -> dict:
-        return self.get("blocks", height)["hash"]
+    def getblockhash(self, height: str) -> dict:
+        return self.get("block_index", height, None)
 
-    def getutreexoproof(self, height: str):
-        return self.get("proofs", height)["hash"]
+    def getheaders(self, locator, verbosity) -> dict:
+        fBlocks = open("tests/data/rpc/blocks.json")
+        blocks_data = json.load(fBlocks)
+        headers = []
+        for i in blocks_data:
+            headers.append(blocks_data[i]["0"][0:160])
+        return headers
+
+    def getutreexoproof(self, height: str, verbosity=None):
+        return self.get("proofs", height)

tests/test_framework/test_framework.py

@@ -1,22 +1,26 @@
 import subprocess
+
 from threading import Thread
-from mock_rpc import MockUtreexod
-from json_rpc import Server
+from test_framework.mock_rpc import MockUtreexod
 
 
 class TestFramework:
     tests = []
     nodes: list() = []
     rpc: MockUtreexod = None
 
-    def run_node(self, datadir: str, net: Network):
-        node = subprocess.Popen(["cargo", "run", "--", "--network",
-                                 net, "run", "--data-dir", datadir])
+    def run_node(self, datadir: str, net: str):
+        node = subprocess.Popen([
+            "cargo", "run", "--", "--network",
+            net, "run", "--rpc-host", "http://localhost:8080",
+            "--data-dir", datadir
+        ])
         self.nodes.append(node)
 
     def run_rpc(self):
         # Run as a thread
-        self.rpc = Thread(target=MockUtreexod)
+        self.rpc = Thread(target=MockUtreexod().run)
+        self.rpc.daemon = True
         self.rpc.start()
 
     def stop_rpc(self):