Updated e2e tests to use deterministic xprivs in bitcoin-core

tests/conftest.py

@@ -1,32 +1,32 @@
-import sys
-import os
 
-absolute_path = os.path.dirname(os.path.abspath(__file__))
-relative_bitcoin_path = ('../bitcoin_client')
-absolute_bitcoin_client_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../')
-sys.path.append(os.path.join(absolute_path, relative_bitcoin_path))
+import sys  # noqa: E402
+import os  # noqa: E402
 
-import random
-from typing import Tuple
-
-from test_utils.fixtures import *
-from test_utils.authproxy import AuthServiceProxy, JSONRPCException
-from test_utils import segwit_addr
-
-import shutil
-import subprocess
-from time import sleep
-from decimal import Decimal
-from pathlib import Path
+absolute_path = os.path.dirname(os.path.abspath(__file__))  # noqa: E402
+relative_bitcoin_path = ('../bitcoin_client')  # noqa: E402
+absolute_bitcoin_client_path = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), '../')   # noqa: E402
+sys.path.append(os.path.join(absolute_path, relative_bitcoin_path))  # noqa: E402
 
-from ledger_bitcoin import Chain
-from ledger_bitcoin.common import sha256
+from ragger_bitcoin import createRaggerClient, RaggerClient
+from ragger.backend import RaisePolicy
+from ragger.backend.interface import BackendInterface
+from ragger.conftest import configuration
 import ledger_bitcoin._base58 as base58
+from ledger_bitcoin.common import sha256
+from ledger_bitcoin import Chain
+from pathlib import Path
+from decimal import Decimal
+from time import sleep
+import subprocess
+import shutil
+from test_utils import segwit_addr
+from test_utils.authproxy import AuthServiceProxy, JSONRPCException
+from test_utils.fixtures import *
+from typing import Tuple
+import random
+from bip32 import BIP32
 
-from ragger.conftest import configuration
-from ragger.backend.interface import BackendInterface
-from ragger.backend import RaisePolicy
-from ragger_bitcoin import createRaggerClient, RaggerClient
 
 ###########################
 ### CONFIGURATION START ###
@@ -105,7 +105,8 @@ def run_bitcoind():
 
     bitcoind = os.getenv("BITCOIND", "/bitcoin/bin/bitcoind")
 
-    shutil.copy(os.path.join(os.path.dirname(__file__), "bitcoin.conf"), BITCOIN_DIRNAME)
+    shutil.copy(os.path.join(os.path.dirname(__file__),
+                "bitcoin.conf"), BITCOIN_DIRNAME)
     subprocess.Popen([bitcoind, f"--datadir={BITCOIN_DIRNAME}"])
 
     # Make sure the node is ready, and generate some initial blocks
@@ -165,29 +166,63 @@ def get_unique_wallet_name() -> str:
     return result
 
 
+def get_pseudorandom_keypair(wallet_name: str) -> Tuple[str, str]:
+    """
+    Generates a tpub and tpriv deterministically from the wallet name
+    Used in tests to have deterministic wallets in bitcoin-core instances.
+    """
+
+    bip32 = BIP32.from_seed(wallet_name.encode(), network="test")
+
+    xpub = bip32.get_xpub_from_path("m")
+    xpriv = bip32.get_xpriv_from_path("m")
+
+    return xpub, xpriv
+
+
 def create_new_wallet() -> Tuple[str, str]:
     """Creates a new descriptor-enabled wallet in bitcoin-core. Each new wallet has an increasing counter as
-    part of it's name in order to avoid conflicts. Returns the wallet name and the xpub (dropping the key origin
+    part of it's name in order to avoid conflicts. Returns the wallet name and the xpub (with no key origin
     information)."""
 
     wallet_name = get_unique_wallet_name()
 
-    # TODO: derive seed from wallet_count, and use it to create a descriptor wallet (how?)
-    #       this would help to have repeatable tests, generating always the same seeds
-
     get_rpc().createwallet(wallet_name=wallet_name, descriptors=True)
-    wallet_rpc = get_wallet_rpc(wallet_name)
-
-    all_descriptors = wallet_rpc.listdescriptors()["descriptors"]
-    descriptor: str = next(filter(lambda d: d["desc"].startswith(
-        "pkh") and "/0/*" in d["desc"], all_descriptors))["desc"]
 
-    core_xpub_orig = descriptor[descriptor.index("(")+1: descriptor.index("/0/*")]
-    core_xpub = core_xpub_orig[core_xpub_orig.find("]") + 1:]
+    core_xpub, _ = get_pseudorandom_keypair(wallet_name)
 
     return wallet_name, core_xpub
 
 
+def recompute_checksum(rpc: AuthServiceProxy, descriptor: str) -> str:
+    # remove "#" and everything after it, if present
+    if '#' in descriptor:
+        descriptor = descriptor[:descriptor.index('#')]
+    descriptor_info = rpc.getdescriptorinfo(descriptor)
+    return descriptor + '#' + descriptor_info["checksum"]
+
+
+def import_descriptors_with_privkeys(core_wallet_name: str, receive_desc: str, change_desc: str):
+    wallet = get_wallet_rpc(core_wallet_name)
+    wallet_xpub, wallet_xpriv = get_pseudorandom_keypair(core_wallet_name)
+
+    assert wallet_xpub in receive_desc and wallet_xpub in change_desc
+
+    import_desc = [{
+        "desc": recompute_checksum(wallet, receive_desc.replace(wallet_xpub, wallet_xpriv)),
+        "active": True,
+        "internal": False,
+        "timestamp": "now"
+    }, {
+        "desc": recompute_checksum(wallet, change_desc.replace(wallet_xpub, wallet_xpriv)),
+        "active": True,
+        "internal": True,
+        "timestamp": "now"
+    }]
+    import_res = wallet.importdescriptors(import_desc)
+    assert import_res[0]["success"] and import_res[1]["success"]
+
+
 def generate_blocks(n):
     return get_rpc().generatetoaddress(n, btc_addr)
 

tests/test_e2e_miniscript.py

@@ -1,6 +1,6 @@
 import pytest
 
-from typing import List, Union
+from typing import List
 
 import hmac
 from hashlib import sha256
@@ -15,13 +15,13 @@
 
 from ragger_bitcoin import RaggerClient
 from ragger_bitcoin.ragger_instructions import Instructions
-from ragger.navigator import Navigator, NavInsID
+from ragger.navigator import Navigator
 from ragger.firmware import Firmware
 from ragger.error import ExceptionRAPDU
 
 from .instructions import e2e_register_wallet_instruction, e2e_sign_psbt_instruction
 
-from .conftest import create_new_wallet, generate_blocks, get_unique_wallet_name, get_wallet_rpc, testnet_to_regtest_addr as T
+from .conftest import create_new_wallet, generate_blocks, get_unique_wallet_name, get_wallet_rpc, import_descriptors_with_privkeys, testnet_to_regtest_addr as T
 from .conftest import AuthServiceProxy
 
 
@@ -34,11 +34,13 @@ def run_test_e2e(navigator: Navigator, client: RaggerClient, wallet_policy: Wall
     assert wallet_id == wallet_policy.id
 
     assert hmac.compare_digest(
-        hmac.new(speculos_globals.wallet_registration_key, wallet_id, sha256).digest(),
+        hmac.new(speculos_globals.wallet_registration_key,
+                 wallet_id, sha256).digest(),
         wallet_hmac,
     )
 
-    address_hww = client.get_wallet_address(wallet_policy, wallet_hmac, 0, 3, False)
+    address_hww = client.get_wallet_address(
+        wallet_policy, wallet_hmac, 0, 3, False)
 
     # ==> verify the address matches what bitcoin-core computes
     receive_descriptor = wallet_policy.get_descriptor(change=False)
@@ -91,7 +93,8 @@ def run_test_e2e(navigator: Navigator, client: RaggerClient, wallet_policy: Wall
             out_address: Decimal("0.01")
         },
         options={
-            "changePosition": 1 # We need a fixed position to be able to know how to navigate in the flows
+            # We need a fixed position to be able to know how to navigate in the flows
+            "changePosition": 1
         }
     )
 
@@ -106,19 +109,27 @@ def run_test_e2e(navigator: Navigator, client: RaggerClient, wallet_policy: Wall
                                 instructions=instructions_sign_psbt,
                                 testname=f"{test_name}_sign")
 
-    n_internal_keys = count_internal_key_placeholders(speculos_globals.seed, "test", wallet_policy)
-    assert len(hww_sigs) == n_internal_keys * len(psbt.inputs)  # should be true as long as all inputs are internal
+    n_internal_keys = count_internal_key_placeholders(
+        speculos_globals.seed, "test", wallet_policy)
+    # should be true as long as all inputs are internal
+    assert len(hww_sigs) == n_internal_keys * len(psbt.inputs)
 
     for i, part_sig in hww_sigs:
         psbt.inputs[i].partial_sigs[part_sig.pubkey] = part_sig.signature
 
     signed_psbt_hww_b64 = psbt.serialize()
 
+    # ==> import descriptor for each bitcoin-core wallet
+    for core_wallet_name in core_wallet_names:
+        import_descriptors_with_privkeys(
+            core_wallet_name, receive_descriptor_chk, change_descriptor_chk)
+
     # ==> sign it with bitcoin-core
     partial_psbts = [signed_psbt_hww_b64]
 
     for core_wallet_name in core_wallet_names:
-        partial_psbts.append(get_wallet_rpc(core_wallet_name).walletprocesspsbt(psbt_b64)["psbt"])
+        partial_psbts.append(get_wallet_rpc(
+            core_wallet_name).walletprocesspsbt(psbt_b64)["psbt"])
 
     # ==> finalize the psbt, extract tx and broadcast
     combined_psbt = rpc.combinepsbt(partial_psbts)
@@ -404,15 +415,18 @@ def test_invalid_miniscript(navigator: Navigator, firmware: Firmware, client: Ra
     run_test_invalid(client, "wsh(wsh(pkh(@0/**)))", [internal_xpub_orig])
 
     # sh(wsh(...)) is meaningful with valid miniscript, but current implementation of miniscript assumes wsh(...)
-    run_test_invalid(client, "sh(wsh(or_d(pk(@0/**),pkh(@1/**))))", [internal_xpub_orig, core_xpub_orig1])
+    run_test_invalid(client, "sh(wsh(or_d(pk(@0/**),pkh(@1/**))))",
+                     [internal_xpub_orig, core_xpub_orig1])
 
     # tr must be top-level
     run_test_invalid(client, "wsh(tr(pk(@0/**)))", [internal_xpub_orig])
     run_test_invalid(client, "sh(tr(pk(@0/**)))", [internal_xpub_orig])
 
     # valid miniscript must be inside wsh()
-    run_test_invalid(client, "or_d(pk(@0/**),pkh(@1/**))", [internal_xpub_orig, core_xpub_orig1])
-    run_test_invalid(client, "sh(or_d(pk(@0/**),pkh(@1/**)))", [internal_xpub_orig, core_xpub_orig1])
+    run_test_invalid(client, "or_d(pk(@0/**),pkh(@1/**))",
+                     [internal_xpub_orig, core_xpub_orig1])
+    run_test_invalid(client, "sh(or_d(pk(@0/**),pkh(@1/**)))",
+                     [internal_xpub_orig, core_xpub_orig1])
 
     # sortedmulti is not valid miniscript, can only be used as a descriptor inside sh or wsh
     run_test_invalid(client, "wsh(or_d(pk(@0/**),sortedmulti(3,@1/**,@2/**,@3/**,@4/**,@5/**)))",

tests/test_e2e_multisig.py

@@ -1,28 +1,25 @@
 import pytest
 
-from typing import List, Union
+from typing import List
 
 import hmac
 from hashlib import sha256
 from decimal import Decimal
 
-from ledger_bitcoin import Client, MultisigWallet, AddressType
-from ledger_bitcoin.client_base import TransportClient
+from ledger_bitcoin import MultisigWallet, AddressType
 from ledger_bitcoin.psbt import PSBT
 from ledger_bitcoin.wallet import WalletPolicy
 
 from test_utils import SpeculosGlobals, get_internal_xpub, count_internal_key_placeholders
 
-from speculos.client import SpeculosClient
-
 from ragger_bitcoin import RaggerClient
 from ragger_bitcoin.ragger_instructions import Instructions
-from ragger.navigator import Navigator, NavInsID
+from ragger.navigator import Navigator
 from ragger.firmware import Firmware
 
 from .instructions import e2e_register_wallet_instruction, e2e_sign_psbt_instruction
 
-from .conftest import create_new_wallet, generate_blocks, get_unique_wallet_name, get_wallet_rpc, testnet_to_regtest_addr as T
+from .conftest import create_new_wallet, generate_blocks, get_unique_wallet_name, get_wallet_rpc, import_descriptors_with_privkeys, testnet_to_regtest_addr as T
 from .conftest import AuthServiceProxy
 
 
@@ -122,6 +119,11 @@ def run_test(navigator: Navigator, client: RaggerClient, wallet_policy: WalletPo
 
     signed_psbt_hww_b64 = psbt.serialize()
 
+    # ==> import descriptor for each bitcoin-core wallet
+    for core_wallet_name in core_wallet_names:
+        import_descriptors_with_privkeys(
+            core_wallet_name, receive_descriptor_chk, change_descriptor_chk)
+
     # ==> sign it with bitcoin-core
     partial_psbts = [signed_psbt_hww_b64]
 

tests/test_e2e_tapscripts.py

@@ -1,13 +1,11 @@
 import pytest
 
-from typing import List, Union
+from typing import List
 
 import hmac
 from hashlib import sha256
 from decimal import Decimal
 
-from ledger_bitcoin import Client
-from ledger_bitcoin.client_base import TransportClient
 from ledger_bitcoin.exception.errors import IncorrectDataError, NotSupportedError
 from ledger_bitcoin.exception.device_exception import DeviceException
 from ledger_bitcoin.psbt import PSBT
@@ -17,12 +15,12 @@
 
 from ragger_bitcoin import RaggerClient
 from ragger_bitcoin.ragger_instructions import Instructions
-from ragger.navigator import Navigator, NavInsID
+from ragger.navigator import Navigator
 from ragger.firmware import Firmware
 from ragger.error import ExceptionRAPDU
 
 from .instructions import e2e_register_wallet_instruction, e2e_sign_psbt_instruction
-from .conftest import AuthServiceProxy
+from .conftest import AuthServiceProxy, import_descriptors_with_privkeys
 from .conftest import create_new_wallet, generate_blocks, get_unique_wallet_name, get_wallet_rpc, testnet_to_regtest_addr as T
 
 
@@ -122,6 +120,11 @@ def run_test_e2e(navigator: Navigator, client: RaggerClient, wallet_policy: Wall
     n_internal_keys = count_internal_key_placeholders(speculos_globals.seed, "test", wallet_policy)
     assert len(hww_sigs) == n_internal_keys * len(psbt.inputs)  # should be true as long as all inputs are internal
 
+    # ==> import descriptor for each bitcoin-core wallet
+    for core_wallet_name in core_wallet_names:
+        import_descriptors_with_privkeys(
+            core_wallet_name, receive_descriptor_chk, change_descriptor_chk)
+
     # ==> sign it with bitcoin-core
 
     partial_psbts = [signed_psbt_hww_b64]