diff --git a/.github/scripts/setup.sh b/.github/scripts/setup.sh index 59a092051897..dc57215668aa 100755 --- a/.github/scripts/setup.sh +++ b/.github/scripts/setup.sh @@ -15,6 +15,7 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ build-essential \ clang \ cppcheck \ + curl \ docbook-xml \ eatmydata \ gcc-aarch64-linux-gnu \ diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml deleted file mode 100644 index 71e51cabb837..000000000000 --- a/.github/workflows/ci_build.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: CI Compilation testing - -on: [push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - include: - - { OS: alpine } - steps: - - uses: actions/checkout@v2 - - name: Integration testing - run: | - docker build -f contrib/docker/Dockerfile.${{matrix.OS}} -t clightning-${{matrix.OS}} . diff --git a/CHANGELOG.md b/CHANGELOG.md index 53a716574669..603f96786f17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,108 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [24.02] - 2024-02-27: "uint needs signature" + +This release named by Erik de Smedt (@ErikDeSmedt). + +### Added + + - JSON-RPC: `listpeerchannels` field `last_stable_connection` and `listclosedchannels` field `last_stable_connection` showing when we last held an established channel for a minute or more. ([#6904]) + - JSON-RPC: `listpeerchannels` new field `reestablished` set once we've exchanged `channel_reestablish` messages. ([#6904]) + - JSON-RPC: `fundchannel`, `multifundchannel`, `fundchannel_start` and `openchannel_init`: new field `channel_type`. ([#6864]) + - JSON-RPC: `fundchannel` and `multifundchannel` now take an optional `channel_type` parameter. ([#6864]) + - JSON-RPC: `fundchannel_start` and `openchannel_init` now take an optional `channel_type` parameter. ([#6864]) + - Plugin: options and commands can specify deprecation start (and optional end) versions. ([#6936]) + - Plugins: rpcmethods and options can set `deprecated` to a pair of version strings, not just a boolean. ([#6936]) + - JSON-RPC: `deprecations` to enable/disable deprecated APIs from this caller. ([#6936]) + - config: `i-promise-to-fix-broken-api-user` allows for a one-release re-enablement of long-deprecated features. ([#6936]) + - Protocol: `option_anchors_zero_fee_htlc_tx` enabled, no longer experimental. ([#6785]) + - Plugins: `deprecated_oneshot` notifiction subscription to change deprecated status for a single command. ([#6936]) + - hsmd: Added hsmd_forget_channel to enable explicit channel deletion. ([#6987]) ([#6988]) + - tracing: It is now possible to inject a parent for the startup trace by setting the `CLN_TRACEPARENT` envvar ([#6912]) + - Plugins: notification custommsg for receiving an unknown protocol message ([#6899]) + - JSON-RPC: `listpeerchannels` now shows gossip update contents (even if channel unannounced). ([#6869]) + + +### Changed + + - lightningd: Speed up blocksync by not parsing unused parts of the transactions ([#6984]) + - Config: `experimental-anchors` now does nothing (it's enabled by default). ([#6785]) + - reckless installs python plugins into virtual environments ([#7018]) + - Protocol: `option_gossip_queries` is now required (advertized by all but 16 nodes) ([#6864]) + - Protocol: `option_gossip_queries` is now required (advertized by all but 11 nodes) ([#6864]) + - Protocol: `option_data_loss_protect` is now required (advertized by all but 11 nodes) ([#6864]) + - core: Processing blocks should now be faster ([#6983]) + - Enable optimizations for libwally/libsecp256k1-zkp ([#6983]) + - Update libwally to 1.2.0 ([#6983]) + - pyln-client: no longer autoconverts _msat field to Millisatoshi class (leaves as ints). ([#6865]) + - JSON-RPC: `listnodes` no longer shows private (peer) nodes: use listpeers ([#6869]) + - startup_regtest.sh: `fund_nodes` will now make balanced channels ([#6898]) + - startup_regtest.sh PATH_TO_LIGHTNING + PATH_TO_BITCOIN are no more. Use LIGHTNING_BIN and BITCOIN_DIR ([#6898]) + + +### Deprecated + +Note: You should always set `allow-deprecated-apis=false` to test for changes. + + - `listchannels` no longer uses local knowledge to set `active` to false if disconnected. ([#6869]) + - JSON-RPC: `listchannels` listing private channels: use listpeerchannels ([#6869]) + + +### Removed + + - Protocol: we no longer ratelimit gossip messages by channel, making our code far simpler. ([#6941]) + - Config: `disable-ip-discovery` (deprecated in v23.02): use `announce-addr-discovered` ([#6936]) + - wallet: removal of p2sh-segwit addresses; newaddr won't issue them, we won't watch them for new funds (deprecated in *23.02*) ([#6936]) + - JSON-RPC: `invoice`, `sendonion`, `sendpay`, `pay`, `keysend`, `fetchinvoice`, `sendinvoice`: `msatoshi` argument (deprecated 0.12.0). Use `amount_msat`. ([#6936]) + + +### Fixed + + - JSON-RPC: `close` with `destination` works even if prior `destination` was rejected. ([#7072]) + - JSON-RPC: `channel_type` reflects option_zeroconf if explicitly negotiated. ([#6864]) + - configure: We now respect the `PKG_CONFIG_PATH` environment variable ([#6967]) + - Default bolt11 invoices are payable by LND nodes. ([#6957]) + - channeld: We could crash `closingd` by sending it a `channeld` message ([#6937]) + - `bkpr-listbalances` would crash for nodes on signet with payments in channels, because onchain events were using a different currency than inchannel events. ([#6888]) + - Hsmtool: Fix segmentation fault when calling `getcodexsecret` without id. ([#6895]) + + +### EXPERIMENTAL + + - JSON-RPC: Deprecated `offer` parameter `recurrence_base` with `@` prefix: use `recurrence_start_any_period`. ([#7034]) + - JSON-RPC: Added `offer` parameter `recurrence_start_any_period`. ([#7034]) + - Plugins: `funder` option "lease-fee-base-msat" removed (deprecated in v0.11, use "lease-fee-base-sat") ([#6936]) + + + +[#6983]: https://github.com/ElementsProject/lightning/pull/6983 +[#7034]: https://github.com/ElementsProject/lightning/pull/7034 +[#6864]: https://github.com/ElementsProject/lightning/pull/6864 +[#7072]: https://github.com/ElementsProject/lightning/pull/7072 +[#6899]: https://github.com/ElementsProject/lightning/pull/6899 +[#6936]: https://github.com/ElementsProject/lightning/pull/6936 +[#6895]: https://github.com/ElementsProject/lightning/pull/6895 +[#6888]: https://github.com/ElementsProject/lightning/pull/6888 +[#6898]: https://github.com/ElementsProject/lightning/pull/6898 +[#6957]: https://github.com/ElementsProject/lightning/pull/6957 +[#6912]: https://github.com/ElementsProject/lightning/pull/6912 +[#6967]: https://github.com/ElementsProject/lightning/pull/6967 +[#6937]: https://github.com/ElementsProject/lightning/pull/6937 +[#6988]: https://github.com/ElementsProject/lightning/pull/6988 +[#6785]: https://github.com/ElementsProject/lightning/pull/6785 +[#7010]: https://github.com/ElementsProject/lightning/pull/7010 +[#6992]: https://github.com/ElementsProject/lightning/pull/6992 +[#6904]: https://github.com/ElementsProject/lightning/pull/6904 +[#7018]: https://github.com/ElementsProject/lightning/pull/7018 +[#6869]: https://github.com/ElementsProject/lightning/pull/6869 +[#6984]: https://github.com/ElementsProject/lightning/pull/6984 +[#6865]: https://github.com/ElementsProject/lightning/pull/6865 +[#6941]: https://github.com/ElementsProject/lightning/pull/6941 +[24.02]: https://github.com/ElementsProject/lightning/releases/tag/v24.02 +[24.02rc2]: https://github.com/ElementsProject/lightning/releases/tag/v24.02rc2 + + ## [23.11] - 2023-11-28: "Bitcoin Orangepaper" This release named by Shahana Farooqui @@ -57,7 +159,7 @@ This release named by Shahana Farooqui Note: You should always set `allow-deprecated-apis=false` to test for changes. -- Plugins: `clnrest` parameters `rest-port`, `rest-protocol`, `rest-host` and `rest-certs`: prefix `cln` to them ([#6876]) + - Plugins: `clnrest` parameters `rest-port`, `rest-protocol`, `rest-host` and `rest-certs`: prefix `cln` to them ([#6876]) ### Removed @@ -149,7 +251,7 @@ Bugfix release for bad issues found since 23.08 which can't wait for 23.11, and - Protocol: Fixed a wrong number type being used in routes ([#6642]) - JSON-RPC: `showrunes` on a specific rune would always say `stored`: false. ([#6640]) - - MacOS: `clnrest` now works ([#6605]) + - MacOS: `clnrest` now works ([#6605]) - Build: test for `python3` or `python`, rather than assuming `python3` ([#6630]) @@ -417,7 +519,7 @@ Note: You should always set `allow-deprecated-apis=false` to test for changes. - JSON-RPC: `close`, `fundchannel`, `fundpsbt`, `multifundchannel`, `multiwithdraw`, `txprepare`, `upgradewallet`, `withdraw` `feerate` (`feerange` for `close`) expressed as, "delayed_to_us", "htlc_resolution", "max_acceptable" or "min_acceptable". Use explicit block counts or *slow*/*normal*/*urgent*/*minimum*. ([#6120]) - Plugins: `estimatefees` returning feerates by name (e.g. "opening"); use `fee_floor` and `feerates`. ([#6120]) - Protocol: Not setting `option_scid_alias` in `option_channel` `channel_type` for unannounced channels. ([#6136]) - + ### Removed @@ -2759,6 +2861,7 @@ There predate the BOLT specifications, and are only of vague historic interest: 6. [0.5.1] - 2016-10-21 7. [0.5.2] - 2016-11-21: "Bitcoin Savings & Trust Daily Interest II" +[24.02]: https://github.com/ElementsProject/lightning/releases/tag/v24.02 [23.11]: https://github.com/ElementsProject/lightning/releases/tag/v23.11 [23.05]: https://github.com/ElementsProject/lightning/releases/tag/v23.05 [23.02.1]: https://github.com/ElementsProject/lightning/releases/tag/v23.02.1 diff --git a/Makefile b/Makefile index bd1a3f723448..4fa4da3bc373 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -D endif # (method=thread to support xdist) -PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS) +PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS) $(PYTEST_MOREOPTS) MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7 # Collect generated python files to be excluded from lint checks PYTHON_GENERATED= \ diff --git a/common/gossmap.c b/common/gossmap.c index 9957bc4ddc14..a4f6fa6bdf5f 100644 --- a/common/gossmap.c +++ b/common/gossmap.c @@ -690,9 +690,13 @@ static bool load_gossip_store(struct gossmap *map) { map->map_size = lseek(map->fd, 0, SEEK_END); map->local = NULL; + + /* gossipd uses pwritev(), which is not consistent with mmap on OpenBSD! */ +#ifndef __OpenBSD__ /* If this fails, we fall back to read */ map->mmap = mmap(NULL, map->map_size, PROT_READ, MAP_SHARED, map->fd, 0); if (map->mmap == MAP_FAILED) +#endif /* __OpenBSD__ */ map->mmap = NULL; /* We only support major version 0 */ @@ -994,8 +998,11 @@ bool gossmap_refresh_mayfail(struct gossmap *map, bool *updated) if (map->mmap) munmap(map->mmap, map->map_size); map->map_size = len; + /* gossipd uses pwritev(), which is not consistent with mmap on OpenBSD! */ +#ifndef __OpenBSD__ map->mmap = mmap(NULL, map->map_size, PROT_READ, MAP_SHARED, map->fd, 0); if (map->mmap == MAP_FAILED) +#endif /* __OpenBSD__ */ map->mmap = NULL; return map_catchup(map, updated); diff --git a/common/hsm_version.h b/common/hsm_version.h index 5a2c2deac84a..7d3f8fb80da2 100644 --- a/common/hsm_version.h +++ b/common/hsm_version.h @@ -22,7 +22,8 @@ * v4 with forget_channel: d87c6934ea188f92785d38d7cd0b13ed7f76aa7417f3200baf0c7b5aa832fe29 * v5 with hsmd_revoke_commitment_tx: 5742538f87ef5d5bf55b66dc19e52c8683cfeb1b887d3e64ba530ba9a4d8e638 * v5 with sign_any_cannouncement: 5fdb9068c43a21887dc03f7dce410d2e3eeff6277f0d49b4fc56595a798fd4a4 + * v5 drop init v2: 5024454532fe5a78bb7558000cb344190888b9915360d3d56ddca22eaba9b872 */ -#define HSM_MIN_VERSION 3 +#define HSM_MIN_VERSION 5 #define HSM_MAX_VERSION 5 #endif /* LIGHTNING_COMMON_HSM_VERSION_H */ diff --git a/contrib/docker/Dockerfile.builder.fedora b/contrib/docker/Dockerfile.builder.fedora index 3cb6b9c7a972..22cb8c22bf67 100644 --- a/contrib/docker/Dockerfile.builder.fedora +++ b/contrib/docker/Dockerfile.builder.fedora @@ -12,7 +12,8 @@ RUN dnf update -y && \ libsq3-devel \ python3-devel \ python3-mako \ - python3-pip \ + python3-pip \ + python3-virtualenv \ python3-setuptools \ redhat-lsb \ net-tools \ @@ -30,5 +31,7 @@ RUN wget https://bitcoin.org/bin/bitcoin-core-$BITCOIN_VERSION/bitcoin-$BITCOIN_ mv bitcoin-$BITCOIN_VERSION/share/man/man1/* /usr/share/man/man1 && \ rm -rf bitcoin.tar.gz bitcoin-$BITCOIN_VERSION -RUN python3 -m pip install --force-reinstall -U pip setuptools && \ - python3 -m pip install python-bitcoinlib pytest pytest-test-groups flake8 pytest-rerunfailures ephemeral-port-reserve +ENV PATH=/opt/venv/bin:${PATH} +RUN python3 -m pip install pip wheel && \ + python3 -m virtualenv /opt/venv && \ + /opt/venv/bin/python3 -m pip install --force-reinstall -U pip poetry wheel diff --git a/contrib/pyln-client/pyln/client/__init__.py b/contrib/pyln-client/pyln/client/__init__.py index 36ca27e272e5..7eb21eae1637 100644 --- a/contrib/pyln-client/pyln/client/__init__.py +++ b/contrib/pyln-client/pyln/client/__init__.py @@ -3,7 +3,7 @@ from .gossmap import Gossmap, GossmapNode, GossmapChannel, GossmapHalfchannel, GossmapNodeId, LnFeatureBits from .gossmapstats import GossmapStats -__version__ = "23.11" +__version__ = "24.02" __all__ = [ "LightningRpc", diff --git a/contrib/pyln-client/pyproject.toml b/contrib/pyln-client/pyproject.toml index e1ef3d3ad4e7..0e1742a41836 100644 --- a/contrib/pyln-client/pyproject.toml +++ b/contrib/pyln-client/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyln-client" -version = "23.11" +version = "24.02" description = "Client library and plugin library for Core Lightning" authors = ["Christian Decker "] license = "BSD-MIT" diff --git a/contrib/pyln-proto/pyln/proto/__init__.py b/contrib/pyln-proto/pyln/proto/__init__.py index 332f49564831..ca4ef4640cfa 100644 --- a/contrib/pyln-proto/pyln/proto/__init__.py +++ b/contrib/pyln-proto/pyln/proto/__init__.py @@ -4,7 +4,7 @@ from .onion import OnionPayload, TlvPayload, LegacyOnionPayload from .wire import LightningConnection, LightningServerSocket -__version__ = "23.11" +__version__ = "24.02" __all__ = [ "Invoice", diff --git a/contrib/pyln-proto/pyproject.toml b/contrib/pyln-proto/pyproject.toml index 3f4bb74b1f21..bd1561041325 100644 --- a/contrib/pyln-proto/pyproject.toml +++ b/contrib/pyln-proto/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyln-proto" -version = "23.11" +version = "24.02" description = "This package implements some of the Lightning Network protocol in pure python. It is intended for protocol testing and some minor tooling only. It is not deemed secure enough to handle any amount of real funds (you have been warned!)." authors = ["Christian Decker "] license = "BSD-MIT" diff --git a/contrib/pyln-testing/pyln/testing/__init__.py b/contrib/pyln-testing/pyln/testing/__init__.py index 1b6207511657..966b3099b296 100644 --- a/contrib/pyln-testing/pyln/testing/__init__.py +++ b/contrib/pyln-testing/pyln/testing/__init__.py @@ -1,4 +1,4 @@ -__version__ = "23.11" +__version__ = "24.02" __all__ = [ "__version__", diff --git a/contrib/pyln-testing/pyln/testing/fixtures.py b/contrib/pyln-testing/pyln/testing/fixtures.py index 7b89de6ee7fc..1af599754813 100644 --- a/contrib/pyln-testing/pyln/testing/fixtures.py +++ b/contrib/pyln-testing/pyln/testing/fixtures.py @@ -1,6 +1,6 @@ from concurrent import futures from pyln.testing.db import SqliteDbProvider, PostgresDbProvider -from pyln.testing.utils import NodeFactory, BitcoinD, ElementsD, env, LightningNode, TEST_DEBUG, TEST_NETWORK +from pyln.testing.utils import NodeFactory, BitcoinD, ElementsD, env, LightningNode, TEST_DEBUG, TEST_NETWORK, LssD from pyln.client import Millisatoshi from typing import Dict @@ -32,6 +32,9 @@ def test_base_dir(): yield directory + if bool(int(os.getenv('TEST_KEEPDIR', '0'))): + return + # Now check if any test directory is left because the corresponding test # failed. If there are no such tests we can clean up the root test # directory. @@ -93,7 +96,7 @@ def directory(request, test_base_dir, test_name): outcome = 'passed' if rep_call is None else rep_call.outcome failed = not outcome or request.node.has_errors or outcome != 'passed' - if not failed: + if not failed and not bool(int(os.getenv('TEST_KEEPDIR', '0'))): try: shutil.rmtree(directory) except OSError: @@ -165,6 +168,25 @@ def bitcoind(directory, teardown_checks): bitcoind.proc.wait() +@pytest.fixture +def lssd(directory, teardown_checks): + lssd = LssD(directory) + + try: + lssd.start() + except Exception: + lssd.stop() + raise + + yield lssd + + try: + lssd.stop() + except Exception: + lssd.proc.kill() + lssd.proc.wait() + + class TeardownErrors(object): def __init__(self): self.errors = [] @@ -447,11 +469,12 @@ def jsonschemas(): @pytest.fixture -def node_factory(request, directory, test_name, bitcoind, executor, db_provider, teardown_checks, node_cls, jsonschemas): +def node_factory(request, directory, test_name, bitcoind, lssd, executor, db_provider, teardown_checks, node_cls, jsonschemas): nf = NodeFactory( request, test_name, bitcoind, + lssd, executor, directory=directory, db_provider=db_provider, diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index f0a4e2336bb6..1ddcc473cf35 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -78,6 +78,7 @@ def env(name, default=None): SLOW_MACHINE = env("SLOW_MACHINE", "0") == "1" DEPRECATED_APIS = env("DEPRECATED_APIS", "0") == "1" TIMEOUT = int(env("TIMEOUT", 180 if SLOW_MACHINE else 60)) +SUBDAEMON = env("SUBDAEMON", "") EXPERIMENTAL_DUAL_FUND = env("EXPERIMENTAL_DUAL_FUND", "0") == "1" EXPERIMENTAL_SPLICING = env("EXPERIMENTAL_SPLICING", "0") == "1" @@ -383,6 +384,45 @@ def f(*args): return f +class LssD(TailableProc): + def __init__(self, directory, rpcport=None): + lss_dir = os.path.join(directory, 'lss') + TailableProc.__init__(self, lss_dir, verbose=False) + + if rpcport is None: + self.reserved_rpcport = reserve_unused_port() + rpcport = self.reserved_rpcport + else: + self.reserved_rpcport = None + + self.rpcport = rpcport + self.prefix = 'lss' + + if not os.path.exists(lss_dir): + os.makedirs(lss_dir) + + self.cmd_line = [ + 'lssd', + '--datadir={}'.format(lss_dir), + '--port={}'.format(rpcport), + ] + + def __del__(self): + if self.reserved_rpcport is not None: + drop_unused_port(self.reserved_rpcport) + + def start(self): + self.env['RUST_LOG'] = 'debug' + TailableProc.start(self) + self.wait_for_log("ready on", timeout=TIMEOUT) + + logging.info("LssD started") + + def stop(self): + logging.info("Stopping LssD") + return TailableProc.stop(self) + + class BitcoinD(TailableProc): def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None): @@ -578,11 +618,50 @@ def getnewaddress(self): return info['unconfidential'] +class ValidatingLightningSignerD(TailableProc): + def __init__(self, vlsd_dir, vlsd_port, vlsd_rpc_port, node_id, network): + TailableProc.__init__(self, vlsd_dir, verbose=True) + self.executable = env("REMOTE_SIGNER_CMD", 'vlsd2') + os.environ['ALLOWLIST'] = env( + 'REMOTE_SIGNER_ALLOWLIST', + 'contrib/remote_hsmd/TESTING_ALLOWLIST') + self.opts = [ + '--network={}'.format(network), + '--datadir={}'.format(vlsd_dir), + '--connect=http://localhost:{}'.format(vlsd_port), + '--rpc-server-port={}'.format(vlsd_rpc_port), + '--integration-test', + ] + self.prefix = 'vlsd2-%d' % (node_id) + self.vlsd_port = vlsd_port + + @property + def cmd_line(self): + return [self.executable] + self.opts + + def start(self, stdin=None, stdout_redir=True, stderr_redir=True, + wait_for_initialized=True): + TailableProc.start(self, stdin, stdout_redir, stderr_redir) + # We need to always wait for initialization + self.wait_for_log("vlsd2 git_desc") + logging.info("vlsd2 started") + + def stop(self, timeout=10): + logging.info("stopping vlsd2") + rc = TailableProc.stop(self, timeout) + logging.info("vlsd2 stopped") + self.logs_catchup() + return rc + + def __del__(self): + self.logs_catchup() + class LightningD(TailableProc): def __init__( self, lightning_dir, bitcoindproxy, + lssd_port, port=9735, random_hsm=False, node_id=0, @@ -594,9 +673,17 @@ def __init__( self.lightning_dir = lightning_dir self.port = port self.cmd_prefix = [] + self.lightning_dir = lightning_dir + self.use_vlsd = False + self.vlsd_dir = os.path.join(lightning_dir, "vlsd") + self.vlsd_port = None + self.vlsd_rpc_server_port = None + self.vlsd = None + self.node_id = node_id self.rpcproxy = bitcoindproxy self.env['CLN_PLUGIN_LOG'] = "cln_plugin=trace,cln_rpc=trace,cln_grpc=trace,debug" + self.lssd_port = lssd_port self.opts = LIGHTNINGD_CONFIG.copy() opts = { @@ -616,12 +703,38 @@ def __init__( if grpc_port is not None: opts['grpc-port'] = grpc_port + if SUBDAEMON: + assert node_id > 0 + subdaemons = SUBDAEMON.split(',') + # VLS_SERIAL_SELECT "swaps" the selected item with the first item + select = env("VLS_SERIAL_SELECT", '1') + if node_id == int(select): + ndx = 1 + elif node_id == 1: + ndx = int(select) + else: + ndx = node_id + if ndx > len(subdaemons): + # use the last element if not as many specifiers as nodes + opts['subdaemon'] = subdaemons[-1] + else: + # use the matching specifier + opts['subdaemon'] = subdaemons[ndx - 1] + + print(f"starting node {node_id} with subdaemon {opts['subdaemon']}") + if SUBDAEMON == 'hsmd:remote_hsmd_socket': + self.use_vlsd = True + for k, v in opts.items(): self.opts[k] = v if not os.path.exists(os.path.join(lightning_dir, TEST_NETWORK)): os.makedirs(os.path.join(lightning_dir, TEST_NETWORK)) + if self.use_vlsd: + if not os.path.exists(self.vlsd_dir): + os.makedirs(self.vlsd_dir) + # Last 32-bytes of final part of dir -> seed. seed = (bytes(re.search('([^/]+)/*$', lightning_dir).group(1), encoding='utf-8') + bytes(32))[:32] if not random_hsm: @@ -637,6 +750,10 @@ def __init__( self.early_opts = ['--developer'] def cleanup(self): + if self.use_vlsd: + # Make sure the remotesigner is shutdown + self.vlsd.stop() + # To force blackhole to exit, disconnect file must be truncated! if 'dev-disconnect' in self.opts: with open(self.opts['dev-disconnect'], "w") as f: @@ -657,12 +774,66 @@ def cmd_line(self): return self.cmd_prefix + [self.executable] + self.early_opts + opts + def __del__(self): + if self.vlsd_port is not None: + drop_unused_port(self.vlsd_port) + def start(self, stdin=None, wait_for_initialized=True, stderr_redir=False): - self.opts['bitcoin-rpcport'] = self.rpcproxy.rpcport - TailableProc.start(self, stdin, stdout_redir=False, stderr_redir=stderr_redir) - if wait_for_initialized: - self.wait_for_log("Server started with public key") - logging.info("LightningD started") + try: + self.env['VLS_LSS'] = f"http://localhost:{self.lssd_port}" + self.env['RUST_LOG'] = 'debug' + # Some of the remote hsmd proxies need a bitcoind RPC connection + self.env['BITCOIND_RPC_URL'] = 'http://{}:{}@localhost:{}'.format( + BITCOIND_CONFIG['rpcuser'], + BITCOIND_CONFIG['rpcpassword'], + BITCOIND_CONFIG['rpcport']) + + # The remote hsmd proxies need to know which network we are using + if 'network' in self.opts: + self.env['VLS_NETWORK'] = self.opts['network'] + + self.opts['bitcoin-rpcport'] = self.rpcproxy.rpcport + + if self.use_vlsd: + self.vlsd_port = reserve_unused_port() + self.vlsd_rpc_server_port = reserve_unused_port() + # We can't do this in the constructor because we need a new port on each restart. + self.env['VLS_PORT'] = str(self.vlsd_port) + # Kill any previous vlsd (we may have been restarted) + if self.vlsd is not None: + logging.info("killing prior vlsd") + self.vlsd.kill() + + TailableProc.start(self, stdin, stdout_redir=False, stderr_redir=stderr_redir) + + if self.use_vlsd: + # Start the remote signer first + self.vlsd = ValidatingLightningSignerD( + self.vlsd_dir, self.vlsd_port, self.vlsd_rpc_server_port, self.node_id, self.opts['network']) + self.vlsd.start( + stdin, stdout_redir=True, stderr_redir=True, + wait_for_initialized=wait_for_initialized) + + if wait_for_initialized: + self.wait_for_log("Server started with public key") + logging.info("LightningD started") + except Exception: + if self.use_vlsd: + # LightningD didn't start, stop the remotesigner + self.vlsd.stop() + raise + + def stop(self, timeout=10): + if self.use_vlsd: + # Stop the remote signer first + self.vlsd.stop(timeout) + return TailableProc.stop(self, timeout) + + def kill(self): + if self.use_vlsd: + # Kill the remote signer first + self.vlsd.kill() + TailableProc.kill(self) def wait(self, timeout=TIMEOUT): """Wait for the daemon to stop for up to timeout seconds @@ -744,7 +915,7 @@ def call(self, method, payload=None, cmdprefix=None, filter=None): class LightningNode(object): - def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fail=False, + def __init__(self, node_id, lightning_dir, bitcoind, lssd, executor, valgrind, may_fail=False, may_reconnect=False, allow_broken_log=False, allow_warning=False, @@ -754,6 +925,7 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai valgrind_plugins=True, **kwargs): self.bitcoin = bitcoind + self.lssd = lssd self.executor = executor self.may_fail = may_fail self.may_reconnect = may_reconnect @@ -773,6 +945,7 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai self.daemon = LightningD( lightning_dir, bitcoindproxy=bitcoind.get_proxy(), + lssd_port=lssd.rpcport, port=port, random_hsm=random_hsm, node_id=node_id, grpc_port=self.grpc_port, ) @@ -1221,6 +1394,9 @@ def pay(self, dst, amt, label=None, route=False): 'channel': scid } + # let the signer know this payment is coming + self.rpc.preapproveinvoice(bolt11=inv['bolt11']) + # sendpay is async now self.rpc.sendpay([routestep], rhash, payment_secret=psecret, bolt11=inv['bolt11']) # wait for sendpay to comply @@ -1496,7 +1672,7 @@ def flock(directory: Path): class NodeFactory(object): """A factory to setup and start `lightningd` daemons. """ - def __init__(self, request, testname, bitcoind, executor, directory, + def __init__(self, request, testname, bitcoind, lssd, executor, directory, db_provider, node_cls, jsonschemas): if request.node.get_closest_marker("slow_test") and SLOW_MACHINE: self.valgrind = False @@ -1508,6 +1684,7 @@ def __init__(self, request, testname, bitcoind, executor, directory, self.reserved_ports = [] self.executor = executor self.bitcoind = bitcoind + self.lssd = lssd self.directory = directory self.lock = threading.Lock() self.db_provider = db_provider @@ -1593,7 +1770,7 @@ def get_node(self, node_id=None, options=None, dbfile=None, db = self.db_provider.get_db(os.path.join(lightning_dir, TEST_NETWORK), self.testname, node_id) db.provider = self.db_provider node = self.node_cls( - node_id, lightning_dir, self.bitcoind, self.executor, self.valgrind, db=db, + node_id, lightning_dir, self.bitcoind, self.lssd, self.executor, self.valgrind, db=db, port=port, options=options, may_fail=may_fail or expect_fail, jsonschemas=self.jsonschemas, **kwargs diff --git a/contrib/pyln-testing/pyproject.toml b/contrib/pyln-testing/pyproject.toml index debc003d44de..02239e7eceab 100644 --- a/contrib/pyln-testing/pyproject.toml +++ b/contrib/pyln-testing/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyln-testing" -version = "23.11" +version = "24.02" description = "Test your Core Lightning integration, plugins or whatever you want" authors = ["Christian Decker "] license = "BSD-MIT" diff --git a/hsmd/hsmd.c b/hsmd/hsmd.c index 1aec53028500..41150da75984 100644 --- a/hsmd/hsmd.c +++ b/hsmd/hsmd.c @@ -436,7 +436,7 @@ static struct io_plan *init_hsm(struct io_conn *conn, struct secret *hsm_encryption_key; struct bip32_key_version bip32_key_version; u32 minversion, maxversion; - const u32 our_minversion = 2, our_maxversion = 3; + const u32 our_minversion = 4, our_maxversion = 5; /* This must be lightningd. */ assert(is_lightningd(c)); @@ -489,7 +489,10 @@ static struct io_plan *init_hsm(struct io_conn *conn, if (hsm_encryption_key) discard_key(take(hsm_encryption_key)); - return req_reply(conn, c, hsmd_init(hsm_secret, bip32_key_version)); + /* Define the minimum common max version for the hsmd one */ + u64 mutual_version = maxversion < our_maxversion ? maxversion : our_maxversion; + return req_reply(conn, c, hsmd_init(hsm_secret, mutual_version, + bip32_key_version)); } /*~ Since we process requests then service them in strict order, and because @@ -701,7 +704,6 @@ static struct io_plan *handle_client(struct io_conn *conn, struct client *c) case WIRE_HSMD_NODE_ANNOUNCEMENT_SIG_REPLY: case WIRE_HSMD_SIGN_WITHDRAWAL_REPLY: case WIRE_HSMD_SIGN_INVOICE_REPLY: - case WIRE_HSMD_INIT_REPLY_V2: case WIRE_HSMD_INIT_REPLY_V4: case WIRE_HSMD_DERIVE_SECRET_REPLY: case WIRE_HSMSTATUS_CLIENT_BAD_REQUEST: diff --git a/hsmd/hsmd_wire.csv b/hsmd/hsmd_wire.csv index 98e3bd021596..cd75835622c3 100644 --- a/hsmd/hsmd_wire.csv +++ b/hsmd/hsmd_wire.csv @@ -19,12 +19,6 @@ msgdata,hsmd_init,hsm_wire_min_version,u32, msgdata,hsmd_init,hsm_wire_max_version,u32, #include -# DEPRECATED after 23.05, remove in two versions! -msgtype,hsmd_init_reply_v2,113 -msgdata,hsmd_init_reply_v2,node_id,node_id, -msgdata,hsmd_init_reply_v2,bip32,ext_key, -msgdata,hsmd_init_reply_v2,bolt12,pubkey, - # Sorry: I should have put version in v2 :( msgtype,hsmd_init_reply_v4,114 # This gets upgraded when the wire protocol changes in incompatible diff --git a/hsmd/libhsmd.c b/hsmd/libhsmd.c index a6cacdb90e38..3dfcf183876a 100644 --- a/hsmd/libhsmd.c +++ b/hsmd/libhsmd.c @@ -157,7 +157,6 @@ bool hsmd_check_client_capabilities(struct hsmd_client *client, case WIRE_HSMD_NODE_ANNOUNCEMENT_SIG_REPLY: case WIRE_HSMD_SIGN_WITHDRAWAL_REPLY: case WIRE_HSMD_SIGN_INVOICE_REPLY: - case WIRE_HSMD_INIT_REPLY_V2: case WIRE_HSMD_INIT_REPLY_V4: case WIRE_HSMSTATUS_CLIENT_BAD_REQUEST: case WIRE_HSMD_SIGN_COMMITMENT_TX_REPLY: @@ -2124,7 +2123,6 @@ u8 *hsmd_handle_client_message(const tal_t *ctx, struct hsmd_client *client, case WIRE_HSMD_NODE_ANNOUNCEMENT_SIG_REPLY: case WIRE_HSMD_SIGN_WITHDRAWAL_REPLY: case WIRE_HSMD_SIGN_INVOICE_REPLY: - case WIRE_HSMD_INIT_REPLY_V2: case WIRE_HSMD_INIT_REPLY_V4: case WIRE_HSMSTATUS_CLIENT_BAD_REQUEST: case WIRE_HSMD_SIGN_COMMITMENT_TX_REPLY: @@ -2151,7 +2149,7 @@ u8 *hsmd_handle_client_message(const tal_t *ctx, struct hsmd_client *client, return hsmd_status_bad_request(client, msg, "Unknown request"); } -u8 *hsmd_init(struct secret hsm_secret, +u8 *hsmd_init(struct secret hsm_secret, const u64 hsmd_version, struct bip32_key_version bip32_key_version) { u8 bip32_seed[BIP32_ENTROPY_LEN_256]; @@ -2297,7 +2295,7 @@ u8 *hsmd_init(struct secret hsm_secret, * incompatibility detection) with alternate implementations. */ return take(towire_hsmd_init_reply_v4( - NULL, 4, + NULL, hsmd_version, /* Capabilities arg needs to be a tal array */ tal_dup_arr(tmpctx, u32, capabilities, ARRAY_SIZE(capabilities), 0), diff --git a/hsmd/libhsmd.h b/hsmd/libhsmd.h index 6388193912ac..756c6c2f526a 100644 --- a/hsmd/libhsmd.h +++ b/hsmd/libhsmd.h @@ -47,7 +47,7 @@ struct hsmd_client { * Returns the `hsmd_init_reply` with the information required by * `lightningd`. */ -u8 *hsmd_init(struct secret hsm_secret, +u8 *hsmd_init(struct secret hsm_secret, const u64 hsmd_version, struct bip32_key_version bip32_key_version); struct hsmd_client *hsmd_client_new_main(const tal_t *ctx, u64 capabilities, diff --git a/lightningd/.gitignore b/lightningd/.gitignore index 4ba7a1a9cec2..576e635ee243 100644 --- a/lightningd/.gitignore +++ b/lightningd/.gitignore @@ -8,3 +8,5 @@ lightning_hsmd lightning_onchaind lightning_openingd lightning_websocketd +remote_hsmd_serial +remote_hsmd_socket diff --git a/lightningd/hsm_control.c b/lightningd/hsm_control.c index 0765af142691..1b636bdb6c0d 100644 --- a/lightningd/hsm_control.c +++ b/lightningd/hsm_control.c @@ -130,12 +130,6 @@ struct ext_key *hsm_init(struct lightningd *ld) &ld->id, bip32_base, &ld->bolt12_base)) { /* nothing to do. */ - } else if (fromwire_hsmd_init_reply_v2(msg, - &ld->id, bip32_base, - &ld->bolt12_base)) { - /* implicit version */ - hsm_version = 3; - ld->hsm_capabilities = NULL; } else { if (ld->config.keypass) errx(EXITCODE_HSM_BAD_PASSWORD, "Wrong password for encrypted hsm_secret."); diff --git a/plugins/clnrest/pyproject.toml b/plugins/clnrest/pyproject.toml index 56546eaf2c45..66a9a62870d3 100644 --- a/plugins/clnrest/pyproject.toml +++ b/plugins/clnrest/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "clnrest" -version = "23.11" +version = "24.02" description = "Transforms RPC calls into REST APIs" authors = ["ShahanaFarooqui "] @@ -10,7 +10,7 @@ json5 = "^0.9.14" flask = "^2.3.3" flask-restx = "^1.1.0" gunicorn = "^21.2.0" -pyln-client = "^23.8" +pyln-client = "^24.2" flask-socketio = "^5.3.6" gevent = "^23.9.0.post1" gevent-websocket = "^0.10.1" diff --git a/plugins/src/lib.rs b/plugins/src/lib.rs index 1dd19a4acc00..ee50ccbe419e 100644 --- a/plugins/src/lib.rs +++ b/plugins/src/lib.rs @@ -136,9 +136,9 @@ where } } - pub fn option( + pub fn option<'a, V: options::OptionType<'a>>( mut self, - opt: options::ConfigOption, + opt: options::ConfigOption<'a, V>, ) -> Builder { self.options.insert(opt.name().to_string(), opt.build()); self @@ -514,9 +514,9 @@ where .map(|c| c.clone()) } - pub fn option( + pub fn option<'a, OV: OptionType<'a>>( &self, - config_option: &options::ConfigOption, + config_option: &options::ConfigOption<'a, OV>, ) -> Result { let value = self.option_str(config_option.name())?; Ok(OV::from_value(&value)) @@ -611,9 +611,9 @@ where .map(|c| c.clone()) } - pub fn option( + pub fn option<'a, OV: OptionType<'a>>( &self, - config_option: &options::ConfigOption, + config_option: &options::ConfigOption<'a, OV>, ) -> Result { let value = self.option_str(config_option.name())?; Ok(OV::from_value(&value)) diff --git a/plugins/src/options.rs b/plugins/src/options.rs index 843ee6e95021..241b761d7fef 100644 --- a/plugins/src/options.rs +++ b/plugins/src/options.rs @@ -134,12 +134,19 @@ use serde::ser::Serializer; use serde::Serialize; pub mod config_type { + #[derive(Clone, Debug)] pub struct Integer; + #[derive(Clone, Debug)] pub struct DefaultInteger; + #[derive(Clone, Debug)] pub struct String; + #[derive(Clone, Debug)] pub struct DefaultString; + #[derive(Clone, Debug)] pub struct Boolean; + #[derive(Clone, Debug)] pub struct DefaultBoolean; + #[derive(Clone, Debug)] pub struct Flag; } @@ -158,7 +165,7 @@ pub type DefaultBooleanConfigOption<'a> = ConfigOption<'a, config_type::DefaultB /// Config value is represented as a flag pub type FlagConfigOption<'a> = ConfigOption<'a, config_type::Flag>; -pub trait OptionType { +pub trait OptionType<'a> { type OutputValue; type DefaultValue; @@ -169,9 +176,9 @@ pub trait OptionType { fn get_value_type() -> ValueType; } -impl OptionType for config_type::DefaultString { +impl<'a> OptionType<'a> for config_type::DefaultString { type OutputValue = String; - type DefaultValue = &'static str; + type DefaultValue = &'a str; fn convert_default(value: &Self::DefaultValue) -> Option { Some(Value::String(value.to_string())) @@ -189,7 +196,7 @@ impl OptionType for config_type::DefaultString { } } -impl OptionType for config_type::DefaultInteger { +impl<'a> OptionType<'a> for config_type::DefaultInteger { type OutputValue = i64; type DefaultValue = i64; @@ -209,7 +216,7 @@ impl OptionType for config_type::DefaultInteger { } } -impl OptionType for config_type::DefaultBoolean { +impl<'a> OptionType<'a> for config_type::DefaultBoolean { type OutputValue = bool; type DefaultValue = bool; @@ -228,7 +235,7 @@ impl OptionType for config_type::DefaultBoolean { } } -impl OptionType for config_type::Flag { +impl<'a> OptionType<'a> for config_type::Flag { type OutputValue = bool; type DefaultValue = (); @@ -248,7 +255,7 @@ impl OptionType for config_type::Flag { } } -impl OptionType for config_type::String { +impl<'a> OptionType<'a> for config_type::String { type OutputValue = Option; type DefaultValue = (); @@ -272,7 +279,7 @@ impl OptionType for config_type::String { } } -impl OptionType for config_type::Integer { +impl<'a> OptionType<'a> for config_type::Integer { type OutputValue = Option; type DefaultValue = (); @@ -295,7 +302,7 @@ impl OptionType for config_type::Integer { ValueType::Integer } } -impl OptionType for config_type::Boolean { +impl<'a> OptionType<'a> for config_type::Boolean { type OutputValue = Option; type DefaultValue = (); @@ -406,7 +413,7 @@ impl Value { } #[derive(Clone, Debug)] -pub struct ConfigOption<'a, V: OptionType> { +pub struct ConfigOption<'a, V: OptionType<'a>> { /// The name of the `ConfigOption`. pub name: &'a str, /// The default value of the `ConfigOption` @@ -415,7 +422,7 @@ pub struct ConfigOption<'a, V: OptionType> { pub deprecated: bool, } -impl ConfigOption<'_, V> { +impl<'a, V: OptionType<'a>> ConfigOption<'a, V> { pub fn build(&self) -> UntypedConfigOption { UntypedConfigOption { name: self.name.to_string(), @@ -427,11 +434,11 @@ impl ConfigOption<'_, V> { } } -impl DefaultStringConfigOption<'_> { +impl<'a> DefaultStringConfigOption<'a> { pub const fn new_str_with_default( - name: &'static str, - default: &'static str, - description: &'static str, + name: &'a str, + default: &'a str, + description: &'a str, ) -> Self { Self { name: name, @@ -442,23 +449,19 @@ impl DefaultStringConfigOption<'_> { } } -impl StringConfigOption<'_> { - pub const fn new_str_no_default(name: &'static str, description: &'static str) -> Self { +impl<'a> StringConfigOption<'a> { + pub const fn new_str_no_default(name: &'a str, description: &'a str) -> Self { Self { name, default: (), - description : description, + description: description, deprecated: false, } } } -impl DefaultIntegerConfigOption<'_> { - pub const fn new_i64_with_default( - name: &'static str, - default: i64, - description: &'static str, - ) -> Self { +impl<'a> DefaultIntegerConfigOption<'a> { + pub const fn new_i64_with_default(name: &'a str, default: i64, description: &'a str) -> Self { Self { name: name, default: default, @@ -468,8 +471,8 @@ impl DefaultIntegerConfigOption<'_> { } } -impl IntegerConfigOption<'_> { - pub const fn new_i64_no_default(name: &'static str, description: &'static str) -> Self { +impl<'a> IntegerConfigOption<'a> { + pub const fn new_i64_no_default(name: &'a str, description: &'a str) -> Self { Self { name: name, default: (), @@ -479,8 +482,8 @@ impl IntegerConfigOption<'_> { } } -impl BooleanConfigOption<'_> { - pub const fn new_bool_no_default(name: &'static str, description: &'static str) -> Self { +impl<'a> BooleanConfigOption<'a> { + pub const fn new_bool_no_default(name: &'a str, description: &'a str) -> Self { Self { name, description, @@ -490,12 +493,8 @@ impl BooleanConfigOption<'_> { } } -impl DefaultBooleanConfigOption<'_> { - pub const fn new_bool_with_default( - name: &'static str, - default: bool, - description: &'static str, - ) -> Self { +impl<'a> DefaultBooleanConfigOption<'a> { + pub const fn new_bool_with_default(name: &'a str, default: bool, description: &'a str) -> Self { Self { name, description, @@ -505,8 +504,8 @@ impl DefaultBooleanConfigOption<'_> { } } -impl FlagConfigOption<'_> { - pub const fn new_flag(name: &'static str, description: &'static str) -> Self { +impl<'a> FlagConfigOption<'a> { + pub const fn new_flag(name: &'a str, description: &'a str) -> Self { Self { name, description, @@ -542,9 +541,9 @@ impl UntypedConfigOption { } } -impl ConfigOption<'_, V> +impl<'a, V> ConfigOption<'a, V> where - V: OptionType, + V: OptionType<'a>, { pub fn name(&self) -> &str { &self.name diff --git a/poetry.lock b/poetry.lock index 70a2cd75c385..8d72ab611e66 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aniso8601" @@ -438,7 +438,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "clnrest" -version = "23.11" +version = "24.02" description = "Transforms RPC calls into REST APIs" optional = false python-versions = "^3.8" @@ -454,7 +454,7 @@ gevent = "^23.9.0.post1" gevent-websocket = "^0.10.1" gunicorn = "^21.2.0" json5 = "^0.9.14" -pyln-client = "^23.8" +pyln-client = "^24.2" [package.source] type = "directory" @@ -1580,6 +1580,7 @@ files = [ {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, @@ -1684,7 +1685,7 @@ files = [ [[package]] name = "pyln-client" -version = "23.11" +version = "24.02" description = "Client library and plugin library for Core Lightning" optional = false python-versions = "^3.8" @@ -1706,6 +1707,7 @@ description = "The compiled GRPC proto for CLN" optional = false python-versions = ">=3.7,<4.0" files = [ + {file = "pyln_grpc_proto-0.1.1-py3-none-any.whl", hash = "sha256:7a34d3ee9ce62abb382fc0e77d12c1c9ffdc50f13409670a7a757d01f10ad118"}, {file = "pyln_grpc_proto-0.1.1.tar.gz", hash = "sha256:544c1c4a0e80b9ac57ce9d0daf27de8337814cf45e98e2f730d01e3095be8829"}, ] @@ -1715,7 +1717,7 @@ protobuf3 = "*" [[package]] name = "pyln-proto" -version = "23.11" +version = "24.02" description = "This package implements some of the Lightning Network protocol in pure python. It is intended for protocol testing and some minor tooling only. It is not deemed secure enough to handle any amount of real funds (you have been warned!)." optional = false python-versions = "^3.8" @@ -1735,7 +1737,7 @@ url = "contrib/pyln-proto" [[package]] name = "pyln-testing" -version = "23.11" +version = "24.02" description = "Test your Core Lightning integration, plugins or whatever you want" optional = false python-versions = "^3.8" @@ -1743,7 +1745,7 @@ files = [] develop = true [package.dependencies] -cheroot = "^8" +cheroot = ">=8 <=10" ephemeral-port-reserve = "^1.1.4" Flask = "^2" grpcio = {version = "^1", optional = true} diff --git a/tests/fixtures.py b/tests/fixtures.py index ea23f10ad0a9..d9a6874080f6 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,5 @@ from utils import TEST_NETWORK, VALGRIND # noqa: F401,F403 -from pyln.testing.fixtures import directory, test_base_dir, test_name, chainparams, node_factory, bitcoind, teardown_checks, db_provider, executor, setup_logging, jsonschemas # noqa: F401,F403 +from pyln.testing.fixtures import directory, test_base_dir, test_name, chainparams, node_factory, bitcoind, lssd, teardown_checks, db_provider, executor, setup_logging, jsonschemas # noqa: F401,F403 from pyln.testing import utils from utils import COMPAT from pathlib import Path diff --git a/tests/test_bookkeeper.py b/tests/test_bookkeeper.py index 6a63c6c3927e..e69beee38065 100644 --- a/tests/test_bookkeeper.py +++ b/tests/test_bookkeeper.py @@ -343,6 +343,7 @@ def test_bookkeeping_rbf_withdraw(node_factory, bitcoind): assert len(fees) == 1 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "hsmd_sign_option_will_fund_offer not supported") @pytest.mark.openchannel('v2') @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start") @unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded") @@ -418,6 +419,7 @@ def _check_events(node, channel_id, exp_events): @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start") @unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "remote_hsmd doesn't allow push of non-trivial amount") @pytest.mark.openchannel('v1', 'Uses push-msat') def test_bookkeeping_missed_chans_pushed(node_factory, bitcoind): """ @@ -540,6 +542,7 @@ def test_bookkeeping_inspect_multifundchannel(node_factory, bitcoind): assert bkpr_total_fee_msat == int(getblock_fee_btc * 100000000000) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded") @pytest.mark.openchannel('v2') def test_bookkeeping_inspect_mfc_dual_funded(node_factory, bitcoind): diff --git a/tests/test_closing.py b/tests/test_closing.py index b949f19accdc..85a62167786f 100644 --- a/tests/test_closing.py +++ b/tests/test_closing.py @@ -496,6 +496,7 @@ def test_closing_negotiation_step_700sat(node_factory, bitcoind, chainparams): closing_negotiation_step(node_factory, bitcoind, chainparams, opts) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "dev_sign_last_tx causes subsequent validate_holder_commitment_tx failure") @pytest.mark.parametrize("anchors", [False, True]) def test_penalty_inhtlc(node_factory, bitcoind, executor, chainparams, anchors): """Test penalty transaction with an incoming HTLC""" @@ -628,6 +629,7 @@ def test_penalty_inhtlc(node_factory, bitcoind, executor, chainparams, anchors): check_utxos_channel(l2, [channel_id], expected_2, tags) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "dev_sign_last_tx causes subsequent validate_holder_commitment_tx failure") @pytest.mark.parametrize("anchors", [False, True]) def test_penalty_outhtlc(node_factory, bitcoind, executor, chainparams, anchors): """Test penalty transaction with an outgoing HTLC""" @@ -764,6 +766,7 @@ def test_penalty_outhtlc(node_factory, bitcoind, executor, chainparams, anchors) @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "handle_sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') @pytest.mark.slow_test def test_channel_lease_falls_behind(node_factory, bitcoind): @@ -803,6 +806,7 @@ def test_channel_lease_falls_behind(node_factory, bitcoind): @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "handle_sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') @pytest.mark.slow_test def test_channel_lease_post_expiry(node_factory, bitcoind, chainparams): @@ -903,6 +907,7 @@ def test_channel_lease_post_expiry(node_factory, bitcoind, chainparams): @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "handle_sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') @pytest.mark.slow_test def test_channel_lease_unilat_closes(node_factory, bitcoind): @@ -1014,6 +1019,7 @@ def test_channel_lease_unilat_closes(node_factory, bitcoind): @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "handle_sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "Makes use of the sqlite3 db") def test_channel_lease_lessor_cheat(node_factory, bitcoind, chainparams): @@ -1088,6 +1094,7 @@ def test_channel_lease_lessor_cheat(node_factory, bitcoind, chainparams): @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "handle_sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "Makes use of the sqlite3 db") def test_channel_lease_lessee_cheat(node_factory, bitcoind, chainparams): @@ -1163,6 +1170,8 @@ def test_channel_lease_lessee_cheat(node_factory, bitcoind, chainparams): 'Unknown spend of OUR_UNILATERAL/DELAYED_OUTPUT_TO_US by']) +# VLS_PERMISSIVE generates invalid signature bassed on bad commitnum +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "invalid next holder commitment number: 4 != 6") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "Makes use of the sqlite3 db") @pytest.mark.slow_test @pytest.mark.parametrize("anchors", [False, True]) @@ -1351,6 +1360,8 @@ def test_penalty_htlc_tx_fulfill(node_factory, bitcoind, chainparams, anchors): check_balance_snaps(l2, expected_bals_2) +# VLS_PERMISSIVE generates invalid signature bassed on bad commitnum +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "invalid next holder commitment number: 4 != 6") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "Makes use of the sqlite3 db") @pytest.mark.slow_test @pytest.mark.parametrize("anchors", [False, True]) @@ -1580,6 +1591,7 @@ def test_penalty_htlc_tx_timeout(node_factory, bitcoind, chainparams, anchors): assert acc['resolved_at_block'] > 0 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "exceeds max fee policy") @pytest.mark.parametrize("anchors", [False, True]) def test_penalty_rbf_normal(node_factory, bitcoind, executor, chainparams, anchors): ''' @@ -1824,6 +1836,7 @@ def test_onchaind_replay(node_factory, bitcoind): 'delay': 101, 'channel': first_scid(l1, l2) } + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret']) l1.daemon.wait_for_log('sendrawtx exit 0') bitcoind.generate_block(1, wait_for_mempool=1) @@ -1967,6 +1980,7 @@ def test_onchain_timeout(node_factory, bitcoind, executor, chainparams, anchors) 'channel': first_scid(l1, l2) } + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'], groupid=1) with pytest.raises(RpcError): l1.rpc.waitsendpay(rhash) @@ -2101,6 +2115,7 @@ def test_onchain_middleman_simple(node_factory, bitcoind, chainparams, anchors): q = queue.Queue() + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming def try_pay(): try: l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) @@ -2240,6 +2255,7 @@ def test_onchain_middleman_their_unilateral_in(node_factory, bitcoind, chainpara q = queue.Queue() + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming def try_pay(): try: l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) @@ -2352,6 +2368,8 @@ def try_pay(): try: # rhash is fake (so is payment_secret) rhash = 'B1' * 32 + # let the signer know this payment is coming + l1.rpc.preapprovekeysend(l2.info['id'], rhash, 10**8) l1.rpc.sendpay(route, rhash, payment_secret=rhash) q.put(None) except Exception as err: @@ -2495,6 +2513,7 @@ def test_onchain_feechange(node_factory, bitcoind, executor): 'channel': first_scid(l1, l2) } + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming executor.submit(l1.rpc.sendpay, [routestep], rhash, payment_secret=inv['payment_secret']) # l2 will drop to chain. @@ -2575,6 +2594,7 @@ def test_onchain_all_dust(node_factory, bitcoind, executor): 'channel': first_scid(l1, l2) } + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming executor.submit(l1.rpc.sendpay, [routestep], rhash, payment_secret=inv['payment_secret']) # l2 will drop to chain. @@ -2888,6 +2908,7 @@ def route_to_l1(src): @pytest.mark.slow_test +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "frequently flakes") def test_onchain_multihtlc_our_unilateral(node_factory, bitcoind): """Node pushes a channel onchain with multiple HTLCs with same payment_hash """ h, l1, l2, l3, l4, l5, l6, l7 = setup_multihtlc_test(node_factory, bitcoind) @@ -2943,6 +2964,7 @@ def test_onchain_multihtlc_our_unilateral(node_factory, bitcoind): @pytest.mark.slow_test +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "frequently flakes") def test_onchain_multihtlc_their_unilateral(node_factory, bitcoind): """Node pushes a channel onchain with multiple HTLCs with same payment_hash """ h, l1, l2, l3, l4, l5, l6, l7 = setup_multihtlc_test(node_factory, bitcoind) @@ -3207,6 +3229,7 @@ def test_shutdown(node_factory): l1.rpc.stop() +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "validate_setup_channel: holder_shutdown_script is not in wallet or allowlist") def test_option_upfront_shutdown_script(node_factory, bitcoind, executor, chainparams): l1 = node_factory.get_node(start=False, allow_warning=True) # Insist on upfront script we're not going to match. @@ -3775,6 +3798,7 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind): 'delay': 12, 'channel': first_scid(l1, l2) } + l1.rpc.preapprovekeysend(routestep['id'], rhash, routestep['amount_msat']) l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret']) l2.daemon.wait_for_log('dev_disconnect') l2.stop() @@ -3842,6 +3866,7 @@ def test_htlc_no_force_close(node_factory, bitcoind, anchors): 'id': l3.info['id'], 'delay': 10, 'channel': first_scid(l2, l3)}] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret']) l3.daemon.wait_for_log('dev_disconnect') @@ -3961,6 +3986,7 @@ def test_peer_anchor_push(node_factory, bitcoind, executor, chainparams): amt = 100_000_000 sticky_inv = l3.rpc.invoice(amt, 'sticky', 'sticky') route = l1.rpc.getroute(l3.info['id'], amt, 1)['route'] + l1.rpc.preapproveinvoice(bolt11=sticky_inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, sticky_inv['payment_hash'], payment_secret=sticky_inv['payment_secret']) l3.daemon.wait_for_log('dev_disconnect: -WIRE_UPDATE_FULFILL_HTLC') diff --git a/tests/test_connection.py b/tests/test_connection.py index c0f0586b2c76..ec17b09b0fba 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -571,6 +571,7 @@ def test_disconnect_fundee(node_factory): assert len(l2.rpc.listpeers()) == 1 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_disconnect_fundee_v2(node_factory): @@ -647,6 +648,7 @@ def test_disconnect_half_signed_v2(node_factory): assert len(l1.rpc.listpeerchannels(l2.info['id'])['channels']) == 1 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') def test_reconnect_signed(node_factory): @@ -831,6 +833,7 @@ def test_reconnect_sender_add1(node_factory): l1.daemon.wait_for_log('Already have funding locked in') # This will send commit, so will reconnect as required. + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) @@ -861,6 +864,7 @@ def test_reconnect_sender_add(node_factory): route = [{'amount_msat': amt, 'id': l2.info['id'], 'delay': 5, 'channel': first_scid(l1, l2)}] # This will send commit, so will reconnect as required. + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) # Should have printed this for every reconnect. for i in range(0, len(disconnects)): @@ -892,6 +896,7 @@ def test_reconnect_receiver_add(node_factory): assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'unpaid' route = [{'amount_msat': amt, 'id': l2.info['id'], 'delay': 5, 'channel': first_scid(l1, l2)}] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) for i in range(len(disconnects)): l1.daemon.wait_for_log('Already have funding locked in') @@ -921,6 +926,7 @@ def test_reconnect_receiver_fulfill(node_factory): assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'unpaid' route = [{'amount_msat': amt, 'id': l2.info['id'], 'delay': 5, 'channel': first_scid(l1, l2)}] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) for i in range(len(disconnects)): l1.daemon.wait_for_log('Already have funding locked in') @@ -1207,6 +1213,8 @@ def test_v2_open(node_factory, bitcoind, chainparams): assert(result['status'] == 'complete') +# policy failed: policy-onchain-no-channel-push validate_onchain_tx: channel push not allowed: dual-funding not supported yet +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') def test_funding_push(node_factory, bitcoind, chainparams): """ Try to push peer some sats """ @@ -1634,6 +1642,7 @@ def test_funding_v2_cancel_race(node_factory, bitcoind, executor): executor.map(lambda n: n.stop(), node_factory.nodes) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "validate_setup_channel: holder_shutdown_script is not in wallet or allowlist") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', "External wallet support doesn't work with elements yet.") @@ -1830,6 +1839,7 @@ def test_multifunding_v1_v2_mixed(node_factory, bitcoind): l1.rpc.pay(inv) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_multifunding_v2_exclusive(node_factory, bitcoind): @@ -2049,6 +2059,7 @@ def test_multifunding_wumbo(node_factory): l1.rpc.multifundchannel(destinations) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "flakes too frequently w/ VLS") @unittest.skipIf(TEST_NETWORK == 'liquid-regtest', "Fees on elements are different") @pytest.mark.openchannel('v1') # v2 the weight calculation is off by 3 @pytest.mark.parametrize("anchors", [False, True]) @@ -2448,6 +2459,7 @@ def test_update_fee(node_factory, bitcoind): l2.daemon.wait_for_log('onchaind complete, forgetting peer') +@pytest.mark.developer def test_fee_limits(node_factory, bitcoind): l1, l2, l3, l4 = node_factory.get_nodes(4, opts=[{'dev-max-fee-multiplier': 5, 'may_reconnect': True, 'allow_warning': True}, @@ -3599,6 +3611,9 @@ def test_channel_features(node_factory, bitcoind, anchors): assert only_one(l2.rpc.listpeerchannels()['channels'])['features'] == chan['features'] +# Still fails w/ VLS_PERMISSIVE with "recomposed tx mismatch" +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "Legacy channel type not supported") +@pytest.mark.developer("need dev-force-features") def test_nonstatic_channel(node_factory, bitcoind): """Smoke test for a channel without option_static_remotekey""" l1, l2 = node_factory.get_nodes(2, @@ -3718,6 +3733,8 @@ def test_openchannel_init_alternate(node_factory, executor): print("nothing to do") +# Still fails w/ VLS_PERMISSIVE with "recomposed tx mismatch" +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "Legacy channel type not supported") def test_upgrade_statickey(node_factory, executor): """l1 doesn't have option_static_remotekey, l2 offers it.""" l1, l2 = node_factory.get_nodes(2, opts=[{'may_reconnect': True, @@ -3756,6 +3773,8 @@ def test_upgrade_statickey(node_factory, executor): l2.daemon.wait_for_log(r"They sent desired_channel_type \[12\]") +# Still fails w/ VLS_PERMISSIVE with "recomposed tx mismatch" +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "Legacy channel type not supported") def test_upgrade_statickey_onchaind(node_factory, executor, bitcoind): """We test penalty before/after, and unilateral before/after""" l1, l2 = node_factory.get_nodes(2, opts=[{'may_reconnect': True, @@ -3913,6 +3932,8 @@ def test_upgrade_statickey_onchaind(node_factory, executor, bitcoind): wait_for(lambda: len(l2.rpc.listpeerchannels()['channels']) == 0) +# Still fails w/ VLS_PERMISSIVE with "recomposed tx mismatch" +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "Legacy channel type not supported") def test_upgrade_statickey_fail(node_factory, executor, bitcoind): """We reconnect at all points during retransmit, and we won't upgrade.""" l1_disconnects = ['-WIRE_COMMITMENT_SIGNED', @@ -4234,6 +4255,7 @@ def test_multichan(node_factory, executor, bitcoind): before = l2.rpc.listpeerchannels(l3.info['id'])['channels'] inv1 = l3.rpc.invoice(100000000, "invoice", "invoice") + l1.rpc.preapproveinvoice(bolt11=inv1['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, inv1['payment_hash'], payment_secret=inv1['payment_secret']) l1.rpc.waitsendpay(inv1['payment_hash']) @@ -4261,6 +4283,7 @@ def test_multichan(node_factory, executor, bitcoind): before = l2.rpc.listpeerchannels(l3.info['id'])['channels'] route[1]['channel'] = scid23b inv2 = l3.rpc.invoice(100000000, "invoice2", "invoice2") + l1.rpc.preapproveinvoice(bolt11=inv2['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, inv2['payment_hash'], payment_secret=inv2['payment_secret']) l1.rpc.waitsendpay(inv2['payment_hash']) # Wait until HTLCs fully settled @@ -4305,6 +4328,7 @@ def test_multichan(node_factory, executor, bitcoind): # We can actually pay by *closed* scid (at least until it's completely forgotten) route[1]['channel'] = scid23a inv3 = l3.rpc.invoice(100000000, "invoice3", "invoice3") + l1.rpc.preapproveinvoice(bolt11=inv3['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, inv3['payment_hash'], payment_secret=inv3['payment_secret']) l1.rpc.waitsendpay(inv3['payment_hash']) diff --git a/tests/test_db.py b/tests/test_db.py index 7db5a38daaa5..c1276bbf6cbe 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -136,6 +136,7 @@ def test_max_channel_id(node_factory, bitcoind): @unittest.skipIf(not COMPAT, "needs COMPAT to convert obsolete db") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "This test is based on a sqlite3 snapshot") @unittest.skipIf(TEST_NETWORK != 'regtest', "The network must match the DB snapshot") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't like channel_nonce changing") def test_scid_upgrade(node_factory, bitcoind): bitcoind.generate_block(1) @@ -167,6 +168,7 @@ def test_scid_upgrade(node_factory, bitcoind): @unittest.skipIf(not COMPAT, "needs COMPAT to convert obsolete db") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "This test is based on a sqlite3 snapshot") @unittest.skipIf(TEST_NETWORK != 'regtest', "The network must match the DB snapshot") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't like channel_nonce changing") def test_last_tx_inflight_psbt_upgrade(node_factory, bitcoind): bitcoind.generate_block(12) @@ -185,6 +187,7 @@ def test_last_tx_inflight_psbt_upgrade(node_factory, bitcoind): @unittest.skipIf(not COMPAT, "needs COMPAT to convert obsolete db") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "This test is based on a sqlite3 snapshot") @unittest.skipIf(TEST_NETWORK != 'regtest', "The network must match the DB snapshot") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't like channel_nonce changing") def test_last_tx_psbt_upgrade(node_factory, bitcoind): bitcoind.generate_block(12) @@ -225,6 +228,7 @@ def test_last_tx_psbt_upgrade(node_factory, bitcoind): @pytest.mark.slow_test @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "This test is based on a sqlite3 snapshot") @unittest.skipIf(TEST_NETWORK != 'regtest', "The network must match the DB snapshot") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't support backfill yet") def test_backfill_scriptpubkeys(node_factory, bitcoind): bitcoind.generate_block(214) @@ -358,6 +362,7 @@ def get_dsn(self): os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "This test is based on a sqlite3 snapshot" ) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't support DB migration") def test_local_basepoints_cache(bitcoind, node_factory): """XXX started caching the local basepoints as well as the remote ones. @@ -446,6 +451,7 @@ def test_sqlite3_builtin_backup(bitcoind, node_factory): @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "Don't know how to swap dbs in Postgres") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "vlsd chokes on allowlist when started on wrong network") def test_db_sanity_checks(bitcoind, node_factory): l1, l2 = node_factory.get_nodes(2, opts=[{'allow_broken_log': True, 'may_fail': True}, {}]) @@ -472,6 +478,7 @@ def test_db_sanity_checks(bitcoind, node_factory): assert l1.daemon.is_in_stderr('Wallet sanity check failed') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "no such channel") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "Canned db used") @unittest.skipIf(not COMPAT, "needs COMPAT to convert obsolete db") @unittest.skipIf(TEST_NETWORK != 'regtest', "The DB migration is network specific due to the chain var.") diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 1d1e6d2c34e7..abf5ceb20c02 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -177,6 +177,7 @@ def test_announce_dns_suppressed(node_factory, bitcoind): assert addresses[0]['port'] == 1236 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "trouble with IPv6 in VLS CI runner") def test_announce_and_connect_via_dns(node_factory, bitcoind): """ Test that DNS announcements propagate and can be used when connecting. @@ -1920,6 +1921,7 @@ def test_close_12_block_delay(node_factory, bitcoind): wait_for(lambda: l4.rpc.listchannels(source=l2.info['id'])['channels'] == []) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "skip until v24.02rc2") def test_gossip_not_dying(node_factory, bitcoind): l1 = node_factory.get_node() l2, l3 = node_factory.line_graph(2, wait_for_announce=True) diff --git a/tests/test_invoices.py b/tests/test_invoices.py index 59639134614f..703a654d2787 100644 --- a/tests/test_invoices.py +++ b/tests/test_invoices.py @@ -130,6 +130,8 @@ def test_invoice_weirdstring(node_factory): l1.rpc.delinvoice(weird_label, "unpaid") +# Can't be filtered with VLS_PERMISSIVE, re-using preimage corrupts internal data ... +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "already have a different invoice for same secret") def test_invoice_preimage(node_factory): """Test explicit invoice 'preimage'. """ @@ -958,6 +960,7 @@ def test_expiry_startup_crash(node_factory, bitcoind): l1.start() +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "no such channel") @unittest.skipIf(TEST_NETWORK != 'regtest', "The DB migration is network specific due to the chain var.") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "This test is based on a sqlite3 snapshot") def test_invoices_wait_db_migration(node_factory, bitcoind): diff --git a/tests/test_misc.py b/tests/test_misc.py index 37d2c023c39c..27b3de70169d 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -1164,6 +1164,7 @@ def test_cli_commando(node_factory): assert only_one(j['invoices'])['label'] == 'l"[]{}' +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd integration test job control fails here") def test_daemon_option(node_factory): """ Make sure --daemon at least vaguely works! @@ -2191,6 +2192,7 @@ def test_bitcoind_feerate_floor(node_factory, bitcoind, anchors): @unittest.skipIf(TEST_NETWORK != 'regtest', "Addresses are network specific") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't support forced secrets") def test_dev_force_bip32_seed(node_factory): l1 = node_factory.get_node(options={'dev-force-bip32-seed': '0000000000000000000000000000000000000000000000000000000000000001'}) # First is m/0/0/1 .. @@ -2530,6 +2532,7 @@ def test_regtest_upgrade(node_factory): @unittest.skipIf(VALGRIND, "valgrind files can't be written since we rmdir") @unittest.skipIf(TEST_NETWORK != "regtest", "needs bitcoin mainnet") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't create hsm_secret file") def test_new_node_is_mainnet(node_factory): """Test that an empty directory causes us to be on mainnet""" l1 = node_factory.get_node(start=False, may_fail=True) @@ -2568,13 +2571,13 @@ def test_unicode_rpc(node_factory, executor, bitcoind): @unittest.skipIf(VALGRIND, "Testing pyln doesn't exercise anything interesting in the c code.") -def test_unix_socket_path_length(node_factory, bitcoind, directory, executor, db_provider, test_base_dir): +def test_unix_socket_path_length(node_factory, bitcoind, lssd, directory, executor, db_provider, test_base_dir): lightning_dir = os.path.join(directory, "anode" + "far" * 30 + "away") os.makedirs(lightning_dir) db = db_provider.get_db(lightning_dir, "test_unix_socket_path_length", 1) db.provider = db_provider - l1 = LightningNode(1, lightning_dir, bitcoind, executor, VALGRIND, db=db, port=reserve()) + l1 = LightningNode(1, lightning_dir, bitcoind, lssd, executor, VALGRIND, db=db, port=reserve()) # `LightningNode.start()` internally calls `LightningRpc.getinfo()` which # exercises the socket logic, and raises an issue if it fails. @@ -2720,6 +2723,7 @@ def test_custommsg_triggers_notification(node_factory): l1.daemon.wait_for_log(f"payload=77770012") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't support dev-force-privkey") def test_makesecret(node_factory): """ Test makesecret command. @@ -2757,6 +2761,7 @@ def test_staticbackup(node_factory): and l1.rpc.staticbackup()["scb"][0][16: 16 + 64] == _["channel_id"]) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd says no such channel") def test_recoverchannel(node_factory): """ Test recoverchannel @@ -3858,6 +3863,7 @@ def test_setconfig(node_factory, bitcoind): assert len(lines) == 3 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't support this command") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "deletes database, which is assumed sqlite3") def test_recover_command(node_factory, bitcoind): l1, l2 = node_factory.get_nodes(2) diff --git a/tests/test_opening.py b/tests/test_opening.py index 1569aff10f9e..acbc06b98a2d 100644 --- a/tests/test_opening.py +++ b/tests/test_opening.py @@ -8,6 +8,7 @@ from pathlib import Path from pprint import pprint +import os import pytest import re import unittest @@ -19,6 +20,7 @@ def find_next_feerate(node, peer): @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "handle_sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') def test_queryrates(node_factory, bitcoind): @@ -172,6 +174,7 @@ def test_v2_open_sigs_reconnect_2(node_factory, bitcoind): l2.daemon.wait_for_log(r'to CHANNELD_NORMAL') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_v2_open_sigs_reconnect_1(node_factory, bitcoind): @@ -334,6 +337,7 @@ def test_v2_open_sigs_restart_while_dead(node_factory, bitcoind): l1.daemon.wait_for_log(r'to CHANNELD_NORMAL') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_v2_rbf_single(node_factory, bitcoind, chainparams): @@ -434,6 +438,7 @@ def test_v2_rbf_single(node_factory, bitcoind, chainparams): @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "handle_sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') def test_v2_rbf_abort_retry(node_factory, bitcoind, chainparams): l1, l2 = node_factory.get_nodes(2, opts={'allow_warning': True}) @@ -515,6 +520,7 @@ def test_v2_rbf_abort_retry(node_factory, bitcoind, chainparams): assert not l2.daemon.is_in_log('WIRE_CHANNEL_REESTABLISH') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_v2_rbf_abort_channel_opens(node_factory, bitcoind, chainparams): @@ -574,6 +580,7 @@ def test_v2_rbf_abort_channel_opens(node_factory, bitcoind, chainparams): l1.daemon.wait_for_log(' to CHANNELD_NORMAL') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "handle_sign_option_will_fund_offer unimplemented") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_v2_rbf_liquidity_ad(node_factory, bitcoind, chainparams): @@ -669,6 +676,7 @@ def test_v2_rbf_liquidity_ad(node_factory, bitcoind, chainparams): l1.daemon.wait_for_log('State changed from CLOSINGD_SIGEXCHANGE to CLOSINGD_COMPLETE') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_v2_rbf_multi(node_factory, bitcoind, chainparams): @@ -859,6 +867,7 @@ def test_rbf_reconnect_ack(node_factory, bitcoind, chainparams): l1.rpc.openchannel_bump(chan_id, chan_amount, initpsbt['psbt']) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_rbf_reconnect_tx_construct(node_factory, bitcoind, chainparams): @@ -980,6 +989,7 @@ def test_rbf_reconnect_tx_construct(node_factory, bitcoind, chainparams): l2.daemon.wait_for_log(r'to CHANNELD_NORMAL') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_rbf_reconnect_tx_sigs(node_factory, bitcoind, chainparams): @@ -1061,6 +1071,7 @@ def test_rbf_reconnect_tx_sigs(node_factory, bitcoind, chainparams): assert l1_funding_txid == l2_funding_txid +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_rbf_to_chain_before_commit(node_factory, bitcoind, chainparams): @@ -1162,6 +1173,7 @@ def test_rbf_no_overlap(node_factory, bitcoind, chainparams): l1.rpc.openchannel_update(chan_id, bump['psbt']) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_rbf_fails_to_broadcast(node_factory, bitcoind, chainparams): @@ -1243,6 +1255,7 @@ def run_retry(): assert last_txs['tx'] +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_rbf_broadcast_close_inflights(node_factory, bitcoind, chainparams): @@ -1318,6 +1331,7 @@ def run_retry(): assert inflights[1]['scratch_txid'] not in bitcoind.rpc.getrawmempool() +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_rbf_non_last_mined(node_factory, bitcoind, chainparams): @@ -1418,6 +1432,7 @@ def censoring_sendrawtx(r): l1.daemon.wait_for_log(r'to ONCHAIN') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @pytest.mark.openchannel('v2') def test_funder_options(node_factory, bitcoind): @@ -1484,6 +1499,8 @@ def test_funder_options(node_factory, bitcoind): assert chan_info['funding']['local_funds_msat'] == Millisatoshi('1000000000msat') +# policy-onchain-no-fund-inbound validate_onchain_tx: can't sign for inbound channel: dual-funding not supported yet +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') def test_funder_contribution_limits(node_factory, bitcoind): opts = {'experimental-dual-fund': None, @@ -1546,6 +1563,7 @@ def test_funder_contribution_limits(node_factory, bitcoind): assert l3.daemon.is_in_log(r'calling `signpsbt` .* 6 inputs') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') def test_inflight_dbload(node_factory, bitcoind): """Bad db field access breaks Postgresql on startup with opening leases""" @@ -1622,6 +1640,7 @@ def test_zeroconf_mindepth(bitcoind, node_factory): wait_for(lambda: only_one(l2.rpc.listpeerchannels()['channels'])['state'] == "CHANNELD_NORMAL") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "ensure_funding_buried_and_unspent: tried commitment 1 when funding is not buried at depth 0") def test_zeroconf_open(bitcoind, node_factory): """Let's open a zeroconf channel @@ -1699,6 +1718,7 @@ def test_zeroconf_open(bitcoind, node_factory): l2.rpc.pay(inv) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "missing WIRE_HSMD_CUPDATE_SIG_REQ log messages") def test_zeroconf_public(bitcoind, node_factory, chainparams): """Test that we transition correctly from zeroconf to public @@ -1801,6 +1821,7 @@ def test_zeroconf_public(bitcoind, node_factory, chainparams): wait_for(lambda: only_one([x for x in n.rpc.bkpr_listbalances()['accounts'] if x['account'] == channel_id])['account_resolved']) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "tried commitment when funding is not buried ") def test_zeroconf_forward(node_factory, bitcoind): """Ensure that we can use zeroconf channels in forwards. @@ -1867,6 +1888,7 @@ def test_buy_liquidity_ad_no_v2(node_factory, bitcoind): compact_lease='029a002d000000004b2003e8') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') def test_v2_replay_bookkeeping(node_factory, bitcoind): """ Test that your bookkeeping for a liquidity ad is good @@ -1929,6 +1951,7 @@ def test_v2_replay_bookkeeping(node_factory, bitcoind): l1.rpc.bkpr_listbalances() +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "sign_option_will_fund_offer unimplemented") @pytest.mark.openchannel('v2') def test_buy_liquidity_ad_check_bookkeeping(node_factory, bitcoind): """ Test that your bookkeeping for a liquidity ad is good.""" @@ -2047,6 +2070,7 @@ def test_scid_alias_private(node_factory, bitcoind): l1.rpc.waitsendpay(inv['payment_hash']) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "tried commitment when funding is not buried ") def test_zeroconf_multichan_forward(node_factory): """The freedom to choose the forward channel bytes us when it is 0conf @@ -2334,6 +2358,7 @@ def _no_utxo_response(r): wait_for(lambda: l2.rpc.listpeerchannels()['channels'] == []) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v2') def test_openchannel_no_unconfirmed_inputs_accepter(node_factory, bitcoind): """ If the accepter flags 'require-confirmed-inputs' for an open, diff --git a/tests/test_pay.py b/tests/test_pay.py index 24bb1f88aa4b..3081ff4a3bd6 100644 --- a/tests/test_pay.py +++ b/tests/test_pay.py @@ -61,7 +61,9 @@ def test_pay(node_factory): # Must provide an amount! with pytest.raises(RpcError): l1.rpc.pay(inv2) - l1.dev_pay(inv2, random.randint(1000, 999999), dev_use_shadow=False) + # WORKAROUND - reduce the possible amount to avoid HTLC creation which + # fails the VLS balance check because the prior invoice was 0 amount. + l1.dev_pay(inv2, random.randint(1000, 499999), dev_use_shadow=False) # Should see 6 completed payments assert len(l1.rpc.listsendpays()['payments']) == 6 @@ -245,6 +247,7 @@ def test_pay0(node_factory): l1.rpc.waitsendpay(rhash) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "feerate above maximum: 880782 > 151000") def test_pay_disconnect(node_factory, bitcoind): """If the remote node has disconnected, we fail payment, but can try again when it reconnects""" l1, l2 = node_factory.line_graph(2, opts={'dev-max-fee-multiplier': 5, @@ -555,6 +558,7 @@ def test_pay_maxfee_shadow(node_factory): assert pay_status["amount_msat"] == Millisatoshi(amount) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "fails multiple policy checks") def test_sendpay(node_factory): l1, l2 = node_factory.line_graph(2, fundamount=10**6) @@ -577,6 +581,7 @@ def invoice_unpaid(dst, label): with pytest.raises(RpcError): rs = copy.deepcopy(routestep) rs['amount_msat'] = rs['amount_msat'] - 1 + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(rhash) assert invoice_unpaid(l2, 'testpayment2') @@ -585,6 +590,7 @@ def invoice_unpaid(dst, label): with pytest.raises(RpcError): rs = copy.deepcopy(routestep) rs['amount_msat'] = rs['amount_msat'] * 2 + 1 + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(rhash) assert invoice_unpaid(l2, 'testpayment2') @@ -593,6 +599,7 @@ def invoice_unpaid(dst, label): with pytest.raises(RpcError): rs = copy.deepcopy(routestep) rs['delay'] = rs['delay'] - 2 + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(rhash) assert invoice_unpaid(l2, 'testpayment2') @@ -602,17 +609,20 @@ def invoice_unpaid(dst, label): with pytest.raises(RpcError): rs = copy.deepcopy(routestep) rs['id'] = '00000000000000000000000000000000' + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret']) assert invoice_unpaid(l2, 'testpayment2') l1.rpc.check_request_schemas = True # Bad payment_secret + l1.rpc.preapprovekeysend(l2.info['id'], rhash, routestep['amount_msat']) l1.rpc.sendpay([routestep], rhash, payment_secret="00" * 32) with pytest.raises(RpcError): l1.rpc.waitsendpay(rhash) assert invoice_unpaid(l2, 'testpayment2') # Missing payment_secret + l1.rpc.preapprovekeysend(l2.info['id'], rhash, routestep['amount_msat']) l1.rpc.sendpay([routestep], rhash) with pytest.raises(RpcError): l1.rpc.waitsendpay(rhash) @@ -628,6 +638,7 @@ def invoice_unpaid(dst, label): # This works. before = int(time.time()) + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming details = l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret']) after = int(time.time()) preimage = l1.rpc.waitsendpay(rhash)['payment_preimage'] @@ -670,6 +681,7 @@ def check_balances(): rhash = inv['payment_hash'] assert only_one(l2.rpc.listinvoices('testpayment3')['invoices'])['status'] == 'unpaid' routestep = {'amount_msat': amt * 2, 'id': l2.info['id'], 'delay': 5, 'channel': first_scid(l1, l2)} + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret']) preimage3 = l1.rpc.waitsendpay(rhash)['payment_preimage'] assert only_one(l2.rpc.listinvoices('testpayment3')['invoices'])['status'] == 'paid' @@ -705,6 +717,7 @@ def test_repay(node_factory): 'delay': 5, 'channel': first_scid(l1, l2) } + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([routestep], inv['payment_hash'], payment_secret=inv['payment_secret']) l1.daemon.wait_for_log("Sending 200000000msat over 1 hops to deliver 200000000msat") l1.rpc.waitsendpay(inv['payment_hash'])['payment_preimage'] @@ -733,6 +746,7 @@ def test_wait_sendpay(node_factory, executor): 'delay': 5, 'channel': first_scid(l1, l2) } + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay([routestep], inv['payment_hash'], payment_secret=inv['payment_secret']) assert wait_created.result(TIMEOUT) == {'subsystem': 'sendpays', 'created': 1, @@ -1160,6 +1174,7 @@ def test_forward(node_factory, bitcoind): # Unknown other peer route = copy.deepcopy(baseroute) route[1]['id'] = '031a8dc444e41bb989653a4501e11175a488a57439b0c4947704fd6e3de5dca607' + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) with pytest.raises(RpcError): l1.rpc.waitsendpay(rhash) @@ -1305,6 +1320,7 @@ def test_forward_different_fees_and_cltv(node_factory, bitcoind): assert only_one(l3.rpc.listinvoices('test_forward_different_fees_and_cltv')['invoices'])['status'] == 'unpaid' # This should work. + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(rhash) @@ -1373,6 +1389,7 @@ def test_forward_pad_fees_and_cltv(node_factory, bitcoind): # This should work. inv = l3.rpc.invoice(4999999, 'test_forward_pad_fees_and_cltv', 'desc') rhash = inv['payment_hash'] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(rhash) assert only_one(l3.rpc.listinvoices('test_forward_pad_fees_and_cltv')['invoices'])['status'] == 'paid' @@ -1563,6 +1580,7 @@ def test_forward_local_failed_stats(node_factory, bitcoind, executor): l2.rpc.close(c23, 1) + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming with pytest.raises(RpcError): l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(payment_hash) @@ -1588,6 +1606,7 @@ def test_forward_local_failed_stats(node_factory, bitcoind, executor): 'delay': 6, 'channel': c24}] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming with pytest.raises(RpcError): l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(payment_hash) @@ -1614,6 +1633,7 @@ def test_forward_local_failed_stats(node_factory, bitcoind, executor): 'delay': 6, 'channel': c25}] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming with pytest.raises(RpcError): l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(payment_hash) @@ -1637,6 +1657,7 @@ def test_forward_local_failed_stats(node_factory, bitcoind, executor): # Replace id with a different pubkey, so onion encoded badly at l2 hop. route[1]['id'] = mangled_nodeid + l6.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming with pytest.raises(RpcError): l6.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret']) l6.rpc.waitsendpay(payment_hash) @@ -1664,6 +1685,7 @@ def test_forward_local_failed_stats(node_factory, bitcoind, executor): 'delay': 5, 'channel': c24}] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming executor.submit(l1.rpc.sendpay, route, payment_hash, payment_secret=inv['payment_secret']) l4.daemon.wait_for_log('permfail') @@ -1719,6 +1741,7 @@ def test_htlcs_cltv_only_difference(node_factory, bitcoind): # L2 tries to pay r = l2.rpc.getroute(l4.info['id'], 10**8, 1)["route"] + l2.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l2.rpc.sendpay(r, h, payment_secret=inv['payment_secret']) # Now increment CLTV @@ -1727,6 +1750,7 @@ def test_htlcs_cltv_only_difference(node_factory, bitcoind): # L1 tries to pay r = l1.rpc.getroute(l4.info['id'], 10**8, 1)["route"] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(r, h, payment_secret=inv['payment_secret']) # Now increment CLTV @@ -1735,6 +1759,7 @@ def test_htlcs_cltv_only_difference(node_factory, bitcoind): # L3 tries to pay r = l3.rpc.getroute(l4.info['id'], 10**8, 1)["route"] + l3.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l3.rpc.sendpay(r, h, payment_secret=inv['payment_secret']) # Give them time to go through. @@ -1800,6 +1825,7 @@ def exhaust_channel(opener, peer, scid, already_spent=0): 'delay': 10, 'channel': scid } + opener.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming opener.rpc.sendpay([routestep], inv['payment_hash'], payment_secret=inv['payment_secret']) opener.rpc.waitsendpay(inv['payment_hash']) @@ -2298,6 +2324,7 @@ def test_setchannel_routing(node_factory, bitcoind): assert decoded['routes'] == [[{'pubkey': l2.info['id'], 'short_channel_id': scid, 'fee_base_msat': 1337, 'fee_proportional_millionths': 137, 'cltv_expiry_delta': 6}]] # This will fail. + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route_bad, inv['payment_hash'], payment_secret=inv['payment_secret']) with pytest.raises(RpcError, match='WIRE_TEMPORARY_CHANNEL_FAILURE'): l1.rpc.waitsendpay(inv['payment_hash']) @@ -2507,6 +2534,7 @@ def test_setchannel_startup_opts(node_factory, bitcoind): assert result[1]['htlc_maximum_msat'] == Millisatoshi(5) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "invoice with any amount") def test_channel_spendable(node_factory, bitcoind): """Test that spendable_msat is accurate""" sats = 10**6 @@ -2527,6 +2555,7 @@ def test_channel_spendable(node_factory, bitcoind): # Exact amount should succeed. route = l1.rpc.getroute(l2.info['id'], amount, riskfactor=1, fuzzpercent=0)['route'] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret']) # Amount should drop to 0 once HTLC is sent; we have time, thanks to @@ -2552,6 +2581,7 @@ def test_channel_spendable(node_factory, bitcoind): # Exact amount should succeed. route = l2.rpc.getroute(l1.info['id'], amount, riskfactor=1, fuzzpercent=0)['route'] + l2.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret']) # Amount should drop to 0 once HTLC is sent; we have time, thanks to @@ -2561,6 +2591,7 @@ def test_channel_spendable(node_factory, bitcoind): l2.rpc.waitsendpay(payment_hash, TIMEOUT) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "invoice with any amount") def test_channel_receivable(node_factory, bitcoind): """Test that receivable_msat is accurate""" sats = 10**6 @@ -2616,6 +2647,7 @@ def test_channel_receivable(node_factory, bitcoind): l2.rpc.waitsendpay(payment_hash, TIMEOUT) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "invoice with any amount") def test_channel_spendable_large(node_factory, bitcoind): """Test that spendable_msat is accurate for large channels""" # This is almost the max allowable spend. @@ -2710,6 +2742,7 @@ def test_htlc_too_dusty_outgoing(node_factory, bitcoind, chainparams): route = l1.rpc.getroute(l2.info['id'], non_dust_htlc_val_sat * 1000, 1)['route'] for i in range(0, 3): inv = l2.rpc.invoice((non_dust_htlc_val_sat * 1000), str(i + 100), str(i + 100)) + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret']) l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs') res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments']) @@ -2719,6 +2752,7 @@ def test_htlc_too_dusty_outgoing(node_factory, bitcoind, chainparams): route = l1.rpc.getroute(l2.info['id'], htlc_val_msat, 1)['route'] for i in range(0, num_dusty_htlcs): inv = l2.rpc.invoice(htlc_val_msat, str(i), str(i)) + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret']) l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs') res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments']) @@ -2726,6 +2760,7 @@ def test_htlc_too_dusty_outgoing(node_factory, bitcoind, chainparams): # one more should tip it over, and return a payment failure inv = l2.rpc.invoice(htlc_val_msat, str(num_dusty_htlcs), str(num_dusty_htlcs)) + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret']) l1.daemon.wait_for_log('CHANNEL_ERR_DUST_FAILURE') wait_for(lambda: only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])['status'] == 'failed') @@ -2733,12 +2768,14 @@ def test_htlc_too_dusty_outgoing(node_factory, bitcoind, chainparams): # but we can still add a non dust htlc route = l1.rpc.getroute(l2.info['id'], non_dust_htlc_val_sat * 1000, 1)['route'] inv = l2.rpc.invoice((10000 * 1000), str(120), str(120)) + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret']) l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs') res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments']) assert res['status'] == 'pending' +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "feerate above maximum (escalates)") def test_htlc_too_dusty_incoming(node_factory, bitcoind): """ Try to hit the 'too much dust' limit, should fail the HTLC """ l1, l2, l3 = node_factory.line_graph(3, opts=[{'may_reconnect': True, @@ -2798,6 +2835,7 @@ def test_error_returns_blockheight(node_factory, bitcoind): == '400f{:016x}{:08x}'.format(100, bitcoind.rpc.getblockcount())) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "inv_nosecret: The invoice is missing the mandatory payment secret") @unittest.skipIf(TEST_NETWORK != 'regtest', "Invoice is network specific") def test_pay_no_secret(node_factory, bitcoind): l1, l2 = node_factory.line_graph(2, wait_for_announce=True) @@ -4171,6 +4209,7 @@ def test_mpp_interference_2(node_factory, bitcoind, executor): @pytest.mark.slow_test +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "remote_hsmd doesn't allow push of greater than 20k sat") def test_mpp_overload_payee(node_factory, bitcoind): """ We had a bug where if the payer is unusually well-connected compared @@ -4449,6 +4488,7 @@ def test_fetchinvoice_3hop(node_factory, bitcoind): l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12']}) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "invoice not bolt12: InvalidSemantics(UnsupportedCurrency) and not bolt11: Bech32Error(InvalidChecksum)") def test_fetchinvoice(node_factory, bitcoind): # We remove the conversion plugin on l3, causing it to get upset. l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True, @@ -4574,6 +4614,7 @@ def test_fetchinvoice(node_factory, bitcoind): l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'timeout': 10}) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "invoice not bolt12: Decode(UnknownRequiredFeature) and not bolt11: Bech32Error(InvalidChecksum)") def test_fetchinvoice_recurrence(node_factory, bitcoind): """Test for our recurrence extension""" l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True, @@ -5159,6 +5200,7 @@ def test_payerkey(node_factory): assert n.rpc.decode(b12)['invreq_payer_id'] == k +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "tried commitment when funding is not buried ") def test_pay_multichannel_use_zeroconf(bitcoind, node_factory): """Check that we use the zeroconf direct channel to pay when we need to""" # 0. Setup normal channel, 200k sats. diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 7082baea6aeb..d92c40ef304c 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -708,6 +708,7 @@ def test_openchannel_hook(node_factory, bitcoind): l1.rpc.fundchannel(l2.info['id'], 100001) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "this test frequently hangs w/ VLSD") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') def test_openchannel_hook_error_handling(node_factory, bitcoind): @@ -1344,12 +1345,14 @@ def test_forward_event_notification(node_factory, bitcoind, executor): route = l1.rpc.getroute(l3.info['id'], amount, 1)['route'] # status: offered -> settled + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, payment_hash13, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(payment_hash13) # status: offered -> failed route = l1.rpc.getroute(l4.info['id'], amount, 1)['route'] payment_hash14 = "f" * 64 + l1.rpc.preapprovekeysend(l4.info['id'], payment_hash14, amount) with pytest.raises(RpcError): l1.rpc.sendpay(route, payment_hash14, payment_secret="f" * 64) l1.rpc.waitsendpay(payment_hash14) @@ -1369,6 +1372,7 @@ def test_forward_event_notification(node_factory, bitcoind, executor): 'delay': 5, 'channel': c25}] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming executor.submit(l1.rpc.sendpay, route, payment_hash15, payment_secret=inv['payment_secret']) l5.daemon.wait_for_log('permfail') @@ -1455,11 +1459,13 @@ def test_sendpay_notifications(node_factory, bitcoind): payment_hash2 = inv2['payment_hash'] route = l1.rpc.getroute(l3.info['id'], amount, 1)['route'] + l1.rpc.preapproveinvoice(bolt11=inv1['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, payment_hash1, payment_secret=inv1['payment_secret']) response1 = l1.rpc.waitsendpay(payment_hash1) l2.rpc.close(chanid23, 1) + l1.rpc.preapproveinvoice(bolt11=inv2['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, payment_hash2, payment_secret=inv2['payment_secret']) with pytest.raises(RpcError) as err: l1.rpc.waitsendpay(payment_hash2) @@ -1486,11 +1492,13 @@ def test_sendpay_notifications_nowaiter(node_factory): payment_hash2 = inv2['payment_hash'] route = l1.rpc.getroute(l3.info['id'], amount, 1)['route'] + l1.rpc.preapproveinvoice(bolt11=inv1['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, payment_hash1, payment_secret=inv1['payment_secret']) l1.daemon.wait_for_log(r'Received a sendpay_success') l2.rpc.close(chanid23, 1) + l1.rpc.preapproveinvoice(bolt11=inv2['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, payment_hash2, payment_secret=inv2['payment_secret']) l1.daemon.wait_for_log(r'Received a sendpay_failure') @@ -1783,6 +1791,7 @@ def test_bitcoin_backend(node_factory, bitcoind): " bitcoind") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "channel too big, then feerate above maximum") def test_bitcoin_bad_estimatefee(node_factory, bitcoind): """ This tests that we don't crash if bitcoind backend gives bad estimatefees. @@ -1961,6 +1970,7 @@ def test_replacement_payload(node_factory): assert l2.daemon.wait_for_log("Attempt to pay.*with wrong secret") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "dev_sign_last_tx causes subsequent validate_holder_commitment_tx failure") def test_watchtower(node_factory, bitcoind, directory, chainparams): """Test watchtower hook. @@ -2100,12 +2110,14 @@ def test_coin_movement_notices(node_factory, bitcoind, chainparams): route = l1.rpc.getroute(l3.info['id'], amount, 1)['route'] # status: offered -> settled + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, payment_hash13, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(payment_hash13) # status: offered -> failed route = l1.rpc.getroute(l3.info['id'], amount, 1)['route'] payment_hash13 = "f" * 64 + l1.rpc.preapprovekeysend(l3.info['id'], payment_hash13, amount) with pytest.raises(RpcError): l1.rpc.sendpay(route, payment_hash13, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(payment_hash13) @@ -2114,6 +2126,7 @@ def test_coin_movement_notices(node_factory, bitcoind, chainparams): inv = l1.rpc.invoice(amount // 2, "first", "desc") payment_hash31 = inv['payment_hash'] route = l3.rpc.getroute(l1.info['id'], amount // 2, 1)['route'] + l3.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l3.rpc.sendpay(route, payment_hash31, payment_secret=inv['payment_secret']) l3.rpc.waitsendpay(payment_hash31) @@ -2121,6 +2134,7 @@ def test_coin_movement_notices(node_factory, bitcoind, chainparams): inv = l2.rpc.invoice(amount, "first", "desc") payment_hash12 = inv['payment_hash'] route = l1.rpc.getroute(l2.info['id'], amount, 1)['route'] + l1.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l1.rpc.sendpay(route, payment_hash12, payment_secret=inv['payment_secret']) l1.rpc.waitsendpay(payment_hash12) @@ -2130,6 +2144,7 @@ def test_coin_movement_notices(node_factory, bitcoind, chainparams): # Make sure previous completely settled wait_for(lambda: only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])['htlcs'] == []) route = l2.rpc.getroute(l1.info['id'], amount // 2, 1)['route'] + l2.rpc.preapproveinvoice(bolt11=inv['bolt11']) # let the signer know this payment is coming l2.rpc.sendpay(route, payment_hash21, payment_secret=inv['payment_secret']) l2.rpc.waitsendpay(payment_hash21) @@ -2174,6 +2189,7 @@ def test_coin_movement_notices(node_factory, bitcoind, chainparams): check_coin_moves(l2, chanid_3, l2_l3_mvts, chainparams) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "trouble managing remotesigner when node killed this way") def test_important_plugin(node_factory): # Cache it here. pluginsdir = os.path.join(os.path.dirname(__file__), "plugins") @@ -3404,6 +3420,7 @@ def test_block_added_notifications(node_factory, bitcoind): assert len(ret) == 3 and ret[1] == next_l2_base + 1 and ret[2] == next_l2_base + 2 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "timeout waiting for hold_invoice plugin") @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') def test_sql(node_factory, bitcoind): opts = {'experimental-offers': None, diff --git a/tests/test_reckless.py b/tests/test_reckless.py index bca922e013f7..3e087498e56d 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -184,6 +184,8 @@ def test_search(node_factory): assert 'found testplugpass in source: https://github.com/lightningd/plugins' in r.stdout +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "no canned github server in gitlab CI") def test_install(node_factory): """test search, git clone, and installation to folder.""" n = get_reckless_node(node_factory) @@ -199,6 +201,7 @@ def test_install(node_factory): @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "no canned github server in gitlab CI") def test_poetry_install(node_factory): """test search, git clone, and installation to folder.""" n = get_reckless_node(node_factory) @@ -218,6 +221,7 @@ def test_poetry_install(node_factory): @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "no canned github server in gitlab CI") def test_local_dir_install(node_factory): """Test search and install from local directory source.""" n = get_reckless_node(node_factory) @@ -235,6 +239,7 @@ def test_local_dir_install(node_factory): @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "no canned github server in gitlab CI") def test_disable_enable(node_factory): """test search, git clone, and installation to folder.""" n = get_reckless_node(node_factory) @@ -265,6 +270,8 @@ def test_disable_enable(node_factory): assert test_plugin in n.rpc.plugin_list()['plugins'] +@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "no canned github server in gitlab CI") @unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection") def test_tag_install(node_factory): "install a plugin from a specific commit hash or tag" diff --git a/tests/test_renepay.py b/tests/test_renepay.py index c5831885eab2..c75320311109 100644 --- a/tests/test_renepay.py +++ b/tests/test_renepay.py @@ -13,6 +13,7 @@ import json import subprocess import os +import unittest def test_simple(node_factory): @@ -86,6 +87,8 @@ def test_mpp(node_factory): send_amount = Millisatoshi("1200000sat") inv = l6.rpc.invoice(send_amount, "test_renepay", "description")["bolt11"] + # FIXME - This shouldn't be necessary, renepay should automatically generate ... vls-hsmd:#6 + l1.rpc.preapproveinvoice(bolt11=inv) # let the signer know this payment is coming details = l1.rpc.call("renepay", {"invstring": inv}) assert details["status"] == "complete" assert details["amount_msat"] == send_amount @@ -183,17 +186,21 @@ def test_pay(node_factory): # Check payment of any-amount invoice. for i in range(5): label = "any{}".format(i) - inv2 = l2.rpc.invoice("any", label, "description")["bolt11"] + inv2x = l2.rpc.invoice("any", label, "description") + rhash = inv2x["payment_hash"] + inv2 = inv2x["bolt11"] # Must provide an amount! with pytest.raises(RpcError): l1.rpc.call("renepay", {"invstring": inv2, "dev_use_shadow": False}) + amt = random.randint(1000, 999999) + l1.rpc.preapprovekeysend(l2.info["id"], rhash, amt) l1.rpc.call( "renepay", { "invstring": inv2, "dev_use_shadow": False, - "amount_msat": random.randint(1000, 999999), + "amount_msat": amt, }, ) @@ -242,6 +249,7 @@ def test_amounts(node_factory): assert invoice["amount_received_msat"] >= Millisatoshi(123456) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "invoice with any amount") def test_limits(node_factory): """ Topology: @@ -350,6 +358,7 @@ def start_channels(connections): ) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "invoice with any amount") def test_hardmpp(node_factory): """ Topology: @@ -446,6 +455,7 @@ def test_fee_allocation(node_factory): ) inv = l4.rpc.invoice("1500000sat", "inv", "description") + l1.rpc.preapprovekeysend(l4.info["id"], inv["payment_hash"], 1500000000 + 75000000) l1.rpc.call("renepay", {"invstring": inv["bolt11"], "maxfee": "75000sat"}) l1.wait_for_htlcs() invoice = only_one(l4.rpc.listinvoices("inv")["invoices"]) @@ -491,7 +501,8 @@ def test_htlc_max(node_factory): inv = l6.rpc.invoice("1800000sat", "inv", "description") - l1.rpc.call("renepay", {"invstring": inv["bolt11"]}) + l1.rpc.preapproveinvoice(bolt11=inv["bolt11"]) # let the signer know this payment is coming + l1.rpc.call("renepay", {'invstring': inv['bolt11']}) l1.wait_for_htlcs() invoice = only_one(l6.rpc.listinvoices("inv")["invoices"]) assert invoice["amount_received_msat"] >= Millisatoshi("1800000sat") @@ -509,6 +520,7 @@ def test_previous_sendpays(node_factory, bitcoind): # First case, do not overpay a pending MPP payment invstr = l3.rpc.invoice("100000sat", "inv1", "description")["bolt11"] inv = l1.rpc.decode(invstr) + l1.rpc.preapproveinvoice(bolt11=invstr) # let the signer know this payment is coming route = l1.rpc.call( "getroute", {"id": inv["payee"], "amount_msat": "50000sat", "riskfactor": 10} ) @@ -533,6 +545,7 @@ def test_previous_sendpays(node_factory, bitcoind): # Second case, do not collide with failed sendpays invstr = l3.rpc.invoice("100000sat", "inv2", "description")["bolt11"] inv = l1.rpc.decode(invstr) + l1.rpc.preapproveinvoice(bolt11=invstr) # let the signer know this payment is coming route = l1.rpc.call( "getroute", {"id": inv["payee"], "amount_msat": "50000sat", "riskfactor": 10} ) @@ -609,6 +622,7 @@ def test_fees(node_factory): # check that once gossip is in sync, fees are paid correctly invstr = dest.rpc.invoice("100000sat", "inv1", "description")["bolt11"] + source.rpc.preapproveinvoice(bolt11=invstr) # let the signer know this payment is coming source.rpc.call("renepay", {"invstring": invstr}) invoice = only_one(dest.rpc.listinvoices("inv1")["invoices"]) assert invoice["amount_received_msat"] == Millisatoshi("100000sat") @@ -621,6 +635,7 @@ def test_fees(node_factory): nodes[3].rpc.setchannel(nodes[4].info["id"], 3000, 350, enforcedelay=0) invstr = dest.rpc.invoice("150000sat", "inv2", "description")["bolt11"] + source.rpc.preapproveinvoice(bolt11=invstr) # let the signer know this payment is coming source.rpc.call("renepay", {"invstring": invstr}) invoice = only_one(dest.rpc.listinvoices("inv2")["invoices"]) assert invoice["amount_received_msat"] == Millisatoshi("150000sat") diff --git a/tests/test_splicing.py b/tests/test_splicing.py index bfeb46bb1864..498825076744 100644 --- a/tests/test_splicing.py +++ b/tests/test_splicing.py @@ -9,10 +9,12 @@ ) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') -@flaky +# incompatible w/ pytest-timeout +# @flaky def test_splice(node_factory, bitcoind): l1, l2 = node_factory.line_graph(2, fundamount=1000000, wait_for_announce=True, opts={'experimental-splicing': None}) @@ -46,6 +48,7 @@ def test_splice(node_factory, bitcoind): assert l1.db_query("SELECT count(*) as c FROM channeltxs;")[0]['c'] == 0 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @@ -104,6 +107,7 @@ def test_splice_gossip(node_factory, bitcoind): assert not l2.daemon.is_in_log("invalid local_channel_announcement") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @@ -138,6 +142,7 @@ def test_splice_listnodes(node_factory, bitcoind): wait_for(lambda: len(l2.rpc.listnodes()['nodes']) == 2) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @@ -173,6 +178,7 @@ def test_splice_out(node_factory, bitcoind): assert l1.db_query("SELECT count(*) as c FROM channeltxs;")[0]['c'] == 0 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @@ -282,6 +288,7 @@ def test_commit_crash_splice(node_factory, bitcoind): assert l1.db_query("SELECT count(*) as c FROM channeltxs;")[0]['c'] == 0 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') diff --git a/tests/test_splicing_disconnect.py b/tests/test_splicing_disconnect.py index f20668498b2f..cecbf71e76b0 100644 --- a/tests/test_splicing_disconnect.py +++ b/tests/test_splicing_disconnect.py @@ -8,6 +8,7 @@ ) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') @@ -62,6 +63,7 @@ def test_splice_disconnect_sig(node_factory, bitcoind): assert l1.db_query("SELECT count(*) as c FROM channeltxs;")[0]['c'] == 0 +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') diff --git a/tests/test_splicing_insane.py b/tests/test_splicing_insane.py index ca7883252f12..6aa9bc04381c 100644 --- a/tests/test_splicing_insane.py +++ b/tests/test_splicing_insane.py @@ -43,7 +43,7 @@ def wait_for_restart(l1, l2): l1.daemon.wait_for_log(r'peer_in WIRE_CHANNEL_REESTABLISH') l2.daemon.wait_for_log(r'peer_in WIRE_CHANNEL_REESTABLISH') - +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_SKIP_SPLICE_TESTS') == '1', "test expected to fail before VLS dual-funding / splicing support") @pytest.mark.openchannel('v1') @pytest.mark.openchannel('v2') @unittest.skipIf(TEST_NETWORK != 'regtest', 'elementsd doesnt yet support PSBT features we need') diff --git a/tests/test_wallet.py b/tests/test_wallet.py index 40d078cd338d..aac20e8fd17a 100644 --- a/tests/test_wallet.py +++ b/tests/test_wallet.py @@ -782,6 +782,7 @@ def test_utxopsbt(node_factory, bitcoind, chainparams): reservedok=True) +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd') and os.getenv('VLS_PERMISSIVE') != '1', "non-beneficial value considered as fees is above maximum feerate") def test_sign_external_psbt(node_factory, bitcoind, chainparams): """ A PSBT w/ one of our inputs should be signable (we can fill @@ -1005,7 +1006,12 @@ def test_sign_and_send_psbt(node_factory, bitcoind, chainparams): # But it can sign all the valid ones at once. half_signed_psbt = l1.rpc.signpsbt(joint_psbt, signonly=sign_success)['signed_psbt'] for s in sign_success: - assert bitcoind.rpc.decodepsbt(half_signed_psbt)['inputs'][s]['partial_signatures'] is not None + decoded_input = bitcoind.rpc.decodepsbt(half_signed_psbt)['inputs'][s] + if 'partial_signatures' in decoded_input: + assert decoded_input['partial_signatures'] is not None + else: + # VLS returns signatures in final_scriptwitness instead + assert decoded_input['final_scriptwitness'] is not None totally_signed = l2.rpc.signpsbt(half_signed_psbt)['signed_psbt'] @@ -1117,6 +1123,7 @@ def write_all(fd, bytestr): @unittest.skipIf(VALGRIND, "It does not play well with prompt and key derivation.") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't support hsm_secret file") def test_hsm_secret_encryption(node_factory): l1 = node_factory.get_node(may_fail=True) # May fail when started without key password = "reckful&é🍕\n" @@ -1180,6 +1187,7 @@ def __init__(self, directory, *args): @unittest.skipIf(VALGRIND, "It does not play well with prompt and key derivation.") +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't support hsm_secret file") def test_hsmtool_secret_decryption(node_factory): l1 = node_factory.get_node() password = "reckless123#{ù}\n" @@ -1316,6 +1324,7 @@ def test_hsmtool_dump_descriptors(node_factory, bitcoind): assert res["total_amount"] == Decimal('0.00001000') +@unittest.skipIf(os.getenv('SUBDAEMON').startswith('hsmd:remote_hsmd'), "remote_hsmd doesn't support generatehsm") def test_hsmtool_generatehsm(node_factory): l1 = node_factory.get_node(start=False) hsm_path = os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, diff --git a/tools/build-release.sh b/tools/build-release.sh index 81dc210d16e3..924ac7af1c70 100755 --- a/tools/build-release.sh +++ b/tools/build-release.sh @@ -8,7 +8,8 @@ if [ "$1" = "--inside-docker" ]; then PLTFM="$3" git clone /src /build cd /build - pip3 install -r plugins/clnrest/requirements.txt + poetry export --without-hashes > /tmp/requirements.txt + python3 -m pip install -r /tmp/requirements.txt ./configure make VERSION="$VER" make install DESTDIR=/"$VER-$PLTFM" RUST_PROFILE=release diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index d940d6caa8c8..73c51bd37988 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -306,9 +306,6 @@ bool fromwire_hsmd_forget_channel_reply(const void *p UNNEEDED) /* Generated stub for fromwire_hsmd_get_output_scriptpubkey_reply */ bool fromwire_hsmd_get_output_scriptpubkey_reply(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **script UNNEEDED) { fprintf(stderr, "fromwire_hsmd_get_output_scriptpubkey_reply called!\n"); abort(); } -/* Generated stub for fromwire_hsmd_init_reply_v2 */ -bool fromwire_hsmd_init_reply_v2(const void *p UNNEEDED, struct node_id *node_id UNNEEDED, struct ext_key *bip32 UNNEEDED, struct pubkey *bolt12 UNNEEDED) -{ fprintf(stderr, "fromwire_hsmd_init_reply_v2 called!\n"); abort(); } /* Generated stub for fromwire_hsmd_init_reply_v4 */ bool fromwire_hsmd_init_reply_v4(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u32 *hsm_version UNNEEDED, u32 **hsm_capabilities UNNEEDED, struct node_id *node_id UNNEEDED, struct ext_key *bip32 UNNEEDED, struct pubkey *bolt12 UNNEEDED) { fprintf(stderr, "fromwire_hsmd_init_reply_v4 called!\n"); abort(); }