From e256b3498ef35192cfa407e5fa6475cf62b37728 Mon Sep 17 00:00:00 2001 From: Duncan Dean Date: Fri, 22 Jul 2022 11:34:25 +0200 Subject: [PATCH 01/91] Reuse `BlockHash` from `validate_pow()` The `validate_pow()` method now returns the BlockHash since rust-bitcoin v0.26.2 thanks to jkczyz's PR (rust-bitcoin/rust-bitcoin/pull/572). --- lightning-block-sync/src/poll.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lightning-block-sync/src/poll.rs b/lightning-block-sync/src/poll.rs index 6e30d2e86d2..4c6cb0c0600 100644 --- a/lightning-block-sync/src/poll.rs +++ b/lightning-block-sync/src/poll.rs @@ -59,12 +59,11 @@ impl Validate for BlockHeaderData { type T = ValidatedBlockHeader; fn validate(self, block_hash: BlockHash) -> BlockSourceResult { - self.header + let pow_valid_block_hash = self.header .validate_pow(&self.header.target()) .or_else(|e| Err(BlockSourceError::persistent(e)))?; - // TODO: Use the result of validate_pow instead of recomputing the block hash once upstream. - if self.header.block_hash() != block_hash { + if pow_valid_block_hash != block_hash { return Err(BlockSourceError::persistent("invalid block hash")); } @@ -76,12 +75,11 @@ impl Validate for Block { type T = ValidatedBlock; fn validate(self, block_hash: BlockHash) -> BlockSourceResult { - self.header + let pow_valid_block_hash = self.header .validate_pow(&self.header.target()) .or_else(|e| Err(BlockSourceError::persistent(e)))?; - // TODO: Use the result of validate_pow instead of recomputing the block hash once upstream. - if self.block_hash() != block_hash { + if pow_valid_block_hash != block_hash { return Err(BlockSourceError::persistent("invalid block hash")); } From 17e6c374c513f2eca810fa4e931be65f0d4fc29f Mon Sep 17 00:00:00 2001 From: jurvis Date: Mon, 18 Jul 2022 20:29:40 -0500 Subject: [PATCH 02/91] Add HTLCHandlingFailed event Adds a HTLCHandlingFailed that expresses failure by our node to process a specific HTLC. A HTLCDestination enum is defined to express the possible cases that causes the handling to fail. --- fuzz/src/chanmon_consistency.rs | 1 + lightning/src/util/events.rs | 69 +++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/fuzz/src/chanmon_consistency.rs b/fuzz/src/chanmon_consistency.rs index 9d5ccaa8105..bfaa6501d1e 100644 --- a/fuzz/src/chanmon_consistency.rs +++ b/fuzz/src/chanmon_consistency.rs @@ -860,6 +860,7 @@ pub fn do_test(data: &[u8], underlying_out: Out) { events::Event::PendingHTLCsForwardable { .. } => { nodes[$node].process_pending_htlc_forwards(); }, + events::Event::HTLCHandlingFailed { .. } => {}, _ => if out.may_fail.load(atomic::Ordering::Acquire) { return; } else { diff --git a/lightning/src/util/events.rs b/lightning/src/util/events.rs index 2282ea55ab3..42b55163325 100644 --- a/lightning/src/util/events.rs +++ b/lightning/src/util/events.rs @@ -152,6 +152,50 @@ impl_writeable_tlv_based_enum_upgradable!(ClosureReason, (12, OutdatedChannelManager) => {}, ); +/// Intended destination of a failed HTLC as indicated in [`Event::HTLCHandlingFailed`]. +#[derive(Clone, Debug, PartialEq)] +pub enum HTLCDestination { + /// We tried forwarding to a channel but failed to do so. An example of such an instance is when + /// there is insufficient capacity in our outbound channel. + NextHopChannel { + /// The `node_id` of the next node. For backwards compatibility, this field is + /// marked as optional, versions prior to 0.0.110 may not always be able to provide + /// counterparty node information. + node_id: Option, + /// The outgoing `channel_id` between us and the next node. + channel_id: [u8; 32], + }, + /// Scenario where we are unsure of the next node to forward the HTLC to. + UnknownNextHop { + /// Short channel id we are requesting to forward an HTLC to. + requested_forward_scid: u64, + }, + /// Failure scenario where an HTLC may have been forwarded to be intended for us, + /// but is invalid for some reason, so we reject it. + /// + /// Some of the reasons may include: + /// * HTLC Timeouts + /// * Expected MPP amount to claim does not equal HTLC total + /// * Claimable amount does not match expected amount + FailedPayment { + /// The payment hash of the payment we attempted to process. + payment_hash: PaymentHash + }, +} + +impl_writeable_tlv_based_enum_upgradable!(HTLCDestination, + (0, NextHopChannel) => { + (0, node_id, required), + (2, channel_id, required), + }, + (2, UnknownNextHop) => { + (0, requested_forward_scid, required), + }, + (4, FailedPayment) => { + (0, payment_hash, required), + } +); + /// An Event which you should probably take some action in response to. /// /// Note that while Writeable and Readable are implemented for Event, you probably shouldn't use @@ -540,6 +584,24 @@ pub enum Event { /// [`ChannelManager`]: crate::ln::channelmanager::ChannelManager channel_type: ChannelTypeFeatures, }, + /// Indicates that the HTLC was accepted, but could not be processed when or after attempting to + /// forward it. + /// + /// Some scenarios where this event may be sent include: + /// * Insufficient capacity in the outbound channel + /// * While waiting to forward the HTLC, the channel it is meant to be forwarded through closes + /// * When an unknown SCID is requested for forwarding a payment. + /// * Claiming an amount for an MPP payment that exceeds the HTLC total + /// * The HTLC has timed out + /// + /// This event, however, does not get generated if an HTLC fails to meet the forwarding + /// requirements (i.e. insufficient fees paid, or a CLTV that is too soon). + HTLCHandlingFailed { + /// The channel over which the HTLC was received. + prev_channel_id: [u8; 32], + /// Destination of the HTLC that failed to be processed. + failed_next_destination: HTLCDestination, + }, } impl Writeable for Event { @@ -684,6 +746,13 @@ impl Writeable for Event { (6, short_channel_id, option), }) }, + &Event::HTLCHandlingFailed { ref prev_channel_id, ref failed_next_destination } => { + 25u8.write(writer)?; + write_tlv_fields!(writer, { + (0, prev_channel_id, required), + (2, failed_next_destination, required), + }) + }, // Note that, going forward, all new events must only write data inside of // `write_tlv_fields`. Versions 0.0.101+ will ignore odd-numbered events that write // data via `write_tlv_fields`. From 5bccd2eee2a5756b32fa1bc18557978b4a5a95c1 Mon Sep 17 00:00:00 2001 From: jurvis Date: Mon, 18 Jul 2022 20:32:05 -0500 Subject: [PATCH 03/91] Add utils to handle HTLC handling failure reason We add `HTLCHandlingFailedConditions` to express the failure parameters, that will be enforced by a new macro, `expect_pending_htlcs_forwardable_conditions`. --- lightning/src/ln/functional_test_utils.rs | 83 +++++++++++++++++++++-- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/lightning/src/ln/functional_test_utils.rs b/lightning/src/ln/functional_test_utils.rs index fe78395e2f9..ee34b590f47 100644 --- a/lightning/src/ln/functional_test_utils.rs +++ b/lightning/src/ln/functional_test_utils.rs @@ -24,7 +24,7 @@ use util::enforcing_trait_impls::EnforcingSigner; use util::scid_utils; use util::test_utils; use util::test_utils::{panicking, TestChainMonitor}; -use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose}; +use util::events::{Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose}; use util::errors::APIError; use util::config::UserConfig; use util::ser::{ReadableArgs, Writeable}; @@ -1254,24 +1254,95 @@ macro_rules! get_route_and_payment_hash { }} } +pub struct HTLCHandlingFailedConditions { + pub expected_destinations: Vec, +} + +impl HTLCHandlingFailedConditions { + pub fn new() -> Self { + Self { + expected_destinations: vec![], + } + } + + pub fn with_reason(mut self, reason: HTLCDestination) -> Self { + self.expected_destinations = vec![reason]; + self + } + + pub fn with_reasons(mut self, reasons: Vec) -> Self { + self.expected_destinations = reasons; + self + } +} + #[macro_export] -/// Clears (and ignores) a PendingHTLCsForwardable event -macro_rules! expect_pending_htlcs_forwardable_ignore { - ($node: expr) => {{ +macro_rules! expect_pending_htlcs_forwardable_conditions { + ($node: expr, $conditions: expr) => {{ + let conditions = $conditions; let events = $node.node.get_and_clear_pending_events(); - assert_eq!(events.len(), 1); match events[0] { $crate::util::events::Event::PendingHTLCsForwardable { .. } => { }, _ => panic!("Unexpected event"), }; + + let count = conditions.expected_destinations.len() + 1; + assert_eq!(events.len(), count); + + if conditions.expected_destinations.len() > 0 { + expect_htlc_handling_failed_destinations!(events, conditions.expected_destinations) + } + }} +} + +#[macro_export] +macro_rules! expect_htlc_handling_failed_destinations { + ($events: expr, $destinations: expr) => {{ + for event in $events { + match event { + $crate::util::events::Event::PendingHTLCsForwardable { .. } => { }, + $crate::util::events::Event::HTLCHandlingFailed { ref failed_next_destination, .. } => { + assert!($destinations.contains(&failed_next_destination)) + }, + _ => panic!("Unexpected destination"), + } + } }} } +#[macro_export] +/// Clears (and ignores) a PendingHTLCsForwardable event +macro_rules! expect_pending_htlcs_forwardable_ignore { + ($node: expr) => {{ + expect_pending_htlcs_forwardable_conditions!($node, $crate::ln::functional_test_utils::HTLCHandlingFailedConditions::new()); + }}; +} + +#[macro_export] +/// Clears (and ignores) PendingHTLCsForwardable and HTLCHandlingFailed events +macro_rules! expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore { + ($node: expr, $conditions: expr) => {{ + expect_pending_htlcs_forwardable_conditions!($node, $conditions); + }}; +} + #[macro_export] /// Handles a PendingHTLCsForwardable event macro_rules! expect_pending_htlcs_forwardable { ($node: expr) => {{ - $crate::expect_pending_htlcs_forwardable_ignore!($node); + expect_pending_htlcs_forwardable_ignore!($node); + $node.node.process_pending_htlc_forwards(); + + // Ensure process_pending_htlc_forwards is idempotent. + $node.node.process_pending_htlc_forwards(); + }}; +} + +#[macro_export] +/// Handles a PendingHTLCsForwardable and HTLCHandlingFailed event +macro_rules! expect_pending_htlcs_forwardable_and_htlc_handling_failed { + ($node: expr, $conditions: expr) => {{ + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!($node, $conditions); $node.node.process_pending_htlc_forwards(); // Ensure process_pending_htlc_forwards is idempotent. From 19b5a48dde8dbf3c1905499a8f153b3f3e62964a Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Fri, 22 Jul 2022 00:32:17 +0000 Subject: [PATCH 04/91] Remove scary disconenct warnings on PeerManager new connection fns In 4703d4e72565ddfd150b9368ea036f4973fd7590 we changed PeerManager::socket_disconnected to no longer require that sockets which the PeerManager decided to disconnect not be disconnected. However, we forgot to remove the scary warnings on the `new_{inbound,outbound}_connection` functions which warned of the old behavior. We do so here. --- lightning/src/ln/peer_handler.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index f8339f0eae9..d34fdfeb52a 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -622,8 +622,7 @@ impl P /// peer using the init message. /// The user should pass the remote network address of the host they are connected to. /// - /// Note that if an Err is returned here you MUST NOT call socket_disconnected for the new - /// descriptor but must disconnect the connection immediately. + /// If an `Err` is returned here you must disconnect the connection immediately. /// /// Returns a small number of bytes to send to the remote node (currently always 50). /// @@ -671,9 +670,8 @@ impl P /// The user should pass the remote network address of the host they are connected to. /// /// May refuse the connection by returning an Err, but will never write bytes to the remote end - /// (outbound connector always speaks first). Note that if an Err is returned here you MUST NOT - /// call socket_disconnected for the new descriptor but must disconnect the connection - /// immediately. + /// (outbound connector always speaks first). If an `Err` is returned here you must disconnect + /// the connection immediately. /// /// Panics if descriptor is duplicative with some other descriptor which has not yet been /// [`socket_disconnected()`]. From ac842ed9dd7a36a4a26eb6b856d80ab04eecf750 Mon Sep 17 00:00:00 2001 From: jurvis Date: Mon, 25 Jul 2022 11:28:51 -0700 Subject: [PATCH 05/91] Send failure event if we fail to handle a HTLC In `ChannelManager::fail_htlc_backwards_internal`, we push a `HTLCHandlingFailed` containing some information about the HTLC --- fuzz/src/chanmon_consistency.rs | 2 +- lightning/src/chain/chainmonitor.rs | 16 +- lightning/src/chain/channelmonitor.rs | 4 + lightning/src/chain/mod.rs | 3 +- lightning/src/ln/chanmon_update_fail_tests.rs | 18 +- lightning/src/ln/channel.rs | 5 +- lightning/src/ln/channelmanager.rs | 157 +++++++++++------- lightning/src/ln/functional_test_utils.rs | 54 ++---- lightning/src/ln/functional_tests.rs | 155 ++++++++++------- lightning/src/ln/monitor_tests.rs | 4 +- lightning/src/ln/onion_route_tests.rs | 16 +- lightning/src/ln/payment_tests.rs | 50 +++--- lightning/src/ln/priv_short_conf_tests.rs | 6 +- lightning/src/ln/reorg_tests.rs | 4 +- lightning/src/util/test_utils.rs | 2 +- 15 files changed, 287 insertions(+), 209 deletions(-) diff --git a/fuzz/src/chanmon_consistency.rs b/fuzz/src/chanmon_consistency.rs index bfaa6501d1e..dc9504878f6 100644 --- a/fuzz/src/chanmon_consistency.rs +++ b/fuzz/src/chanmon_consistency.rs @@ -148,7 +148,7 @@ impl chain::Watch for TestChainMonitor { self.chain_monitor.update_channel(funding_txo, update) } - fn release_pending_monitor_events(&self) -> Vec<(OutPoint, Vec)> { + fn release_pending_monitor_events(&self) -> Vec<(OutPoint, Vec, Option)> { return self.chain_monitor.release_pending_monitor_events(); } } diff --git a/lightning/src/chain/chainmonitor.rs b/lightning/src/chain/chainmonitor.rs index e6b5733520a..eff81f1484a 100644 --- a/lightning/src/chain/chainmonitor.rs +++ b/lightning/src/chain/chainmonitor.rs @@ -43,6 +43,7 @@ use prelude::*; use sync::{RwLock, RwLockReadGuard, Mutex, MutexGuard}; use core::ops::Deref; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use bitcoin::secp256k1::PublicKey; #[derive(Clone, Copy, Hash, PartialEq, Eq)] /// A specific update's ID stored in a `MonitorUpdateId`, separated out to make the contents @@ -235,7 +236,7 @@ pub struct ChainMonitor)>>, + pending_monitor_events: Mutex, Option)>>, /// The best block height seen, used as a proxy for the passage of time. highest_chain_height: AtomicUsize, } @@ -299,7 +300,7 @@ where C::Target: chain::Filter, log_trace!(self.logger, "Finished syncing Channel Monitor for channel {}", log_funding_info!(monitor)), Err(ChannelMonitorUpdateErr::PermanentFailure) => { monitor_state.channel_perm_failed.store(true, Ordering::Release); - self.pending_monitor_events.lock().unwrap().push((*funding_outpoint, vec![MonitorEvent::UpdateFailed(*funding_outpoint)])); + self.pending_monitor_events.lock().unwrap().push((*funding_outpoint, vec![MonitorEvent::UpdateFailed(*funding_outpoint)], monitor.get_counterparty_node_id())); }, Err(ChannelMonitorUpdateErr::TemporaryFailure) => { log_debug!(self.logger, "Channel Monitor sync for channel {} in progress, holding events until completion!", log_funding_info!(monitor)); @@ -458,7 +459,7 @@ where C::Target: chain::Filter, self.pending_monitor_events.lock().unwrap().push((funding_txo, vec![MonitorEvent::UpdateCompleted { funding_txo, monitor_update_id: monitor_data.monitor.get_latest_update_id(), - }])); + }], monitor_data.monitor.get_counterparty_node_id())); }, MonitorUpdateId { contents: UpdateOrigin::ChainSync(_) } => { if !monitor_data.has_pending_chainsync_updates(&pending_monitor_updates) { @@ -476,10 +477,12 @@ where C::Target: chain::Filter, /// channel_monitor_updated once with the highest ID. #[cfg(any(test, fuzzing))] pub fn force_channel_monitor_updated(&self, funding_txo: OutPoint, monitor_update_id: u64) { + let monitors = self.monitors.read().unwrap(); + let counterparty_node_id = monitors.get(&funding_txo).and_then(|m| m.monitor.get_counterparty_node_id()); self.pending_monitor_events.lock().unwrap().push((funding_txo, vec![MonitorEvent::UpdateCompleted { funding_txo, monitor_update_id, - }])); + }], counterparty_node_id)); } #[cfg(any(test, fuzzing, feature = "_test_utils"))] @@ -666,7 +669,7 @@ where C::Target: chain::Filter, } } - fn release_pending_monitor_events(&self) -> Vec<(OutPoint, Vec)> { + fn release_pending_monitor_events(&self) -> Vec<(OutPoint, Vec, Option)> { let mut pending_monitor_events = self.pending_monitor_events.lock().unwrap().split_off(0); for monitor_state in self.monitors.read().unwrap().values() { let is_pending_monitor_update = monitor_state.has_pending_chainsync_updates(&monitor_state.pending_monitor_updates.lock().unwrap()); @@ -695,7 +698,8 @@ where C::Target: chain::Filter, let monitor_events = monitor_state.monitor.get_and_clear_pending_monitor_events(); if monitor_events.len() > 0 { let monitor_outpoint = monitor_state.monitor.get_funding_txo().0; - pending_monitor_events.push((monitor_outpoint, monitor_events)); + let counterparty_node_id = monitor_state.monitor.get_counterparty_node_id(); + pending_monitor_events.push((monitor_outpoint, monitor_events, counterparty_node_id)); } } } diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 80cd9cb9d45..220951e07f9 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1209,6 +1209,10 @@ impl ChannelMonitor { self.inner.lock().unwrap().get_cur_holder_commitment_number() } + pub(crate) fn get_counterparty_node_id(&self) -> Option { + self.inner.lock().unwrap().counterparty_node_id + } + /// Used by ChannelManager deserialization to broadcast the latest holder state if its copy of /// the Channel was out-of-date. You may use it to get a broadcastable holder toxic tx in case of /// fallen-behind, i.e when receiving a channel_reestablish with a proof that our counterparty side knows diff --git a/lightning/src/chain/mod.rs b/lightning/src/chain/mod.rs index 5959508de3b..a0eb17c6be0 100644 --- a/lightning/src/chain/mod.rs +++ b/lightning/src/chain/mod.rs @@ -15,6 +15,7 @@ use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::{Transaction, TxOut}; use bitcoin::hash_types::{BlockHash, Txid}; use bitcoin::network::constants::Network; +use bitcoin::secp256k1::PublicKey; use chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate, MonitorEvent}; use chain::keysinterface::Sign; @@ -302,7 +303,7 @@ pub trait Watch { /// /// For details on asynchronous [`ChannelMonitor`] updating and returning /// [`MonitorEvent::UpdateCompleted`] here, see [`ChannelMonitorUpdateErr::TemporaryFailure`]. - fn release_pending_monitor_events(&self) -> Vec<(OutPoint, Vec)>; + fn release_pending_monitor_events(&self) -> Vec<(OutPoint, Vec, Option)>; } /// The `Filter` trait defines behavior for indicating chain activity of interest pertaining to diff --git a/lightning/src/ln/chanmon_update_fail_tests.rs b/lightning/src/ln/chanmon_update_fail_tests.rs index 0d2b3b6a9f9..02ac7b48137 100644 --- a/lightning/src/ln/chanmon_update_fail_tests.rs +++ b/lightning/src/ln/chanmon_update_fail_tests.rs @@ -26,7 +26,7 @@ use ln::msgs; use ln::msgs::{ChannelMessageHandler, RoutingMessageHandler}; use util::config::UserConfig; use util::enforcing_trait_impls::EnforcingSigner; -use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason}; +use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason, HTLCDestination}; use util::errors::APIError; use util::ser::{ReadableArgs, Writeable}; use util::test_utils::TestBroadcaster; @@ -832,7 +832,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) { // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA nodes[2].node.fail_htlc_backwards(&payment_hash_1); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash: payment_hash_1 }]); check_added_monitors!(nodes[2], 1); let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id()); @@ -913,7 +913,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) { let (outpoint, latest_update, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2.2).unwrap().clone(); nodes[1].chain_monitor.chain_monitor.force_channel_monitor_updated(outpoint, latest_update); check_added_monitors!(nodes[1], 0); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); check_added_monitors!(nodes[1], 1); let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events(); @@ -1690,14 +1690,14 @@ fn test_monitor_update_on_pending_forwards() { let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]); let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs); let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known()); + let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known()); // Rebalance a bit so that we can send backwards from 3 to 1. send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000); let (_, payment_hash_1, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000); nodes[2].node.fail_htlc_backwards(&payment_hash_1); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash: payment_hash_1 }]); check_added_monitors!(nodes[2], 1); let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id()); @@ -1718,7 +1718,7 @@ fn test_monitor_update_on_pending_forwards() { commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false); chanmon_cfgs[1].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure)); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); check_added_monitors!(nodes[1], 1); assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty()); nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1); @@ -2106,7 +2106,7 @@ fn test_fail_htlc_on_broadcast_after_claim() { check_closed_broadcast!(nodes[1], true); connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1); check_added_monitors!(nodes[1], 1); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]); nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]); expect_payment_sent_without_paths!(nodes[0], payment_preimage); @@ -2469,7 +2469,7 @@ fn do_test_reconnect_dup_htlc_claims(htlc_status: HTLCStatusAtDupClaim, second_f }; if second_fails { nodes[2].node.fail_htlc_backwards(&payment_hash); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash }]); check_added_monitors!(nodes[2], 1); get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id()); } else { @@ -2505,7 +2505,7 @@ fn do_test_reconnect_dup_htlc_claims(htlc_status: HTLCStatusAtDupClaim, second_f if second_fails { reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (1, 0), (0, 0), (0, 0), (false, false)); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]); } else { reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (1, 0), (0, 0), (0, 0), (0, 0), (false, false)); } diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index f612cb97943..ed5bff63d47 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -5765,7 +5765,7 @@ impl Channel { /// those explicitly stated to be allowed after shutdown completes, eg some simple getters). /// Also returns the list of payment_hashes for channels which we can safely fail backwards /// immediately (others we will have to allow to time out). - pub fn force_shutdown(&mut self, should_broadcast: bool) -> (Option<(OutPoint, ChannelMonitorUpdate)>, Vec<(HTLCSource, PaymentHash)>) { + pub fn force_shutdown(&mut self, should_broadcast: bool) -> (Option<(OutPoint, ChannelMonitorUpdate)>, Vec<(HTLCSource, PaymentHash, PublicKey, [u8; 32])>) { // Note that we MUST only generate a monitor update that indicates force-closure - we're // called during initialization prior to the chain_monitor in the encompassing ChannelManager // being fully configured in some cases. Thus, its likely any monitor events we generate will @@ -5775,10 +5775,11 @@ impl Channel { // We go ahead and "free" any holding cell HTLCs or HTLCs we haven't yet committed to and // return them to fail the payment. let mut dropped_outbound_htlcs = Vec::with_capacity(self.holding_cell_htlc_updates.len()); + let counterparty_node_id = self.get_counterparty_node_id(); for htlc_update in self.holding_cell_htlc_updates.drain(..) { match htlc_update { HTLCUpdateAwaitingACK::AddHTLC { source, payment_hash, .. } => { - dropped_outbound_htlcs.push((source, payment_hash)); + dropped_outbound_htlcs.push((source, payment_hash, counterparty_node_id, self.channel_id)); }, _ => {} } diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index bae408d29f0..504ed00ab38 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -52,7 +52,7 @@ use ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, MAX_VALUE_MSA use ln::wire::Encode; use chain::keysinterface::{Sign, KeysInterface, KeysManager, InMemorySigner, Recipient}; use util::config::{UserConfig, ChannelConfig}; -use util::events::{EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason}; +use util::events::{EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination}; use util::{byte_utils, events}; use util::scid_utils::fake_scid; use util::ser::{BigSize, FixedLengthReader, Readable, ReadableArgs, MaybeReadable, Writeable, Writer, VecWriter}; @@ -281,7 +281,7 @@ enum ClaimFundsFromHop { DuplicateClaim, } -type ShutdownResult = (Option<(OutPoint, ChannelMonitorUpdate)>, Vec<(HTLCSource, PaymentHash)>); +type ShutdownResult = (Option<(OutPoint, ChannelMonitorUpdate)>, Vec<(HTLCSource, PaymentHash, PublicKey, [u8; 32])>); /// Error type returned across the channel_state mutex boundary. When an Err is generated for a /// Channel, we generally end up with a ChannelError::Close for which we have to close the channel @@ -1890,7 +1890,8 @@ impl ChannelMana }; for htlc_source in failed_htlcs.drain(..) { - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_source.0, &htlc_source.1, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }); + let receiver = HTLCDestination::NextHopChannel { node_id: Some(*counterparty_node_id), channel_id: *channel_id }; + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_source.0, &htlc_source.1, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver); } let _ = handle_error!(self, result, *counterparty_node_id); @@ -1946,7 +1947,9 @@ impl ChannelMana let (monitor_update_option, mut failed_htlcs) = shutdown_res; log_debug!(self.logger, "Finishing force-closure of channel with {} HTLCs to fail", failed_htlcs.len()); for htlc_source in failed_htlcs.drain(..) { - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_source.0, &htlc_source.1, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }); + let (source, payment_hash, counterparty_node_id, channel_id) = htlc_source; + let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id), channel_id: channel_id }; + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), source, &payment_hash, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver); } if let Some((funding_txo, monitor_update)) = monitor_update_option { // There isn't anything we can do if we get an update failure - we're already @@ -3107,21 +3110,42 @@ impl ChannelMana HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo { routing, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value }, prev_funding_outpoint } => { + macro_rules! failure_handler { + ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => { + log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg); + + let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData { + short_channel_id: prev_short_channel_id, + outpoint: prev_funding_outpoint, + htlc_id: prev_htlc_id, + incoming_packet_shared_secret: incoming_shared_secret, + phantom_shared_secret: $phantom_ss, + }); + + let reason = if $next_hop_unknown { + HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id } + } else { + HTLCDestination::FailedPayment{ payment_hash } + }; + + failed_forwards.push((htlc_source, payment_hash, + HTLCFailReason::Reason { failure_code: $err_code, data: $err_data }, + reason + )); + continue; + } + } macro_rules! fail_forward { ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => { { - log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg); - let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData { - short_channel_id: prev_short_channel_id, - outpoint: prev_funding_outpoint, - htlc_id: prev_htlc_id, - incoming_packet_shared_secret: incoming_shared_secret, - phantom_shared_secret: $phantom_ss, - }); - failed_forwards.push((htlc_source, payment_hash, - HTLCFailReason::Reason { failure_code: $err_code, data: $err_data } - )); - continue; + failure_handler!($msg, $err_code, $err_data, $phantom_ss, true); + } + } + } + macro_rules! failed_payment { + ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => { + { + failure_handler!($msg, $err_code, $err_data, $phantom_ss, false); } } } @@ -3137,17 +3161,17 @@ impl ChannelMana // `update_fail_malformed_htlc`, meaning here we encrypt the error as // if it came from us (the second-to-last hop) but contains the sha256 // of the onion. - fail_forward!(err_msg, err_code, sha256_of_onion.to_vec(), None); + failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None); }, Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => { - fail_forward!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret)); + failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret)); }, }; match next_hop { onion_utils::Hop::Receive(hop_data) => { match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value, Some(phantom_shared_secret)) { Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])), - Err(ReceiveError { err_code, err_data, msg }) => fail_forward!(msg, err_code, err_data, Some(phantom_shared_secret)) + Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret)) } }, _ => panic!(), @@ -3198,7 +3222,8 @@ impl ChannelMana } let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get()); failed_forwards.push((htlc_source, payment_hash, - HTLCFailReason::Reason { failure_code, data } + HTLCFailReason::Reason { failure_code, data }, + HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id } )); continue; }, @@ -3327,7 +3352,7 @@ impl ChannelMana }; macro_rules! fail_htlc { - ($htlc: expr) => { + ($htlc: expr, $payment_hash: expr) => { let mut htlc_msat_height_data = byte_utils::be64_to_array($htlc.value).to_vec(); htlc_msat_height_data.extend_from_slice( &byte_utils::be32_to_array(self.best_block.read().unwrap().height()), @@ -3339,7 +3364,8 @@ impl ChannelMana incoming_packet_shared_secret: $htlc.prev_hop.incoming_packet_shared_secret, phantom_shared_secret, }), payment_hash, - HTLCFailReason::Reason { failure_code: 0x4000 | 15, data: htlc_msat_height_data } + HTLCFailReason::Reason { failure_code: 0x4000 | 15, data: htlc_msat_height_data }, + HTLCDestination::FailedPayment { payment_hash: $payment_hash }, )); } } @@ -3358,7 +3384,7 @@ impl ChannelMana if htlcs.len() == 1 { if let OnionPayload::Spontaneous(_) = htlcs[0].onion_payload { log_trace!(self.logger, "Failing new HTLC with payment_hash {} as we already had an existing keysend HTLC with the same payment hash", log_bytes!(payment_hash.0)); - fail_htlc!(claimable_htlc); + fail_htlc!(claimable_htlc, payment_hash); continue } } @@ -3380,7 +3406,7 @@ impl ChannelMana if total_value >= msgs::MAX_VALUE_MSAT || total_value > $payment_data.total_msat { log_trace!(self.logger, "Failing HTLCs with payment_hash {} as the total value {} ran over expected value {} (or HTLCs were inconsistent)", log_bytes!(payment_hash.0), total_value, $payment_data.total_msat); - fail_htlc!(claimable_htlc); + fail_htlc!(claimable_htlc, payment_hash); } else if total_value == $payment_data.total_msat { htlcs.push(claimable_htlc); new_events.push(events::Event::PaymentReceived { @@ -3414,7 +3440,7 @@ impl ChannelMana let payment_preimage = match inbound_payment::verify(payment_hash, &payment_data, self.highest_seen_timestamp.load(Ordering::Acquire) as u64, &self.inbound_payment_key, &self.logger) { Ok(payment_preimage) => payment_preimage, Err(()) => { - fail_htlc!(claimable_htlc); + fail_htlc!(claimable_htlc, payment_hash); continue } }; @@ -3433,7 +3459,7 @@ impl ChannelMana }, hash_map::Entry::Occupied(_) => { log_trace!(self.logger, "Failing new keysend HTLC with payment_hash {} for a duplicative payment hash", log_bytes!(payment_hash.0)); - fail_htlc!(claimable_htlc); + fail_htlc!(claimable_htlc, payment_hash); } } } @@ -3442,17 +3468,17 @@ impl ChannelMana hash_map::Entry::Occupied(inbound_payment) => { if payment_data.is_none() { log_trace!(self.logger, "Failing new keysend HTLC with payment_hash {} because we already have an inbound payment with the same payment hash", log_bytes!(payment_hash.0)); - fail_htlc!(claimable_htlc); + fail_htlc!(claimable_htlc, payment_hash); continue }; let payment_data = payment_data.unwrap(); if inbound_payment.get().payment_secret != payment_data.payment_secret { log_trace!(self.logger, "Failing new HTLC with payment_hash {} as it didn't match our expected payment secret.", log_bytes!(payment_hash.0)); - fail_htlc!(claimable_htlc); + fail_htlc!(claimable_htlc, payment_hash); } else if inbound_payment.get().min_value_msat.is_some() && payment_data.total_msat < inbound_payment.get().min_value_msat.unwrap() { log_trace!(self.logger, "Failing new HTLC with payment_hash {} as it didn't match our minimum value (had {}, needed {}).", log_bytes!(payment_hash.0), payment_data.total_msat, inbound_payment.get().min_value_msat.unwrap()); - fail_htlc!(claimable_htlc); + fail_htlc!(claimable_htlc, payment_hash); } else { let payment_received_generated = check_total_value!(payment_data, inbound_payment.get().payment_preimage); if payment_received_generated { @@ -3471,8 +3497,8 @@ impl ChannelMana } } - for (htlc_source, payment_hash, failure_reason) in failed_forwards.drain(..) { - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_source, &payment_hash, failure_reason); + for (htlc_source, payment_hash, failure_reason, destination) in failed_forwards.drain(..) { + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_source, &payment_hash, failure_reason, destination); } self.forward_htlcs(&mut phantom_receives); @@ -3695,7 +3721,8 @@ impl ChannelMana } for htlc_source in timed_out_mpp_htlcs.drain(..) { - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), HTLCSource::PreviousHopData(htlc_source.0), &htlc_source.1, HTLCFailReason::Reason { failure_code: 23, data: Vec::new() }); + let receiver = HTLCDestination::FailedPayment { payment_hash: htlc_source.1 }; + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), HTLCSource::PreviousHopData(htlc_source.0.clone()), &htlc_source.1, HTLCFailReason::Reason { failure_code: 23, data: Vec::new() }, receiver ); } for (err, counterparty_node_id) in handle_errors.drain(..) { @@ -3731,7 +3758,8 @@ impl ChannelMana self.best_block.read().unwrap().height())); self.fail_htlc_backwards_internal(channel_state.take().unwrap(), HTLCSource::PreviousHopData(htlc.prev_hop), payment_hash, - HTLCFailReason::Reason { failure_code: 0x4000 | 15, data: htlc_msat_height_data }); + HTLCFailReason::Reason { failure_code: 0x4000 | 15, data: htlc_msat_height_data }, + HTLCDestination::FailedPayment { payment_hash: *payment_hash }); } } } @@ -3787,7 +3815,7 @@ impl ChannelMana // be surfaced to the user. fn fail_holding_cell_htlcs( &self, mut htlcs_to_fail: Vec<(HTLCSource, PaymentHash)>, channel_id: [u8; 32], - _counterparty_node_id: &PublicKey + counterparty_node_id: &PublicKey ) { for (htlc_src, payment_hash) in htlcs_to_fail.drain(..) { match htlc_src { @@ -3800,8 +3828,9 @@ impl ChannelMana hash_map::Entry::Vacant(_) => (0x4000|10, Vec::new()) }; let channel_state = self.channel_state.lock().unwrap(); - self.fail_htlc_backwards_internal(channel_state, - htlc_src, &payment_hash, HTLCFailReason::Reason { failure_code, data: onion_failure_data}); + + let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id.clone()), channel_id }; + self.fail_htlc_backwards_internal(channel_state, htlc_src, &payment_hash, HTLCFailReason::Reason { failure_code, data: onion_failure_data }, receiver) }, HTLCSource::OutboundRoute { session_priv, payment_id, path, payment_params, .. } => { let mut session_priv_bytes = [0; 32]; @@ -3854,7 +3883,7 @@ impl ChannelMana /// to fail and take the channel_state lock for each iteration (as we take ownership and may /// drop it). In other words, no assumptions are made that entries in claimable_htlcs point to /// still-available channels. - fn fail_htlc_backwards_internal(&self, mut channel_state_lock: MutexGuard>, source: HTLCSource, payment_hash: &PaymentHash, onion_error: HTLCFailReason) { + fn fail_htlc_backwards_internal(&self, mut channel_state_lock: MutexGuard>, source: HTLCSource, payment_hash: &PaymentHash, onion_error: HTLCFailReason, destination: HTLCDestination) { //TODO: There is a timing attack here where if a node fails an HTLC back to us they can //identify whether we sent it or not based on the (I presume) very different runtime //between the branches here. We should make this async and move it into the forward HTLCs @@ -3979,7 +4008,7 @@ impl ChannelMana pending_events.push(path_failure); if let Some(ev) = full_failure_ev { pending_events.push(ev); } }, - HTLCSource::PreviousHopData(HTLCPreviousHopData { short_channel_id, htlc_id, incoming_packet_shared_secret, phantom_shared_secret, .. }) => { + HTLCSource::PreviousHopData(HTLCPreviousHopData { short_channel_id, htlc_id, incoming_packet_shared_secret, phantom_shared_secret, outpoint }) => { let err_packet = match onion_error { HTLCFailReason::Reason { failure_code, data } => { log_trace!(self.logger, "Failing HTLC with payment_hash {} backwards from us with code {}", log_bytes!(payment_hash.0), failure_code); @@ -4011,12 +4040,16 @@ impl ChannelMana } } mem::drop(channel_state_lock); + let mut pending_events = self.pending_events.lock().unwrap(); if let Some(time) = forward_event { - let mut pending_events = self.pending_events.lock().unwrap(); pending_events.push(events::Event::PendingHTLCsForwardable { time_forwardable: time }); } + pending_events.push(events::Event::HTLCHandlingFailed { + prev_channel_id: outpoint.to_channel_id(), + failed_next_destination: destination + }); }, } } @@ -4108,7 +4141,8 @@ impl ChannelMana self.best_block.read().unwrap().height())); self.fail_htlc_backwards_internal(channel_state.take().unwrap(), HTLCSource::PreviousHopData(htlc.prev_hop), &payment_hash, - HTLCFailReason::Reason { failure_code: 0x4000|15, data: htlc_msat_height_data }); + HTLCFailReason::Reason { failure_code: 0x4000|15, data: htlc_msat_height_data }, + HTLCDestination::FailedPayment { payment_hash } ); } else { match self.claim_funds_from_hop(channel_state.as_mut().unwrap(), htlc.prev_hop, payment_preimage) { ClaimFundsFromHop::MonitorUpdateFail(pk, err, _) => { @@ -4352,7 +4386,7 @@ impl ChannelMana let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier); let chan_restoration_res; - let (mut pending_failures, finalized_claims) = { + let (mut pending_failures, finalized_claims, counterparty_node_id) = { let mut channel_lock = self.channel_state.lock().unwrap(); let channel_state = &mut *channel_lock; let mut channel = match channel_state.by_id.entry(funding_txo.to_channel_id()) { @@ -4363,6 +4397,7 @@ impl ChannelMana return; } + let counterparty_node_id = channel.get().get_counterparty_node_id(); let updates = channel.get_mut().monitor_updating_restored(&self.logger, self.get_our_node_id(), self.genesis_hash, self.best_block.read().unwrap().height()); let channel_update = if updates.channel_ready.is_some() && channel.get().is_usable() { // We only send a channel_update in the case where we are just now sending a @@ -4381,12 +4416,14 @@ impl ChannelMana if let Some(upd) = channel_update { channel_state.pending_msg_events.push(upd); } - (updates.failed_htlcs, updates.finalized_claimed_htlcs) + + (updates.failed_htlcs, updates.finalized_claimed_htlcs, counterparty_node_id) }; post_handle_chan_restoration!(self, chan_restoration_res); self.finalize_claims(finalized_claims); for failure in pending_failures.drain(..) { - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), failure.0, &failure.1, failure.2); + let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id), channel_id: funding_txo.to_channel_id() }; + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), failure.0, &failure.1, failure.2, receiver); } } @@ -4745,7 +4782,8 @@ impl ChannelMana } }; for htlc_source in dropped_htlcs.drain(..) { - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_source.0, &htlc_source.1, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }); + let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id.clone()), channel_id: msg.channel_id }; + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_source.0, &htlc_source.1, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver); } let _ = handle_error!(self, result, *counterparty_node_id); @@ -5031,7 +5069,8 @@ impl ChannelMana short_channel_id, channel_outpoint)) => { for failure in pending_failures.drain(..) { - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), failure.0, &failure.1, failure.2); + let receiver = HTLCDestination::NextHopChannel { node_id: Some(*counterparty_node_id), channel_id: channel_outpoint.to_channel_id() }; + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), failure.0, &failure.1, failure.2, receiver); } self.forward_htlcs(&mut [(short_channel_id, channel_outpoint, pending_forwards)]); self.finalize_claims(finalized_claim_htlcs); @@ -5178,7 +5217,7 @@ impl ChannelMana let mut failed_channels = Vec::new(); let mut pending_monitor_events = self.chain_monitor.release_pending_monitor_events(); let has_pending_monitor_events = !pending_monitor_events.is_empty(); - for (funding_outpoint, mut monitor_events) in pending_monitor_events.drain(..) { + for (funding_outpoint, mut monitor_events, counterparty_node_id) in pending_monitor_events.drain(..) { for monitor_event in monitor_events.drain(..) { match monitor_event { MonitorEvent::HTLCEvent(htlc_update) => { @@ -5187,7 +5226,8 @@ impl ChannelMana self.claim_funds_internal(self.channel_state.lock().unwrap(), htlc_update.source, preimage, htlc_update.htlc_value_satoshis.map(|v| v * 1000), true, funding_outpoint.to_channel_id()); } else { log_trace!(self.logger, "Failing HTLC with hash {} from our monitor", log_bytes!(htlc_update.payment_hash.0)); - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_update.source, &htlc_update.payment_hash, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }); + let receiver = HTLCDestination::NextHopChannel { node_id: counterparty_node_id, channel_id: funding_outpoint.to_channel_id() }; + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_update.source, &htlc_update.payment_hash, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver); } }, MonitorEvent::CommitmentTxConfirmed(funding_outpoint) | @@ -5829,7 +5869,7 @@ where let (failure_code, data) = self.get_htlc_inbound_temp_fail_err_and_data(0x1000|14 /* expiry_too_soon */, &channel); timed_out_htlcs.push((source, payment_hash, HTLCFailReason::Reason { failure_code, data, - })); + }, HTLCDestination::NextHopChannel { node_id: Some(channel.get_counterparty_node_id()), channel_id: channel.channel_id() })); } if let Some(channel_ready) = channel_ready_opt { send_channel_ready!(short_to_chan_info, pending_msg_events, channel, channel_ready); @@ -5910,10 +5950,11 @@ where if height >= htlc.cltv_expiry - HTLC_FAIL_BACK_BUFFER { let mut htlc_msat_height_data = byte_utils::be64_to_array(htlc.value).to_vec(); htlc_msat_height_data.extend_from_slice(&byte_utils::be32_to_array(height)); + timed_out_htlcs.push((HTLCSource::PreviousHopData(htlc.prev_hop.clone()), payment_hash.clone(), HTLCFailReason::Reason { failure_code: 0x4000 | 15, data: htlc_msat_height_data - })); + }, HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() })); false } else { true } }); @@ -5924,8 +5965,8 @@ where self.handle_init_event_channel_failures(failed_channels); - for (source, payment_hash, reason) in timed_out_htlcs.drain(..) { - self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), source, &payment_hash, reason); + for (source, payment_hash, reason, destination) in timed_out_htlcs.drain(..) { + self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), source, &payment_hash, reason, destination); } } @@ -7281,7 +7322,9 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> }; for htlc_source in failed_htlcs.drain(..) { - channel_manager.fail_htlc_backwards_internal(channel_manager.channel_state.lock().unwrap(), htlc_source.0, &htlc_source.1, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }); + let (source, payment_hash, counterparty_node_id, channel_id) = htlc_source; + let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id), channel_id }; + channel_manager.fail_htlc_backwards_internal(channel_manager.channel_state.lock().unwrap(), source, &payment_hash, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver); } //TODO: Broadcast channel update for closed channels, but only after we've made a @@ -7306,7 +7349,7 @@ mod tests { use ln::msgs::ChannelMessageHandler; use routing::router::{PaymentParameters, RouteParameters, find_route}; use util::errors::APIError; - use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason}; + use util::events::{Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, ClosureReason}; use util::test_utils; use chain::keysinterface::KeysInterface; @@ -7469,7 +7512,7 @@ mod tests { check_added_monitors!(nodes[1], 0); commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false); expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }]); check_added_monitors!(nodes[1], 1); let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); assert!(updates.update_add_htlcs.is_empty()); @@ -7589,8 +7632,10 @@ mod tests { nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]); check_added_monitors!(nodes[1], 0); commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false); + // We have to forward pending HTLCs twice - once tries to forward the payment forward (and + // fails), the second will process the resulting failure and fail the HTLC backward expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]); check_added_monitors!(nodes[1], 1); let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); assert!(updates.update_add_htlcs.is_empty()); @@ -7631,7 +7676,7 @@ mod tests { check_added_monitors!(nodes[1], 0); commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false); expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]); check_added_monitors!(nodes[1], 1); let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); assert!(updates.update_add_htlcs.is_empty()); diff --git a/lightning/src/ln/functional_test_utils.rs b/lightning/src/ln/functional_test_utils.rs index ee34b590f47..38a593141f4 100644 --- a/lightning/src/ln/functional_test_utils.rs +++ b/lightning/src/ln/functional_test_utils.rs @@ -46,6 +46,7 @@ use core::cell::RefCell; use alloc::rc::Rc; use sync::{Arc, Mutex}; use core::mem; +use core::iter::repeat; pub const CHAN_CONFIRM_DEPTH: u32 = 10; @@ -1187,7 +1188,7 @@ macro_rules! commitment_signed_dance { { commitment_signed_dance!($node_a, $node_b, $commitment_signed, $fail_backwards, true); if $fail_backwards { - $crate::expect_pending_htlcs_forwardable!($node_a); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!($node_a, vec![$crate::util::events::HTLCDestination::NextHopChannel{ node_id: Some($node_b.node.get_our_node_id()), channel_id: $commitment_signed.channel_id }]); check_added_monitors!($node_a, 1); let channel_state = $node_a.node.channel_state.lock().unwrap(); @@ -1254,55 +1255,33 @@ macro_rules! get_route_and_payment_hash { }} } -pub struct HTLCHandlingFailedConditions { - pub expected_destinations: Vec, -} - -impl HTLCHandlingFailedConditions { - pub fn new() -> Self { - Self { - expected_destinations: vec![], - } - } - - pub fn with_reason(mut self, reason: HTLCDestination) -> Self { - self.expected_destinations = vec![reason]; - self - } - - pub fn with_reasons(mut self, reasons: Vec) -> Self { - self.expected_destinations = reasons; - self - } -} - #[macro_export] macro_rules! expect_pending_htlcs_forwardable_conditions { - ($node: expr, $conditions: expr) => {{ - let conditions = $conditions; + ($node: expr, $expected_failures: expr) => {{ + let expected_failures = $expected_failures; let events = $node.node.get_and_clear_pending_events(); match events[0] { $crate::util::events::Event::PendingHTLCsForwardable { .. } => { }, _ => panic!("Unexpected event"), }; - let count = conditions.expected_destinations.len() + 1; + let count = expected_failures.len() + 1; assert_eq!(events.len(), count); - if conditions.expected_destinations.len() > 0 { - expect_htlc_handling_failed_destinations!(events, conditions.expected_destinations) + if expected_failures.len() > 0 { + expect_htlc_handling_failed_destinations!(events, expected_failures) } }} } #[macro_export] macro_rules! expect_htlc_handling_failed_destinations { - ($events: expr, $destinations: expr) => {{ + ($events: expr, $expected_failures: expr) => {{ for event in $events { match event { $crate::util::events::Event::PendingHTLCsForwardable { .. } => { }, $crate::util::events::Event::HTLCHandlingFailed { ref failed_next_destination, .. } => { - assert!($destinations.contains(&failed_next_destination)) + assert!($expected_failures.contains(&failed_next_destination)) }, _ => panic!("Unexpected destination"), } @@ -1314,15 +1293,15 @@ macro_rules! expect_htlc_handling_failed_destinations { /// Clears (and ignores) a PendingHTLCsForwardable event macro_rules! expect_pending_htlcs_forwardable_ignore { ($node: expr) => {{ - expect_pending_htlcs_forwardable_conditions!($node, $crate::ln::functional_test_utils::HTLCHandlingFailedConditions::new()); + expect_pending_htlcs_forwardable_conditions!($node, vec![]); }}; } #[macro_export] /// Clears (and ignores) PendingHTLCsForwardable and HTLCHandlingFailed events macro_rules! expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore { - ($node: expr, $conditions: expr) => {{ - expect_pending_htlcs_forwardable_conditions!($node, $conditions); + ($node: expr, $expected_failures: expr) => {{ + expect_pending_htlcs_forwardable_conditions!($node, $expected_failures); }}; } @@ -1341,8 +1320,8 @@ macro_rules! expect_pending_htlcs_forwardable { #[macro_export] /// Handles a PendingHTLCsForwardable and HTLCHandlingFailed event macro_rules! expect_pending_htlcs_forwardable_and_htlc_handling_failed { - ($node: expr, $conditions: expr) => {{ - expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!($node, $conditions); + ($node: expr, $expected_failures: expr) => {{ + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!($node, $expected_failures); $node.node.process_pending_htlc_forwards(); // Ensure process_pending_htlc_forwards is idempotent. @@ -1884,7 +1863,8 @@ pub fn fail_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe assert_eq!(path.last().unwrap().node.get_our_node_id(), expected_paths[0].last().unwrap().node.get_our_node_id()); } expected_paths[0].last().unwrap().node.fail_htlc_backwards(&our_payment_hash); - expect_pending_htlcs_forwardable!(expected_paths[0].last().unwrap()); + let expected_destinations: Vec = repeat(HTLCDestination::FailedPayment { payment_hash: our_payment_hash }).take(expected_paths.len()).collect(); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(expected_paths[0].last().unwrap(), expected_destinations); pass_failed_payment_back(origin_node, expected_paths, skip_last, our_payment_hash); } @@ -1925,7 +1905,7 @@ pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe node.node.handle_update_fail_htlc(&prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0); commitment_signed_dance!(node, prev_node, next_msgs.as_ref().unwrap().1, update_next_node); if !update_next_node { - expect_pending_htlcs_forwardable!(node); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(node, vec![HTLCDestination::NextHopChannel { node_id: Some(prev_node.node.get_our_node_id()), channel_id: next_msgs.as_ref().unwrap().0.channel_id }]); } } let events = node.node.get_and_clear_pending_msg_events(); diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index 4894ed02b3b..8e7b3e8fbbd 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -31,7 +31,7 @@ use ln::msgs; use ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, OptionalField, ErrorAction}; use util::enforcing_trait_impls::EnforcingSigner; use util::{byte_utils, test_utils}; -use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason}; +use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason, HTLCDestination}; use util::errors::APIError; use util::ser::{Writeable, ReadableArgs}; use util::config::UserConfig; @@ -54,6 +54,7 @@ use io; use prelude::*; use alloc::collections::BTreeSet; use core::default::Default; +use core::iter::repeat; use sync::{Arc, Mutex}; use ln::functional_test_utils::*; @@ -1150,7 +1151,7 @@ fn holding_cell_htlc_counting() { // We have to forward pending HTLCs twice - once tries to forward the payment forward (and // fails), the second will process the resulting failure and fail the HTLC backward. expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); check_added_monitors!(nodes[1], 1); let bs_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); @@ -2616,7 +2617,7 @@ fn claim_htlc_outputs_single_tx() { check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed); let mut events = nodes[0].node.get_and_clear_pending_events(); expect_pending_htlcs_forwardable_from_events!(nodes[0], events[0..1], true); - match events[1] { + match events.last().unwrap() { Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {} _ => panic!("Unexpected event"), } @@ -2928,7 +2929,7 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) { check_spends!(commitment_tx[0], chan_2.3); nodes[2].node.fail_htlc_backwards(&payment_hash); check_added_monitors!(nodes[2], 0); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() }]); check_added_monitors!(nodes[2], 1); let events = nodes[2].node.get_and_clear_pending_msg_events(); @@ -3000,7 +3001,7 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) { } } - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); check_added_monitors!(nodes[1], 1); let events = nodes[1].node.get_and_clear_pending_msg_events(); assert_eq!(events.len(), 1); @@ -3068,7 +3069,7 @@ fn test_simple_commitment_revoked_fail_backward() { check_added_monitors!(nodes[1], 1); check_closed_broadcast!(nodes[1], true); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); check_added_monitors!(nodes[1], 1); let events = nodes[1].node.get_and_clear_pending_msg_events(); assert_eq!(events.len(), 1); @@ -3131,7 +3132,7 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use let (_, third_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], value); nodes[2].node.fail_htlc_backwards(&first_payment_hash); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]); check_added_monitors!(nodes[2], 1); let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id()); assert!(updates.update_add_htlcs.is_empty()); @@ -3144,7 +3145,7 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use // Drop the last RAA from 3 -> 2 nodes[2].node.fail_htlc_backwards(&second_payment_hash); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash: second_payment_hash }]); check_added_monitors!(nodes[2], 1); let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id()); assert!(updates.update_add_htlcs.is_empty()); @@ -3161,7 +3162,7 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use check_added_monitors!(nodes[2], 1); nodes[2].node.fail_htlc_backwards(&third_payment_hash); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash: third_payment_hash }]); check_added_monitors!(nodes[2], 1); let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id()); assert!(updates.update_add_htlcs.is_empty()); @@ -3193,11 +3194,15 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use // commitment transaction for nodes[0] until process_pending_htlc_forwards(). check_added_monitors!(nodes[1], 1); let events = nodes[1].node.get_and_clear_pending_events(); - assert_eq!(events.len(), 1); + assert_eq!(events.len(), 2); match events[0] { Event::PendingHTLCsForwardable { .. } => { }, _ => panic!("Unexpected event"), }; + match events[1] { + Event::HTLCHandlingFailed { .. } => { }, + _ => panic!("Unexpected event"), + } // Deliberately don't process the pending fail-back so they all fail back at once after // block connection just like the !deliver_bs_raa case } @@ -3211,7 +3216,7 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use assert!(ANTI_REORG_DELAY > PAYMENT_EXPIRY_BLOCKS); // We assume payments will also expire let events = nodes[1].node.get_and_clear_pending_events(); - assert_eq!(events.len(), if deliver_bs_raa { 2 } else { 4 }); + assert_eq!(events.len(), if deliver_bs_raa { 2 + (nodes.len() - 1) } else { 4 + nodes.len() }); match events[0] { Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => { }, _ => panic!("Unexepected event"), @@ -4285,7 +4290,7 @@ fn do_test_htlc_timeout(send_partial_mpp: bool) { connect_block(&nodes[1], &block); } - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }]); check_added_monitors!(nodes[1], 1); let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); @@ -4349,7 +4354,7 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) { connect_blocks(&nodes[1], 1); if forwarded_htlc { - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); check_added_monitors!(nodes[1], 1); let fail_commit = nodes[1].node.get_and_clear_pending_msg_events(); assert_eq!(fail_commit.len(), 1); @@ -5397,7 +5402,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() { mine_transaction(&nodes[1], &htlc_timeout_tx); connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); assert!(htlc_updates.update_add_htlcs.is_empty()); assert_eq!(htlc_updates.update_fail_htlcs.len(), 1); @@ -5517,18 +5522,18 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno &[Some(config.clone()), Some(config.clone()), Some(config.clone()), Some(config.clone()), Some(config.clone()), Some(config.clone())]); let nodes = create_network(6, &node_cfgs, &node_chanmgrs); - create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known()); - create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known()); - let chan = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()); - create_announced_chan_between_nodes(&nodes, 3, 4, InitFeatures::known(), InitFeatures::known()); - create_announced_chan_between_nodes(&nodes, 3, 5, InitFeatures::known(), InitFeatures::known()); + let _chan_0_2 = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known()); + let _chan_1_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known()); + let chan_2_3 = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()); + let chan_3_4 = create_announced_chan_between_nodes(&nodes, 3, 4, InitFeatures::known(), InitFeatures::known()); + let chan_3_5 = create_announced_chan_between_nodes(&nodes, 3, 5, InitFeatures::known(), InitFeatures::known()); // Rebalance and check output sanity... send_payment(&nodes[0], &[&nodes[2], &nodes[3], &nodes[4]], 500000); send_payment(&nodes[1], &[&nodes[2], &nodes[3], &nodes[5]], 500000); - assert_eq!(get_local_commitment_txn!(nodes[3], chan.2)[0].output.len(), 2); + assert_eq!(get_local_commitment_txn!(nodes[3], chan_2_3.2)[0].output.len(), 2); - let ds_dust_limit = nodes[3].node.channel_state.lock().unwrap().by_id.get(&chan.2).unwrap().holder_dust_limit_satoshis; + let ds_dust_limit = nodes[3].node.channel_state.lock().unwrap().by_id.get(&chan_2_3.2).unwrap().holder_dust_limit_satoshis; // 0th HTLC: let (_, payment_hash_1, _) = route_payment(&nodes[0], &[&nodes[2], &nodes[3], &nodes[4]], ds_dust_limit*1000); // not added < dust limit + HTLC tx fee // 1st HTLC: @@ -5563,8 +5568,8 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno // Double-check that six of the new HTLC were added // We now have six HTLCs pending over the dust limit and six HTLCs under the dust limit (ie, // with to_local and to_remote outputs, 8 outputs and 6 HTLCs not included). - assert_eq!(get_local_commitment_txn!(nodes[3], chan.2).len(), 1); - assert_eq!(get_local_commitment_txn!(nodes[3], chan.2)[0].output.len(), 8); + assert_eq!(get_local_commitment_txn!(nodes[3], chan_2_3.2).len(), 1); + assert_eq!(get_local_commitment_txn!(nodes[3], chan_2_3.2)[0].output.len(), 8); // Now fail back three of the over-dust-limit and three of the under-dust-limit payments in one go. // Fail 0th below-dust, 4th above-dust, 8th above-dust, 10th below-dust HTLCs @@ -5573,7 +5578,14 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno nodes[4].node.fail_htlc_backwards(&payment_hash_5); nodes[4].node.fail_htlc_backwards(&payment_hash_6); check_added_monitors!(nodes[4], 0); - expect_pending_htlcs_forwardable!(nodes[4]); + + let failed_destinations = vec![ + HTLCDestination::FailedPayment { payment_hash: payment_hash_1 }, + HTLCDestination::FailedPayment { payment_hash: payment_hash_3 }, + HTLCDestination::FailedPayment { payment_hash: payment_hash_5 }, + HTLCDestination::FailedPayment { payment_hash: payment_hash_6 }, + ]; + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[4], failed_destinations); check_added_monitors!(nodes[4], 1); let four_removes = get_htlc_update_msgs!(nodes[4], nodes[3].node.get_our_node_id()); @@ -5587,7 +5599,12 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno nodes[5].node.fail_htlc_backwards(&payment_hash_2); nodes[5].node.fail_htlc_backwards(&payment_hash_4); check_added_monitors!(nodes[5], 0); - expect_pending_htlcs_forwardable!(nodes[5]); + + let failed_destinations_2 = vec![ + HTLCDestination::FailedPayment { payment_hash: payment_hash_2 }, + HTLCDestination::FailedPayment { payment_hash: payment_hash_4 }, + ]; + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[5], failed_destinations_2); check_added_monitors!(nodes[5], 1); let two_removes = get_htlc_update_msgs!(nodes[5], nodes[3].node.get_our_node_id()); @@ -5595,9 +5612,18 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno nodes[3].node.handle_update_fail_htlc(&nodes[5].node.get_our_node_id(), &two_removes.update_fail_htlcs[1]); commitment_signed_dance!(nodes[3], nodes[5], two_removes.commitment_signed, false); - let ds_prev_commitment_tx = get_local_commitment_txn!(nodes[3], chan.2); - - expect_pending_htlcs_forwardable!(nodes[3]); + let ds_prev_commitment_tx = get_local_commitment_txn!(nodes[3], chan_2_3.2); + + // After 4 and 2 removes respectively above in nodes[4] and nodes[5], nodes[3] should receive 6 PaymentForwardedFailed events + let failed_destinations_3 = vec![ + HTLCDestination::NextHopChannel { node_id: Some(nodes[4].node.get_our_node_id()), channel_id: chan_3_4.2 }, + HTLCDestination::NextHopChannel { node_id: Some(nodes[4].node.get_our_node_id()), channel_id: chan_3_4.2 }, + HTLCDestination::NextHopChannel { node_id: Some(nodes[4].node.get_our_node_id()), channel_id: chan_3_4.2 }, + HTLCDestination::NextHopChannel { node_id: Some(nodes[4].node.get_our_node_id()), channel_id: chan_3_4.2 }, + HTLCDestination::NextHopChannel { node_id: Some(nodes[5].node.get_our_node_id()), channel_id: chan_3_5.2 }, + HTLCDestination::NextHopChannel { node_id: Some(nodes[5].node.get_our_node_id()), channel_id: chan_3_5.2 }, + ]; + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], failed_destinations_3); check_added_monitors!(nodes[3], 1); let six_removes = get_htlc_update_msgs!(nodes[3], nodes[2].node.get_our_node_id()); nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &six_removes.update_fail_htlcs[0]); @@ -5623,7 +5649,7 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno // // Alternatively, we may broadcast the previous commitment transaction, which should only // result in failures for the below-dust HTLCs, ie the 0th, 1st, 2nd, 3rd, 9th, and 10th HTLCs. - let ds_last_commitment_tx = get_local_commitment_txn!(nodes[3], chan.2); + let ds_last_commitment_tx = get_local_commitment_txn!(nodes[3], chan_2_3.2); if announce_latest { mine_transaction(&nodes[2], &ds_last_commitment_tx[0]); @@ -5632,11 +5658,11 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno } let events = nodes[2].node.get_and_clear_pending_events(); let close_event = if deliver_last_raa { - assert_eq!(events.len(), 2); - events[1].clone() + assert_eq!(events.len(), 2 + 6); + events.last().clone().unwrap() } else { assert_eq!(events.len(), 1); - events[0].clone() + events.last().clone().unwrap() }; match close_event { Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {} @@ -5647,8 +5673,17 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno check_closed_broadcast!(nodes[2], true); if deliver_last_raa { expect_pending_htlcs_forwardable_from_events!(nodes[2], events[0..1], true); + + let expected_destinations: Vec = repeat(HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_2_3.2 }).take(3).collect(); + expect_htlc_handling_failed_destinations!(nodes[2].node.get_and_clear_pending_events(), expected_destinations); } else { - expect_pending_htlcs_forwardable!(nodes[2]); + let expected_destinations: Vec = if announce_latest { + repeat(HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_2_3.2 }).take(9).collect() + } else { + repeat(HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_2_3.2 }).take(6).collect() + }; + + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], expected_destinations); } check_added_monitors!(nodes[2], 3); @@ -6012,7 +6047,7 @@ fn do_htlc_claim_previous_remote_commitment_only(use_dust: bool, check_revoke_no let htlc_value = if use_dust { 50000 } else { 3000000 }; let (_, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], htlc_value); nodes[1].node.fail_htlc_backwards(&our_payment_hash); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }]); check_added_monitors!(nodes[1], 1); let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); @@ -6472,7 +6507,7 @@ fn test_fail_holding_cell_htlc_upon_free_multihop() { // nodes[1]'s ChannelManager will now signal that we have HTLC forwards to process. let process_htlc_forwards_event = nodes[1].node.get_and_clear_pending_events(); - assert_eq!(process_htlc_forwards_event.len(), 1); + assert_eq!(process_htlc_forwards_event.len(), 2); match &process_htlc_forwards_event[0] { &Event::PendingHTLCsForwardable { .. } => {}, _ => panic!("Unexpected event"), @@ -7097,7 +7132,7 @@ fn test_update_fulfill_htlc_bolt2_after_malformed_htlc_message_must_forward_upda let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]); let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs); create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000, InitFeatures::known(), InitFeatures::known()); - create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 1000000, 1000000, InitFeatures::known(), InitFeatures::known()); + let chan_2 = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 1000000, 1000000, InitFeatures::known(), InitFeatures::known()); let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100000); @@ -7145,7 +7180,7 @@ fn test_update_fulfill_htlc_bolt2_after_malformed_htlc_message_must_forward_upda check_added_monitors!(nodes[1], 0); commitment_signed_dance!(nodes[1], nodes[2], update_msg.1, false, true); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); let events_4 = nodes[1].node.get_and_clear_pending_msg_events(); assert_eq!(events_4.len(), 1); @@ -7189,7 +7224,7 @@ fn do_test_failure_delay_dust_htlc_local_commitment(announce_latest: bool) { // Fail one HTLC to prune it in the will-be-latest-local commitment tx nodes[1].node.fail_htlc_backwards(&payment_hash_2); check_added_monitors!(nodes[1], 0); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash: payment_hash_2 }]); check_added_monitors!(nodes[1], 1); let remove = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); @@ -7590,7 +7625,7 @@ fn test_check_htlc_underpaying() { // Note that we first have to wait a random delay before processing the receipt of the HTLC, // and then will wait a second random delay before failing the HTLC back: expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }]); // Node 3 is expecting payment of 100_000 but received 10_000, // it should fail htlc like we didn't know the preimage. @@ -7868,7 +7903,7 @@ fn test_bump_penalty_txn_on_revoked_htlcs() { connect_block(&nodes[0], &Block { header: header_129, txdata: vec![revoked_htlc_txn[0].clone(), revoked_htlc_txn[2].clone()] }); let events = nodes[0].node.get_and_clear_pending_events(); expect_pending_htlcs_forwardable_from_events!(nodes[0], events[0..1], true); - match events[1] { + match events.last().unwrap() { Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {} _ => panic!("Unexpected event"), } @@ -8144,19 +8179,19 @@ fn test_bump_txn_sanitize_tracking_maps() { let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 59000000, InitFeatures::known(), InitFeatures::known()); // Lock HTLC in both directions - let payment_preimage = route_payment(&nodes[0], &vec!(&nodes[1])[..], 9_000_000).0; - route_payment(&nodes[1], &vec!(&nodes[0])[..], 9_000_000).0; + let (payment_preimage_1, _, _) = route_payment(&nodes[0], &vec!(&nodes[1])[..], 9_000_000); + let (_, payment_hash_2, _) = route_payment(&nodes[1], &vec!(&nodes[0])[..], 9_000_000); let revoked_local_txn = get_local_commitment_txn!(nodes[1], chan.2); assert_eq!(revoked_local_txn[0].input.len(), 1); assert_eq!(revoked_local_txn[0].input[0].previous_output.txid, chan.3.txid()); // Revoke local commitment tx - claim_payment(&nodes[0], &vec!(&nodes[1])[..], payment_preimage); + claim_payment(&nodes[0], &vec!(&nodes[1])[..], payment_preimage_1); // Broadcast set of revoked txn on A connect_blocks(&nodes[0], TEST_FINAL_CLTV + 2 - CHAN_CONFIRM_DEPTH); - expect_pending_htlcs_forwardable_ignore!(nodes[0]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[0], vec![HTLCDestination::FailedPayment { payment_hash: payment_hash_2 }]); assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 0); mine_transaction(&nodes[0], &revoked_local_txn[0]); @@ -8709,7 +8744,7 @@ fn test_bad_secret_hash() { // All the below cases should end up being handled exactly identically, so we macro the // resulting events. macro_rules! handle_unknown_invalid_payment_data { - () => { + ($payment_hash: expr) => { check_added_monitors!(nodes[0], 1); let mut events = nodes[0].node.get_and_clear_pending_msg_events(); let payment_event = SendEvent::from_event(events.pop().unwrap()); @@ -8719,7 +8754,7 @@ fn test_bad_secret_hash() { // We have to forward pending HTLCs once to process the receipt of the HTLC and then // again to process the pending backwards-failure of the HTLC expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment{ payment_hash: $payment_hash }]); check_added_monitors!(nodes[1], 1); // We should fail the payment back @@ -8740,17 +8775,17 @@ fn test_bad_secret_hash() { // Send a payment with the right payment hash but the wrong payment secret nodes[0].node.send_payment(&route, our_payment_hash, &Some(random_payment_secret)).unwrap(); - handle_unknown_invalid_payment_data!(); + handle_unknown_invalid_payment_data!(our_payment_hash); expect_payment_failed!(nodes[0], our_payment_hash, true, expected_error_code, expected_error_data); // Send a payment with a random payment hash, but the right payment secret nodes[0].node.send_payment(&route, random_payment_hash, &Some(our_payment_secret)).unwrap(); - handle_unknown_invalid_payment_data!(); + handle_unknown_invalid_payment_data!(random_payment_hash); expect_payment_failed!(nodes[0], random_payment_hash, true, expected_error_code, expected_error_data); // Send a payment with a random payment hash and random payment secret nodes[0].node.send_payment(&route, random_payment_hash, &Some(random_payment_secret)).unwrap(); - handle_unknown_invalid_payment_data!(); + handle_unknown_invalid_payment_data!(random_payment_hash); expect_payment_failed!(nodes[0], random_payment_hash, true, expected_error_code, expected_error_data); } @@ -9576,7 +9611,7 @@ fn do_test_tx_confirmed_skipping_blocks_immediate_broadcast(test_height_before_t // additional block built on top of the current chain. nodes[1].chain_monitor.chain_monitor.transactions_confirmed( &nodes[1].get_block_header(conf_height + 1), &[(0, &spending_txn[1])], conf_height + 1); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: channel_id }]); check_added_monitors!(nodes[1], 1); let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); @@ -9759,7 +9794,11 @@ fn do_test_dup_htlc_second_rejected(test_for_second_fail_panic: bool) { // Now we go fail back the first HTLC from the user end. nodes[1].node.fail_htlc_backwards(&our_payment_hash); - expect_pending_htlcs_forwardable_ignore!(nodes[1]); + let expected_destinations = vec![ + HTLCDestination::FailedPayment { payment_hash: our_payment_hash }, + HTLCDestination::FailedPayment { payment_hash: our_payment_hash }, + ]; + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], expected_destinations); nodes[1].node.process_pending_htlc_forwards(); check_added_monitors!(nodes[1], 1); @@ -9776,7 +9815,7 @@ fn do_test_dup_htlc_second_rejected(test_for_second_fail_panic: bool) { if let Event::PaymentPathFailed { .. } = failure_events[1] {} else { panic!(); } } else { // Let the second HTLC fail and claim the first - expect_pending_htlcs_forwardable_ignore!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }]); nodes[1].node.process_pending_htlc_forwards(); check_added_monitors!(nodes[1], 1); @@ -9818,7 +9857,7 @@ fn test_inconsistent_mpp_params() { create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0, InitFeatures::known(), InitFeatures::known()); create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0, InitFeatures::known(), InitFeatures::known()); create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0, InitFeatures::known(), InitFeatures::known()); - create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0, InitFeatures::known(), InitFeatures::known()); + let chan_2_3 =create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0, InitFeatures::known(), InitFeatures::known()); let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id()) .with_features(InvoiceFeatures::known()); @@ -9873,7 +9912,7 @@ fn test_inconsistent_mpp_params() { } expect_pending_htlcs_forwardable_ignore!(nodes[3]); nodes[3].node.process_pending_htlc_forwards(); - expect_pending_htlcs_forwardable_ignore!(nodes[3]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }]); nodes[3].node.process_pending_htlc_forwards(); check_added_monitors!(nodes[3], 1); @@ -9882,7 +9921,7 @@ fn test_inconsistent_mpp_params() { nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &fail_updates_1.update_fail_htlcs[0]); commitment_signed_dance!(nodes[2], nodes[3], fail_updates_1.commitment_signed, false); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_2_3.2 }]); check_added_monitors!(nodes[2], 1); let fail_updates_2 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id()); @@ -10002,7 +10041,11 @@ fn test_double_partial_claim() { connect_blocks(&nodes[3], TEST_FINAL_CLTV); connect_blocks(&nodes[0], TEST_FINAL_CLTV); // To get the same height for sending later - expect_pending_htlcs_forwardable!(nodes[3]); + let failed_destinations = vec![ + HTLCDestination::FailedPayment { payment_hash }, + HTLCDestination::FailedPayment { payment_hash }, + ]; + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], failed_destinations); pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash); diff --git a/lightning/src/ln/monitor_tests.rs b/lightning/src/ln/monitor_tests.rs index 7c46f7b6d28..4f36b9a8881 100644 --- a/lightning/src/ln/monitor_tests.rs +++ b/lightning/src/ln/monitor_tests.rs @@ -15,7 +15,7 @@ use ln::channel; use ln::channelmanager::BREAKDOWN_TIMEOUT; use ln::features::InitFeatures; use ln::msgs::ChannelMessageHandler; -use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason}; +use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination}; use bitcoin::blockdata::script::Builder; use bitcoin::blockdata::opcodes; @@ -74,7 +74,7 @@ fn chanmon_fail_from_stale_commitment() { assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty()); connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]); check_added_monitors!(nodes[1], 1); let fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); diff --git a/lightning/src/ln/onion_route_tests.rs b/lightning/src/ln/onion_route_tests.rs index c66f45a4492..b9d86fe3a6f 100644 --- a/lightning/src/ln/onion_route_tests.rs +++ b/lightning/src/ln/onion_route_tests.rs @@ -23,7 +23,7 @@ use ln::features::{InitFeatures, InvoiceFeatures, NodeFeatures}; use ln::msgs; use ln::msgs::{ChannelMessageHandler, ChannelUpdate, OptionalField}; use ln::wire::Encode; -use util::events::{Event, MessageSendEvent, MessageSendEventsProvider}; +use util::events::{Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider}; use util::ser::{ReadableArgs, Writeable, Writer}; use util::{byte_utils, test_utils}; use util::config::{UserConfig, ChannelConfig}; @@ -126,7 +126,7 @@ fn run_onion_failure_test_with_fail_intercept(_name: &str, test_case: expect_htlc_forward!(&nodes[2]); expect_event!(&nodes[2], Event::PaymentReceived); callback_node(); - expect_pending_htlcs_forwardable!(nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() }]); } let update_2_1 = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id()); @@ -1036,7 +1036,7 @@ fn test_phantom_onion_hmac_failure() { }; expect_pending_htlcs_forwardable_ignore!(nodes[1]); nodes[1].node.process_pending_htlc_forwards(); - expect_pending_htlcs_forwardable_ignore!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]); nodes[1].node.process_pending_htlc_forwards(); let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); check_added_monitors!(&nodes[1], 1); @@ -1108,7 +1108,7 @@ fn test_phantom_invalid_onion_payload() { } expect_pending_htlcs_forwardable_ignore!(nodes[1]); nodes[1].node.process_pending_htlc_forwards(); - expect_pending_htlcs_forwardable_ignore!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]); nodes[1].node.process_pending_htlc_forwards(); let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); check_added_monitors!(&nodes[1], 1); @@ -1164,7 +1164,7 @@ fn test_phantom_final_incorrect_cltv_expiry() { } expect_pending_htlcs_forwardable_ignore!(nodes[1]); nodes[1].node.process_pending_htlc_forwards(); - expect_pending_htlcs_forwardable_ignore!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]); nodes[1].node.process_pending_htlc_forwards(); let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); check_added_monitors!(&nodes[1], 1); @@ -1210,7 +1210,7 @@ fn test_phantom_failure_too_low_cltv() { expect_pending_htlcs_forwardable_ignore!(nodes[1]); nodes[1].node.process_pending_htlc_forwards(); - expect_pending_htlcs_forwardable_ignore!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]); nodes[1].node.process_pending_htlc_forwards(); let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); check_added_monitors!(&nodes[1], 1); @@ -1255,7 +1255,7 @@ fn test_phantom_failure_too_low_recv_amt() { nodes[1].node.process_pending_htlc_forwards(); expect_pending_htlcs_forwardable_ignore!(nodes[1]); nodes[1].node.process_pending_htlc_forwards(); - expect_pending_htlcs_forwardable_ignore!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() }]); nodes[1].node.process_pending_htlc_forwards(); let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); check_added_monitors!(&nodes[1], 1); @@ -1352,7 +1352,7 @@ fn test_phantom_failure_reject_payment() { nodes[1].node.process_pending_htlc_forwards(); expect_payment_received!(nodes[1], payment_hash, payment_secret, recv_amt_msat); nodes[1].node.fail_htlc_backwards(&payment_hash); - expect_pending_htlcs_forwardable_ignore!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]); nodes[1].node.process_pending_htlc_forwards(); let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); diff --git a/lightning/src/ln/payment_tests.rs b/lightning/src/ln/payment_tests.rs index d3feb96df11..785edecebbe 100644 --- a/lightning/src/ln/payment_tests.rs +++ b/lightning/src/ln/payment_tests.rs @@ -21,7 +21,7 @@ use ln::features::{InitFeatures, InvoiceFeatures}; use ln::msgs; use ln::msgs::ChannelMessageHandler; use routing::router::{PaymentParameters, get_route}; -use util::events::{ClosureReason, Event, MessageSendEvent, MessageSendEventsProvider}; +use util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider}; use util::test_utils; use util::errors::APIError; use util::enforcing_trait_impls::EnforcingSigner; @@ -43,7 +43,7 @@ fn retry_single_path_payment() { let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs); let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - let _chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1, InitFeatures::known(), InitFeatures::known()); + let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1, InitFeatures::known(), InitFeatures::known()); // Rebalance to find a route send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000); @@ -62,7 +62,7 @@ fn retry_single_path_payment() { check_added_monitors!(nodes[1], 0); commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false); expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(&nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]); let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); assert!(htlc_updates.update_add_htlcs.is_empty()); assert_eq!(htlc_updates.update_fail_htlcs.len(), 1); @@ -120,10 +120,10 @@ fn mpp_retry() { let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]); let nodes = create_network(4, &node_cfgs, &node_chanmgrs); - let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id; - let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id; - let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id; - let chan_4_id = create_announced_chan_between_nodes(&nodes, 3, 2, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id; + let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); + let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known()); + let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()); + let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2, InitFeatures::known(), InitFeatures::known()); // Rebalance send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000); @@ -131,11 +131,11 @@ fn mpp_retry() { let path = route.paths[0].clone(); route.paths.push(path); route.paths[0][0].pubkey = nodes[1].node.get_our_node_id(); - route.paths[0][0].short_channel_id = chan_1_id; - route.paths[0][1].short_channel_id = chan_3_id; + route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id; + route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id; route.paths[1][0].pubkey = nodes[2].node.get_our_node_id(); - route.paths[1][0].short_channel_id = chan_2_id; - route.paths[1][1].short_channel_id = chan_4_id; + route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id; + route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id; // Initiate the MPP payment. let payment_id = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap(); @@ -165,7 +165,7 @@ fn mpp_retry() { // Attempt to forward the payment and complete the 2nd path's failure. expect_pending_htlcs_forwardable!(&nodes[2]); - expect_pending_htlcs_forwardable!(&nodes[2]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]); let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id()); assert!(htlc_updates.update_add_htlcs.is_empty()); assert_eq!(htlc_updates.update_fail_htlcs.len(), 1); @@ -206,20 +206,20 @@ fn do_mpp_receive_timeout(send_partial_mpp: bool) { let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]); let nodes = create_network(4, &node_cfgs, &node_chanmgrs); - let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id; - let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id; - let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id; - let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id; + let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); + let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known()); + let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()); + let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()); let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000); let path = route.paths[0].clone(); route.paths.push(path); route.paths[0][0].pubkey = nodes[1].node.get_our_node_id(); - route.paths[0][0].short_channel_id = chan_1_id; - route.paths[0][1].short_channel_id = chan_3_id; + route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id; + route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id; route.paths[1][0].pubkey = nodes[2].node.get_our_node_id(); - route.paths[1][0].short_channel_id = chan_2_id; - route.paths[1][1].short_channel_id = chan_4_id; + route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id; + route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id; // Initiate the MPP payment. let _ = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap(); @@ -237,7 +237,7 @@ fn do_mpp_receive_timeout(send_partial_mpp: bool) { } // Failed HTLC from node 3 -> 1 - expect_pending_htlcs_forwardable!(nodes[3]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]); let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id()); assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1); nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]); @@ -245,7 +245,7 @@ fn do_mpp_receive_timeout(send_partial_mpp: bool) { commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false); // Failed HTLC from node 1 -> 0 - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]); let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1); nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]); @@ -280,7 +280,7 @@ fn retry_expired_payment() { let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs); let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - let _chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1, InitFeatures::known(), InitFeatures::known()); + let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1, InitFeatures::known(), InitFeatures::known()); // Rebalance to find a route send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000); @@ -299,7 +299,7 @@ fn retry_expired_payment() { check_added_monitors!(nodes[1], 0); commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false); expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(&nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]); let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); assert!(htlc_updates.update_add_htlcs.is_empty()); assert_eq!(htlc_updates.update_fail_htlcs.len(), 1); @@ -803,7 +803,7 @@ fn test_fulfill_restart_failure() { reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false)); nodes[1].node.fail_htlc_backwards(&payment_hash); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]); check_added_monitors!(nodes[1], 1); let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]); diff --git a/lightning/src/ln/priv_short_conf_tests.rs b/lightning/src/ln/priv_short_conf_tests.rs index 508ba414037..5a02d15a9d1 100644 --- a/lightning/src/ln/priv_short_conf_tests.rs +++ b/lightning/src/ln/priv_short_conf_tests.rs @@ -22,7 +22,7 @@ use ln::msgs; use ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, OptionalField, ChannelUpdate, ErrorAction}; use ln::wire::Encode; use util::enforcing_trait_impls::EnforcingSigner; -use util::events::{ClosureReason, Event, MessageSendEvent, MessageSendEventsProvider}; +use util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider}; use util::config::UserConfig; use util::ser::{Writeable, ReadableArgs}; use util::test_utils; @@ -478,7 +478,7 @@ fn test_scid_alias_returned() { let nodes = create_network(3, &node_cfgs, &node_chanmgrs); create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0, InitFeatures::known(), InitFeatures::known()); - create_unannounced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000, 0, InitFeatures::known(), InitFeatures::known()); + let chan = create_unannounced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000, 0, InitFeatures::known(), InitFeatures::known()); let last_hop = nodes[2].node.list_usable_channels(); let mut hop_hints = vec![RouteHint(vec![RouteHintHop { @@ -508,7 +508,7 @@ fn test_scid_alias_returned() { commitment_signed_dance!(nodes[1], nodes[0], &as_updates.commitment_signed, false, true); expect_pending_htlcs_forwardable!(nodes[1]); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan.0.channel_id }]); check_added_monitors!(nodes[1], 1); let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()); diff --git a/lightning/src/ln/reorg_tests.rs b/lightning/src/ln/reorg_tests.rs index 29349392f76..f97bdb1bd95 100644 --- a/lightning/src/ln/reorg_tests.rs +++ b/lightning/src/ln/reorg_tests.rs @@ -16,7 +16,7 @@ use ln::channelmanager::{ChannelManager, ChannelManagerReadArgs}; use ln::features::InitFeatures; use ln::msgs::ChannelMessageHandler; use util::enforcing_trait_impls::EnforcingSigner; -use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason}; +use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination}; use util::test_utils; use util::ser::{ReadableArgs, Writeable}; @@ -147,7 +147,7 @@ fn do_test_onchain_htlc_reorg(local_commitment: bool, claim: bool) { txdata: vec![], }; connect_block(&nodes[1], &block); - expect_pending_htlcs_forwardable!(nodes[1]); + expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]); } check_added_monitors!(nodes[1], 1); diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index 4f3d800be5d..fe99b06c9bc 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -169,7 +169,7 @@ impl<'a> chain::Watch for TestChainMonitor<'a> { update_res } - fn release_pending_monitor_events(&self) -> Vec<(OutPoint, Vec)> { + fn release_pending_monitor_events(&self) -> Vec<(OutPoint, Vec, Option)> { return self.chain_monitor.release_pending_monitor_events(); } } From ca9357147e521a15702d1f05c687c54f9a65bf2b Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Wed, 13 Jul 2022 16:52:27 +0000 Subject: [PATCH 06/91] Use a separate (non-trait) fee-estimation fn in LowerBoundedEstimator This should make it somewhat more difficult to accidentally use a straight fee estimator when we actually want a LowerBoundedFeeEstimator by not having the types be exchangeable at all. --- lightning/src/chain/chaininterface.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lightning/src/chain/chaininterface.rs b/lightning/src/chain/chaininterface.rs index 2ec7f318ff5..ab659cbca13 100644 --- a/lightning/src/chain/chaininterface.rs +++ b/lightning/src/chain/chaininterface.rs @@ -68,7 +68,10 @@ pub const MIN_RELAY_FEE_SAT_PER_1000_WEIGHT: u64 = 4000; pub const FEERATE_FLOOR_SATS_PER_KW: u32 = 253; /// Wraps a `Deref` to a `FeeEstimator` so that any fee estimations provided by it -/// are bounded below by `FEERATE_FLOOR_SATS_PER_KW` (253 sats/KW) +/// are bounded below by `FEERATE_FLOOR_SATS_PER_KW` (253 sats/KW). +/// +/// Note that this does *not* implement [`FeeEstimator`] to make it harder to accidentally mix the +/// two. pub(crate) struct LowerBoundedFeeEstimator(pub F) where F::Target: FeeEstimator; impl LowerBoundedFeeEstimator where F::Target: FeeEstimator { @@ -76,10 +79,8 @@ impl LowerBoundedFeeEstimator where F::Target: FeeEstimator { pub fn new(fee_estimator: F) -> Self { LowerBoundedFeeEstimator(fee_estimator) } -} -impl FeeEstimator for LowerBoundedFeeEstimator where F::Target: FeeEstimator { - fn get_est_sat_per_1000_weight(&self, confirmation_target: ConfirmationTarget) -> u32 { + pub fn get_est_sat_per_1000_weight(&self, confirmation_target: ConfirmationTarget) -> u32 { cmp::max( self.0.get_est_sat_per_1000_weight(confirmation_target), FEERATE_FLOOR_SATS_PER_KW, From a491200f96760315fa420e433d0e60c4e6503a97 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 21 Jul 2022 21:38:00 +0000 Subject: [PATCH 07/91] Avoid blanket implementing FeeEstimator for Deref This simplifies things for bindings (and, to some extent, downstream users) by exploiting the fact that we can always "clone" a reference to a struct by dereferencing and then creating a new reference. --- fuzz/src/chanmon_consistency.rs | 2 +- lightning/src/chain/chaininterface.rs | 8 -------- lightning/src/chain/chainmonitor.rs | 2 +- lightning/src/chain/channelmonitor.rs | 12 ++++++------ 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/fuzz/src/chanmon_consistency.rs b/fuzz/src/chanmon_consistency.rs index 9d5ccaa8105..71c1ebc3304 100644 --- a/fuzz/src/chanmon_consistency.rs +++ b/fuzz/src/chanmon_consistency.rs @@ -140,7 +140,7 @@ impl chain::Watch for TestChainMonitor { }; let deserialized_monitor = <(BlockHash, channelmonitor::ChannelMonitor)>:: read(&mut Cursor::new(&map_entry.get().1), &*self.keys).unwrap().1; - deserialized_monitor.update_monitor(&update, &&TestBroadcaster{}, &&FuzzEstimator { ret_val: atomic::AtomicU32::new(253) }, &self.logger).unwrap(); + deserialized_monitor.update_monitor(&update, &&TestBroadcaster{}, &FuzzEstimator { ret_val: atomic::AtomicU32::new(253) }, &self.logger).unwrap(); let mut ser = VecWriter(Vec::new()); deserialized_monitor.write(&mut ser).unwrap(); map_entry.insert((update.update_id, ser.0)); diff --git a/lightning/src/chain/chaininterface.rs b/lightning/src/chain/chaininterface.rs index ab659cbca13..50570dfdf11 100644 --- a/lightning/src/chain/chaininterface.rs +++ b/lightning/src/chain/chaininterface.rs @@ -52,14 +52,6 @@ pub trait FeeEstimator { fn get_est_sat_per_1000_weight(&self, confirmation_target: ConfirmationTarget) -> u32; } -// We need `FeeEstimator` implemented so that in some places where we only have a shared -// reference to a `Deref` to a `FeeEstimator`, we can still wrap it. -impl FeeEstimator for D where D::Target: FeeEstimator { - fn get_est_sat_per_1000_weight(&self, confirmation_target: ConfirmationTarget) -> u32 { - (**self).get_est_sat_per_1000_weight(confirmation_target) - } -} - /// Minimum relay fee as required by bitcoin network mempool policy. pub const MIN_RELAY_FEE_SAT_PER_1000_WEIGHT: u64 = 4000; /// Minimum feerate that takes a sane approach to bitcoind weight-to-vbytes rounding. diff --git a/lightning/src/chain/chainmonitor.rs b/lightning/src/chain/chainmonitor.rs index e6b5733520a..239affc73dd 100644 --- a/lightning/src/chain/chainmonitor.rs +++ b/lightning/src/chain/chainmonitor.rs @@ -636,7 +636,7 @@ where C::Target: chain::Filter, Some(monitor_state) => { let monitor = &monitor_state.monitor; log_trace!(self.logger, "Updating ChannelMonitor for channel {}", log_funding_info!(monitor)); - let update_res = monitor.update_monitor(&update, &self.broadcaster, &self.fee_estimator, &self.logger); + let update_res = monitor.update_monitor(&update, &self.broadcaster, &*self.fee_estimator, &self.logger); if update_res.is_err() { log_error!(self.logger, "Failed to update ChannelMonitor for channel {}.", log_funding_info!(monitor)); } diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 80cd9cb9d45..f91af7cc66c 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1134,7 +1134,7 @@ impl ChannelMonitor { &self, updates: &ChannelMonitorUpdate, broadcaster: &B, - fee_estimator: &F, + fee_estimator: F, logger: &L, ) -> Result<(), ()> where @@ -1944,10 +1944,10 @@ impl ChannelMonitorImpl { self.pending_monitor_events.push(MonitorEvent::CommitmentTxConfirmed(self.funding_info.0)); } - pub fn update_monitor(&mut self, updates: &ChannelMonitorUpdate, broadcaster: &B, fee_estimator: &F, logger: &L) -> Result<(), ()> + pub fn update_monitor(&mut self, updates: &ChannelMonitorUpdate, broadcaster: &B, fee_estimator: F, logger: &L) -> Result<(), ()> where B::Target: BroadcasterInterface, - F::Target: FeeEstimator, - L::Target: Logger, + F::Target: FeeEstimator, + L::Target: Logger, { log_info!(logger, "Applying update to monitor {}, bringing update_id from {} to {} with {} changes.", log_funding_info!(self), self.latest_update_id, updates.update_id, updates.updates.len()); @@ -1985,7 +1985,7 @@ impl ChannelMonitorImpl { }, ChannelMonitorUpdateStep::PaymentPreimage { payment_preimage } => { log_trace!(logger, "Updating ChannelMonitor with payment preimage"); - let bounded_fee_estimator = LowerBoundedFeeEstimator::new(fee_estimator); + let bounded_fee_estimator = LowerBoundedFeeEstimator::new(&*fee_estimator); self.provide_payment_preimage(&PaymentHash(Sha256::hash(&payment_preimage.0[..]).into_inner()), &payment_preimage, broadcaster, &bounded_fee_estimator, logger) }, ChannelMonitorUpdateStep::CommitmentSecret { idx, secret } => { @@ -3534,7 +3534,7 @@ mod tests { let broadcaster = TestBroadcaster::new(Arc::clone(&nodes[1].blocks)); assert!( - pre_update_monitor.update_monitor(&replay_update, &&broadcaster, &&chanmon_cfgs[1].fee_estimator, &nodes[1].logger) + pre_update_monitor.update_monitor(&replay_update, &&broadcaster, &chanmon_cfgs[1].fee_estimator, &nodes[1].logger) .is_err()); // Even though we error'd on the first update, we should still have generated an HTLC claim // transaction From af7e9b608d7a78dbb1a87a31a5b45c758f15c0ec Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 25 Jul 2022 18:08:44 +0000 Subject: [PATCH 08/91] Change `LowerBoundedFeeEstimator` fn name to make it hard to swap This change the method name on `LowerBoundedFeeEstimator` to further differentiate it from the generic `FeeEstimator` trait. --- lightning/src/chain/chaininterface.rs | 6 +++--- lightning/src/chain/package.rs | 6 +++--- lightning/src/ln/channel.rs | 10 +++++----- lightning/src/ln/channelmanager.rs | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lightning/src/chain/chaininterface.rs b/lightning/src/chain/chaininterface.rs index 50570dfdf11..be86261f0c0 100644 --- a/lightning/src/chain/chaininterface.rs +++ b/lightning/src/chain/chaininterface.rs @@ -72,7 +72,7 @@ impl LowerBoundedFeeEstimator where F::Target: FeeEstimator { LowerBoundedFeeEstimator(fee_estimator) } - pub fn get_est_sat_per_1000_weight(&self, confirmation_target: ConfirmationTarget) -> u32 { + pub fn bounded_sat_per_1000_weight(&self, confirmation_target: ConfirmationTarget) -> u32 { cmp::max( self.0.get_est_sat_per_1000_weight(confirmation_target), FEERATE_FLOOR_SATS_PER_KW, @@ -100,7 +100,7 @@ mod tests { let test_fee_estimator = &TestFeeEstimator { sat_per_kw }; let fee_estimator = LowerBoundedFeeEstimator::new(test_fee_estimator); - assert_eq!(fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Background), FEERATE_FLOOR_SATS_PER_KW); + assert_eq!(fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Background), FEERATE_FLOOR_SATS_PER_KW); } #[test] @@ -109,6 +109,6 @@ mod tests { let test_fee_estimator = &TestFeeEstimator { sat_per_kw }; let fee_estimator = LowerBoundedFeeEstimator::new(test_fee_estimator); - assert_eq!(fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Background), sat_per_kw); + assert_eq!(fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Background), sat_per_kw); } } diff --git a/lightning/src/chain/package.rs b/lightning/src/chain/package.rs index b0cfa9bf000..30530303e59 100644 --- a/lightning/src/chain/package.rs +++ b/lightning/src/chain/package.rs @@ -778,13 +778,13 @@ fn compute_fee_from_spent_amounts(input_amounts: u64, predic where F::Target: FeeEstimator, L::Target: Logger, { - let mut updated_feerate = fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::HighPriority) as u64; + let mut updated_feerate = fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::HighPriority) as u64; let mut fee = updated_feerate * (predicted_weight as u64) / 1000; if input_amounts <= fee { - updated_feerate = fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Normal) as u64; + updated_feerate = fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Normal) as u64; fee = updated_feerate * (predicted_weight as u64) / 1000; if input_amounts <= fee { - updated_feerate = fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Background) as u64; + updated_feerate = fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Background) as u64; fee = updated_feerate * (predicted_weight as u64) / 1000; if input_amounts <= fee { log_error!(logger, "Failed to generate an on-chain punishment tx as even low priority fee ({} sat) was more than the entire claim balance ({} sat)", diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index f612cb97943..b698a919b01 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -917,7 +917,7 @@ impl Channel { return Err(APIError::APIMisuseError { err: format!("Holder selected channel reserve below implemention limit dust_limit_satoshis {}", holder_selected_channel_reserve_satoshis) }); } - let feerate = fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Normal); + let feerate = fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Normal); let value_to_self_msat = channel_value_satoshis * 1000 - push_msat; let commitment_tx_fee = Self::commit_tx_fee_msat(feerate, MIN_AFFORDABLE_HTLC_COUNT, opt_anchors); @@ -1064,11 +1064,11 @@ impl Channel { // We generally don't care too much if they set the feerate to something very high, but it // could result in the channel being useless due to everything being dust. let upper_limit = cmp::max(250 * 25, - fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::HighPriority) as u64 * 10); + fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::HighPriority) as u64 * 10); if feerate_per_kw as u64 > upper_limit { return Err(ChannelError::Close(format!("Peer's feerate much too high. Actual: {}. Our expected upper limit: {}", feerate_per_kw, upper_limit))); } - let lower_limit = fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Background); + let lower_limit = fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Background); // Some fee estimators round up to the next full sat/vbyte (ie 250 sats per kw), causing // occasional issues with feerate disagreements between an initiator that wants a feerate // of 1.1 sat/vbyte and a receiver that wants 1.1 rounded up to 2. Thus, we always add 250 @@ -4022,8 +4022,8 @@ impl Channel { // Propose a range from our current Background feerate to our Normal feerate plus our // force_close_avoidance_max_fee_satoshis. // If we fail to come to consensus, we'll have to force-close. - let mut proposed_feerate = fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Background); - let normal_feerate = fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Normal); + let mut proposed_feerate = fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Background); + let normal_feerate = fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Normal); let mut proposed_max_feerate = if self.is_outbound() { normal_feerate } else { u32::max_value() }; // The spec requires that (when the channel does not have anchors) we only send absolute diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index bae408d29f0..979cea39527 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -3577,7 +3577,7 @@ impl ChannelMana PersistenceNotifierGuard::optionally_notify(&self.total_consistency_lock, &self.persistence_notifier, || { let mut should_persist = NotifyOption::SkipPersist; - let new_feerate = self.fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Normal); + let new_feerate = self.fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Normal); let mut handle_errors = Vec::new(); { @@ -3616,7 +3616,7 @@ impl ChannelMana let mut should_persist = NotifyOption::SkipPersist; if self.process_background_events() { should_persist = NotifyOption::DoPersist; } - let new_feerate = self.fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Normal); + let new_feerate = self.fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Normal); let mut handle_errors = Vec::new(); let mut timed_out_mpp_htlcs = Vec::new(); From b0e8b739b73cc25f3e1ab00695a14d972162a140 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 25 Jul 2022 20:35:51 +0200 Subject: [PATCH 09/91] Make `htlc_maximum_msat` a required field. --- .github/workflows/build.yml | 4 +- lightning-rapid-gossip-sync/src/lib.rs | 2 +- lightning-rapid-gossip-sync/src/processing.rs | 29 +-- lightning/src/ln/channel.rs | 4 +- lightning/src/ln/channelmanager.rs | 4 +- lightning/src/ln/functional_tests.rs | 10 +- lightning/src/ln/msgs.rs | 50 ++-- lightning/src/ln/onion_route_tests.rs | 4 +- lightning/src/ln/priv_short_conf_tests.rs | 4 +- lightning/src/routing/gossip.rs | 182 +++++++++++--- lightning/src/routing/router.rs | 234 +++++++++--------- lightning/src/routing/scoring.rs | 4 +- lightning/src/util/test_utils.rs | 3 +- 13 files changed, 303 insertions(+), 231 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68cf376235f..cfb1b9c8474 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -232,14 +232,14 @@ jobs: EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM: 05a5361278f68ee2afd086cc04a1f927a63924be451f3221d380533acfacc303 - name: Fetch rapid graph sync reference input run: | - curl --verbose -L -o lightning-rapid-gossip-sync/res/full_graph.lngossip https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin + curl --verbose -L -o lightning-rapid-gossip-sync/res/full_graph.lngossip https://bitcoin.ninja/ldk-compressed_graph-285cb27df79-2022-07-21.bin echo "Sha sum: $(sha256sum lightning-rapid-gossip-sync/res/full_graph.lngossip | awk '{ print $1 }')" if [ "$(sha256sum lightning-rapid-gossip-sync/res/full_graph.lngossip | awk '{ print $1 }')" != "${EXPECTED_RAPID_GOSSIP_SHASUM}" ]; then echo "Bad hash" exit 1 fi env: - EXPECTED_RAPID_GOSSIP_SHASUM: 9637b91cea9d64320cf48fc0787c70fe69fc062f90d3512e207044110cadfd7b + EXPECTED_RAPID_GOSSIP_SHASUM: e0f5d11641c11896d7af3a2246d3d6c3f1720b7d2d17aab321ecce82e6b7deb8 - name: Test with Network Graph on Rust ${{ matrix.toolchain }} run: | cd lightning diff --git a/lightning-rapid-gossip-sync/src/lib.rs b/lightning-rapid-gossip-sync/src/lib.rs index 7880e11a60e..6e9280f86a3 100644 --- a/lightning-rapid-gossip-sync/src/lib.rs +++ b/lightning-rapid-gossip-sync/src/lib.rs @@ -240,7 +240,7 @@ mod tests { let sync_result = rapid_sync .sync_network_graph_with_file_path("./res/full_graph.lngossip"); if let Err(crate::error::GraphSyncError::DecodeError(DecodeError::Io(io_error))) = &sync_result { - let error_string = format!("Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin\n\n{:?}", io_error); + let error_string = format!("Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-285cb27df79-2022-07-21.bin\n\n{:?}", io_error); #[cfg(not(require_route_graph_test))] { println!("{}", error_string); diff --git a/lightning-rapid-gossip-sync/src/processing.rs b/lightning-rapid-gossip-sync/src/processing.rs index 09693a76d6f..818e19fe4b4 100644 --- a/lightning-rapid-gossip-sync/src/processing.rs +++ b/lightning-rapid-gossip-sync/src/processing.rs @@ -8,7 +8,7 @@ use bitcoin::BlockHash; use bitcoin::secp256k1::PublicKey; use lightning::ln::msgs::{ - DecodeError, ErrorAction, LightningError, OptionalField, UnsignedChannelUpdate, + DecodeError, ErrorAction, LightningError, UnsignedChannelUpdate, }; use lightning::routing::gossip::NetworkGraph; use lightning::util::logger::Logger; @@ -119,12 +119,7 @@ impl>, L: Deref> RapidGossipSync where L let default_htlc_minimum_msat: u64 = Readable::read(&mut read_cursor)?; let default_fee_base_msat: u32 = Readable::read(&mut read_cursor)?; let default_fee_proportional_millionths: u32 = Readable::read(&mut read_cursor)?; - let tentative_default_htlc_maximum_msat: u64 = Readable::read(&mut read_cursor)?; - let default_htlc_maximum_msat = if tentative_default_htlc_maximum_msat == u64::max_value() { - OptionalField::Absent - } else { - OptionalField::Present(tentative_default_htlc_maximum_msat) - }; + let default_htlc_maximum_msat: u64 = Readable::read(&mut read_cursor)?; for _ in 0..update_count { let scid_delta: BigSize = Readable::read(read_cursor)?; @@ -147,7 +142,7 @@ impl>, L: Deref> RapidGossipSync where L flags: standard_channel_flags, cltv_expiry_delta: default_cltv_expiry_delta, htlc_minimum_msat: default_htlc_minimum_msat, - htlc_maximum_msat: default_htlc_maximum_msat.clone(), + htlc_maximum_msat: default_htlc_maximum_msat, fee_base_msat: default_fee_base_msat, fee_proportional_millionths: default_fee_proportional_millionths, excess_data: vec![], @@ -170,13 +165,6 @@ impl>, L: Deref> RapidGossipSync where L action: ErrorAction::IgnoreError, })?; - let htlc_maximum_msat = - if let Some(htlc_maximum_msat) = directional_info.htlc_maximum_msat { - OptionalField::Present(htlc_maximum_msat) - } else { - OptionalField::Absent - }; - UnsignedChannelUpdate { chain_hash, short_channel_id, @@ -184,7 +172,7 @@ impl>, L: Deref> RapidGossipSync where L flags: standard_channel_flags, cltv_expiry_delta: directional_info.cltv_expiry_delta, htlc_minimum_msat: directional_info.htlc_minimum_msat, - htlc_maximum_msat, + htlc_maximum_msat: directional_info.htlc_maximum_msat, fee_base_msat: directional_info.fees.base_msat, fee_proportional_millionths: directional_info.fees.proportional_millionths, excess_data: vec![], @@ -212,13 +200,8 @@ impl>, L: Deref> RapidGossipSync where L } if channel_flags & 0b_0000_0100 > 0 { - let tentative_htlc_maximum_msat: u64 = Readable::read(read_cursor)?; - synthetic_update.htlc_maximum_msat = if tentative_htlc_maximum_msat == u64::max_value() - { - OptionalField::Absent - } else { - OptionalField::Present(tentative_htlc_maximum_msat) - }; + let htlc_maximum_msat: u64 = Readable::read(read_cursor)?; + synthetic_update.htlc_maximum_msat = htlc_maximum_msat; } network_graph.update_channel_unsigned(&synthetic_update)?; diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index f612cb97943..ca0836f6327 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -6572,7 +6572,7 @@ mod tests { use ln::channel::{Channel, InboundHTLCOutput, OutboundHTLCOutput, InboundHTLCState, OutboundHTLCState, HTLCCandidate, HTLCInitiator}; use ln::channel::{MAX_FUNDING_SATOSHIS_NO_WUMBO, TOTAL_BITCOIN_SUPPLY_SATOSHIS}; use ln::features::{InitFeatures, ChannelTypeFeatures}; - use ln::msgs::{ChannelUpdate, DataLossProtect, DecodeError, OptionalField, UnsignedChannelUpdate}; + use ln::msgs::{ChannelUpdate, DataLossProtect, DecodeError, OptionalField, UnsignedChannelUpdate, MAX_VALUE_MSAT}; use ln::script::ShutdownScript; use ln::chan_utils; use ln::chan_utils::{htlc_success_tx_weight, htlc_timeout_tx_weight}; @@ -6988,7 +6988,7 @@ mod tests { flags: 0, cltv_expiry_delta: 100, htlc_minimum_msat: 5, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 110, fee_proportional_millionths: 11, excess_data: Vec::new(), diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 611e8a562c6..c764c7cafea 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -48,7 +48,7 @@ use routing::router::{PaymentParameters, Route, RouteHop, RoutePath, RouteParame use ln::msgs; use ln::msgs::NetAddress; use ln::onion_utils; -use ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, MAX_VALUE_MSAT, OptionalField}; +use ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, MAX_VALUE_MSAT}; use ln::wire::Encode; use chain::keysinterface::{Sign, KeysInterface, KeysManager, InMemorySigner, Recipient}; use util::config::{UserConfig, ChannelConfig}; @@ -2397,7 +2397,7 @@ impl ChannelMana flags: (!were_node_one) as u8 | ((!chan.is_live() as u8) << 1), cltv_expiry_delta: chan.get_cltv_expiry_delta(), htlc_minimum_msat: chan.get_counterparty_htlc_minimum_msat(), - htlc_maximum_msat: OptionalField::Present(chan.get_announced_htlc_max_msat()), + htlc_maximum_msat: chan.get_announced_htlc_max_msat(), fee_base_msat: chan.get_outbound_forwarding_fee_base_msat(), fee_proportional_millionths: chan.get_fee_proportional_millionths(), excess_data: Vec::new(), diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index 1da45bf2c99..23dfbf36f09 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -28,7 +28,7 @@ use routing::gossip::NetworkGraph; use routing::router::{PaymentParameters, Route, RouteHop, RouteParameters, find_route, get_route}; use ln::features::{ChannelFeatures, InitFeatures, InvoiceFeatures, NodeFeatures}; use ln::msgs; -use ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, OptionalField, ErrorAction}; +use ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction}; use util::enforcing_trait_impls::EnforcingSigner; use util::{byte_utils, test_utils}; use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason}; @@ -8308,19 +8308,19 @@ fn test_channel_update_has_correct_htlc_maximum_msat() { // Assert that `node[0]`'s `ChannelUpdate` is capped at 50 percent of the `channel_value`, as // that's the value of `node[1]`'s `holder_max_htlc_value_in_flight_msat`. - assert_eq!(node_0_chan_update.contents.htlc_maximum_msat, OptionalField::Present(channel_value_50_percent_msat)); + assert_eq!(node_0_chan_update.contents.htlc_maximum_msat, channel_value_50_percent_msat); // Assert that `node[1]`'s `ChannelUpdate` is capped at 30 percent of the `channel_value`, as // that's the value of `node[0]`'s `holder_max_htlc_value_in_flight_msat`. - assert_eq!(node_1_chan_update.contents.htlc_maximum_msat, OptionalField::Present(channel_value_30_percent_msat)); + assert_eq!(node_1_chan_update.contents.htlc_maximum_msat, channel_value_30_percent_msat); // Assert that `node[2]`'s `ChannelUpdate` is capped at 90 percent of the `channel_value`, as // the value of `node[3]`'s `holder_max_htlc_value_in_flight_msat` (100%), exceeds 90% of the // `channel_value`. - assert_eq!(node_2_chan_update.contents.htlc_maximum_msat, OptionalField::Present(channel_value_90_percent_msat)); + assert_eq!(node_2_chan_update.contents.htlc_maximum_msat, channel_value_90_percent_msat); // Assert that `node[3]`'s `ChannelUpdate` is capped at 90 percent of the `channel_value`, as // the value of `node[2]`'s `holder_max_htlc_value_in_flight_msat` (95%), exceeds 90% of the // `channel_value`. - assert_eq!(node_3_chan_update.contents.htlc_maximum_msat, OptionalField::Present(channel_value_90_percent_msat)); + assert_eq!(node_3_chan_update.contents.htlc_maximum_msat, channel_value_90_percent_msat); } #[test] diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 3e44cfcf024..c2925d7354d 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -647,8 +647,8 @@ pub struct UnsignedChannelUpdate { pub cltv_expiry_delta: u16, /// The minimum HTLC size incoming to sender, in milli-satoshi pub htlc_minimum_msat: u64, - /// Optionally, the maximum HTLC value incoming to sender, in milli-satoshi - pub htlc_maximum_msat: OptionalField, + /// The maximum HTLC value incoming to sender, in milli-satoshi. Used to be optional. + pub htlc_maximum_msat: u64, /// The base HTLC fee charged by sender, in milli-satoshi pub fee_base_msat: u32, /// The amount to fee multiplier, in micro-satoshi @@ -1514,14 +1514,12 @@ impl_writeable!(ChannelAnnouncement, { impl Writeable for UnsignedChannelUpdate { fn write(&self, w: &mut W) -> Result<(), io::Error> { - let mut message_flags: u8 = 0; - if let OptionalField::Present(_) = self.htlc_maximum_msat { - message_flags = 1; - } + // `message_flags` used to indicate presence of `htlc_maximum_msat`, but was deprecated in the spec. + const MESSAGE_FLAGS: u8 = 1; self.chain_hash.write(w)?; self.short_channel_id.write(w)?; self.timestamp.write(w)?; - let all_flags = self.flags as u16 | ((message_flags as u16) << 8); + let all_flags = self.flags as u16 | ((MESSAGE_FLAGS as u16) << 8); all_flags.write(w)?; self.cltv_expiry_delta.write(w)?; self.htlc_minimum_msat.write(w)?; @@ -1535,22 +1533,20 @@ impl Writeable for UnsignedChannelUpdate { impl Readable for UnsignedChannelUpdate { fn read(r: &mut R) -> Result { - let has_htlc_maximum_msat; Ok(Self { chain_hash: Readable::read(r)?, short_channel_id: Readable::read(r)?, timestamp: Readable::read(r)?, flags: { let flags: u16 = Readable::read(r)?; - let message_flags = flags >> 8; - has_htlc_maximum_msat = (message_flags as i32 & 1) == 1; + // Note: we ignore the `message_flags` for now, since it was deprecated by the spec. flags as u8 }, cltv_expiry_delta: Readable::read(r)?, htlc_minimum_msat: Readable::read(r)?, fee_base_msat: Readable::read(r)?, fee_proportional_millionths: Readable::read(r)?, - htlc_maximum_msat: if has_htlc_maximum_msat { Readable::read(r)? } else { OptionalField::Absent }, + htlc_maximum_msat: Readable::read(r)?, excess_data: read_to_end(r)?, }) } @@ -2103,7 +2099,7 @@ mod tests { do_encoding_node_announcement(false, false, true, false, true, false, false, false); } - fn do_encoding_channel_update(direction: bool, disable: bool, htlc_maximum_msat: bool, excess_data: bool) { + fn do_encoding_channel_update(direction: bool, disable: bool, excess_data: bool) { let secp_ctx = Secp256k1::new(); let (privkey_1, _) = get_keys_from!("0101010101010101010101010101010101010101010101010101010101010101", secp_ctx); let sig_1 = get_sig_on!(privkey_1, secp_ctx, String::from("01010101010101010101010101010101")); @@ -2114,7 +2110,7 @@ mod tests { flags: if direction { 1 } else { 0 } | if disable { 1 << 1 } else { 0 }, cltv_expiry_delta: 144, htlc_minimum_msat: 1000000, - htlc_maximum_msat: if htlc_maximum_msat { OptionalField::Present(131355275467161) } else { OptionalField::Absent }, + htlc_maximum_msat: 131355275467161, fee_base_msat: 10000, fee_proportional_millionths: 20, excess_data: if excess_data { vec![0, 0, 0, 0, 59, 154, 202, 0] } else { Vec::new() } @@ -2127,11 +2123,7 @@ mod tests { let mut target_value = hex::decode("d977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a").unwrap(); target_value.append(&mut hex::decode("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f").unwrap()); target_value.append(&mut hex::decode("00083a840000034d013413a7").unwrap()); - if htlc_maximum_msat { - target_value.append(&mut hex::decode("01").unwrap()); - } else { - target_value.append(&mut hex::decode("00").unwrap()); - } + target_value.append(&mut hex::decode("01").unwrap()); target_value.append(&mut hex::decode("00").unwrap()); if direction { let flag = target_value.last_mut().unwrap(); @@ -2142,9 +2134,7 @@ mod tests { *flag = *flag | 1 << 1; } target_value.append(&mut hex::decode("009000000000000f42400000271000000014").unwrap()); - if htlc_maximum_msat { - target_value.append(&mut hex::decode("0000777788889999").unwrap()); - } + target_value.append(&mut hex::decode("0000777788889999").unwrap()); if excess_data { target_value.append(&mut hex::decode("000000003b9aca00").unwrap()); } @@ -2153,16 +2143,14 @@ mod tests { #[test] fn encoding_channel_update() { - do_encoding_channel_update(false, false, false, false); - do_encoding_channel_update(false, false, false, true); - do_encoding_channel_update(true, false, false, false); - do_encoding_channel_update(true, false, false, true); - do_encoding_channel_update(false, true, false, false); - do_encoding_channel_update(false, true, false, true); - do_encoding_channel_update(false, false, true, false); - do_encoding_channel_update(false, false, true, true); - do_encoding_channel_update(true, true, true, false); - do_encoding_channel_update(true, true, true, true); + do_encoding_channel_update(false, false, false); + do_encoding_channel_update(false, false, true); + do_encoding_channel_update(true, false, false); + do_encoding_channel_update(true, false, true); + do_encoding_channel_update(false, true, false); + do_encoding_channel_update(false, true, true); + do_encoding_channel_update(true, true, false); + do_encoding_channel_update(true, true, true); } fn do_encoding_open_channel(random_bit: bool, shutdown: bool, incl_chan_type: bool) { diff --git a/lightning/src/ln/onion_route_tests.rs b/lightning/src/ln/onion_route_tests.rs index c66f45a4492..ed1516efcb0 100644 --- a/lightning/src/ln/onion_route_tests.rs +++ b/lightning/src/ln/onion_route_tests.rs @@ -21,7 +21,7 @@ use routing::gossip::{NetworkUpdate, RoutingFees, NodeId}; use routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop}; use ln::features::{InitFeatures, InvoiceFeatures, NodeFeatures}; use ln::msgs; -use ln::msgs::{ChannelMessageHandler, ChannelUpdate, OptionalField}; +use ln::msgs::{ChannelMessageHandler, ChannelUpdate}; use ln::wire::Encode; use util::events::{Event, MessageSendEvent, MessageSendEventsProvider}; use util::ser::{ReadableArgs, Writeable, Writer}; @@ -227,7 +227,7 @@ impl msgs::ChannelUpdate { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: msgs::MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: vec![], diff --git a/lightning/src/ln/priv_short_conf_tests.rs b/lightning/src/ln/priv_short_conf_tests.rs index 508ba414037..d5dd51cac4e 100644 --- a/lightning/src/ln/priv_short_conf_tests.rs +++ b/lightning/src/ln/priv_short_conf_tests.rs @@ -19,7 +19,7 @@ use routing::gossip::RoutingFees; use routing::router::{PaymentParameters, RouteHint, RouteHintHop}; use ln::features::{InitFeatures, InvoiceFeatures, ChannelTypeFeatures}; use ln::msgs; -use ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, OptionalField, ChannelUpdate, ErrorAction}; +use ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ChannelUpdate, ErrorAction}; use ln::wire::Encode; use util::enforcing_trait_impls::EnforcingSigner; use util::events::{ClosureReason, Event, MessageSendEvent, MessageSendEventsProvider}; @@ -523,7 +523,7 @@ fn test_scid_alias_returned() { flags: 1, cltv_expiry_delta: accept_forward_cfg.channel_config.cltv_expiry_delta, htlc_minimum_msat: 1_000, - htlc_maximum_msat: OptionalField::Present(1_000_000), // Defaults to 10% of the channel value + htlc_maximum_msat: 1_000_000, // Defaults to 10% of the channel value fee_base_msat: last_hop[0].counterparty.forwarding_info.as_ref().unwrap().fee_base_msat, fee_proportional_millionths: last_hop[0].counterparty.forwarding_info.as_ref().unwrap().fee_proportional_millionths, excess_data: Vec::new(), diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 10ce74b971e..510dbcfb1c4 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -25,10 +25,10 @@ use chain; use chain::Access; use ln::features::{ChannelFeatures, NodeFeatures}; use ln::msgs::{DecodeError, ErrorAction, Init, LightningError, RoutingMessageHandler, NetAddress, MAX_VALUE_MSAT}; -use ln::msgs::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement, OptionalField, GossipTimestampFilter}; +use ln::msgs::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement, GossipTimestampFilter}; use ln::msgs::{QueryChannelRange, ReplyChannelRange, QueryShortChannelIds, ReplyShortChannelIdsEnd}; use ln::msgs; -use util::ser::{Readable, ReadableArgs, Writeable, Writer}; +use util::ser::{Readable, ReadableArgs, Writeable, Writer, MaybeReadable}; use util::logger::{Logger, Level}; use util::events::{Event, EventHandler, MessageSendEvent, MessageSendEventsProvider}; use util::scid_utils::{block_from_scid, scid_from_parts, MAX_SCID_BLOCK}; @@ -611,7 +611,7 @@ pub struct ChannelUpdateInfo { /// The minimum value, which must be relayed to the next hop via the channel pub htlc_minimum_msat: u64, /// The maximum value which may be relayed to the next hop via the channel. - pub htlc_maximum_msat: Option, + pub htlc_maximum_msat: u64, /// Fees charged when the channel is used for routing pub fees: RoutingFees, /// Most recent update for the channel received from the network @@ -628,15 +628,58 @@ impl fmt::Display for ChannelUpdateInfo { } } -impl_writeable_tlv_based!(ChannelUpdateInfo, { - (0, last_update, required), - (2, enabled, required), - (4, cltv_expiry_delta, required), - (6, htlc_minimum_msat, required), - (8, htlc_maximum_msat, required), - (10, fees, required), - (12, last_update_message, required), -}); +impl Writeable for ChannelUpdateInfo { + fn write(&self, writer: &mut W) -> Result<(), io::Error> { + write_tlv_fields!(writer, { + (0, self.last_update, required), + (2, self.enabled, required), + (4, self.cltv_expiry_delta, required), + (6, self.htlc_minimum_msat, required), + // Writing htlc_maximum_msat as an Option is required to maintain backwards + // compatibility with LDK versions prior to v0.0.110. + (8, Some(self.htlc_maximum_msat), required), + (10, self.fees, required), + (12, self.last_update_message, required), + }); + Ok(()) + } +} + +impl Readable for ChannelUpdateInfo { + fn read(reader: &mut R) -> Result { + init_tlv_field_var!(last_update, required); + init_tlv_field_var!(enabled, required); + init_tlv_field_var!(cltv_expiry_delta, required); + init_tlv_field_var!(htlc_minimum_msat, required); + init_tlv_field_var!(htlc_maximum_msat, option); + init_tlv_field_var!(fees, required); + init_tlv_field_var!(last_update_message, required); + + read_tlv_fields!(reader, { + (0, last_update, required), + (2, enabled, required), + (4, cltv_expiry_delta, required), + (6, htlc_minimum_msat, required), + (8, htlc_maximum_msat, required), + (10, fees, required), + (12, last_update_message, required) + }); + + if let Some(htlc_maximum_msat) = htlc_maximum_msat { + Ok(ChannelUpdateInfo { + last_update: init_tlv_based_struct_field!(last_update, required), + enabled: init_tlv_based_struct_field!(enabled, required), + cltv_expiry_delta: init_tlv_based_struct_field!(cltv_expiry_delta, required), + htlc_minimum_msat: init_tlv_based_struct_field!(htlc_minimum_msat, required), + htlc_maximum_msat, + fees: init_tlv_based_struct_field!(fees, required), + last_update_message: init_tlv_based_struct_field!(last_update_message, required), + }) + } else { + Err(DecodeError::InvalidValue) + } + } +} #[derive(Clone, Debug, PartialEq)] /// Details about a channel (both directions). @@ -715,16 +758,73 @@ impl fmt::Display for ChannelInfo { } } -impl_writeable_tlv_based!(ChannelInfo, { - (0, features, required), - (1, announcement_received_time, (default_value, 0)), - (2, node_one, required), - (4, one_to_two, required), - (6, node_two, required), - (8, two_to_one, required), - (10, capacity_sats, required), - (12, announcement_message, required), -}); +impl Writeable for ChannelInfo { + fn write(&self, writer: &mut W) -> Result<(), io::Error> { + write_tlv_fields!(writer, { + (0, self.features, required), + (1, self.announcement_received_time, (default_value, 0)), + (2, self.node_one, required), + (4, self.one_to_two, required), + (6, self.node_two, required), + (8, self.two_to_one, required), + (10, self.capacity_sats, required), + (12, self.announcement_message, required), + }); + Ok(()) + } +} + +// A wrapper allowing for the optional deseralization of ChannelUpdateInfo. Utilizing this is +// necessary to maintain backwards compatibility with previous serializations of `ChannelUpdateInfo` +// that may have no `htlc_maximum_msat` field set. In case the field is absent, we simply ignore +// the error and continue reading the `ChannelInfo`. Hopefully, we'll then eventually receive newer +// channel updates via the gossip network. +struct ChannelUpdateInfoDeserWrapper(Option); + +impl MaybeReadable for ChannelUpdateInfoDeserWrapper { + fn read(reader: &mut R) -> Result, DecodeError> { + match ::util::ser::Readable::read(reader) { + Ok(channel_update_option) => Ok(Some(Self(channel_update_option))), + Err(DecodeError::ShortRead) => Ok(None), + Err(DecodeError::InvalidValue) => Ok(None), + Err(err) => Err(err), + } + } +} + +impl Readable for ChannelInfo { + fn read(reader: &mut R) -> Result { + init_tlv_field_var!(features, required); + init_tlv_field_var!(announcement_received_time, (default_value, 0)); + init_tlv_field_var!(node_one, required); + let mut one_to_two_wrap: Option = None; + init_tlv_field_var!(node_two, required); + let mut two_to_one_wrap: Option = None; + init_tlv_field_var!(capacity_sats, required); + init_tlv_field_var!(announcement_message, required); + read_tlv_fields!(reader, { + (0, features, required), + (1, announcement_received_time, (default_value, 0)), + (2, node_one, required), + (4, one_to_two_wrap, ignorable), + (6, node_two, required), + (8, two_to_one_wrap, ignorable), + (10, capacity_sats, required), + (12, announcement_message, required), + }); + + Ok(ChannelInfo { + features: init_tlv_based_struct_field!(features, required), + node_one: init_tlv_based_struct_field!(node_one, required), + one_to_two: one_to_two_wrap.map(|w| w.0).unwrap_or(None), + node_two: init_tlv_based_struct_field!(node_two, required), + two_to_one: two_to_one_wrap.map(|w| w.0).unwrap_or(None), + capacity_sats: init_tlv_based_struct_field!(capacity_sats, required), + announcement_message: init_tlv_based_struct_field!(announcement_message, required), + announcement_received_time: init_tlv_based_struct_field!(announcement_received_time, (default_value, 0)), + }) + } +} /// A wrapper around [`ChannelInfo`] representing information about the channel as directed from a /// source node to a target node. @@ -739,7 +839,7 @@ pub struct DirectedChannelInfo<'a> { impl<'a> DirectedChannelInfo<'a> { #[inline] fn new(channel: &'a ChannelInfo, direction: Option<&'a ChannelUpdateInfo>) -> Self { - let htlc_maximum_msat = direction.and_then(|direction| direction.htlc_maximum_msat); + let htlc_maximum_msat = direction.map(|direction| direction.htlc_maximum_msat); let capacity_msat = channel.capacity_sats.map(|capacity_sats| capacity_sats * 1000); let (htlc_maximum_msat, effective_capacity) = match (htlc_maximum_msat, capacity_msat) { @@ -1495,17 +1595,19 @@ impl NetworkGraph where L::Target: Logger { match channels.get_mut(&msg.short_channel_id) { None => return Err(LightningError{err: "Couldn't find channel for update".to_owned(), action: ErrorAction::IgnoreError}), Some(channel) => { - if let OptionalField::Present(htlc_maximum_msat) = msg.htlc_maximum_msat { - if htlc_maximum_msat > MAX_VALUE_MSAT { - return Err(LightningError{err: "htlc_maximum_msat is larger than maximum possible msats".to_owned(), action: ErrorAction::IgnoreError}); - } + if msg.htlc_maximum_msat > MAX_VALUE_MSAT { + return Err(LightningError{err: + "htlc_maximum_msat is larger than maximum possible msats".to_owned(), + action: ErrorAction::IgnoreError}); + } - if let Some(capacity_sats) = channel.capacity_sats { - // It's possible channel capacity is available now, although it wasn't available at announcement (so the field is None). - // Don't query UTXO set here to reduce DoS risks. - if capacity_sats > MAX_VALUE_MSAT / 1000 || htlc_maximum_msat > capacity_sats * 1000 { - return Err(LightningError{err: "htlc_maximum_msat is larger than channel capacity or capacity is bogus".to_owned(), action: ErrorAction::IgnoreError}); - } + if let Some(capacity_sats) = channel.capacity_sats { + // It's possible channel capacity is available now, although it wasn't available at announcement (so the field is None). + // Don't query UTXO set here to reduce DoS risks. + if capacity_sats > MAX_VALUE_MSAT / 1000 || msg.htlc_maximum_msat > capacity_sats * 1000 { + return Err(LightningError{err: + "htlc_maximum_msat is larger than channel capacity or capacity is bogus".to_owned(), + action: ErrorAction::IgnoreError}); } } macro_rules! check_update_latest { @@ -1539,7 +1641,7 @@ impl NetworkGraph where L::Target: Logger { last_update: msg.timestamp, cltv_expiry_delta: msg.cltv_expiry_delta, htlc_minimum_msat: msg.htlc_minimum_msat, - htlc_maximum_msat: if let OptionalField::Present(max_value) = msg.htlc_maximum_msat { Some(max_value) } else { None }, + htlc_maximum_msat: msg.htlc_maximum_msat, fees: RoutingFees { base_msat: msg.fee_base_msat, proportional_millionths: msg.fee_proportional_millionths, @@ -1680,8 +1782,8 @@ mod tests { use chain; use ln::PaymentHash; use ln::features::{ChannelFeatures, InitFeatures, NodeFeatures}; - use routing::gossip::{P2PGossipSync, NetworkGraph, NetworkUpdate, NodeAlias, MAX_EXCESS_BYTES_FOR_RELAY}; - use ln::msgs::{Init, OptionalField, RoutingMessageHandler, UnsignedNodeAnnouncement, NodeAnnouncement, + use routing::gossip::{P2PGossipSync, NetworkGraph, NetworkUpdate, NodeAlias, MAX_EXCESS_BYTES_FOR_RELAY, NodeId, RoutingFees, ChannelUpdateInfo, ChannelInfo, NodeAnnouncementInfo, NodeInfo}; + use ln::msgs::{Init, RoutingMessageHandler, UnsignedNodeAnnouncement, NodeAnnouncement, UnsignedChannelAnnouncement, ChannelAnnouncement, UnsignedChannelUpdate, ChannelUpdate, ReplyChannelRange, QueryChannelRange, QueryShortChannelIds, MAX_VALUE_MSAT}; use util::test_utils; @@ -1805,7 +1907,7 @@ mod tests { flags: 0, cltv_expiry_delta: 144, htlc_minimum_msat: 1_000_000, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: 1_000_000, fee_base_msat: 10_000, fee_proportional_millionths: 20, excess_data: Vec::new() @@ -2026,7 +2128,7 @@ mod tests { let valid_channel_update = get_signed_channel_update(|_| {}, node_1_privkey, &secp_ctx); match gossip_sync.handle_channel_update(&valid_channel_update) { Ok(res) => assert!(res), - _ => panic!() + _ => panic!(), }; { @@ -2059,7 +2161,7 @@ mod tests { }; let valid_channel_update = get_signed_channel_update(|unsigned_channel_update| { - unsigned_channel_update.htlc_maximum_msat = OptionalField::Present(MAX_VALUE_MSAT + 1); + unsigned_channel_update.htlc_maximum_msat = MAX_VALUE_MSAT + 1; unsigned_channel_update.timestamp += 110; }, node_1_privkey, &secp_ctx); match gossip_sync.handle_channel_update(&valid_channel_update) { @@ -2068,7 +2170,7 @@ mod tests { }; let valid_channel_update = get_signed_channel_update(|unsigned_channel_update| { - unsigned_channel_update.htlc_maximum_msat = OptionalField::Present(amount_sats * 1000 + 1); + unsigned_channel_update.htlc_maximum_msat = amount_sats * 1000 + 1; unsigned_channel_update.timestamp += 110; }, node_1_privkey, &secp_ctx); match gossip_sync.handle_channel_update(&valid_channel_update) { diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index 1b14e9c735f..4c00dc5fcf9 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -1940,8 +1940,8 @@ mod tests { use chain::transaction::OutPoint; use chain::keysinterface::KeysInterface; use ln::features::{ChannelFeatures, InitFeatures, InvoiceFeatures, NodeFeatures}; - use ln::msgs::{ErrorAction, LightningError, OptionalField, UnsignedChannelAnnouncement, ChannelAnnouncement, RoutingMessageHandler, - NodeAnnouncement, UnsignedNodeAnnouncement, ChannelUpdate, UnsignedChannelUpdate}; + use ln::msgs::{ErrorAction, LightningError, UnsignedChannelAnnouncement, ChannelAnnouncement, RoutingMessageHandler, + NodeAnnouncement, UnsignedNodeAnnouncement, ChannelUpdate, UnsignedChannelUpdate, MAX_VALUE_MSAT}; use ln::channelmanager; use util::test_utils; use util::chacha20::ChaCha20; @@ -2133,7 +2133,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2145,7 +2145,7 @@ mod tests { flags: 1, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2239,7 +2239,7 @@ mod tests { flags: 1, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2255,7 +2255,7 @@ mod tests { flags: 0, cltv_expiry_delta: (5 << 4) | 3, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: u32::max_value(), fee_proportional_millionths: u32::max_value(), excess_data: Vec::new() @@ -2267,7 +2267,7 @@ mod tests { flags: 1, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2283,7 +2283,7 @@ mod tests { flags: 0, cltv_expiry_delta: (5 << 4) | 3, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: u32::max_value(), fee_proportional_millionths: u32::max_value(), excess_data: Vec::new() @@ -2295,7 +2295,7 @@ mod tests { flags: 1, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2311,7 +2311,7 @@ mod tests { flags: 0, cltv_expiry_delta: (3 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2323,7 +2323,7 @@ mod tests { flags: 1, cltv_expiry_delta: (3 << 4) | 2, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 100, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2337,7 +2337,7 @@ mod tests { flags: 0, cltv_expiry_delta: (4 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 1000000, excess_data: Vec::new() @@ -2349,7 +2349,7 @@ mod tests { flags: 1, cltv_expiry_delta: (4 << 4) | 2, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2363,7 +2363,7 @@ mod tests { flags: 0, cltv_expiry_delta: (13 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 2000000, excess_data: Vec::new() @@ -2375,7 +2375,7 @@ mod tests { flags: 1, cltv_expiry_delta: (13 << 4) | 2, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2391,7 +2391,7 @@ mod tests { flags: 0, cltv_expiry_delta: (6 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2403,7 +2403,7 @@ mod tests { flags: 1, cltv_expiry_delta: (6 << 4) | 2, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new(), @@ -2417,7 +2417,7 @@ mod tests { flags: 0, cltv_expiry_delta: (11 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2429,7 +2429,7 @@ mod tests { flags: 1, cltv_expiry_delta: (11 << 4) | 2, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2447,7 +2447,7 @@ mod tests { flags: 0, cltv_expiry_delta: (7 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 1000000, excess_data: Vec::new() @@ -2459,7 +2459,7 @@ mod tests { flags: 1, cltv_expiry_delta: (7 << 4) | 2, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2544,7 +2544,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2556,7 +2556,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2568,7 +2568,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2580,7 +2580,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2592,7 +2592,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2607,7 +2607,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 200_000_000, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2622,7 +2622,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(199_999_999), + htlc_maximum_msat: 199_999_999, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2641,7 +2641,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2671,7 +2671,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 35_000, - htlc_maximum_msat: OptionalField::Present(40_000), + htlc_maximum_msat: 40_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2683,7 +2683,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 35_000, - htlc_maximum_msat: OptionalField::Present(40_000), + htlc_maximum_msat: 40_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2697,7 +2697,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2709,7 +2709,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2723,7 +2723,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2744,7 +2744,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 65_000, - htlc_maximum_msat: OptionalField::Present(80_000), + htlc_maximum_msat: 80_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2756,7 +2756,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2768,7 +2768,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 100_000, excess_data: Vec::new() @@ -2807,7 +2807,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -2819,7 +2819,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3225,7 +3225,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3237,7 +3237,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3297,7 +3297,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3309,7 +3309,7 @@ mod tests { flags: 2, // to disable cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3642,7 +3642,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3654,7 +3654,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3669,7 +3669,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(1_000_000_000), + htlc_maximum_msat: 1_000_000_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3684,7 +3684,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: 250_000_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3717,7 +3717,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(1_000_000_000), + htlc_maximum_msat: 1_000_000_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3752,7 +3752,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(1_000_000_000), + htlc_maximum_msat: 1_000_000_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3767,7 +3767,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(15_000), + htlc_maximum_msat: 15_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3802,7 +3802,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3825,7 +3825,7 @@ mod tests { flags: 0, cltv_expiry_delta: (3 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: 15_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3837,7 +3837,7 @@ mod tests { flags: 1, cltv_expiry_delta: (3 << 4) | 2, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: 15_000, fee_base_msat: 100, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3869,7 +3869,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(10_000), + htlc_maximum_msat: 10_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3917,7 +3917,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3929,7 +3929,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3944,7 +3944,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3956,7 +3956,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3969,7 +3969,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(50_000), + htlc_maximum_msat: 50_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -3981,7 +3981,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4038,7 +4038,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 1_000_000, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4050,7 +4050,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(50_000), + htlc_maximum_msat: 50_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4095,7 +4095,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4107,7 +4107,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(50_000), + htlc_maximum_msat: 50_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4122,7 +4122,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(60_000), + htlc_maximum_msat: 60_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4134,7 +4134,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(60_000), + htlc_maximum_msat: 60_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4149,7 +4149,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(200_000), + htlc_maximum_msat: 200_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4161,7 +4161,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(180_000), + htlc_maximum_msat: 180_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4252,7 +4252,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4264,7 +4264,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4278,7 +4278,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4290,7 +4290,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4305,7 +4305,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(200_000), + htlc_maximum_msat: 200_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4321,7 +4321,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(200_000), + htlc_maximum_msat: 200_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4333,7 +4333,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(200_000), + htlc_maximum_msat: 200_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4346,7 +4346,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4358,7 +4358,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4420,7 +4420,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4432,7 +4432,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4446,7 +4446,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4458,7 +4458,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4473,7 +4473,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(200_000), + htlc_maximum_msat: 200_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4489,7 +4489,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(200_000), + htlc_maximum_msat: 200_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4501,7 +4501,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(200_000), + htlc_maximum_msat: 200_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4514,7 +4514,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 1_000, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4526,7 +4526,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4587,7 +4587,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4600,7 +4600,7 @@ mod tests { flags: 2, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4614,7 +4614,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4626,7 +4626,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4640,7 +4640,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4663,7 +4663,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(250_000), + htlc_maximum_msat: 250_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4675,7 +4675,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4688,7 +4688,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 150_000, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4700,7 +4700,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4771,7 +4771,7 @@ mod tests { flags: 0, cltv_expiry_delta: (5 << 4) | 5, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(99_000), + htlc_maximum_msat: 99_000, fee_base_msat: u32::max_value(), fee_proportional_millionths: u32::max_value(), excess_data: Vec::new() @@ -4783,7 +4783,7 @@ mod tests { flags: 0, cltv_expiry_delta: (5 << 4) | 3, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(99_000), + htlc_maximum_msat: 99_000, fee_base_msat: u32::max_value(), fee_proportional_millionths: u32::max_value(), excess_data: Vec::new() @@ -4795,7 +4795,7 @@ mod tests { flags: 0, cltv_expiry_delta: (4 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 1, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4807,7 +4807,7 @@ mod tests { flags: 0|2, // Channel disabled cltv_expiry_delta: (13 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 2000000, excess_data: Vec::new() @@ -4860,7 +4860,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(100_000), + htlc_maximum_msat: 100_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4872,7 +4872,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(50_000), + htlc_maximum_msat: 50_000, fee_base_msat: 100, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4886,7 +4886,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(60_000), + htlc_maximum_msat: 60_000, fee_base_msat: 100, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4898,7 +4898,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(60_000), + htlc_maximum_msat: 60_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4912,7 +4912,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(20_000), + htlc_maximum_msat: 20_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -4924,7 +4924,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(20_000), + htlc_maximum_msat: 20_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5011,7 +5011,7 @@ mod tests { flags: 0, cltv_expiry_delta: (6 << 4) | 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5026,7 +5026,7 @@ mod tests { flags: 0, cltv_expiry_delta: (5 << 4) | 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 100, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5041,7 +5041,7 @@ mod tests { flags: 0, cltv_expiry_delta: (4 << 4) | 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5056,7 +5056,7 @@ mod tests { flags: 0, cltv_expiry_delta: (3 << 4) | 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5071,7 +5071,7 @@ mod tests { flags: 0, cltv_expiry_delta: (2 << 4) | 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5085,7 +5085,7 @@ mod tests { flags: 0, cltv_expiry_delta: (1 << 4) | 0, htlc_minimum_msat: 100, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5143,7 +5143,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(85_000), + htlc_maximum_msat: 85_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5156,7 +5156,7 @@ mod tests { flags: 0, cltv_expiry_delta: (4 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(270_000), + htlc_maximum_msat: 270_000, fee_base_msat: 0, fee_proportional_millionths: 1000000, excess_data: Vec::new() @@ -5208,7 +5208,7 @@ mod tests { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(80_000), + htlc_maximum_msat: 80_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5220,7 +5220,7 @@ mod tests { flags: 0, cltv_expiry_delta: (4 << 4) | 1, htlc_minimum_msat: 90_000, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5741,7 +5741,7 @@ mod tests { flags: 0, cltv_expiry_delta: (4 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(250_000_000), + htlc_maximum_msat: 250_000_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() @@ -5753,7 +5753,7 @@ mod tests { flags: 0, cltv_expiry_delta: (13 << 4) | 1, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(250_000_000), + htlc_maximum_msat: 250_000_000, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: Vec::new() diff --git a/lightning/src/routing/scoring.rs b/lightning/src/routing/scoring.rs index e2e0bdd18a6..458a24d9f1b 100644 --- a/lightning/src/routing/scoring.rs +++ b/lightning/src/routing/scoring.rs @@ -1258,7 +1258,7 @@ mod tests { use util::time::tests::SinceEpoch; use ln::features::{ChannelFeatures, NodeFeatures}; - use ln::msgs::{ChannelAnnouncement, ChannelUpdate, OptionalField, UnsignedChannelAnnouncement, UnsignedChannelUpdate}; + use ln::msgs::{ChannelAnnouncement, ChannelUpdate, UnsignedChannelAnnouncement, UnsignedChannelUpdate}; use routing::gossip::{EffectiveCapacity, NetworkGraph, NodeId}; use routing::router::RouteHop; use routing::scoring::{ChannelUsage, Score}; @@ -1385,7 +1385,7 @@ mod tests { flags, cltv_expiry_delta: 18, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Present(1_000), + htlc_maximum_msat: 1_000, fee_base_msat: 1, fee_proportional_millionths: 0, excess_data: Vec::new(), diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index 4f3d800be5d..c2d14073d99 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -19,7 +19,6 @@ use chain::transaction::OutPoint; use chain::keysinterface; use ln::features::{ChannelFeatures, InitFeatures}; use ln::{msgs, wire}; -use ln::msgs::OptionalField; use ln::script::ShutdownScript; use routing::scoring::FixedPenaltyScorer; use util::enforcing_trait_impls::{EnforcingSigner, EnforcementState}; @@ -407,7 +406,7 @@ fn get_dummy_channel_update(short_chan_id: u64) -> msgs::ChannelUpdate { flags: 0, cltv_expiry_delta: 0, htlc_minimum_msat: 0, - htlc_maximum_msat: OptionalField::Absent, + htlc_maximum_msat: msgs::MAX_VALUE_MSAT, fee_base_msat: 0, fee_proportional_millionths: 0, excess_data: vec![], From 8f4c951b1f9a807b28affc9c7cc92ab6f7b42a0a Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 25 Jul 2022 20:38:18 +0200 Subject: [PATCH 10/91] Don't fail read `NodeInfo` for inv. `NetAddress` Fixes a deserialization incompatibility introduced with #1553. --- lightning/src/routing/gossip.rs | 54 ++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 510dbcfb1c4..f943513238a 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -34,6 +34,7 @@ use util::events::{Event, EventHandler, MessageSendEvent, MessageSendEventsProvi use util::scid_utils::{block_from_scid, scid_from_parts, MAX_SCID_BLOCK}; use io; +use io_extras::{copy, sink}; use prelude::*; use alloc::collections::{BTreeMap, btree_map::Entry as BtreeEntry}; use core::{cmp, fmt}; @@ -1089,11 +1090,54 @@ impl fmt::Display for NodeInfo { } } -impl_writeable_tlv_based!(NodeInfo, { - (0, lowest_inbound_channel_fees, option), - (2, announcement_info, option), - (4, channels, vec_type), -}); +impl Writeable for NodeInfo { + fn write(&self, writer: &mut W) -> Result<(), io::Error> { + write_tlv_fields!(writer, { + (0, self.lowest_inbound_channel_fees, option), + (2, self.announcement_info, option), + (4, self.channels, vec_type), + }); + Ok(()) + } +} + +// A wrapper allowing for the optional deseralization of `NodeAnnouncementInfo`. Utilizing this is +// necessary to maintain compatibility with previous serializations of `NetAddress` that have an +// invalid hostname set. We ignore and eat all errors until we are either able to read a +// `NodeAnnouncementInfo` or hit a `ShortRead`, i.e., read the TLV field to the end. +struct NodeAnnouncementInfoDeserWrapper(NodeAnnouncementInfo); + +impl MaybeReadable for NodeAnnouncementInfoDeserWrapper { + fn read(reader: &mut R) -> Result, DecodeError> { + match ::util::ser::Readable::read(reader) { + Ok(node_announcement_info) => return Ok(Some(Self(node_announcement_info))), + Err(_) => { + copy(reader, &mut sink()).unwrap(); + return Ok(None) + }, + }; + } +} + +impl Readable for NodeInfo { + fn read(reader: &mut R) -> Result { + init_tlv_field_var!(lowest_inbound_channel_fees, option); + let mut announcement_info_wrap: Option = None; + init_tlv_field_var!(channels, vec_type); + + read_tlv_fields!(reader, { + (0, lowest_inbound_channel_fees, option), + (2, announcement_info_wrap, ignorable), + (4, channels, vec_type), + }); + + Ok(NodeInfo { + lowest_inbound_channel_fees: init_tlv_based_struct_field!(lowest_inbound_channel_fees, option), + announcement_info: announcement_info_wrap.map(|w| w.0), + channels: init_tlv_based_struct_field!(channels, vec_type), + }) + } +} const SERIALIZATION_VERSION: u8 = 1; const MIN_SERIALIZATION_VERSION: u8 = 1; From 8b86ed7c52a91cd421d0ced5ca5d7fd2560500d5 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 25 Jul 2022 20:39:06 +0200 Subject: [PATCH 11/91] Test serialization of `ChannelInfo` and `NodeInfo` --- lightning/src/routing/gossip.rs | 136 ++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index f943513238a..37bf1ea5a6f 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -2952,6 +2952,142 @@ mod tests { assert_eq!(format_bytes_alias(b"\xFFI \0LDK!"), "\u{FFFD}I "); assert_eq!(format_bytes_alias(b"\xFFI \tLDK!"), "\u{FFFD}I \u{FFFD}LDK!"); } + + #[test] + fn channel_info_is_readable() { + let chanmon_cfgs = ::ln::functional_test_utils::create_chanmon_cfgs(2); + let node_cfgs = ::ln::functional_test_utils::create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = ::ln::functional_test_utils::create_node_chanmgrs(2, &node_cfgs, &[None, None, None, None]); + let nodes = ::ln::functional_test_utils::create_network(2, &node_cfgs, &node_chanmgrs); + + // 1. Test encoding/decoding of ChannelUpdateInfo + let chan_update_info = ChannelUpdateInfo { + last_update: 23, + enabled: true, + cltv_expiry_delta: 42, + htlc_minimum_msat: 1234, + htlc_maximum_msat: 5678, + fees: RoutingFees { base_msat: 9, proportional_millionths: 10 }, + last_update_message: None, + }; + + let mut encoded_chan_update_info: Vec = Vec::new(); + assert!(chan_update_info.write(&mut encoded_chan_update_info).is_ok()); + + // First make sure we can read ChannelUpdateInfos we just wrote + let read_chan_update_info: ChannelUpdateInfo = ::util::ser::Readable::read(&mut encoded_chan_update_info.as_slice()).unwrap(); + assert_eq!(chan_update_info, read_chan_update_info); + + // Check the serialization hasn't changed. + let legacy_chan_update_info_with_some: Vec = hex::decode("340004000000170201010402002a060800000000000004d2080909000000000000162e0a0d0c00040000000902040000000a0c0100").unwrap(); + assert_eq!(encoded_chan_update_info, legacy_chan_update_info_with_some); + + // Check we fail if htlc_maximum_msat is not present in either the ChannelUpdateInfo itself + // or the ChannelUpdate enclosed with `last_update_message`. + let legacy_chan_update_info_with_some_and_fail_update: Vec = hex::decode("b40004000000170201010402002a060800000000000004d2080909000000000000162e0a0d0c00040000000902040000000a0c8181d977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f00083a840000034d013413a70000009000000000000f42400000271000000014").unwrap(); + let read_chan_update_info_res: Result = ::util::ser::Readable::read(&mut legacy_chan_update_info_with_some_and_fail_update.as_slice()); + assert!(read_chan_update_info_res.is_err()); + + let legacy_chan_update_info_with_none: Vec = hex::decode("2c0004000000170201010402002a060800000000000004d20801000a0d0c00040000000902040000000a0c0100").unwrap(); + let read_chan_update_info_res: Result = ::util::ser::Readable::read(&mut legacy_chan_update_info_with_none.as_slice()); + assert!(read_chan_update_info_res.is_err()); + + // 2. Test encoding/decoding of ChannelInfo + // Check we can encode/decode ChannelInfo without ChannelUpdateInfo fields present. + let chan_info_none_updates = ChannelInfo { + features: ChannelFeatures::known(), + node_one: NodeId::from_pubkey(&nodes[0].node.get_our_node_id()), + one_to_two: None, + node_two: NodeId::from_pubkey(&nodes[1].node.get_our_node_id()), + two_to_one: None, + capacity_sats: None, + announcement_message: None, + announcement_received_time: 87654, + }; + + let mut encoded_chan_info: Vec = Vec::new(); + assert!(chan_info_none_updates.write(&mut encoded_chan_info).is_ok()); + + let read_chan_info: ChannelInfo = ::util::ser::Readable::read(&mut encoded_chan_info.as_slice()).unwrap(); + assert_eq!(chan_info_none_updates, read_chan_info); + + // Check we can encode/decode ChannelInfo with ChannelUpdateInfo fields present. + let chan_info_some_updates = ChannelInfo { + features: ChannelFeatures::known(), + node_one: NodeId::from_pubkey(&nodes[0].node.get_our_node_id()), + one_to_two: Some(chan_update_info.clone()), + node_two: NodeId::from_pubkey(&nodes[1].node.get_our_node_id()), + two_to_one: Some(chan_update_info.clone()), + capacity_sats: None, + announcement_message: None, + announcement_received_time: 87654, + }; + + let mut encoded_chan_info: Vec = Vec::new(); + assert!(chan_info_some_updates.write(&mut encoded_chan_info).is_ok()); + + let read_chan_info: ChannelInfo = ::util::ser::Readable::read(&mut encoded_chan_info.as_slice()).unwrap(); + assert_eq!(chan_info_some_updates, read_chan_info); + + // Check the serialization hasn't changed. + let legacy_chan_info_with_some: Vec = hex::decode("ca00020000010800000000000156660221027f921585f2ac0c7c70e36110adecfd8fd14b8a99bfb3d000a283fcac358fce88043636340004000000170201010402002a060800000000000004d2080909000000000000162e0a0d0c00040000000902040000000a0c010006210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c23083636340004000000170201010402002a060800000000000004d2080909000000000000162e0a0d0c00040000000902040000000a0c01000a01000c0100").unwrap(); + assert_eq!(encoded_chan_info, legacy_chan_info_with_some); + + // Check we can decode legacy ChannelInfo, even if the `two_to_one` / `one_to_two` / + // `last_update_message` fields fail to decode due to missing htlc_maximum_msat. + let legacy_chan_info_with_some_and_fail_update = hex::decode("fd01ca00020000010800000000000156660221027f921585f2ac0c7c70e36110adecfd8fd14b8a99bfb3d000a283fcac358fce8804b6b6b40004000000170201010402002a060800000000000004d2080909000000000000162e0a0d0c00040000000902040000000a0c8181d977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f00083a840000034d013413a70000009000000000000f4240000027100000001406210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c2308b6b6b40004000000170201010402002a060800000000000004d2080909000000000000162e0a0d0c00040000000902040000000a0c8181d977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f00083a840000034d013413a70000009000000000000f424000002710000000140a01000c0100").unwrap(); + let read_chan_info: ChannelInfo = ::util::ser::Readable::read(&mut legacy_chan_info_with_some_and_fail_update.as_slice()).unwrap(); + assert_eq!(read_chan_info.announcement_received_time, 87654); + assert_eq!(read_chan_info.one_to_two, None); + assert_eq!(read_chan_info.two_to_one, None); + + let legacy_chan_info_with_none: Vec = hex::decode("ba00020000010800000000000156660221027f921585f2ac0c7c70e36110adecfd8fd14b8a99bfb3d000a283fcac358fce88042e2e2c0004000000170201010402002a060800000000000004d20801000a0d0c00040000000902040000000a0c010006210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c23082e2e2c0004000000170201010402002a060800000000000004d20801000a0d0c00040000000902040000000a0c01000a01000c0100").unwrap(); + let read_chan_info: ChannelInfo = ::util::ser::Readable::read(&mut legacy_chan_info_with_none.as_slice()).unwrap(); + assert_eq!(read_chan_info.announcement_received_time, 87654); + assert_eq!(read_chan_info.one_to_two, None); + assert_eq!(read_chan_info.two_to_one, None); + } + + #[test] + fn node_info_is_readable() { + use std::convert::TryFrom; + + // 1. Check we can read a valid NodeAnnouncementInfo and fail on an invalid one + let valid_netaddr = ::ln::msgs::NetAddress::Hostname { hostname: ::util::ser::Hostname::try_from("A".to_string()).unwrap(), port: 1234 }; + let valid_node_ann_info = NodeAnnouncementInfo { + features: NodeFeatures::known(), + last_update: 0, + rgb: [0u8; 3], + alias: NodeAlias([0u8; 32]), + addresses: vec![valid_netaddr], + announcement_message: None, + }; + + let mut encoded_valid_node_ann_info = Vec::new(); + assert!(valid_node_ann_info.write(&mut encoded_valid_node_ann_info).is_ok()); + let read_valid_node_ann_info: NodeAnnouncementInfo = ::util::ser::Readable::read(&mut encoded_valid_node_ann_info.as_slice()).unwrap(); + assert_eq!(read_valid_node_ann_info, valid_node_ann_info); + + let encoded_invalid_node_ann_info = hex::decode("3f0009000788a000080a51a20204000000000403000000062000000000000000000000000000000000000000000000000000000000000000000a0505014004d2").unwrap(); + let read_invalid_node_ann_info_res: Result = ::util::ser::Readable::read(&mut encoded_invalid_node_ann_info.as_slice()); + assert!(read_invalid_node_ann_info_res.is_err()); + + // 2. Check we can read a NodeInfo anyways, but set the NodeAnnouncementInfo to None if invalid + let valid_node_info = NodeInfo { + channels: Vec::new(), + lowest_inbound_channel_fees: None, + announcement_info: Some(valid_node_ann_info), + }; + + let mut encoded_valid_node_info = Vec::new(); + assert!(valid_node_info.write(&mut encoded_valid_node_info).is_ok()); + let read_valid_node_info: NodeInfo = ::util::ser::Readable::read(&mut encoded_valid_node_info.as_slice()).unwrap(); + assert_eq!(read_valid_node_info, valid_node_info); + + let encoded_invalid_node_info_hex = hex::decode("4402403f0009000788a000080a51a20204000000000403000000062000000000000000000000000000000000000000000000000000000000000000000a0505014004d20400").unwrap(); + let read_invalid_node_info: NodeInfo = ::util::ser::Readable::read(&mut encoded_invalid_node_info_hex.as_slice()).unwrap(); + assert_eq!(read_invalid_node_info.announcement_info, None); + } } #[cfg(all(test, feature = "_bench_unstable"))] From fe39a89ab3b4a7542d68ad33505a36a1e84152d0 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 14 Jul 2022 01:06:10 +0000 Subject: [PATCH 12/91] Add a per-amount base penalty in the `ProbabilisticScorer` There's not much reason to not have a per-hop-per-amount penalty in the `ProbabilisticScorer` to go along with the per-hop penalty to let it scale up to larger amounts, so we add one here. Notably, we use a divisor of 2^30 instead of 2^20 (like the equivalent liquidity penalty) as it allows for more flexibility, and there's not really any reason to worry about us not being able to create high enough penalties. Closes #1616 --- lightning/src/routing/scoring.rs | 72 +++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/lightning/src/routing/scoring.rs b/lightning/src/routing/scoring.rs index e2e0bdd18a6..5e3e7b53109 100644 --- a/lightning/src/routing/scoring.rs +++ b/lightning/src/routing/scoring.rs @@ -324,6 +324,9 @@ where L::Target: Logger { /// /// Used to configure base, liquidity, and amount penalties, the sum of which comprises the channel /// penalty (i.e., the amount in msats willing to be paid to avoid routing through the channel). +/// +/// The penalty applied to any channel by the [`ProbabilisticScorer`] is the sum of each of the +/// parameters here. #[derive(Clone)] pub struct ProbabilisticScoringParameters { /// A fixed penalty in msats to apply to each channel. @@ -331,6 +334,20 @@ pub struct ProbabilisticScoringParameters { /// Default value: 500 msat pub base_penalty_msat: u64, + /// A multiplier used with the payment amount to calculate a fixed penalty applied to each + /// channel, in excess of the [`base_penalty_msat`]. + /// + /// The purpose of the amount penalty is to avoid having fees dominate the channel cost (i.e., + /// fees plus penalty) for large payments. The penalty is computed as the product of this + /// multiplier and `2^30`ths of the payment amount. + /// + /// ie `base_penalty_amount_multiplier_msat * amount_msat / 2^30` + /// + /// Default value: 8,192 msat + /// + /// [`base_penalty_msat`]: Self::base_penalty_msat + pub base_penalty_amount_multiplier_msat: u64, + /// A multiplier used in conjunction with the negative `log10` of the channel's success /// probability for a payment to determine the liquidity penalty. /// @@ -536,6 +553,7 @@ impl ProbabilisticScoringParameters { fn zero_penalty() -> Self { Self { base_penalty_msat: 0, + base_penalty_amount_multiplier_msat: 0, liquidity_penalty_multiplier_msat: 0, liquidity_offset_half_life: Duration::from_secs(3600), amount_penalty_multiplier_msat: 0, @@ -558,6 +576,7 @@ impl Default for ProbabilisticScoringParameters { fn default() -> Self { Self { base_penalty_msat: 500, + base_penalty_amount_multiplier_msat: 8192, liquidity_penalty_multiplier_msat: 40_000, liquidity_offset_half_life: Duration::from_secs(3600), amount_penalty_multiplier_msat: 256, @@ -631,10 +650,11 @@ const PRECISION_LOWER_BOUND_DENOMINATOR: u64 = approx::LOWER_BITS_BOUND; /// The divisor used when computing the amount penalty. const AMOUNT_PENALTY_DIVISOR: u64 = 1 << 20; +const BASE_AMOUNT_PENALTY_DIVISOR: u64 = 1 << 30; impl, T: Time, U: Deref> DirectedChannelLiquidity { - /// Returns a penalty for routing the given HTLC `amount_msat` through the channel in this - /// direction. + /// Returns a liquidity penalty for routing the given HTLC `amount_msat` through the channel in + /// this direction. fn penalty_msat(&self, amount_msat: u64, params: &ProbabilisticScoringParameters) -> u64 { let max_liquidity_msat = self.max_liquidity_msat(); let min_liquidity_msat = core::cmp::min(self.min_liquidity_msat(), max_liquidity_msat); @@ -653,8 +673,8 @@ impl, T: Time, U: Deref> DirectedChannelLiqui if amount_msat - min_liquidity_msat < denominator / PRECISION_LOWER_BOUND_DENOMINATOR { // If the failure probability is < 1.5625% (as 1 - numerator/denominator < 1/64), // don't bother trying to use the log approximation as it gets too noisy to be - // particularly helpful, instead just round down to 0 and return the base penalty. - params.base_penalty_msat + // particularly helpful, instead just round down to 0. + 0 } else { let negative_log10_times_2048 = approx::negative_log10_times_2048(numerator, denominator); @@ -663,7 +683,7 @@ impl, T: Time, U: Deref> DirectedChannelLiqui } } - /// Computes the liquidity and amount penalties and adds them to the base penalty. + /// Computes the liquidity penalty from the penalty multipliers. #[inline(always)] fn combined_penalty_msat( &self, amount_msat: u64, negative_log10_times_2048: u64, @@ -679,9 +699,7 @@ impl, T: Time, U: Deref> DirectedChannelLiqui .saturating_mul(params.amount_penalty_multiplier_msat) .saturating_mul(amount_msat) / 2048 / AMOUNT_PENALTY_DIVISOR; - params.base_penalty_msat - .saturating_add(liquidity_penalty_msat) - .saturating_add(amount_penalty_msat) + liquidity_penalty_msat.saturating_add(amount_penalty_msat) } /// Returns the lower bound of the channel liquidity balance in this direction. @@ -763,13 +781,17 @@ impl>, L: Deref, T: Time> Score for Probabilis return *penalty; } + let base_penalty_msat = self.params.base_penalty_msat.saturating_add( + self.params.base_penalty_amount_multiplier_msat + .saturating_mul(usage.amount_msat) / BASE_AMOUNT_PENALTY_DIVISOR); + let mut anti_probing_penalty_msat = 0; match usage.effective_capacity { EffectiveCapacity::ExactLiquidity { liquidity_msat } => { if usage.amount_msat > liquidity_msat { return u64::max_value(); } else { - return self.params.base_penalty_msat; + return base_penalty_msat; } }, EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: Some(htlc_maximum_msat) } => { @@ -790,6 +812,7 @@ impl>, L: Deref, T: Time> Score for Probabilis .as_directed(source, target, capacity_msat, liquidity_offset_half_life) .penalty_msat(amount_msat, &self.params) .saturating_add(anti_probing_penalty_msat) + .saturating_add(base_penalty_msat) } fn payment_path_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) { @@ -2069,47 +2092,47 @@ mod tests { inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Total { capacity_msat: 950_000_000, htlc_maximum_msat: Some(1_000) }, }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 3613); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 4375); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1977); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 2739); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 2_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1474); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 2236); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 3_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1223); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1985); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 4_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 877); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1639); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 5_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 845); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1607); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 6_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 500); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1262); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 7_450_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 500); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1262); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 7_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 500); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1262); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 8_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 500); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1262); let usage = ChannelUsage { effective_capacity: EffectiveCapacity::Total { capacity_msat: 9_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage }; - assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 500); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1262); } #[test] @@ -2137,6 +2160,15 @@ mod tests { }; let scorer = ProbabilisticScorer::new(params, &network_graph, &logger); assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 558); + + let params = ProbabilisticScoringParameters { + base_penalty_msat: 500, liquidity_penalty_multiplier_msat: 1_000, + base_penalty_amount_multiplier_msat: (1 << 30), + anti_probing_penalty_msat: 0, ..Default::default() + }; + + let scorer = ProbabilisticScorer::new(params, &network_graph, &logger); + assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 558 + 128); } #[test] From 7f80972e1fd91f3743944857026f7b39aa788b9a Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 21 Jul 2022 17:47:19 +0000 Subject: [PATCH 13/91] Rename amount penalty to `liquidity_penalty_amount_multiplier_msat` This makes our `ProbabilisticScorer` field names more consistent, as we add more types of penalties, referring to a penalty as only the "amount penalty" no longer makes sense - we not have several amount multiplier penalties. --- lightning/src/routing/scoring.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lightning/src/routing/scoring.rs b/lightning/src/routing/scoring.rs index 5e3e7b53109..a587f53c556 100644 --- a/lightning/src/routing/scoring.rs +++ b/lightning/src/routing/scoring.rs @@ -386,7 +386,7 @@ pub struct ProbabilisticScoringParameters { /// multiplier and `2^20`ths of the payment amount, weighted by the negative `log10` of the /// success probability. /// - /// `-log10(success_probability) * amount_penalty_multiplier_msat * amount_msat / 2^20` + /// `-log10(success_probability) * liquidity_penalty_amount_multiplier_msat * amount_msat / 2^20` /// /// In practice, this means for 0.1 success probability (`-log10(0.1) == 1`) each `2^20`th of /// the amount will result in a penalty of the multiplier. And, as the success probability @@ -395,7 +395,7 @@ pub struct ProbabilisticScoringParameters { /// fall below `1`. /// /// Default value: 256 msat - pub amount_penalty_multiplier_msat: u64, + pub liquidity_penalty_amount_multiplier_msat: u64, /// Manual penalties used for the given nodes. Allows to set a particular penalty for a given /// node. Note that a manual penalty of `u64::max_value()` means the node would not ever be @@ -416,7 +416,7 @@ pub struct ProbabilisticScoringParameters { /// current estimate of the channel's available liquidity. /// /// Note that in this case all other penalties, including the - /// [`liquidity_penalty_multiplier_msat`] and [`amount_penalty_multiplier_msat`]-based + /// [`liquidity_penalty_multiplier_msat`] and [`liquidity_penalty_amount_multiplier_msat`]-based /// penalties, as well as the [`base_penalty_msat`] and the [`anti_probing_penalty_msat`], if /// applicable, are still included in the overall penalty. /// @@ -426,7 +426,7 @@ pub struct ProbabilisticScoringParameters { /// Default value: 1_0000_0000_000 msat (1 Bitcoin) /// /// [`liquidity_penalty_multiplier_msat`]: Self::liquidity_penalty_multiplier_msat - /// [`amount_penalty_multiplier_msat`]: Self::amount_penalty_multiplier_msat + /// [`liquidity_penalty_amount_multiplier_msat`]: Self::liquidity_penalty_amount_multiplier_msat /// [`base_penalty_msat`]: Self::base_penalty_msat /// [`anti_probing_penalty_msat`]: Self::anti_probing_penalty_msat pub considered_impossible_penalty_msat: u64, @@ -556,7 +556,7 @@ impl ProbabilisticScoringParameters { base_penalty_amount_multiplier_msat: 0, liquidity_penalty_multiplier_msat: 0, liquidity_offset_half_life: Duration::from_secs(3600), - amount_penalty_multiplier_msat: 0, + liquidity_penalty_amount_multiplier_msat: 0, manual_node_penalties: HashMap::new(), anti_probing_penalty_msat: 0, considered_impossible_penalty_msat: 0, @@ -579,7 +579,7 @@ impl Default for ProbabilisticScoringParameters { base_penalty_amount_multiplier_msat: 8192, liquidity_penalty_multiplier_msat: 40_000, liquidity_offset_half_life: Duration::from_secs(3600), - amount_penalty_multiplier_msat: 256, + liquidity_penalty_amount_multiplier_msat: 256, manual_node_penalties: HashMap::new(), anti_probing_penalty_msat: 250, considered_impossible_penalty_msat: 1_0000_0000_000, @@ -696,7 +696,7 @@ impl, T: Time, U: Deref> DirectedChannelLiqui (negative_log10_times_2048.saturating_mul(multiplier_msat) / 2048).min(max_penalty_msat) }; let amount_penalty_msat = negative_log10_times_2048 - .saturating_mul(params.amount_penalty_multiplier_msat) + .saturating_mul(params.liquidity_penalty_amount_multiplier_msat) .saturating_mul(amount_msat) / 2048 / AMOUNT_PENALTY_DIVISOR; liquidity_penalty_msat.saturating_add(amount_penalty_msat) @@ -2185,7 +2185,7 @@ mod tests { let params = ProbabilisticScoringParameters { liquidity_penalty_multiplier_msat: 1_000, - amount_penalty_multiplier_msat: 0, + liquidity_penalty_amount_multiplier_msat: 0, ..ProbabilisticScoringParameters::zero_penalty() }; let scorer = ProbabilisticScorer::new(params, &network_graph, &logger); @@ -2193,7 +2193,7 @@ mod tests { let params = ProbabilisticScoringParameters { liquidity_penalty_multiplier_msat: 1_000, - amount_penalty_multiplier_msat: 256, + liquidity_penalty_amount_multiplier_msat: 256, ..ProbabilisticScoringParameters::zero_penalty() }; let scorer = ProbabilisticScorer::new(params, &network_graph, &logger); From b2a2b1fb02c9efd0c2ddfb09fd8ed9efa127509d Mon Sep 17 00:00:00 2001 From: Duncan Dean Date: Tue, 26 Jul 2022 09:54:09 +0200 Subject: [PATCH 14/91] Specify why flags for `channel_disabled` error are zero We can remove the TODO for this and specify why the flags are zero as it's now fully specified in BOLT 4. See https://github.com/lightning/bolts/blob/341ec84/04-onion-routing.md?plain=1#L1008 --- lightning/src/ln/channelmanager.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 67c7b58e879..d011d6b42c8 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -3794,7 +3794,8 @@ impl ChannelMana if let Ok(upd) = self.get_channel_update_for_onion(scid, chan) { let mut enc = VecWriter(Vec::with_capacity(upd.serialized_length() + 6)); if desired_err_code == 0x1000 | 20 { - // TODO: underspecified, follow https://github.com/lightning/bolts/issues/791 + // No flags for `disabled_flags` are currently defined so they're always two zero bytes. + // See https://github.com/lightning/bolts/blob/341ec84/04-onion-routing.md?plain=1#L1008 0u16.write(&mut enc).expect("Writes cannot fail"); } (upd.serialized_length() as u16 + 2).write(&mut enc).expect("Writes cannot fail"); From 10aa4aa42f9c777ed58468cf80851d8197009d4d Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 26 Jul 2022 20:29:36 +0000 Subject: [PATCH 15/91] Fix new compilation warnings in `debug_sync` module --- lightning/src/debug_sync.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lightning/src/debug_sync.rs b/lightning/src/debug_sync.rs index daec309e9b5..b7466776a6e 100644 --- a/lightning/src/debug_sync.rs +++ b/lightning/src/debug_sync.rs @@ -68,7 +68,8 @@ struct LockMetadata { struct LockDep { lock: Arc, - lockdep_trace: Backtrace, + /// lockdep_trace is unused unless we're building with `backtrace`, so we mark it _ + _lockdep_trace: Backtrace, } #[cfg(feature = "backtrace")] @@ -140,13 +141,13 @@ impl LockMetadata { // of the same lock. debug_assert!(cfg!(feature = "backtrace"), "Tried to acquire a lock while it was held!"); } - for (locked_dep_idx, locked_dep) in locked.locked_before.lock().unwrap().iter() { + for (locked_dep_idx, _locked_dep) in locked.locked_before.lock().unwrap().iter() { if *locked_dep_idx == this.lock_idx && *locked_dep_idx != locked.lock_idx { #[cfg(feature = "backtrace")] panic!("Tried to violate existing lockorder.\nMutex that should be locked after the current lock was created at the following backtrace.\nNote that to get a backtrace for the lockorder violation, you should set RUST_BACKTRACE=1\nLock being taken constructed at: {} ({}):\n{:?}\nLock constructed at: {} ({})\n{:?}\n\nLock dep created at:\n{:?}\n\n", get_construction_location(&this._lock_construction_bt), this.lock_idx, this._lock_construction_bt, get_construction_location(&locked._lock_construction_bt), locked.lock_idx, locked._lock_construction_bt, - locked_dep.lockdep_trace); + _locked_dep._lockdep_trace); #[cfg(not(feature = "backtrace"))] panic!("Tried to violate existing lockorder. Build with the backtrace feature for more info."); } @@ -154,7 +155,7 @@ impl LockMetadata { // Insert any already-held locks in our locked-before set. let mut locked_before = this.locked_before.lock().unwrap(); if !locked_before.contains_key(&locked.lock_idx) { - let lockdep = LockDep { lock: Arc::clone(locked), lockdep_trace: Backtrace::new() }; + let lockdep = LockDep { lock: Arc::clone(locked), _lockdep_trace: Backtrace::new() }; locked_before.insert(lockdep.lock.lock_idx, lockdep); } } @@ -175,7 +176,7 @@ impl LockMetadata { let mut locked_before = this.locked_before.lock().unwrap(); for (locked_idx, locked) in held.borrow().iter() { if !locked_before.contains_key(locked_idx) { - let lockdep = LockDep { lock: Arc::clone(locked), lockdep_trace: Backtrace::new() }; + let lockdep = LockDep { lock: Arc::clone(locked), _lockdep_trace: Backtrace::new() }; locked_before.insert(*locked_idx, lockdep); } } From 4da6f23cfff84467aa0de2fa36ee6300f89b5055 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 26 Jul 2022 20:31:17 +0000 Subject: [PATCH 16/91] Drop unused test code in `lightning-invoice::payment` --- lightning-invoice/src/payment.rs | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/lightning-invoice/src/payment.rs b/lightning-invoice/src/payment.rs index 051893ea0a3..476d12a3475 100644 --- a/lightning-invoice/src/payment.rs +++ b/lightning-invoice/src/payment.rs @@ -1437,8 +1437,6 @@ mod tests { enum TestResult { PaymentFailure { path: Vec, short_channel_id: u64 }, PaymentSuccess { path: Vec }, - ProbeFailure { path: Vec, short_channel_id: u64 }, - ProbeSuccess { path: Vec }, } impl TestScorer { @@ -1474,12 +1472,6 @@ mod tests { Some(TestResult::PaymentSuccess { path }) => { panic!("Unexpected successful payment path: {:?}", path) }, - Some(TestResult::ProbeFailure { path, .. }) => { - panic!("Unexpected failed payment probe: {:?}", path) - }, - Some(TestResult::ProbeSuccess { path }) => { - panic!("Unexpected successful payment probe: {:?}", path) - }, None => panic!("Unexpected payment_path_failed call: {:?}", actual_path), } } @@ -1494,18 +1486,12 @@ mod tests { Some(TestResult::PaymentSuccess { path }) => { assert_eq!(actual_path, &path.iter().collect::>()[..]); }, - Some(TestResult::ProbeFailure { path, .. }) => { - panic!("Unexpected failed payment probe: {:?}", path) - }, - Some(TestResult::ProbeSuccess { path }) => { - panic!("Unexpected successful payment probe: {:?}", path) - }, None => panic!("Unexpected payment_path_successful call: {:?}", actual_path), } } } - fn probe_failed(&mut self, actual_path: &[&RouteHop], actual_short_channel_id: u64) { + fn probe_failed(&mut self, actual_path: &[&RouteHop], _: u64) { if let Some(expectations) = &mut self.expectations { match expectations.pop_front() { Some(TestResult::PaymentFailure { path, .. }) => { @@ -1514,13 +1500,6 @@ mod tests { Some(TestResult::PaymentSuccess { path }) => { panic!("Unexpected successful payment path: {:?}", path) }, - Some(TestResult::ProbeFailure { path, short_channel_id }) => { - assert_eq!(actual_path, &path.iter().collect::>()[..]); - assert_eq!(actual_short_channel_id, short_channel_id); - }, - Some(TestResult::ProbeSuccess { path }) => { - panic!("Unexpected successful payment probe: {:?}", path) - }, None => panic!("Unexpected payment_path_failed call: {:?}", actual_path), } } @@ -1534,12 +1513,6 @@ mod tests { Some(TestResult::PaymentSuccess { path }) => { panic!("Unexpected successful payment path: {:?}", path) }, - Some(TestResult::ProbeFailure { path, .. }) => { - panic!("Unexpected failed payment probe: {:?}", path) - }, - Some(TestResult::ProbeSuccess { path }) => { - assert_eq!(actual_path, &path.iter().collect::>()[..]); - }, None => panic!("Unexpected payment_path_successful call: {:?}", actual_path), } } From a22ba3991606337802051aaa92bd45ae1b7894da Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 25 Jul 2022 23:49:24 +0000 Subject: [PATCH 17/91] Add CHANGELOG entry for 0.0.110 --- CHANGELOG.md | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 961b1121196..c907952ce91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,60 @@ +# 0.0.110 - 2022-XXX + +## API Updates + * `ChannelManager::send_probe` and `Score::probe_{failed,successful}` have + been added to make probing more explicit, as well as new + `Event::Probe{Failed,Successful}` events (#1567). + * `ProbabilisticScoringParameters::banned_nodes` has been renamed + `manual_node_penalties` and changed to take msat penalties (#1592). + * Per-payment tracking of failed paths was added to enable configuration of + `ProbabilisticScoringParameters::considered_impossible_penalty_msat` (#1600) + * `ProbabilisticScoringParameters::base_penalty_amount_multiplier_msat` was + added to allow a penalty that is only amount-dependent (#1617). + * `ProbabilisticScoringParameters::amount_penalty_multiplier_msat` was renamed + `liquidity_penalty_amount_multiplier_msat` (#1617). + * A new `Event::HTLCHandlingFailed` has been added which provides visibility + into failures to forward/claim accepted HTLCs (#1403). + * Support has been added for DNS hostnames in the `NetAddress` type, see + [BOLT PR #911](https://github.com/lightning/bolts/pull/911) (#1553). + * `GossipSync` now has `rapid`, `p2p`, and `none` constructors (#1618). + * `lightning-net-tokio` no longer requires types to be in `Arc`s (#1623). + * The `htlc_maximum_msat` field is now required in `ChannelUpdate` gossip + messages. In tests this rejects < 1% of channels (#1519). + * `ReadOnlyNetworkGraph::{channel,node}` have been added to query for + individual channel/node data, primarily for bindings users (#1543). + * `FeeEstimator` implementations are now wrapped internally to ensure values + below 253 sats/kW are never used (#1552). + * Route selection no longer attempts to randomize path selection. This is + unlikely to lead to a material change in the paths selected (#1610). + +## Bug Fixes + * Fixed a panic when deserializing `ChannelDetails` objects (#1588). + * When routing, channels are no longer fully saturated before MPP splits are + generated, instead a configuration knob was added as + `PaymentParameters::max_channel_saturation_power_of_half` (#1605). + * Fixed a panic which occurred in `ProbabilisticScorer` when wallclock time + goes backwards across a restart (#1603). + +## Serialization Compatibility + * All new fields are ignored by prior versions of LDK. All new fields are not + present when reading objects serialized by prior versions of LDK. + * Channel information written in the `NetworkGraph` which is missing + `htlc_maximum_msat` may be dropped on deserialization (#1519). + * Similarly, node information written in the `NetworkGraph` which contains an + invalid hostname may be dropped on deserialization (#1519). + +In total, this release features 79 files changed, 2935 insertions, 1363 +deletions in 52 commits from 9 authors, in alphabetical order: + * Duncan Dean + * Elias Rohrer + * Jeffrey Czyz + * Matt Corallo + * Max Fang + * Viktor Tigerström + * Willem Van Lint + * Wilmer Paulino + * jurvis + # 0.0.109 - 2022-07-01 ## API Updates From e10dfe4fd071e1c382399dd2caa05848759efdc0 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 25 Jul 2022 23:50:50 +0000 Subject: [PATCH 18/91] Bump crate versions to 0.0.110/invoice 0.18 --- lightning-background-processor/Cargo.toml | 12 ++++++------ lightning-block-sync/Cargo.toml | 4 ++-- lightning-invoice/Cargo.toml | 6 +++--- lightning-net-tokio/Cargo.toml | 4 ++-- lightning-persister/Cargo.toml | 6 +++--- lightning-rapid-gossip-sync/Cargo.toml | 6 +++--- lightning/Cargo.toml | 2 +- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lightning-background-processor/Cargo.toml b/lightning-background-processor/Cargo.toml index 7b2a26f7b71..2df83f2b75a 100644 --- a/lightning-background-processor/Cargo.toml +++ b/lightning-background-processor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightning-background-processor" -version = "0.0.109" +version = "0.0.110" authors = ["Valentine Wallace "] license = "MIT OR Apache-2.0" repository = "http://github.com/lightningdevkit/rust-lightning" @@ -15,10 +15,10 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] bitcoin = "0.28.1" -lightning = { version = "0.0.109", path = "../lightning", features = ["std"] } -lightning-rapid-gossip-sync = { version = "0.0.109", path = "../lightning-rapid-gossip-sync" } +lightning = { version = "0.0.110", path = "../lightning", features = ["std"] } +lightning-rapid-gossip-sync = { version = "0.0.110", path = "../lightning-rapid-gossip-sync" } [dev-dependencies] -lightning = { version = "0.0.109", path = "../lightning", features = ["_test_utils"] } -lightning-invoice = { version = "0.17.0", path = "../lightning-invoice" } -lightning-persister = { version = "0.0.109", path = "../lightning-persister" } +lightning = { version = "0.0.110", path = "../lightning", features = ["_test_utils"] } +lightning-invoice = { version = "0.18.0", path = "../lightning-invoice" } +lightning-persister = { version = "0.0.110", path = "../lightning-persister" } diff --git a/lightning-block-sync/Cargo.toml b/lightning-block-sync/Cargo.toml index d24953f50df..c6650208e26 100644 --- a/lightning-block-sync/Cargo.toml +++ b/lightning-block-sync/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightning-block-sync" -version = "0.0.109" +version = "0.0.110" authors = ["Jeffrey Czyz", "Matt Corallo"] license = "MIT OR Apache-2.0" repository = "http://github.com/lightningdevkit/rust-lightning" @@ -19,7 +19,7 @@ rpc-client = [ "serde", "serde_json", "chunked_transfer" ] [dependencies] bitcoin = "0.28.1" -lightning = { version = "0.0.109", path = "../lightning" } +lightning = { version = "0.0.110", path = "../lightning" } futures = { version = "0.3" } tokio = { version = "1.0", features = [ "io-util", "net", "time" ], optional = true } serde = { version = "1.0", features = ["derive"], optional = true } diff --git a/lightning-invoice/Cargo.toml b/lightning-invoice/Cargo.toml index cae1d9f94d3..cc0e68cfc42 100644 --- a/lightning-invoice/Cargo.toml +++ b/lightning-invoice/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lightning-invoice" description = "Data structures to parse and serialize BOLT11 lightning invoices" -version = "0.17.0" +version = "0.18.0" authors = ["Sebastian Geisler "] documentation = "https://docs.rs/lightning-invoice/" license = "MIT OR Apache-2.0" @@ -20,7 +20,7 @@ std = ["bitcoin_hashes/std", "num-traits/std", "lightning/std", "bech32/std"] [dependencies] bech32 = { version = "0.8", default-features = false } -lightning = { version = "0.0.109", path = "../lightning", default-features = false } +lightning = { version = "0.0.110", path = "../lightning", default-features = false } secp256k1 = { version = "0.22", default-features = false, features = ["recovery", "alloc"] } num-traits = { version = "0.2.8", default-features = false } bitcoin_hashes = { version = "0.10", default-features = false } @@ -29,6 +29,6 @@ core2 = { version = "0.3.0", default-features = false, optional = true } serde = { version = "1.0.118", optional = true } [dev-dependencies] -lightning = { version = "0.0.109", path = "../lightning", default-features = false, features = ["_test_utils"] } +lightning = { version = "0.0.110", path = "../lightning", default-features = false, features = ["_test_utils"] } hex = "0.4" serde_json = { version = "1"} diff --git a/lightning-net-tokio/Cargo.toml b/lightning-net-tokio/Cargo.toml index 050ad4d0121..dd7770cbe8c 100644 --- a/lightning-net-tokio/Cargo.toml +++ b/lightning-net-tokio/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightning-net-tokio" -version = "0.0.109" +version = "0.0.110" authors = ["Matt Corallo"] license = "MIT OR Apache-2.0" repository = "https://github.com/lightningdevkit/rust-lightning/" @@ -16,7 +16,7 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] bitcoin = "0.28.1" -lightning = { version = "0.0.109", path = "../lightning" } +lightning = { version = "0.0.110", path = "../lightning" } tokio = { version = "1.0", features = [ "io-util", "macros", "rt", "sync", "net", "time" ] } [dev-dependencies] diff --git a/lightning-persister/Cargo.toml b/lightning-persister/Cargo.toml index c13dcb169d4..7de0ddc153c 100644 --- a/lightning-persister/Cargo.toml +++ b/lightning-persister/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightning-persister" -version = "0.0.109" +version = "0.0.110" authors = ["Valentine Wallace", "Matt Corallo"] license = "MIT OR Apache-2.0" repository = "https://github.com/lightningdevkit/rust-lightning/" @@ -17,11 +17,11 @@ _bench_unstable = ["lightning/_bench_unstable"] [dependencies] bitcoin = "0.28.1" -lightning = { version = "0.0.109", path = "../lightning" } +lightning = { version = "0.0.110", path = "../lightning" } libc = "0.2" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["winbase"] } [dev-dependencies] -lightning = { version = "0.0.109", path = "../lightning", features = ["_test_utils"] } +lightning = { version = "0.0.110", path = "../lightning", features = ["_test_utils"] } diff --git a/lightning-rapid-gossip-sync/Cargo.toml b/lightning-rapid-gossip-sync/Cargo.toml index 568e399fde4..39518bbbebd 100644 --- a/lightning-rapid-gossip-sync/Cargo.toml +++ b/lightning-rapid-gossip-sync/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightning-rapid-gossip-sync" -version = "0.0.109" +version = "0.0.110" authors = ["Arik Sosman "] license = "MIT OR Apache-2.0" repository = "https://github.com/lightningdevkit/rust-lightning" @@ -13,8 +13,8 @@ Utility to process gossip routing data from Rapid Gossip Sync Server. _bench_unstable = [] [dependencies] -lightning = { version = "0.0.109", path = "../lightning" } +lightning = { version = "0.0.110", path = "../lightning" } bitcoin = { version = "0.28.1", default-features = false } [dev-dependencies] -lightning = { version = "0.0.109", path = "../lightning", features = ["_test_utils"] } +lightning = { version = "0.0.110", path = "../lightning", features = ["_test_utils"] } diff --git a/lightning/Cargo.toml b/lightning/Cargo.toml index 3162a0acdb2..7980a9b1dc8 100644 --- a/lightning/Cargo.toml +++ b/lightning/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightning" -version = "0.0.109" +version = "0.0.110" authors = ["Matt Corallo"] license = "MIT OR Apache-2.0" repository = "https://github.com/lightningdevkit/rust-lightning/" From 7812215a1e1f5d3b78564f6951fc31893864d42b Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 26 Jul 2022 23:43:58 +0000 Subject: [PATCH 19/91] Set release date on 0.0.110 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c907952ce91..46184463975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 0.0.110 - 2022-XXX +# 0.0.110 - 2022-07-26 ## API Updates * `ChannelManager::send_probe` and `Score::probe_{failed,successful}` have From 75b111291faf931f4a203e1d75d2c8ed8b6eb7dd Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Wed, 27 Jul 2022 03:53:47 +0000 Subject: [PATCH 20/91] Expose `ChannelMonitor::get_counterparty_node_id` This fixes an oversight in ac842ed9dd7a36a4a26eb6b856d80ab04eecf750 namely that it left users unable to implement their own `ChainMonitor` from outside of the `rust-lightning` crate. --- lightning/src/chain/channelmonitor.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 38dcd6cf0e7..855263fe53b 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1214,7 +1214,11 @@ impl ChannelMonitor { self.inner.lock().unwrap().get_cur_holder_commitment_number() } - pub(crate) fn get_counterparty_node_id(&self) -> Option { + /// Gets the `node_id` of the counterparty for this channel. + /// + /// Will be `None` for channels constructed on LDK versions prior to 0.0.110 and always `Some` + /// otherwise. + pub fn get_counterparty_node_id(&self) -> Option { self.inner.lock().unwrap().counterparty_node_id } From cf7655a1a41516aff039901856eaf0d033904d56 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 22 Jul 2022 18:28:31 +0200 Subject: [PATCH 21/91] Remove 'slack' (and other things) --- README.md | 106 +++++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 3c34335c244..fea9c35dc0e 100644 --- a/README.md +++ b/README.md @@ -5,74 +5,73 @@ Rust-Lightning [![Documentation](https://img.shields.io/static/v1?logo=read-the-docs&label=docs.rs&message=lightning&color=informational)](https://docs.rs/lightning/) [![Safety Dance](https://img.shields.io/badge/unsafe-forbidden-success.svg)](https://github.com/rust-secure-code/safety-dance/) -Rust-Lightning is a Bitcoin Lightning library written in Rust. The main crate, +`rust-lightning` is a Bitcoin Lightning library written in Rust. The main crate, `lightning`, does not handle networking, persistence, or any other I/O. Thus, it is runtime-agnostic, but users must implement basic networking logic, chain interactions, and disk storage. More information is available in the `About` section. -The `lightning-net-tokio` crate implements Lightning networking using the -[Tokio](https://github.com/tokio-rs/tokio) async runtime. - -The `lightning-persister` crate implements persistence for channel data that -is crucial to avoiding loss of channel funds. Sample modules for persistence of -other Rust-Lightning data is coming soon. - Status ------ - -The project implements all of the BOLT specifications in the 1.0 spec. The +The project implements all of the [BOLT +specifications](https://github.com/lightning/bolts). The implementation has pretty good test coverage that is expected to continue to improve. It is also anticipated that as developers begin using the API, the lessons from that will result in changes to the API, so any developer using this API at this stage should be prepared to embrace that. The current state is -sufficient for a developer or project to experiment with it. Recent increased -contribution rate to the project is expected to lead to a high quality, stable, -production-worthy implementation in 2021. +sufficient for a developer or project to experiment with it. -Communications for Rust-Lightning and Lightning Development Kit happens through our LDK -[slack](https://join.slack.com/t/lightningdevkit/shared_invite/zt-tte36cb7-r5f41MDn3ObFtDu~N9dCrQ) & [discord](https://discord.gg/5AcknnMfBw) channels. +Communications for `rust-lightning` and Lightning Development Kit happen through +our LDK [Discord](https://discord.gg/5AcknnMfBw) channels. Crates ----------- -1. [lightning](./lightning) - The Core of the LDK library, implements the lightning protocol, channel state machine, - and on-chain logic. Supports no-std and exposes on relatively low-level interfaces. +1. [lightning](./lightning) + The core of the LDK library, implements the Lightning protocol, channel state machine, + and on-chain logic. Supports `no-std` and exposes only relatively low-level interfaces. 2. [lightning-background-processor](./lightning-background-processor) Utilities to perform required background tasks for Rust Lightning. 3. [lightning-block-sync](./lightning-block-sync) Utilities to fetch the chain data from a block source and feed them into Rust Lightning. 4. [lightning-invoice](./lightning-invoice) - Data structures to parse and serialize BOLT11 lightning invoices. + Data structures to parse and serialize + [BOLT #11](https://github.com/lightning/bolts/blob/master/11-payment-encoding.md) + Lightning invoices. 5. [lightning-net-tokio](./lightning-net-tokio) - Implementation of the rust-lightning network stack using Tokio. - For Rust-Lightning clients which wish to make direct connections to Lightning P2P nodes, - this is a simple alternative to implementing the required network stack, especially for those already using Tokio. + Implementation of the `rust-lightning` network stack using the + [Tokio](https://github.com/tokio-rs/tokio) `async` runtime. For `rust-lightning` + clients which wish to make direct connections to Lightning P2P nodes, this is + a simple alternative to implementing the required network stack, especially + for those already using Tokio. 6. [lightning-persister](./lightning-persister) - Utilities to manage Rust-Lightning channel data persistence and retrieval. + Implements utilities to manage `rust-lightning` channel data persistence and retrieval. + Persisting channel data is crucial to avoiding loss of channel funds. 7. [lightning-rapid-gossip-sync](./lightning-rapid-gossip-sync) Client for rapid gossip graph syncing, aimed primarily at mobile clients. About ----------- -LDK/Rust-Lightning is a generic library which allows you to build a lightning -node without needing to worry about getting all of the lightning state machine, +LDK/`rust-lightning` is a generic library which allows you to build a Lightning +node without needing to worry about getting all of the Lightning state machine, routing, and on-chain punishment code (and other chain interactions) exactly -correct. Note that Rust-Lightning isn't, in itself, a node. There are various +correct. Note that `rust-lightning` isn't, in itself, a node. There are various working/in progress demos which could be used as a node today, but if you "just" -want a generic lightning node, you're almost certainly better off with -`c-lightning`/`lnd` - if, on the other hand, you want to integrate lightning -with custom features such as your own chain sync, your own key management, your -own data storage/backup logic, etc., LDK is likely your only option. Some -Rust-Lightning utilities such as those in `chan_utils` are also suitable for use -in non-LN Bitcoin applications such as DLCs and bulletin boards. - -We are currently working on a demo node which fetches blockchain data and -on-chain funds via Bitcoin Core RPC/REST. The individual pieces of that demo -are/will be composable, so you can pick the off-the-shelf parts you want and -replace the rest. - -In general, Rust-Lightning does not provide (but LDK has implementations of): +want a generic Lightning node, you're almost certainly better off with [Core +Lightning](https://github.com/ElementsProject/lightning) or +[LND](https://github.com/lightningnetwork/lnd). If, on the other hand, you want +to integrate Lightning with custom features such as your own chain sync, your +own key management, your own data storage/backup logic, etc., LDK is likely your +only option. Some `rust-lightning` utilities such as those in +[`chan_utils`](./lightning/src/ln/chan_utils.rs) are also suitable for use in +non-LN Bitcoin applications such as Discreet Log Contracts (DLCs) and bulletin boards. + +A sample node which fetches blockchain data and manages on-chain funds via the +Bitcoin Core RPC/REST interface is available +[here](https://github.com/lightningdevkit/ldk-sample/). The individual pieces of +that demo are composable, so you can pick the off-the-shelf parts you want +and replace the rest. + +In general, `rust-lightning` does not provide (but LDK has implementations of): * on-disk storage - you can store the channel state any way you want - whether Google Drive/iCloud, a local disk, any key-value store/database/a remote server, or any combination of them - we provide a clean API that provides @@ -84,21 +83,21 @@ In general, Rust-Lightning does not provide (but LDK has implementations of): informed of, which is compatible with Electrum server requests/neutrino filtering/etc. * UTXO management - RL/LDK owns on-chain funds as long as they are claimable as - a part of a lightning output which can be contested - once a channel is closed + part of a Lightning output which can be contested - once a channel is closed and all on-chain outputs are spendable only by the user, we provide users notifications that a UTXO is "theirs" again and it is up to them to spend it as they wish. Additionally, channel funding is accomplished with a generic API which notifies users of the output which needs to appear on-chain, which they can then create a transaction for. Once a transaction is created, we handle the rest. This is a large part of our API's goals - making it easier to - integrate lightning into existing on-chain wallets which have their own + integrate Lightning into existing on-chain wallets which have their own on-chain logic - without needing to move funds in and out of a separate - lightning wallet with on-chain transactions and a separate private key system. -* networking - to enable a user to run a full lightning node on an embedded + Lightning wallet with on-chain transactions and a separate private key system. +* networking - to enable a user to run a full Lightning node on an embedded machine, we don't specify exactly how to connect to another node at all! We provide a default implementation which uses TCP sockets, but, e.g., if you - wanted to run your full lightning node on a hardware wallet, you could, by - piping the lightning network messages over USB/serial and then sending them in + wanted to run your full Lightning node on a hardware wallet, you could, by + piping the Lightning network messages over USB/serial and then sending them in a TCP socket from another machine. * private keys - again we have "default implementations", but users can chose to provide private keys to RL/LDK in any way they wish following a simple API. We @@ -111,25 +110,24 @@ https://vimeo.com/showcase/8372504/video/412818125 Design Goal ----------- - -The goal is to provide a full-featured but also incredibly flexible lightning +The goal is to provide a full-featured but also incredibly flexible Lightning implementation, allowing the user to decide how they wish to use it. With that in mind, everything should be exposed via simple, composable APIs. More -information about Rust-Lightning's flexibility is provided in the `About` +information about `rust-lightning`'s flexibility is provided in the `About` section above. For security reasons, do not add new dependencies. Really do not add new non-optional/non-test/non-library dependencies. Really really do not add dependencies with dependencies. Do convince Andrew to cut down dependency usage -in rust-bitcoin. +in `rust-bitcoin`. Rust-Lightning vs. LDK (Lightning Development Kit) ------------- -Rust-Lightning refers to the core `lightning` crate within this repo, whereas -LDK encompasses Rust-Lightning and all of its sample modules and crates (e.g. +`rust-lightning` refers to the core `lightning` crate within this repo, whereas +LDK encompasses `rust-lightning` and all of its sample modules and crates (e.g. the `lightning-persister` crate), language bindings, sample node -implementation(s), and other tools built around using Rust-Lightning for -lightning integration or building a lightning node. +implementation(s), and other tools built around using `rust-lightning` for +Lightning integration or building a Lightning node. Tagline ------- @@ -144,7 +142,7 @@ Contributors are warmly welcome, see [CONTRIBUTING.md](CONTRIBUTING.md). Project Architecture --------------------- -For a Rust-Lightning high-level API introduction, see [ARCH.md](ARCH.md). +For a `rust-lightning` high-level API introduction, see [ARCH.md](ARCH.md). License is either Apache-2.0 or MIT, at the option of the user (ie dual-license Apache-2.0 and MIT). From 4e5381a50fa524cb183a3d92dee05cb4f2950255 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 27 May 2022 16:25:15 -0700 Subject: [PATCH 22/91] Add onion messages module + enable the construction of blinded routes Blinded routes can be provided as destinations for onion messages, when the recipient prefers to remain anonymous. We also add supporting utilities for constructing blinded path keys, and control TLVs structs representing blinded payloads prior to being encoded/encrypted. These utilities and struct will be re-used in upcoming commits for sending and receiving/forwarding onion messages. Finally, add utilities for reading the padding from an onion message's encrypted TLVs without an intermediate Vec. --- lightning/src/lib.rs | 4 +- lightning/src/ln/mod.rs | 2 +- lightning/src/ln/onion_utils.rs | 8 + lightning/src/onion_message/blinded_route.rs | 153 +++++++++++++++++++ lightning/src/onion_message/mod.rs | 16 ++ lightning/src/onion_message/utils.rs | 79 ++++++++++ 6 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 lightning/src/onion_message/blinded_route.rs create mode 100644 lightning/src/onion_message/mod.rs create mode 100644 lightning/src/onion_message/utils.rs diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index ba6d6bc7191..2e6b3ab3c0a 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -17,7 +17,7 @@ //! figure out how best to make networking happen/timers fire/things get written to disk/keys get //! generated/etc. This makes it a good candidate for tight integration into an existing wallet //! instead of having a rather-separate lightning appendage to a wallet. -//! +//! //! `default` features are: //! //! * `std` - enables functionalities which require `std`, including `std::io` trait implementations and things which utilize time @@ -76,6 +76,8 @@ pub mod util; pub mod chain; pub mod ln; pub mod routing; +#[allow(unused)] +mod onion_message; // To be exposed after sending/receiving OMs is supported in PeerManager. #[cfg(feature = "std")] /// Re-export of either `core2::io` or `std::io`, depending on the `std` feature flag. diff --git a/lightning/src/ln/mod.rs b/lightning/src/ln/mod.rs index 9522e83d18d..c2190d62a11 100644 --- a/lightning/src/ln/mod.rs +++ b/lightning/src/ln/mod.rs @@ -43,7 +43,7 @@ pub mod channel; #[cfg(not(fuzzing))] pub(crate) mod channel; -mod onion_utils; +pub(crate) mod onion_utils; pub mod wire; // Older rustc (which we support) refuses to let us call the get_payment_preimage_hash!() macro diff --git a/lightning/src/ln/onion_utils.rs b/lightning/src/ln/onion_utils.rs index b223a344dbe..57228b92d04 100644 --- a/lightning/src/ln/onion_utils.rs +++ b/lightning/src/ln/onion_utils.rs @@ -43,6 +43,14 @@ pub(super) struct OnionKeys { pub(super) mu: [u8; 32], } +#[inline] +pub(crate) fn gen_rho_from_shared_secret(shared_secret: &[u8]) -> [u8; 32] { + assert_eq!(shared_secret.len(), 32); + let mut hmac = HmacEngine::::new(&[0x72, 0x68, 0x6f]); // rho + hmac.input(&shared_secret); + Hmac::from_engine(hmac).into_inner() +} + #[inline] pub(super) fn gen_rho_mu_from_shared_secret(shared_secret: &[u8]) -> ([u8; 32], [u8; 32]) { assert_eq!(shared_secret.len(), 32); diff --git a/lightning/src/onion_message/blinded_route.rs b/lightning/src/onion_message/blinded_route.rs new file mode 100644 index 00000000000..be6e2a01ca5 --- /dev/null +++ b/lightning/src/onion_message/blinded_route.rs @@ -0,0 +1,153 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! Creating blinded routes and related utilities live here. + +use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey}; + +use chain::keysinterface::{KeysInterface, Sign}; +use super::utils; +use util::chacha20poly1305rfc::ChaChaPolyWriteAdapter; +use util::ser::{VecWriter, Writeable, Writer}; + +use core::iter::FromIterator; +use io; +use prelude::*; + +/// Onion messages can be sent and received to blinded routes, which serve to hide the identity of +/// the recipient. +pub struct BlindedRoute { + /// To send to a blinded route, the sender first finds a route to the unblinded + /// `introduction_node_id`, which can unblind its [`encrypted_payload`] to find out the onion + /// message's next hop and forward it along. + /// + /// [`encrypted_payload`]: BlindedHop::encrypted_payload + introduction_node_id: PublicKey, + /// Used by the introduction node to decrypt its [`encrypted_payload`] to forward the onion + /// message. + /// + /// [`encrypted_payload`]: BlindedHop::encrypted_payload + blinding_point: PublicKey, + /// The hops composing the blinded route. + blinded_hops: Vec, +} + +/// Used to construct the blinded hops portion of a blinded route. These hops cannot be identified +/// by outside observers and thus can be used to hide the identity of the recipient. +pub struct BlindedHop { + /// The blinded node id of this hop in a blinded route. + blinded_node_id: PublicKey, + /// The encrypted payload intended for this hop in a blinded route. + // The node sending to this blinded route will later encode this payload into the onion packet for + // this hop. + encrypted_payload: Vec, +} + +impl BlindedRoute { + /// Create a blinded route to be forwarded along `node_pks`. The last node pubkey in `node_pks` + /// will be the destination node. + /// + /// Errors if less than two hops are provided or if `node_pk`(s) are invalid. + // TODO: make all payloads the same size with padding + add dummy hops + pub fn new + (node_pks: &[PublicKey], keys_manager: &K, secp_ctx: &Secp256k1) -> Result + { + if node_pks.len() < 2 { return Err(()) } + let blinding_secret_bytes = keys_manager.get_secure_random_bytes(); + let blinding_secret = SecretKey::from_slice(&blinding_secret_bytes[..]).expect("RNG is busted"); + let introduction_node_id = node_pks[0]; + + Ok(BlindedRoute { + introduction_node_id, + blinding_point: PublicKey::from_secret_key(secp_ctx, &blinding_secret), + blinded_hops: blinded_hops(secp_ctx, node_pks, &blinding_secret).map_err(|_| ())?, + }) + } +} + +/// Construct blinded hops for the given `unblinded_path`. +fn blinded_hops( + secp_ctx: &Secp256k1, unblinded_path: &[PublicKey], session_priv: &SecretKey +) -> Result, secp256k1::Error> { + let mut blinded_hops = Vec::with_capacity(unblinded_path.len()); + + let mut prev_ss_and_blinded_node_id = None; + utils::construct_keys_callback(secp_ctx, unblinded_path, session_priv, |blinded_node_id, _, _, encrypted_payload_ss, unblinded_pk| { + if let Some((prev_ss, prev_blinded_node_id)) = prev_ss_and_blinded_node_id { + if let Some(pk) = unblinded_pk { + let payload = ForwardTlvs { + next_node_id: pk, + next_blinding_override: None, + }; + blinded_hops.push(BlindedHop { + blinded_node_id: prev_blinded_node_id, + encrypted_payload: encrypt_payload(payload, prev_ss), + }); + } else { debug_assert!(false); } + } + prev_ss_and_blinded_node_id = Some((encrypted_payload_ss, blinded_node_id)); + })?; + + if let Some((final_ss, final_blinded_node_id)) = prev_ss_and_blinded_node_id { + let final_payload = ReceiveTlvs { path_id: None }; + blinded_hops.push(BlindedHop { + blinded_node_id: final_blinded_node_id, + encrypted_payload: encrypt_payload(final_payload, final_ss), + }); + } else { debug_assert!(false) } + + Ok(blinded_hops) +} + +/// Encrypt TLV payload to be used as a [`BlindedHop::encrypted_payload`]. +fn encrypt_payload(payload: P, encrypted_tlvs_ss: [u8; 32]) -> Vec { + let mut writer = VecWriter(Vec::new()); + let write_adapter = ChaChaPolyWriteAdapter::new(encrypted_tlvs_ss, &payload); + write_adapter.write(&mut writer).expect("In-memory writes cannot fail"); + writer.0 +} + +/// TLVs to encode in an intermediate onion message packet's hop data. When provided in a blinded +/// route, they are encoded into [`BlindedHop::encrypted_payload`]. +pub(crate) struct ForwardTlvs { + /// The node id of the next hop in the onion message's path. + next_node_id: PublicKey, + /// Senders to a blinded route use this value to concatenate the route they find to the + /// introduction node with the blinded route. + next_blinding_override: Option, +} + +/// Similar to [`ForwardTlvs`], but these TLVs are for the final node. +pub(crate) struct ReceiveTlvs { + /// If `path_id` is `Some`, it is used to identify the blinded route that this onion message is + /// sending to. This is useful for receivers to check that said blinded route is being used in + /// the right context. + path_id: Option<[u8; 32]>, +} + +impl Writeable for ForwardTlvs { + fn write(&self, writer: &mut W) -> Result<(), io::Error> { + // TODO: write padding + encode_tlv_stream!(writer, { + (4, self.next_node_id, required), + (8, self.next_blinding_override, option) + }); + Ok(()) + } +} + +impl Writeable for ReceiveTlvs { + fn write(&self, writer: &mut W) -> Result<(), io::Error> { + // TODO: write padding + encode_tlv_stream!(writer, { + (6, self.path_id, option), + }); + Ok(()) + } +} diff --git a/lightning/src/onion_message/mod.rs b/lightning/src/onion_message/mod.rs new file mode 100644 index 00000000000..fabe8d58eda --- /dev/null +++ b/lightning/src/onion_message/mod.rs @@ -0,0 +1,16 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! Onion Messages: sending, receiving, forwarding, and ancillary utilities live here + +mod blinded_route; +mod utils; + +// Re-export structs so they can be imported with just the `onion_message::` module prefix. +pub use self::blinded_route::{BlindedRoute, BlindedHop}; diff --git a/lightning/src/onion_message/utils.rs b/lightning/src/onion_message/utils.rs new file mode 100644 index 00000000000..785a373caa4 --- /dev/null +++ b/lightning/src/onion_message/utils.rs @@ -0,0 +1,79 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! Onion message utility methods live here. + +use bitcoin::hashes::{Hash, HashEngine}; +use bitcoin::hashes::hmac::{Hmac, HmacEngine}; +use bitcoin::hashes::sha256::Hash as Sha256; +use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey}; +use bitcoin::secp256k1::ecdh::SharedSecret; + +use ln::onion_utils; + +use prelude::*; + +// TODO: DRY with onion_utils::construct_onion_keys_callback +#[inline] +pub(super) fn construct_keys_callback)>( + secp_ctx: &Secp256k1, unblinded_path: &[PublicKey], + session_priv: &SecretKey, mut callback: FType +) -> Result<(), secp256k1::Error> { + let mut msg_blinding_point_priv = session_priv.clone(); + let mut msg_blinding_point = PublicKey::from_secret_key(secp_ctx, &msg_blinding_point_priv); + let mut onion_packet_pubkey_priv = msg_blinding_point_priv.clone(); + let mut onion_packet_pubkey = msg_blinding_point.clone(); + + macro_rules! build_keys { + ($pk: expr, $blinded: expr) => { + let encrypted_data_ss = SharedSecret::new(&$pk, &msg_blinding_point_priv); + + let blinded_hop_pk = if $blinded { $pk } else { + let hop_pk_blinding_factor = { + let mut hmac = HmacEngine::::new(b"blinded_node_id"); + hmac.input(encrypted_data_ss.as_ref()); + Hmac::from_engine(hmac).into_inner() + }; + let mut unblinded_pk = $pk; + unblinded_pk.mul_assign(secp_ctx, &hop_pk_blinding_factor)?; + unblinded_pk + }; + let onion_packet_ss = SharedSecret::new(&blinded_hop_pk, &onion_packet_pubkey_priv); + + let rho = onion_utils::gen_rho_from_shared_secret(encrypted_data_ss.as_ref()); + let unblinded_pk_opt = if $blinded { None } else { Some($pk) }; + callback(blinded_hop_pk, onion_packet_ss, onion_packet_pubkey, rho, unblinded_pk_opt); + + let msg_blinding_point_blinding_factor = { + let mut sha = Sha256::engine(); + sha.input(&msg_blinding_point.serialize()[..]); + sha.input(encrypted_data_ss.as_ref()); + Sha256::from_engine(sha).into_inner() + }; + + msg_blinding_point_priv.mul_assign(&msg_blinding_point_blinding_factor)?; + msg_blinding_point = PublicKey::from_secret_key(secp_ctx, &msg_blinding_point_priv); + + let onion_packet_pubkey_blinding_factor = { + let mut sha = Sha256::engine(); + sha.input(&onion_packet_pubkey.serialize()[..]); + sha.input(onion_packet_ss.as_ref()); + Sha256::from_engine(sha).into_inner() + }; + onion_packet_pubkey_priv.mul_assign(&onion_packet_pubkey_blinding_factor)?; + onion_packet_pubkey = PublicKey::from_secret_key(secp_ctx, &onion_packet_pubkey_priv); + }; + } + + for pk in unblinded_path { + build_keys!(*pk, false); + } + Ok(()) +} From afdad4921c337e9c9888fe0011ee34bd978b46fc Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 28 Jul 2022 09:26:02 +0200 Subject: [PATCH 23/91] Remove Slack and update `CONTRIBUTING` --- CONTRIBUTING.md | 148 +++++++++++++++++++++++++++--------------------- 1 file changed, 82 insertions(+), 66 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93666dd2297..f7cf8c4e699 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,27 +1,35 @@ Contributing to Rust-Lightning ============================== -The Rust-Lightning project operates an open contributor model where anyone is -welcome to contribute towards development in the form of peer review, documentation, -testing and patches. - -Anyone is invited to contribute without regard to technical experience, "expertise", OSS -experience, age, or other concern. However, the development of cryptocurrencies demands a -high-level of rigor, adversarial thinking, thorough testing and risk-minimization. -Any bug may cost users real money. That being said, we deeply welcome people contributing -for the first time to an open source project or pick up Rust while contributing. Don't be shy, -you'll learn. - -Communications Channels +The `rust-lightning` project operates an open contributor model where anyone is +welcome to contribute towards development in the form of peer review, +documentation, testing and patches. + +Anyone is invited to contribute without regard to technical experience, +"expertise", OSS experience, age, or other concern. However, the development of +cryptocurrencies demands a high-level of rigor, adversarial thinking, thorough +testing and risk-minimization. Any bug may cost users real money. That being +said, we deeply welcome people contributing for the first time to an open source +project or pick up Rust while contributing. Don't be shy, you'll learn. + +Communication Channels ----------------------- -Communication about Rust-Lightning happens primarily on #ldk-dev on the -[LDK slack](http://www.lightningdevkit.org/), but also #rust-bitcoin on IRC Freenode. +Communication about the development of LDK and `rust-lightning` happens +primarily on the [LDK Discord](https://discord.gg/5AcknnMfBw) in the `#ldk-dev` +channel. Additionally, live LDK devevelopment meetings are held every other +Monday 19:00 UTC in the [LDK Dev Jitsi Meeting +Room](https://meet.jit.si/ldkdevmeeting). Upcoming events can be found in the +[LDK calendar](https://calendar.google.com/calendar/embed?src=c_e6fv6vlshbpoob2mmbvblkkoj4%40group.calendar.google.com). + +Contributors starting out with the Rust language are welcome to discuss and ask +for help in the `#rust-101` channel on LDK Discord. Discussion about code base improvements happens in GitHub issues and on pull requests. -Major projects are tracked [here](https://github.com/lightningdevkit/rust-lightning/projects). +The LDK roadmap is tracked [here](https://github.com/orgs/lightningdevkit/projects/2). + Major milestones are tracked [here](https://github.com/lightningdevkit/rust-lightning/milestones?direction=asc&sort=title&state=open). Getting Started @@ -29,25 +37,28 @@ Getting Started First and foremost, start small. -This doesn't mean don't be ambitious with the breadth and depth of your contributions but rather -understand the project culture before investing an asymmetric number of hours on -development compared to your merged work. +This doesn't mean don't be ambitious with the breadth and depth of your +contributions but rather understand the project culture before investing an +asymmetric number of hours on development compared to your merged work. Browsing through the [meeting minutes](https://github.com/lightningdevkit/rust-lightning/wiki/Meetings) -is a good first step. You will learn who is working on what, how releases are drafted, what are the -pending tasks to deliver, where you can contribute review bandwidth, etc. +is a good first step. You will learn who is working on what, how releases are +drafted, what are the pending tasks to deliver, where you can contribute review +bandwidth, etc. -Even if you have an extensive open source background or sound software engineering skills, consider -that the reviewers' comprehension of the code is as much important as technical correctness. +Even if you have an extensive open source background or sound software +engineering skills, consider that the reviewers' comprehension of the code is as +much important as technical correctness. -It's very welcome to ask for review, either on IRC or LDK Slack. And also for reviewers, it's nice -to provide timelines when you hope to fulfill the request while bearing in mind for both sides that's -a "soft" commitment. +It's very welcome to ask for review on LDK Discord. And also for reviewers, it's +nice to provide timelines when you hope to fulfill the request while bearing in +mind for both sides that's a "soft" commitment. -If you're eager to increase the velocity of the dev process, reviewing other contributors work is -the best you can do while waiting review on yours. +If you're eager to increase the velocity of the dev process, reviewing other +contributors work is the best you can do while waiting review on yours. -Also, getting familiar with the [glossary](GLOSSARY.md) will streamline discussions with regular contributors. +Also, getting familiar with the [glossary](GLOSSARY.md) will streamline +discussions with regular contributors. Contribution Workflow --------------------- @@ -80,14 +91,14 @@ our GitHub Actions). Also, the compatibility for LDK object serialization is currently ensured back to and including crate version 0.0.99 (see the [changelog](CHANGELOG.md)). -Commits should cover both the issue fixed and the solution's rationale. -These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind. +Commits should cover both the issue fixed and the solution's rationale. These +[guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind. -To facilitate communication with other contributors, the project is making use of -GitHub's "assignee" field. First check that no one is assigned and then comment -suggesting that you're working on it. If someone is already assigned, don't hesitate -to ask if the assigned party or previous commenters are still working on it if it has -been awhile. +To facilitate communication with other contributors, the project is making use +of GitHub's "assignee" field. First check that no one is assigned and then +comment suggesting that you're working on it. If someone is already assigned, +don't hesitate to ask if the assigned party or previous commenters are still +working on it if it has been awhile. Peer review ----------- @@ -95,8 +106,8 @@ Peer review Anyone may participate in peer review which is expressed by comments in the pull request. Typically reviewers will review the code for obvious errors, as well as test out the patch set and opine on the technical merits of the patch. PR should -be reviewed first on the conceptual level before focusing on code style or grammar -fixes. +be reviewed first on the conceptual level before focusing on code style or +grammar fixes. Coding Conventions ------------------ @@ -104,65 +115,70 @@ Coding Conventions Use tabs. If you want to align lines, use spaces. Any desired alignment should display fine at any tab-length display setting. -Our CI enforces [clippy's](https://github.com/rust-lang/rust-clippy) default linting -[settings](https://rust-lang.github.io/rust-clippy/rust-1.39.0/index.html). -This includes all lint groups except for nursery, pedantic, and cargo in addition to allowing the following lints: -`erasing_op`, `never_loop`, `if_same_then_else`. +Our CI enforces [clippy's](https://github.com/rust-lang/rust-clippy) default +linting +[settings](https://rust-lang.github.io/rust-clippy/rust-1.39.0/index.html). This +includes all lint groups except for nursery, pedantic, and cargo in addition to +allowing the following lints: `erasing_op`, `never_loop`, `if_same_then_else`. -If you use rustup, feel free to lint locally, otherwise you can just push to CI for automated linting. +If you use rustup, feel free to lint locally, otherwise you can just push to CI +for automated linting. ```bash rustup component add clippy cargo clippy ``` -Significant structures that users persist should always have their serialization methods (usually -`Writeable::write` and `ReadableArgs::read`) begin with +Significant structures that users persist should always have their serialization +methods (usually `Writeable::write` and `ReadableArgs::read`) begin with `write_ver_prefix!()`/`read_ver_prefix!()` calls, and end with calls to `write_tlv_fields!()`/`read_tlv_fields!()`. -Updates to the serialized format which has implications for backwards or forwards compatibility -must be included in release notes. +Updates to the serialized format which has implications for backwards or +forwards compatibility must be included in release notes. Security -------- -Security is the primary focus of Rust-Lightning; disclosure of security vulnerabilites -helps prevent user loss of funds. If you believe a vulnerability may affect other Lightning -implementations, please inform them. +Security is the primary focus of `rust-lightning`; disclosure of security +vulnerabilites helps prevent user loss of funds. If you believe a vulnerability +may affect other Lightning implementations, please inform them. -Note that Rust-Lightning is currently considered "pre-production" during this time, there -is no special handling of security issues. Please simply open an issue on Github. +You can find further information on submitting (possible) vulnerabilities in the +[security policy](SECURITY.md). Testing ------- -Related to the security aspect, Rust-Lightning developers take testing -very seriously. Due to the modular nature of the project, writing new functional -tests is easy and good test coverage of the codebase is an important goal. Refactoring -the project to enable fine-grained unit testing is also an ongoing effort. +Related to the security aspect, `rust-lightning` developers take testing very +seriously. Due to the modular nature of the project, writing new functional +tests is easy and good test coverage of the codebase is an important goal. +Refactoring the project to enable fine-grained unit testing is also an ongoing +effort. Fuzzing is heavily encouraged: you will find all related material under `fuzz/` -Mutation testing is work-in-progress; any contribution there would be warmly welcomed. +Mutation testing is work-in-progress; any contribution there would be warmly +welcomed. C/C++ Bindings -------------- -You can learn more about the C/C++ bindings that are made available by reading the -[C/C++ Bindings README](lightning-c-bindings/README.md). If you are not using the C/C++ bindings, -you likely don't need to worry about them, and during their early experimental phase we are not -requiring that pull requests keep the bindings up to date (and, thus, pass the bindings_check CI -run). If you wish to ensure your PR passes the bindings generation phase, you should run the -`genbindings.sh` script in the top of the directory tree to generate, build, and test C bindings on -your local system. +You can learn more about the C/C++ bindings that are made available by reading +the [C/C++ Bindings README](https://github.com/lightningdevkit/ldk-c-bindings/blob/main/lightning-c-bindings/README.md). +If you are not using the C/C++ bindings, you likely don't need to worry about +them, and during their early experimental phase we are not requiring that pull +requests keep the bindings up to date (and, thus, pass the `bindings_check` CI +run). If you wish to ensure your PR passes the bindings generation phase, you +should run the `genbindings.sh` script in the top of the directory tree to +generate, build, and test C bindings on your local system. Going further ------------- -You may be interested by Jon Atack guide on [How to review Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-review-bitcoin-core-prs.md) +You may be interested by Jon Atack's guide on [How to review Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-review-bitcoin-core-prs.md) and [How to make Bitcoin Core PRs](https://github.com/jonatack/bitcoin-development/blob/master/how-to-make-bitcoin-core-prs.md). -While there are differences between the projects in terms of context and maturity, many -of the suggestions offered apply to this project. +While there are differences between the projects in terms of context and +maturity, many of the suggestions offered apply to this project. Overall, have fun :) From 65e6fb746735aa71f912eca0b65015313e2e6c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= <11711198+ViktorTigerstrom@users.noreply.github.com> Date: Sun, 24 Jul 2022 00:10:48 +0200 Subject: [PATCH 24/91] Don't return `channel_state` from `decode_update_add_htlc_onion` Currently `decode_update_add_htlc_onion` returns the `channel_state` lock to ensure that `internal_update_add_htlc` holds a single `channel_state` lock in when the entire function execution. This is unnecessary, and since we are moving the channel storage to the `per_peer_state`, this no longer achieves the goal it was intended for. We therefore avoid returning the `channel_state` from `decode_update_add_htlc_onion`, and just retake the lock in `internal_update_add_htlc` instead. --- lightning/src/ln/channelmanager.rs | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index d011d6b42c8..771449e6cfe 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -2144,17 +2144,17 @@ impl ChannelMana }) } - fn decode_update_add_htlc_onion(&self, msg: &msgs::UpdateAddHTLC) -> (PendingHTLCStatus, MutexGuard>) { + fn decode_update_add_htlc_onion(&self, msg: &msgs::UpdateAddHTLC) -> PendingHTLCStatus { macro_rules! return_malformed_err { ($msg: expr, $err_code: expr) => { { log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg); - return (PendingHTLCStatus::Fail(HTLCFailureMsg::Malformed(msgs::UpdateFailMalformedHTLC { + return PendingHTLCStatus::Fail(HTLCFailureMsg::Malformed(msgs::UpdateFailMalformedHTLC { channel_id: msg.channel_id, htlc_id: msg.htlc_id, sha256_of_onion: Sha256::hash(&msg.onion_routing_packet.hop_data).into_inner(), failure_code: $err_code, - })), self.channel_state.lock().unwrap()); + })); } } } @@ -2174,20 +2174,15 @@ impl ChannelMana //node knows the HMAC matched, so they already know what is there... return_malformed_err!("Unknown onion packet version", 0x8000 | 0x4000 | 4); } - - let mut channel_state = None; macro_rules! return_err { ($msg: expr, $err_code: expr, $data: expr) => { { log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg); - if channel_state.is_none() { - channel_state = Some(self.channel_state.lock().unwrap()); - } - return (PendingHTLCStatus::Fail(HTLCFailureMsg::Relay(msgs::UpdateFailHTLC { + return PendingHTLCStatus::Fail(HTLCFailureMsg::Relay(msgs::UpdateFailHTLC { channel_id: msg.channel_id, htlc_id: msg.htlc_id, reason: onion_utils::build_first_hop_failure_packet(&shared_secret, $err_code, $data), - })), channel_state.unwrap()); + })); } } } @@ -2246,14 +2241,14 @@ impl ChannelMana } }; - channel_state = Some(self.channel_state.lock().unwrap()); if let &PendingHTLCStatus::Forward(PendingHTLCInfo { ref routing, ref amt_to_forward, ref outgoing_cltv_value, .. }) = &pending_forward_info { // If short_channel_id is 0 here, we'll reject the HTLC as there cannot be a channel // with a short_channel_id of 0. This is important as various things later assume // short_channel_id is non-0 in any ::Forward. if let &PendingHTLCRouting::Forward { ref short_channel_id, .. } = routing { - let id_option = channel_state.as_ref().unwrap().short_to_chan_info.get(&short_channel_id).cloned(); if let Some((err, code, chan_update)) = loop { + let mut channel_state = self.channel_state.lock().unwrap(); + let id_option = channel_state.short_to_chan_info.get(&short_channel_id).cloned(); let forwarding_id_opt = match id_option { None => { // unknown_next_peer // Note that this is likely a timing oracle for detecting whether an scid is a @@ -2267,7 +2262,7 @@ impl ChannelMana Some((_cp_id, chan_id)) => Some(chan_id.clone()), }; let chan_update_opt = if let Some(forwarding_id) = forwarding_id_opt { - let chan = channel_state.as_mut().unwrap().by_id.get_mut(&forwarding_id).unwrap(); + let chan = channel_state.by_id.get_mut(&forwarding_id).unwrap(); if !chan.should_announce() && !self.default_configuration.accept_forwards_to_priv_channels { // Note that the behavior here should be identical to the above block - we // should NOT reveal the existence or non-existence of a private channel if @@ -2353,7 +2348,7 @@ impl ChannelMana } } - (pending_forward_info, channel_state.unwrap()) + pending_forward_info } /// Gets the current channel_update for the given channel. This first checks if the channel is @@ -4850,7 +4845,8 @@ impl ChannelMana //encrypted with the same key. It's not immediately obvious how to usefully exploit that, //but we should prevent it anyway. - let (pending_forward_info, mut channel_state_lock) = self.decode_update_add_htlc_onion(msg); + let pending_forward_info = self.decode_update_add_htlc_onion(msg); + let mut channel_state_lock = self.channel_state.lock().unwrap(); let channel_state = &mut *channel_state_lock; match channel_state.by_id.entry(msg.channel_id) { From 092d1c1f0df67b9a7c17d0424f1307d7395a7d22 Mon Sep 17 00:00:00 2001 From: Gursharan Singh Date: Mon, 18 Jul 2022 16:58:10 -0700 Subject: [PATCH 25/91] Add config support for 'their_channel_reserve_proportional_millionths' [#1498] It is proportion of the channel value to configure as the `their_channel_reserve_satoshis` for both outbound and inbound channels. It decides the minimum balance that the other node has to maintain on their side, at all times. --- .gitignore | 2 + lightning/src/ln/channel.rs | 99 ++++++++++++++++++++++++---- lightning/src/ln/channelmanager.rs | 2 +- lightning/src/ln/functional_tests.rs | 28 ++++---- lightning/src/util/config.rs | 25 +++++++ 5 files changed, 131 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 10e905af514..d99c5a4ddf6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ lightning-c-bindings/a.out Cargo.lock .idea lightning/target +lightning/ldk-net_graph-*.bin no-std-check/target + diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index f612cb97943..da770c7a1e4 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -795,6 +795,9 @@ pub const MAX_CHAN_DUST_LIMIT_SATOSHIS: u64 = MAX_STD_OUTPUT_DUST_LIMIT_SATOSHIS /// See https://github.com/lightning/bolts/issues/905 for more details. pub const MIN_CHAN_DUST_LIMIT_SATOSHIS: u64 = 354; +// Just a reasonable implementation-specific safe lower bound, higher than the dust limit. +pub const MIN_THEIR_CHAN_RESERVE_SATOSHIS: u64 = 1000; + /// Used to return a simple Error back to ChannelManager. Will get converted to a /// msgs::ErrorAction::SendErrorMessage or msgs::ErrorAction::IgnoreError as appropriate with our /// channel_id in ChannelManager. @@ -843,16 +846,25 @@ impl Channel { } /// Returns a minimum channel reserve value the remote needs to maintain, - /// required by us. + /// required by us according to the configured or default + /// [`ChannelHandshakeConfig::their_channel_reserve_proportional_millionths`] /// /// Guaranteed to return a value no larger than channel_value_satoshis /// - /// This is used both for new channels and to figure out what reserve value we sent to peers - /// for channels serialized before we included our selected reserve value in the serialized - /// data explicitly. - pub(crate) fn get_holder_selected_channel_reserve_satoshis(channel_value_satoshis: u64) -> u64 { + /// This is used both for outbound and inbound channels and has lower bound + /// of `MIN_THEIR_CHAN_RESERVE_SATOSHIS`. + pub(crate) fn get_holder_selected_channel_reserve_satoshis(channel_value_satoshis: u64, config: &UserConfig) -> u64 { + let calculated_reserve = channel_value_satoshis.saturating_mul(config.channel_handshake_config.their_channel_reserve_proportional_millionths as u64) / 1_000_000; + cmp::min(channel_value_satoshis, cmp::max(calculated_reserve, MIN_THEIR_CHAN_RESERVE_SATOSHIS)) + } + + /// This is for legacy reasons, present for forward-compatibility. + /// LDK versions older than 0.0.104 don't know how read/handle values other than default + /// from storage. Hence, we use this function to not persist default values of + /// `holder_selected_channel_reserve_satoshis` for channels into storage. + pub(crate) fn get_legacy_default_holder_selected_channel_reserve_satoshis(channel_value_satoshis: u64) -> u64 { let (q, _) = channel_value_satoshis.overflowing_div(100); - cmp::min(channel_value_satoshis, cmp::max(q, 1000)) //TODO + cmp::min(channel_value_satoshis, cmp::max(q, 1000)) } pub(crate) fn opt_anchors(&self) -> bool { @@ -912,8 +924,10 @@ impl Channel { if holder_selected_contest_delay < BREAKDOWN_TIMEOUT { return Err(APIError::APIMisuseError {err: format!("Configured with an unreasonable our_to_self_delay ({}) putting user funds at risks", holder_selected_contest_delay)}); } - let holder_selected_channel_reserve_satoshis = Channel::::get_holder_selected_channel_reserve_satoshis(channel_value_satoshis); + let holder_selected_channel_reserve_satoshis = Channel::::get_holder_selected_channel_reserve_satoshis(channel_value_satoshis, config); if holder_selected_channel_reserve_satoshis < MIN_CHAN_DUST_LIMIT_SATOSHIS { + // Protocol level safety check in place, although it should never happen because + // of `MIN_THEIR_CHAN_RESERVE_SATOSHIS` return Err(APIError::APIMisuseError { err: format!("Holder selected channel reserve below implemention limit dust_limit_satoshis {}", holder_selected_channel_reserve_satoshis) }); } @@ -1204,12 +1218,14 @@ impl Channel { } } - let holder_selected_channel_reserve_satoshis = Channel::::get_holder_selected_channel_reserve_satoshis(msg.funding_satoshis); + let holder_selected_channel_reserve_satoshis = Channel::::get_holder_selected_channel_reserve_satoshis(msg.funding_satoshis, config); if holder_selected_channel_reserve_satoshis < MIN_CHAN_DUST_LIMIT_SATOSHIS { + // Protocol level safety check in place, although it should never happen because + // of `MIN_THEIR_CHAN_RESERVE_SATOSHIS` return Err(ChannelError::Close(format!("Suitable channel reserve not found. remote_channel_reserve was ({}). dust_limit_satoshis is ({}).", holder_selected_channel_reserve_satoshis, MIN_CHAN_DUST_LIMIT_SATOSHIS))); } if holder_selected_channel_reserve_satoshis * 1000 >= full_channel_value_msat { - return Err(ChannelError::Close(format!("Suitable channel reserve not found. remote_channel_reserve was ({}). Channel value is ({} - {}).", holder_selected_channel_reserve_satoshis, full_channel_value_msat, msg.push_msat))); + return Err(ChannelError::Close(format!("Suitable channel reserve not found. remote_channel_reserve was ({})msats. Channel value is ({} - {})msats.", holder_selected_channel_reserve_satoshis * 1000, full_channel_value_msat, msg.push_msat))); } if msg.channel_reserve_satoshis < MIN_CHAN_DUST_LIMIT_SATOSHIS { log_debug!(logger, "channel_reserve_satoshis ({}) is smaller than our dust limit ({}). We can broadcast stale states without any risk, implying this channel is very insecure for our counterparty.", @@ -6106,7 +6122,7 @@ impl Writeable for Channel { // a different percentage of the channel value then 10%, which older versions of LDK used // to set it to before the percentage was made configurable. let serialized_holder_selected_reserve = - if self.holder_selected_channel_reserve_satoshis != Self::get_holder_selected_channel_reserve_satoshis(self.channel_value_satoshis) + if self.holder_selected_channel_reserve_satoshis != Self::get_legacy_default_holder_selected_channel_reserve_satoshis(self.channel_value_satoshis) { Some(self.holder_selected_channel_reserve_satoshis) } else { None }; let mut old_max_in_flight_percent_config = UserConfig::default().channel_handshake_config; @@ -6381,7 +6397,7 @@ impl<'a, Signer: Sign, K: Deref> ReadableArgs<(&'a K, u32)> for Channel let mut announcement_sigs = None; let mut target_closing_feerate_sats_per_kw = None; let mut monitor_pending_finalized_fulfills = Some(Vec::new()); - let mut holder_selected_channel_reserve_satoshis = Some(Self::get_holder_selected_channel_reserve_satoshis(channel_value_satoshis)); + let mut holder_selected_channel_reserve_satoshis = Some(Self::get_legacy_default_holder_selected_channel_reserve_satoshis(channel_value_satoshis)); let mut holder_max_htlc_value_in_flight_msat = Some(Self::get_holder_max_htlc_value_in_flight_msat(channel_value_satoshis, &UserConfig::default().channel_handshake_config)); // Prior to supporting channel type negotiation, all of our channels were static_remotekey // only, so we default to that if none was written. @@ -6561,6 +6577,7 @@ impl<'a, Signer: Sign, K: Deref> ReadableArgs<(&'a K, u32)> for Channel #[cfg(test)] mod tests { + use std::cmp; use bitcoin::blockdata::script::{Script, Builder}; use bitcoin::blockdata::transaction::{Transaction, TxOut}; use bitcoin::blockdata::constants::genesis_block; @@ -6570,7 +6587,7 @@ mod tests { use ln::PaymentHash; use ln::channelmanager::{HTLCSource, PaymentId}; use ln::channel::{Channel, InboundHTLCOutput, OutboundHTLCOutput, InboundHTLCState, OutboundHTLCState, HTLCCandidate, HTLCInitiator}; - use ln::channel::{MAX_FUNDING_SATOSHIS_NO_WUMBO, TOTAL_BITCOIN_SUPPLY_SATOSHIS}; + use ln::channel::{MAX_FUNDING_SATOSHIS_NO_WUMBO, TOTAL_BITCOIN_SUPPLY_SATOSHIS, MIN_THEIR_CHAN_RESERVE_SATOSHIS}; use ln::features::{InitFeatures, ChannelTypeFeatures}; use ln::msgs::{ChannelUpdate, DataLossProtect, DecodeError, OptionalField, UnsignedChannelUpdate}; use ln::script::ShutdownScript; @@ -6962,6 +6979,64 @@ mod tests { assert_eq!(chan_8.holder_max_htlc_value_in_flight_msat, chan_8_value_msat); } + #[test] + fn test_configured_holder_selected_channel_reserve_satoshis() { + + // Test that `new_outbound` and `new_from_req` create a channel with the correct + // channel reserves, when `their_channel_reserve_proportional_millionths` is configured. + test_self_and_counterparty_channel_reserve(10_000_000, 0.02, 0.02); + + // Test with valid but unreasonably high channel reserves + // Requesting and accepting parties have requested for 49%-49% and 60%-30% channel reserve + test_self_and_counterparty_channel_reserve(10_000_000, 0.49, 0.49); + test_self_and_counterparty_channel_reserve(10_000_000, 0.60, 0.30); + + // Test with calculated channel reserve less than lower bound + // i.e `MIN_THEIR_CHAN_RESERVE_SATOSHIS` + test_self_and_counterparty_channel_reserve(100_000, 0.00002, 0.30); + + // Test with invalid channel reserves since sum of both is greater than or equal + // to channel value + test_self_and_counterparty_channel_reserve(10_000_000, 0.50, 0.50); + test_self_and_counterparty_channel_reserve(10_000_000, 0.60, 0.50); + } + + fn test_self_and_counterparty_channel_reserve(channel_value_satoshis: u64, outbound_selected_channel_reserve_perc: f64, inbound_selected_channel_reserve_perc: f64) { + let fee_est = LowerBoundedFeeEstimator::new(&TestFeeEstimator { fee_est: 15_000 }); + let logger = test_utils::TestLogger::new(); + let secp_ctx = Secp256k1::new(); + let seed = [42; 32]; + let network = Network::Testnet; + let keys_provider = test_utils::TestKeysInterface::new(&seed, network); + let outbound_node_id = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()); + let inbound_node_id = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[7; 32]).unwrap()); + + + let mut outbound_node_config = UserConfig::default(); + outbound_node_config.channel_handshake_config.their_channel_reserve_proportional_millionths = (outbound_selected_channel_reserve_perc * 1_000_000.0) as u32; + let chan = Channel::::new_outbound(&&fee_est, &&keys_provider, outbound_node_id, &InitFeatures::known(), channel_value_satoshis, 100_000, 42, &outbound_node_config, 0, 42).unwrap(); + + let expected_outbound_selected_chan_reserve = cmp::max(MIN_THEIR_CHAN_RESERVE_SATOSHIS, (chan.channel_value_satoshis as f64 * outbound_selected_channel_reserve_perc) as u64); + assert_eq!(chan.holder_selected_channel_reserve_satoshis, expected_outbound_selected_chan_reserve); + + let chan_open_channel_msg = chan.get_open_channel(genesis_block(network).header.block_hash()); + let mut inbound_node_config = UserConfig::default(); + inbound_node_config.channel_handshake_config.their_channel_reserve_proportional_millionths = (inbound_selected_channel_reserve_perc * 1_000_000.0) as u32; + + if outbound_selected_channel_reserve_perc + inbound_selected_channel_reserve_perc < 1.0 { + let chan_inbound_node = Channel::::new_from_req(&&fee_est, &&keys_provider, inbound_node_id, &InitFeatures::known(), &chan_open_channel_msg, 7, &inbound_node_config, 0, &&logger, 42).unwrap(); + + let expected_inbound_selected_chan_reserve = cmp::max(MIN_THEIR_CHAN_RESERVE_SATOSHIS, (chan.channel_value_satoshis as f64 * inbound_selected_channel_reserve_perc) as u64); + + assert_eq!(chan_inbound_node.holder_selected_channel_reserve_satoshis, expected_inbound_selected_chan_reserve); + assert_eq!(chan_inbound_node.counterparty_selected_channel_reserve_satoshis.unwrap(), expected_outbound_selected_chan_reserve); + } else { + // Channel Negotiations failed + let result = Channel::::new_from_req(&&fee_est, &&keys_provider, inbound_node_id, &InitFeatures::known(), &chan_open_channel_msg, 7, &inbound_node_config, 0, &&logger, 42); + assert!(result.is_err()); + } + } + #[test] fn channel_update() { let feeest = LowerBoundedFeeEstimator::new(&TestFeeEstimator{fee_est: 15000}); diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 611e8a562c6..568df11dfa9 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -1635,7 +1635,7 @@ impl ChannelMana } } - /// Gets the current configuration applied to all new channels, as + /// Gets the current configuration applied to all new channels. pub fn get_current_default_configuration(&self) -> &UserConfig { &self.default_configuration } diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index 4894ed02b3b..e1050727c1e 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -73,7 +73,7 @@ fn test_insane_channel_opens() { // Instantiate channel parameters where we push the maximum msats given our // funding satoshis let channel_value_sat = 31337; // same as funding satoshis - let channel_reserve_satoshis = Channel::::get_holder_selected_channel_reserve_satoshis(channel_value_sat); + let channel_reserve_satoshis = Channel::::get_holder_selected_channel_reserve_satoshis(channel_value_sat, &cfg); let push_msat = (channel_value_sat - channel_reserve_satoshis) * 1000; // Have node0 initiate a channel to node1 with aforementioned parameters @@ -148,13 +148,14 @@ fn do_test_counterparty_no_reserve(send_from_initiator: bool) { let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + let default_config = UserConfig::default(); // Have node0 initiate a channel to node1 with aforementioned parameters let mut push_amt = 100_000_000; let feerate_per_kw = 253; let opt_anchors = false; push_amt -= feerate_per_kw as u64 * (commitment_tx_base_weight(opt_anchors) + 4 * COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000 * 1000; - push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000) * 1000; + push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000, &default_config) * 1000; let temp_channel_id = nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100_000, if send_from_initiator { 0 } else { push_amt }, 42, None).unwrap(); let mut open_channel_message = get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()); @@ -633,7 +634,8 @@ fn test_update_fee_that_funder_cannot_afford() { let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, channel_value, push_sats * 1000, InitFeatures::known(), InitFeatures::known()); let channel_id = chan.2; let secp_ctx = Secp256k1::new(); - let bs_channel_reserve_sats = Channel::::get_holder_selected_channel_reserve_satoshis(channel_value); + let default_config = UserConfig::default(); + let bs_channel_reserve_sats = Channel::::get_holder_selected_channel_reserve_satoshis(channel_value, &default_config); let opt_anchors = false; @@ -1520,12 +1522,13 @@ fn test_chan_reserve_violation_outbound_htlc_inbound_chan() { let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs); - + let default_config = UserConfig::default(); let opt_anchors = false; let mut push_amt = 100_000_000; push_amt -= commit_tx_fee_msat(feerate_per_kw, MIN_AFFORDABLE_HTLC_COUNT as u64, opt_anchors); - push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000) * 1000; + + push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000, &default_config) * 1000; let _ = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, push_amt, InitFeatures::known(), InitFeatures::known()); @@ -1549,7 +1552,7 @@ fn test_chan_reserve_violation_inbound_htlc_outbound_channel() { let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs); - + let default_config = UserConfig::default(); let opt_anchors = false; // Set nodes[0]'s balance such that they will consider any above-dust received HTLC to be a @@ -1557,7 +1560,7 @@ fn test_chan_reserve_violation_inbound_htlc_outbound_channel() { // transaction fee with 0 HTLCs (183 sats)). let mut push_amt = 100_000_000; push_amt -= commit_tx_fee_msat(feerate_per_kw, MIN_AFFORDABLE_HTLC_COUNT as u64, opt_anchors); - push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000) * 1000; + push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000, &default_config) * 1000; let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, push_amt, InitFeatures::known(), InitFeatures::known()); // Send four HTLCs to cover the initial push_msat buffer we're required to include @@ -1602,7 +1605,7 @@ fn test_chan_reserve_dust_inbound_htlcs_outbound_chan() { let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]); let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs); - + let default_config = UserConfig::default(); let opt_anchors = false; // Set nodes[0]'s balance such that they will consider any above-dust received HTLC to be a @@ -1610,7 +1613,7 @@ fn test_chan_reserve_dust_inbound_htlcs_outbound_chan() { // transaction fee with 0 HTLCs (183 sats)). let mut push_amt = 100_000_000; push_amt -= commit_tx_fee_msat(feerate_per_kw, MIN_AFFORDABLE_HTLC_COUNT as u64, opt_anchors); - push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000) * 1000; + push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000, &default_config) * 1000; create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, push_amt, InitFeatures::known(), InitFeatures::known()); let dust_amt = crate::ln::channel::MIN_CHAN_DUST_LIMIT_SATOSHIS * 1000 @@ -1640,7 +1643,7 @@ fn test_chan_init_feerate_unaffordability() { let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs); - + let default_config = UserConfig::default(); let opt_anchors = false; // Set the push_msat amount such that nodes[0] will not be able to afford to add even a single @@ -1652,7 +1655,7 @@ fn test_chan_init_feerate_unaffordability() { // During open, we don't have a "counterparty channel reserve" to check against, so that // requirement only comes into play on the open_channel handling side. - push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000) * 1000; + push_amt -= Channel::::get_holder_selected_channel_reserve_satoshis(100_000, &default_config) * 1000; nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100_000, push_amt, 42, None).unwrap(); let mut open_channel_msg = get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()); open_channel_msg.push_msat += 1; @@ -1771,10 +1774,11 @@ fn test_inbound_outbound_capacity_is_not_zero() { let _ = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 95000000, InitFeatures::known(), InitFeatures::known()); let channels0 = node_chanmgrs[0].list_channels(); let channels1 = node_chanmgrs[1].list_channels(); + let default_config = UserConfig::default(); assert_eq!(channels0.len(), 1); assert_eq!(channels1.len(), 1); - let reserve = Channel::::get_holder_selected_channel_reserve_satoshis(100000); + let reserve = Channel::::get_holder_selected_channel_reserve_satoshis(100_000, &default_config); assert_eq!(channels0[0].inbound_capacity_msat, 95000000 - reserve*1000); assert_eq!(channels1[0].outbound_capacity_msat, 95000000 - reserve*1000); diff --git a/lightning/src/util/config.rs b/lightning/src/util/config.rs index 2f65e958f8b..54ea6178798 100644 --- a/lightning/src/util/config.rs +++ b/lightning/src/util/config.rs @@ -126,6 +126,30 @@ pub struct ChannelHandshakeConfig { /// /// [`KeysInterface::get_shutdown_scriptpubkey`]: crate::chain::keysinterface::KeysInterface::get_shutdown_scriptpubkey pub commit_upfront_shutdown_pubkey: bool, + + /// The Proportion of the channel value to configure as counterparty's channel reserve, + /// i.e., `their_channel_reserve_satoshis` for both outbound and inbound channels. + /// + /// `their_channel_reserve_satoshis` is the minimum balance that the other node has to maintain + /// on their side, at all times. + /// This ensures that if our counterparty broadcasts a revoked state, we can punish them by + /// claiming at least this value on chain. + /// + /// Channel reserve values greater than 30% could be considered highly unreasonable, since that + /// amount can never be used for payments. + /// Also, if our selected channel reserve for counterparty and counterparty's selected + /// channel reserve for us sum up to equal or greater than channel value, channel negotiations + /// will fail. + /// + /// Note: Versions of LDK earlier than v0.0.104 will fail to read channels with any channel reserve + /// other than the default value. + /// + /// Default value: 1% of channel value, i.e., configured as 10,000 millionths. + /// Minimum value: If the calculated proportional value is less than 1000 sats, it will be treated + /// as 1000 sats instead, which is a safe implementation-specific lower bound. + /// Maximum value: 1,000,000, any values larger than 1 Million will be treated as 1 Million (or 100%) + /// instead, although channel negotiations will fail in that case. + pub their_channel_reserve_proportional_millionths: u32 } impl Default for ChannelHandshakeConfig { @@ -138,6 +162,7 @@ impl Default for ChannelHandshakeConfig { negotiate_scid_privacy: false, announced_channel: false, commit_upfront_shutdown_pubkey: true, + their_channel_reserve_proportional_millionths: 10_000, } } } From 33ff2746ef3d1a36aab4f07cd1a7b512b6e411da Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 27 May 2022 17:47:15 -0700 Subject: [PATCH 26/91] Add onion_message::Packet and adapt construct_onion_packet_with_init_noise for it We need to add a new Packet struct because onion message packet hop_data fields can be of variable length, whereas regular payment packets are always 1366 bytes. Co-authored-by: Valentine Wallace Co-authored-by: Jeffrey Czyz --- lightning/src/ln/msgs.rs | 13 +++++ lightning/src/ln/onion_utils.rs | 66 ++++++++++++++------- lightning/src/onion_message/mod.rs | 1 + lightning/src/onion_message/packet.rs | 82 +++++++++++++++++++++++++++ lightning/src/util/ser.rs | 8 +++ 5 files changed, 148 insertions(+), 22 deletions(-) create mode 100644 lightning/src/onion_message/packet.rs diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 3e44cfcf024..424dbafe661 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -31,6 +31,7 @@ use bitcoin::blockdata::script::Script; use bitcoin::hash_types::{Txid, BlockHash}; use ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures}; +use ln::onion_utils; use prelude::*; use core::fmt; @@ -993,6 +994,18 @@ pub(crate) struct OnionPacket { pub(crate) hmac: [u8; 32], } +impl onion_utils::Packet for OnionPacket { + type Data = onion_utils::FixedSizeOnionPacket; + fn new(pubkey: PublicKey, hop_data: onion_utils::FixedSizeOnionPacket, hmac: [u8; 32]) -> Self { + Self { + version: 0, + public_key: Ok(pubkey), + hop_data: hop_data.0, + hmac, + } + } +} + impl PartialEq for OnionPacket { fn eq(&self, other: &OnionPacket) -> bool { for (i, j) in self.hop_data.iter().zip(other.hop_data.iter()) { diff --git a/lightning/src/ln/onion_utils.rs b/lightning/src/ln/onion_utils.rs index 57228b92d04..914c8e03c80 100644 --- a/lightning/src/ln/onion_utils.rs +++ b/lightning/src/ln/onion_utils.rs @@ -30,7 +30,7 @@ use bitcoin::secp256k1; use prelude::*; use io::{Cursor, Read}; -use core::convert::TryInto; +use core::convert::{AsMut, TryInto}; use core::ops::Deref; pub(super) struct OnionKeys { @@ -195,8 +195,8 @@ pub(super) fn build_onion_payloads(path: &Vec, total_msat: u64, paymen pub(crate) const ONION_DATA_LEN: usize = 20*65; #[inline] -fn shift_arr_right(arr: &mut [u8; ONION_DATA_LEN], amt: usize) { - for i in (amt..ONION_DATA_LEN).rev() { +fn shift_slice_right(arr: &mut [u8], amt: usize) { + for i in (amt..arr.len()).rev() { arr[i] = arr[i-amt]; } for i in 0..amt { @@ -218,14 +218,15 @@ pub(super) fn route_size_insane(payloads: &Vec) -> bool { false } -/// panics if route_size_insane(paylods) +/// panics if route_size_insane(payloads) pub(super) fn construct_onion_packet(payloads: Vec, onion_keys: Vec, prng_seed: [u8; 32], associated_data: &PaymentHash) -> msgs::OnionPacket { let mut packet_data = [0; ONION_DATA_LEN]; let mut chacha = ChaCha20::new(&prng_seed, &[0; 8]); chacha.process(&[0; ONION_DATA_LEN], &mut packet_data); - construct_onion_packet_with_init_noise(payloads, onion_keys, packet_data, associated_data) + construct_onion_packet_with_init_noise::<_, _>( + payloads, onion_keys, FixedSizeOnionPacket(packet_data), Some(associated_data)) } #[cfg(test)] @@ -237,12 +238,34 @@ pub(super) fn construct_onion_packet_bogus_hopdata(payloads: Vec< let mut chacha = ChaCha20::new(&prng_seed, &[0; 8]); chacha.process(&[0; ONION_DATA_LEN], &mut packet_data); - construct_onion_packet_with_init_noise(payloads, onion_keys, packet_data, associated_data) + construct_onion_packet_with_init_noise::<_, _>( + payloads, onion_keys, FixedSizeOnionPacket(packet_data), Some(associated_data)) } -/// panics if route_size_insane(paylods) -fn construct_onion_packet_with_init_noise(mut payloads: Vec, onion_keys: Vec, mut packet_data: [u8; ONION_DATA_LEN], associated_data: &PaymentHash) -> msgs::OnionPacket { +/// Since onion message packets and onion payment packets have different lengths but are otherwise +/// identical, we use this trait to allow `construct_onion_packet_with_init_noise` to return either +/// type. +pub(crate) trait Packet { + type Data: AsMut<[u8]>; + fn new(pubkey: PublicKey, hop_data: Self::Data, hmac: [u8; 32]) -> Self; +} + +// Needed for rustc versions older than 1.47 to avoid E0277: "arrays only have std trait +// implementations for lengths 0..=32". +pub(crate) struct FixedSizeOnionPacket(pub(crate) [u8; ONION_DATA_LEN]); + +impl AsMut<[u8]> for FixedSizeOnionPacket { + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0 + } +} + +/// panics if route_size_insane(payloads) +fn construct_onion_packet_with_init_noise( + mut payloads: Vec, onion_keys: Vec, mut packet_data: P::Data, associated_data: Option<&PaymentHash>) -> P +{ let filler = { + let packet_data = packet_data.as_mut(); const ONION_HOP_DATA_LEN: usize = 65; // We may decrease this eventually after TLV is common let mut res = Vec::with_capacity(ONION_HOP_DATA_LEN * (payloads.len() - 1)); @@ -251,7 +274,7 @@ fn construct_onion_packet_with_init_noise(mut payloads: Vec, if i == payloads.len() - 1 { break; } let mut chacha = ChaCha20::new(&keys.rho, &[0u8; 8]); - for _ in 0..(ONION_DATA_LEN - pos) { // TODO: Batch this. + for _ in 0..(packet_data.len() - pos) { // TODO: Batch this. let mut dummy = [0; 1]; chacha.process_in_place(&mut dummy); // We don't have a seek function :( } @@ -259,7 +282,7 @@ fn construct_onion_packet_with_init_noise(mut payloads: Vec, let mut payload_len = LengthCalculatingWriter(0); payload.write(&mut payload_len).expect("Failed to calculate length"); pos += payload_len.0 + 32; - assert!(pos <= ONION_DATA_LEN); + assert!(pos <= packet_data.len()); res.resize(pos, 0u8); chacha.process_in_place(&mut res); @@ -271,29 +294,28 @@ fn construct_onion_packet_with_init_noise(mut payloads: Vec, for (i, (payload, keys)) in payloads.iter_mut().zip(onion_keys.iter()).rev().enumerate() { let mut payload_len = LengthCalculatingWriter(0); payload.write(&mut payload_len).expect("Failed to calculate length"); - shift_arr_right(&mut packet_data, payload_len.0 + 32); + + let packet_data = packet_data.as_mut(); + shift_slice_right(packet_data, payload_len.0 + 32); packet_data[0..payload_len.0].copy_from_slice(&payload.encode()[..]); packet_data[payload_len.0..(payload_len.0 + 32)].copy_from_slice(&hmac_res); let mut chacha = ChaCha20::new(&keys.rho, &[0u8; 8]); - chacha.process_in_place(&mut packet_data); + chacha.process_in_place(packet_data); if i == 0 { packet_data[ONION_DATA_LEN - filler.len()..ONION_DATA_LEN].copy_from_slice(&filler[..]); } let mut hmac = HmacEngine::::new(&keys.mu); - hmac.input(&packet_data); - hmac.input(&associated_data.0[..]); + hmac.input(packet_data); + if let Some(associated_data) = associated_data { + hmac.input(&associated_data.0[..]); + } hmac_res = Hmac::from_engine(hmac).into_inner(); } - msgs::OnionPacket { - version: 0, - public_key: Ok(onion_keys.first().unwrap().ephemeral_pubkey), - hop_data: packet_data, - hmac: hmac_res, - } + P::new(onion_keys.first().unwrap().ephemeral_pubkey, packet_data, hmac_res) } /// Encrypts a failure packet. raw_packet can either be a @@ -783,7 +805,7 @@ mod tests { }, ); - let packet = super::construct_onion_packet_with_init_noise(payloads, onion_keys, [0; super::ONION_DATA_LEN], &PaymentHash([0x42; 32])); + let packet: msgs::OnionPacket = super::construct_onion_packet_with_init_noise::<_, _>(payloads, onion_keys, super::FixedSizeOnionPacket([0; super::ONION_DATA_LEN]), Some(&PaymentHash([0x42; 32]))); // Just check the final packet encoding, as it includes all the per-hop vectors in it // anyway... assert_eq!(packet.encode(), hex::decode("0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a716a996c7845c93d90e4ecbb9bde4ece2f69425c99e4bc820e44485455f135edc0d10f7d61ab590531cf08000179a333a347f8b4072f216400406bdf3bf038659793d4a1fd7b246979e3150a0a4cb052c9ec69acf0f48c3d39cd55675fe717cb7d80ce721caad69320c3a469a202f1e468c67eaf7a7cd8226d0fd32f7b48084dca885d56047694762b67021713ca673929c163ec36e04e40ca8e1c6d17569419d3039d9a1ec866abe044a9ad635778b961fc0776dc832b3a451bd5d35072d2269cf9b040f6b7a7dad84fb114ed413b1426cb96ceaf83825665ed5a1d002c1687f92465b49ed4c7f0218ff8c6c7dd7221d589c65b3b9aaa71a41484b122846c7c7b57e02e679ea8469b70e14fe4f70fee4d87b910cf144be6fe48eef24da475c0b0bcc6565ae82cd3f4e3b24c76eaa5616c6111343306ab35c1fe5ca4a77c0e314ed7dba39d6f1e0de791719c241a939cc493bea2bae1c1e932679ea94d29084278513c77b899cc98059d06a27d171b0dbdf6bee13ddc4fc17a0c4d2827d488436b57baa167544138ca2e64a11b43ac8a06cd0c2fba2d4d900ed2d9205305e2d7383cc98dacb078133de5f6fb6bed2ef26ba92cea28aafc3b9948dd9ae5559e8bd6920b8cea462aa445ca6a95e0e7ba52961b181c79e73bd581821df2b10173727a810c92b83b5ba4a0403eb710d2ca10689a35bec6c3a708e9e92f7d78ff3c5d9989574b00c6736f84c199256e76e19e78f0c98a9d580b4a658c84fc8f2096c2fbea8f5f8c59d0fdacb3be2802ef802abbecb3aba4acaac69a0e965abd8981e9896b1f6ef9d60f7a164b371af869fd0e48073742825e9434fc54da837e120266d53302954843538ea7c6c3dbfb4ff3b2fdbe244437f2a153ccf7bdb4c92aa08102d4f3cff2ae5ef86fab4653595e6a5837fa2f3e29f27a9cde5966843fb847a4a61f1e76c281fe8bb2b0a181d096100db5a1a5ce7a910238251a43ca556712eaadea167fb4d7d75825e440f3ecd782036d7574df8bceacb397abefc5f5254d2722215c53ff54af8299aaaad642c6d72a14d27882d9bbd539e1cc7a527526ba89b8c037ad09120e98ab042d3e8652b31ae0e478516bfaf88efca9f3676ffe99d2819dcaeb7610a626695f53117665d267d3f7abebd6bbd6733f645c72c389f03855bdf1e4b8075b516569b118233a0f0971d24b83113c0b096f5216a207ca99a7cddc81c130923fe3d91e7508c9ac5f2e914ff5dccab9e558566fa14efb34ac98d878580814b94b73acbfde9072f30b881f7f0fff42d4045d1ace6322d86a97d164aa84d93a60498065cc7c20e636f5862dc81531a88c60305a2e59a985be327a6902e4bed986dbf4a0b50c217af0ea7fdf9ab37f9ea1a1aaa72f54cf40154ea9b269f1a7c09f9f43245109431a175d50e2db0132337baa0ef97eed0fcf20489da36b79a1172faccc2f7ded7c60e00694282d93359c4682135642bc81f433574aa8ef0c97b4ade7ca372c5ffc23c7eddd839bab4e0f14d6df15c9dbeab176bec8b5701cf054eb3072f6dadc98f88819042bf10c407516ee58bce33fbe3b3d86a54255e577db4598e30a135361528c101683a5fcde7e8ba53f3456254be8f45fe3a56120ae96ea3773631fcb3873aa3abd91bcff00bd38bd43697a2e789e00da6077482e7b1b1a677b5afae4c54e6cbdf7377b694eb7d7a5b913476a5be923322d3de06060fd5e819635232a2cf4f0731da13b8546d1d6d4f8d75b9fce6c2341a71b0ea6f780df54bfdb0dd5cd9855179f602f9172307c7268724c3618e6817abd793adc214a0dc0bc616816632f27ea336fb56dfd").unwrap()); @@ -862,7 +884,7 @@ mod tests { }), ); - let packet = super::construct_onion_packet_with_init_noise(payloads, onion_keys, [0; super::ONION_DATA_LEN], &PaymentHash([0x42; 32])); + let packet: msgs::OnionPacket = super::construct_onion_packet_with_init_noise::<_, _>(payloads, onion_keys, super::FixedSizeOnionPacket([0; super::ONION_DATA_LEN]), Some(&PaymentHash([0x42; 32]))); // Just check the final packet encoding, as it includes all the per-hop vectors in it // anyway... assert_eq!(packet.encode(), hex::decode("0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a71a060daf367132b378b3a3883c0e2c0e026b8900b2b5cdbc784e1a3bb913f88a9c50f7d61ab590531cf08000178a333a347f8b4072ed056f820f77774345e183a342ec4729f3d84accf515e88adddb85ecc08daba68404bae9a8e8d7178977d7094a1ae549f89338c0777551f874159eb42d3a59fb9285ad4e24883f27de23942ec966611e99bee1cee503455be9e8e642cef6cef7b9864130f692283f8a973d47a8f1c1726b6e59969385975c766e35737c8d76388b64f748ee7943ffb0e2ee45c57a1abc40762ae598723d21bd184e2b338f68ebff47219357bd19cd7e01e2337b806ef4d717888e129e59cd3dc31e6201ccb2fd6d7499836f37a993262468bcb3a4dcd03a22818aca49c6b7b9b8e9e870045631d8e039b066ff86e0d1b7291f71cefa7264c70404a8e538b566c17ccc5feab231401e6c08a01bd5edfc1aa8e3e533b96e82d1f91118d508924b923531929aea889fcdf050597c681185f336b1da63b0939aa2b7c50b21b5eb7b6ad66c81fab98a3cdf73f658149e7e9ced4edde5d38c9b8f92e16f6b4ab13d7fca6a0e4ecc9f9de611a90da6e99c39551094c56e3196f282c5dffd9fc4b2fc12f3bca8e6fe47eb45fbdd3be21a8a8d200797eae3c9a0497132f92410d804977408494dff49dd3d8bce248e0b74fd9e6f0f7102c25ddfa02bd9ad9f746abbfa337ef811d5345a9e16b60de1767b209645ba40bd1f9a5f75bc04feca9b27c5554be4fe83fac2cb83aa447a817bb85ae966c68b420063833fada375e2f515965e687a45699632902672c654d1d18d7bcbf55e8fa57f63f2da449f8e1e606e8722df081e5f193fc4179feb99ad22819afdeef211f7c54afdba92aeef0c00b7bc2b65a4813c01f907a8377585708f2d4c940a25328e585714c8ded0a9a4d7a6de1027c1cb7a0198cd3db68b58c0704dfd0cfbe624e9cd18cc0ae5d96697bb476708b9ee0403d211e64e0d5a7683a7a9a140c02f0ff1c6e67a302941b4052bdea8a63e70a3ad62c5b89c698f1fd3c7685cb49705096cad702d02d93bcb1c27a409f4c9bddec001205ca4a2740f19b50900be81c7e847f1a863deea8d35701f1355cad8db57b1d4eb2ab4e29587734785abfb46ddede71928213d7d089dfdeda052827f459f1688cc0935bd47e7bcec27427c8376dcce7e22699567c0d145f8a7db33f6758815f1f15f9f7a9760dec4f34ae095edda4c64e9735bdd029c4e32c2ee31ba47ec5e6bdb97813d52dbd15b4e0b7a2c7f790ae64104d99f38c127f0a093288fa34144adb16b8968d4fa7656fcec99de8503dd46d3b03620a71c7cd085364abd30dccf7fbda25a1cdc102600149c9af1c97aa0372cd2e1909f28ac5c686f432b310e79528c9b8b9e8f314c1e74621ce6308ad2278b81d460892e0d9dd38b7c76d58be6dfd10ae7583ee1e7ef5b3f6f78dc60af0950df1b00cc55b6d178ba2e476bea0eaeef49323b83f05804159e7aef4eed4cc60dd07be76f067dfd0bcfb0b806b69ba921336a20c43c832d0cab8fa3ddeb29e3bf07b0d98a112eb07802756235a49d44a8b82a950d84e95e01971f0e106ccb337f07384e21620e0ad39e16ed9edca123226cf55ac44f449eeb53e38a7f27d101806e4823e4efcc887414240ee6826c4a5cb1c6443ad36ebf905a435c1d9054e54173911b17b5b40f60b3d9fd5f12eac54ca1e20191f5f18544d5fd3d665e9bcef96fb44b76110aa64d9db4c86c9513cbdad546538e8aec521fbe83ceac5e74a15629f1ed0b870a1d0d1e5680b6d6100d1bd3f3b9043bd35b8919c4088f1949b8be89e4701eb870f8ed64fafa446c78df3ea").unwrap()); diff --git a/lightning/src/onion_message/mod.rs b/lightning/src/onion_message/mod.rs index fabe8d58eda..37858c1a4a6 100644 --- a/lightning/src/onion_message/mod.rs +++ b/lightning/src/onion_message/mod.rs @@ -10,6 +10,7 @@ //! Onion Messages: sending, receiving, forwarding, and ancillary utilities live here mod blinded_route; +mod packet; mod utils; // Re-export structs so they can be imported with just the `onion_message::` module prefix. diff --git a/lightning/src/onion_message/packet.rs b/lightning/src/onion_message/packet.rs new file mode 100644 index 00000000000..37f178f5521 --- /dev/null +++ b/lightning/src/onion_message/packet.rs @@ -0,0 +1,82 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! Structs and enums useful for constructing and reading an onion message packet. + +use bitcoin::secp256k1::PublicKey; + +use ln::msgs::DecodeError; +use ln::onion_utils; +use util::ser::{LengthRead, LengthReadable, Readable, Writeable, Writer}; + +use core::cmp; +use io; +use prelude::*; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Packet { + version: u8, + public_key: PublicKey, + // Unlike the onion packets used for payments, onion message packets can have payloads greater + // than 1300 bytes. + // TODO: if 1300 ends up being the most common size, optimize this to be: + // enum { ThirteenHundred([u8; 1300]), VarLen(Vec) } + hop_data: Vec, + hmac: [u8; 32], +} + +impl onion_utils::Packet for Packet { + type Data = Vec; + fn new(public_key: PublicKey, hop_data: Vec, hmac: [u8; 32]) -> Packet { + Self { + version: 0, + public_key, + hop_data, + hmac, + } + } +} + +impl Writeable for Packet { + fn write(&self, w: &mut W) -> Result<(), io::Error> { + self.version.write(w)?; + self.public_key.write(w)?; + w.write_all(&self.hop_data)?; + self.hmac.write(w)?; + Ok(()) + } +} + +impl LengthReadable for Packet { + fn read(r: &mut R) -> Result { + const READ_BUFFER_SIZE: usize = 4096; + + let version = Readable::read(r)?; + let public_key = Readable::read(r)?; + + let mut hop_data = Vec::new(); + let hop_data_len = r.total_bytes() as usize - 66; // 1 (version) + 33 (pubkey) + 32 (HMAC) = 66 + let mut read_idx = 0; + while read_idx < hop_data_len { + let mut read_buffer = [0; READ_BUFFER_SIZE]; + let read_amt = cmp::min(hop_data_len - read_idx, READ_BUFFER_SIZE); + r.read_exact(&mut read_buffer[..read_amt]); + hop_data.extend_from_slice(&read_buffer[..read_amt]); + read_idx += read_amt; + } + + let hmac = Readable::read(r)?; + Ok(Packet { + version, + public_key, + hop_data, + hmac, + }) + } +} diff --git a/lightning/src/util/ser.rs b/lightning/src/util/ser.rs index 5b1a86a6a95..ecf85839a5a 100644 --- a/lightning/src/util/ser.rs +++ b/lightning/src/util/ser.rs @@ -244,6 +244,14 @@ pub(crate) trait LengthReadableArgs

where Self: Sized fn read(reader: &mut R, params: P) -> Result; } +/// A trait that various higher-level rust-lightning types implement allowing them to be read in +/// from a Read, requiring the implementer to provide the total length of the read. +pub(crate) trait LengthReadable where Self: Sized +{ + /// Reads a Self in from the given LengthRead + fn read(reader: &mut R) -> Result; +} + /// A trait that various rust-lightning types implement allowing them to (maybe) be read in from a Read /// /// (C-not exported) as we only export serialization to/from byte arrays instead From 6017379b8e7943a1967bbc2cabcdb37d07c12531 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Mon, 11 Jul 2022 16:27:10 -0400 Subject: [PATCH 27/91] KeysInterface: add new ecdh method This method will help us avoid retrieving our node secret, something we want to get rid of entirely. It will be used in upcoming commits when decoding the onion message packet, and in future PRs to help us get rid of KeysInterface::get_node_secret usages across the codebase --- fuzz/src/chanmon_consistency.rs | 9 +++++++++ fuzz/src/full_stack.rs | 9 +++++++++ lightning/src/chain/keysinterface.rs | 23 +++++++++++++++++++++++ lightning/src/ln/channel.rs | 2 ++ lightning/src/util/test_utils.rs | 5 +++++ 5 files changed, 48 insertions(+) diff --git a/fuzz/src/chanmon_consistency.rs b/fuzz/src/chanmon_consistency.rs index 615f3aad995..c511e4ff31c 100644 --- a/fuzz/src/chanmon_consistency.rs +++ b/fuzz/src/chanmon_consistency.rs @@ -54,6 +54,7 @@ use utils::test_logger::{self, Output}; use utils::test_persister::TestPersister; use bitcoin::secp256k1::{PublicKey,SecretKey}; +use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; use bitcoin::secp256k1::Secp256k1; @@ -165,6 +166,14 @@ impl KeysInterface for KeyProvider { Ok(SecretKey::from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, self.node_id]).unwrap()) } + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + let mut node_secret = self.get_node_secret(recipient)?; + if let Some(tweak) = tweak { + node_secret.mul_assign(tweak).map_err(|_| ())?; + } + Ok(SharedSecret::new(other_key, &node_secret)) + } + fn get_inbound_payment_key_material(&self) -> KeyMaterial { KeyMaterial([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, self.node_id]) } diff --git a/fuzz/src/full_stack.rs b/fuzz/src/full_stack.rs index b4ca316ed0d..40c77f06662 100644 --- a/fuzz/src/full_stack.rs +++ b/fuzz/src/full_stack.rs @@ -51,6 +51,7 @@ use utils::test_logger; use utils::test_persister::TestPersister; use bitcoin::secp256k1::{PublicKey,SecretKey}; +use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; use bitcoin::secp256k1::Secp256k1; @@ -269,6 +270,14 @@ impl KeysInterface for KeyProvider { Ok(self.node_secret.clone()) } + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + let mut node_secret = self.get_node_secret(recipient)?; + if let Some(tweak) = tweak { + node_secret.mul_assign(tweak).map_err(|_| ())?; + } + Ok(SharedSecret::new(other_key, &node_secret)) + } + fn get_inbound_payment_key_material(&self) -> KeyMaterial { self.inbound_payment_key.clone() } diff --git a/lightning/src/chain/keysinterface.rs b/lightning/src/chain/keysinterface.rs index 4231d08259a..9a3baea8bb4 100644 --- a/lightning/src/chain/keysinterface.rs +++ b/lightning/src/chain/keysinterface.rs @@ -27,6 +27,7 @@ use bitcoin::hash_types::WPubkeyHash; use bitcoin::secp256k1::{SecretKey, PublicKey}; use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature, Signing}; +use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; use bitcoin::{secp256k1, Witness}; @@ -404,6 +405,12 @@ pub trait KeysInterface { /// This method must return the same value each time it is called with a given `Recipient` /// parameter. fn get_node_secret(&self, recipient: Recipient) -> Result; + /// Gets the ECDH shared secret of our [`node secret`] and `other_key`, multiplying by `tweak` if + /// one is provided. Note that this tweak can be applied to `other_key` instead of our node + /// secret, though this is less efficient. + /// + /// [`node secret`]: Self::get_node_secret + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result; /// Get a script pubkey which we send funds to when claiming on-chain contestable outputs. /// /// This method should return a different value each time it is called, to avoid linking @@ -1133,6 +1140,14 @@ impl KeysInterface for KeysManager { } } + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + let mut node_secret = self.get_node_secret(recipient)?; + if let Some(tweak) = tweak { + node_secret.mul_assign(tweak).map_err(|_| ())?; + } + Ok(SharedSecret::new(other_key, &node_secret)) + } + fn get_inbound_payment_key_material(&self) -> KeyMaterial { self.inbound_payment_key.clone() } @@ -1217,6 +1232,14 @@ impl KeysInterface for PhantomKeysManager { } } + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + let mut node_secret = self.get_node_secret(recipient)?; + if let Some(tweak) = tweak { + node_secret.mul_assign(tweak).map_err(|_| ())?; + } + Ok(SharedSecret::new(other_key, &node_secret)) + } + fn get_inbound_payment_key_material(&self) -> KeyMaterial { self.inbound_payment_key.clone() } diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index c5133a85058..2da743033d3 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -6583,6 +6583,7 @@ mod tests { use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature}; use bitcoin::secp256k1::ffi::Signature as FFISignature; use bitcoin::secp256k1::{SecretKey,PublicKey}; + use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::hashes::Hash; @@ -6621,6 +6622,7 @@ mod tests { type Signer = InMemorySigner; fn get_node_secret(&self, _recipient: Recipient) -> Result { panic!(); } + fn ecdh(&self, _recipient: Recipient, _other_key: &PublicKey, _tweak: Option<&[u8; 32]>) -> Result { panic!(); } fn get_inbound_payment_key_material(&self) -> KeyMaterial { panic!(); } fn get_destination_script(&self) -> Script { let secp_ctx = Secp256k1::signing_only(); diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index 4f3d800be5d..92383c91e8f 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -36,6 +36,7 @@ use bitcoin::network::constants::Network; use bitcoin::hash_types::{BlockHash, Txid}; use bitcoin::secp256k1::{SecretKey, PublicKey, Secp256k1, ecdsa::Signature}; +use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; use regex; @@ -74,6 +75,7 @@ impl keysinterface::KeysInterface for OnlyReadsKeysInterface { type Signer = EnforcingSigner; fn get_node_secret(&self, _recipient: Recipient) -> Result { unreachable!(); } + fn ecdh(&self, _recipient: Recipient, _other_key: &PublicKey, _tweak: Option<&[u8; 32]>) -> Result { unreachable!(); } fn get_inbound_payment_key_material(&self) -> KeyMaterial { unreachable!(); } fn get_destination_script(&self) -> Script { unreachable!(); } fn get_shutdown_scriptpubkey(&self) -> ShutdownScript { unreachable!(); } @@ -599,6 +601,9 @@ impl keysinterface::KeysInterface for TestKeysInterface { fn get_node_secret(&self, recipient: Recipient) -> Result { self.backing.get_node_secret(recipient) } + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + self.backing.ecdh(recipient, other_key, tweak) + } fn get_inbound_payment_key_material(&self) -> keysinterface::KeyMaterial { self.backing.get_inbound_payment_key_material() } From 4c8dc2c2a0b9589298d937bf16061ae0ac99b31e Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Wed, 22 Jun 2022 17:03:06 -0400 Subject: [PATCH 28/91] Add baseline OnionMessenger and msgs::OnionMessage and its serialization OnionMessenger will be hooked up to the PeerManager to send and receive OMs in a follow-up PR. --- lightning/src/ln/msgs.rs | 34 +++++++++++- lightning/src/onion_message/messenger.rs | 71 ++++++++++++++++++++++++ lightning/src/onion_message/mod.rs | 3 + 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 lightning/src/onion_message/messenger.rs diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 424dbafe661..950d6a35947 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -32,6 +32,7 @@ use bitcoin::hash_types::{Txid, BlockHash}; use ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures}; use ln::onion_utils; +use onion_message; use prelude::*; use core::fmt; @@ -41,7 +42,7 @@ use io_extras::read_to_end; use util::events::MessageSendEventsProvider; use util::logger; -use util::ser::{Readable, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt, Hostname}; +use util::ser::{LengthReadable, Readable, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt, Hostname}; use ln::{PaymentPreimage, PaymentHash, PaymentSecret}; @@ -305,6 +306,14 @@ pub struct UpdateAddHTLC { pub(crate) onion_routing_packet: OnionPacket, } + /// An onion message to be sent or received from a peer +#[derive(Clone, Debug, PartialEq)] +pub struct OnionMessage { + /// Used in decrypting the onion packet's payload. + pub blinding_point: PublicKey, + pub(crate) onion_routing_packet: onion_message::Packet, +} + /// An update_fulfill_htlc message to be sent or received from a peer #[derive(Clone, Debug, PartialEq)] pub struct UpdateFulfillHTLC { @@ -1340,6 +1349,29 @@ impl_writeable_msg!(UpdateAddHTLC, { onion_routing_packet }, {}); +impl Readable for OnionMessage { + fn read(r: &mut R) -> Result { + let blinding_point: PublicKey = Readable::read(r)?; + let len: u16 = Readable::read(r)?; + let mut packet_reader = FixedLengthReader::new(r, len as u64); + let onion_routing_packet: onion_message::Packet = ::read(&mut packet_reader)?; + Ok(Self { + blinding_point, + onion_routing_packet, + }) + } +} + +impl Writeable for OnionMessage { + fn write(&self, w: &mut W) -> Result<(), io::Error> { + self.blinding_point.write(w)?; + let onion_packet_len = self.onion_routing_packet.serialized_length(); + (onion_packet_len as u16).write(w)?; + self.onion_routing_packet.write(w)?; + Ok(()) + } +} + impl Writeable for FinalOnionHopData { fn write(&self, w: &mut W) -> Result<(), io::Error> { self.payment_secret.0.write(w)?; diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs new file mode 100644 index 00000000000..e2d7b51a9b5 --- /dev/null +++ b/lightning/src/onion_message/messenger.rs @@ -0,0 +1,71 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! LDK sends, receives, and forwards onion messages via the [`OnionMessenger`]. See its docs for +//! more information. + +use bitcoin::secp256k1::{self, PublicKey, Secp256k1}; + +use chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Sign}; +use ln::msgs; +use util::logger::Logger; + +use core::ops::Deref; +use sync::{Arc, Mutex}; +use prelude::*; + +/// A sender, receiver and forwarder of onion messages. In upcoming releases, this object will be +/// used to retrieve invoices and fulfill invoice requests from [offers]. +/// +/// [offers]: +pub struct OnionMessenger + where K::Target: KeysInterface, + L::Target: Logger, +{ + keys_manager: K, + logger: L, + pending_messages: Mutex>>, + secp_ctx: Secp256k1, + // Coming soon: + // invoice_handler: InvoiceHandler, + // custom_handler: CustomHandler, // handles custom onion messages +} + +impl OnionMessenger + where K::Target: KeysInterface, + L::Target: Logger, +{ + /// Constructs a new `OnionMessenger` to send, forward, and delegate received onion messages to + /// their respective handlers. + pub fn new(keys_manager: K, logger: L) -> Self { + let mut secp_ctx = Secp256k1::new(); + secp_ctx.seeded_randomize(&keys_manager.get_secure_random_bytes()); + OnionMessenger { + keys_manager, + pending_messages: Mutex::new(HashMap::new()), + secp_ctx, + logger, + } + } +} + +// TODO: parameterize the below Simple* types with OnionMessenger and handle the messages it +// produces +/// Useful for simplifying the parameters of [`SimpleArcChannelManager`] and +/// [`SimpleArcPeerManager`]. See their docs for more details. +/// +///[`SimpleArcChannelManager`]: crate::ln::channelmanager::SimpleArcChannelManager +///[`SimpleArcPeerManager`]: crate::ln::peer_handler::SimpleArcPeerManager +pub type SimpleArcOnionMessenger = OnionMessenger, Arc>; +/// Useful for simplifying the parameters of [`SimpleRefChannelManager`] and +/// [`SimpleRefPeerManager`]. See their docs for more details. +/// +///[`SimpleRefChannelManager`]: crate::ln::channelmanager::SimpleRefChannelManager +///[`SimpleRefPeerManager`]: crate::ln::peer_handler::SimpleRefPeerManager +pub type SimpleRefOnionMessenger<'a, 'b, L> = OnionMessenger; diff --git a/lightning/src/onion_message/mod.rs b/lightning/src/onion_message/mod.rs index 37858c1a4a6..291030f4e26 100644 --- a/lightning/src/onion_message/mod.rs +++ b/lightning/src/onion_message/mod.rs @@ -10,8 +10,11 @@ //! Onion Messages: sending, receiving, forwarding, and ancillary utilities live here mod blinded_route; +mod messenger; mod packet; mod utils; // Re-export structs so they can be imported with just the `onion_message::` module prefix. pub use self::blinded_route::{BlindedRoute, BlindedHop}; +pub use self::messenger::{OnionMessenger, SimpleArcOnionMessenger, SimpleRefOnionMessenger}; +pub(crate) use self::packet::Packet; From 9051c38ebe42e171fd0fcfa22d2b9ff6a1607b3b Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 27 May 2022 18:31:27 -0700 Subject: [PATCH 29/91] Support sending onion messages This adds several utilities in service of then adding OnionMessenger::send_onion_message, which can send to either an unblinded pubkey or a blinded route. Sending custom TLVs and sending an onion message containing a reply path are not yet supported. We also need to split the construct_keys_callback macro into two macros to avoid an unused assignment warning. --- lightning/src/ln/onion_utils.rs | 32 +++-- lightning/src/onion_message/blinded_route.rs | 18 +-- lightning/src/onion_message/messenger.rs | 140 ++++++++++++++++++- lightning/src/onion_message/mod.rs | 2 +- lightning/src/onion_message/packet.rs | 76 ++++++++++ lightning/src/onion_message/utils.rs | 31 +++- 6 files changed, 275 insertions(+), 24 deletions(-) diff --git a/lightning/src/ln/onion_utils.rs b/lightning/src/ln/onion_utils.rs index 914c8e03c80..ce91d0d04f7 100644 --- a/lightning/src/ln/onion_utils.rs +++ b/lightning/src/ln/onion_utils.rs @@ -33,14 +33,14 @@ use io::{Cursor, Read}; use core::convert::{AsMut, TryInto}; use core::ops::Deref; -pub(super) struct OnionKeys { +pub(crate) struct OnionKeys { #[cfg(test)] - pub(super) shared_secret: SharedSecret, + pub(crate) shared_secret: SharedSecret, #[cfg(test)] - pub(super) blinding_factor: [u8; 32], - pub(super) ephemeral_pubkey: PublicKey, - pub(super) rho: [u8; 32], - pub(super) mu: [u8; 32], + pub(crate) blinding_factor: [u8; 32], + pub(crate) ephemeral_pubkey: PublicKey, + pub(crate) rho: [u8; 32], + pub(crate) mu: [u8; 32], } #[inline] @@ -52,7 +52,7 @@ pub(crate) fn gen_rho_from_shared_secret(shared_secret: &[u8]) -> [u8; 32] { } #[inline] -pub(super) fn gen_rho_mu_from_shared_secret(shared_secret: &[u8]) -> ([u8; 32], [u8; 32]) { +pub(crate) fn gen_rho_mu_from_shared_secret(shared_secret: &[u8]) -> ([u8; 32], [u8; 32]) { assert_eq!(shared_secret.len(), 32); ({ let mut hmac = HmacEngine::::new(&[0x72, 0x68, 0x6f]); // rho @@ -260,7 +260,23 @@ impl AsMut<[u8]> for FixedSizeOnionPacket { } } -/// panics if route_size_insane(payloads) +pub(crate) fn payloads_serialized_length(payloads: &Vec) -> usize { + payloads.iter().map(|p| p.serialized_length() + 32 /* HMAC */).sum() +} + +/// panics if payloads_serialized_length(payloads) > packet_data_len +pub(crate) fn construct_onion_message_packet>>( + payloads: Vec, onion_keys: Vec, prng_seed: [u8; 32], packet_data_len: usize) -> P +{ + let mut packet_data = vec![0; packet_data_len]; + + let mut chacha = ChaCha20::new(&prng_seed, &[0; 8]); + chacha.process_in_place(&mut packet_data); + + construct_onion_packet_with_init_noise::<_, _>(payloads, onion_keys, packet_data, None) +} + +/// panics if payloads_serialized_length(payloads) > packet_data.len() fn construct_onion_packet_with_init_noise( mut payloads: Vec, onion_keys: Vec, mut packet_data: P::Data, associated_data: Option<&PaymentHash>) -> P { diff --git a/lightning/src/onion_message/blinded_route.rs b/lightning/src/onion_message/blinded_route.rs index be6e2a01ca5..d18372e3b00 100644 --- a/lightning/src/onion_message/blinded_route.rs +++ b/lightning/src/onion_message/blinded_route.rs @@ -28,25 +28,25 @@ pub struct BlindedRoute { /// message's next hop and forward it along. /// /// [`encrypted_payload`]: BlindedHop::encrypted_payload - introduction_node_id: PublicKey, + pub(super) introduction_node_id: PublicKey, /// Used by the introduction node to decrypt its [`encrypted_payload`] to forward the onion /// message. /// /// [`encrypted_payload`]: BlindedHop::encrypted_payload - blinding_point: PublicKey, + pub(super) blinding_point: PublicKey, /// The hops composing the blinded route. - blinded_hops: Vec, + pub(super) blinded_hops: Vec, } /// Used to construct the blinded hops portion of a blinded route. These hops cannot be identified /// by outside observers and thus can be used to hide the identity of the recipient. pub struct BlindedHop { /// The blinded node id of this hop in a blinded route. - blinded_node_id: PublicKey, + pub(super) blinded_node_id: PublicKey, /// The encrypted payload intended for this hop in a blinded route. // The node sending to this blinded route will later encode this payload into the onion packet for // this hop. - encrypted_payload: Vec, + pub(super) encrypted_payload: Vec, } impl BlindedRoute { @@ -78,7 +78,7 @@ fn blinded_hops( let mut blinded_hops = Vec::with_capacity(unblinded_path.len()); let mut prev_ss_and_blinded_node_id = None; - utils::construct_keys_callback(secp_ctx, unblinded_path, session_priv, |blinded_node_id, _, _, encrypted_payload_ss, unblinded_pk| { + utils::construct_keys_callback(secp_ctx, unblinded_path, None, session_priv, |blinded_node_id, _, _, encrypted_payload_ss, unblinded_pk, _| { if let Some((prev_ss, prev_blinded_node_id)) = prev_ss_and_blinded_node_id { if let Some(pk) = unblinded_pk { let payload = ForwardTlvs { @@ -117,10 +117,10 @@ fn encrypt_payload(payload: P, encrypted_tlvs_ss: [u8; 32]) -> Vec /// route, they are encoded into [`BlindedHop::encrypted_payload`]. pub(crate) struct ForwardTlvs { /// The node id of the next hop in the onion message's path. - next_node_id: PublicKey, + pub(super) next_node_id: PublicKey, /// Senders to a blinded route use this value to concatenate the route they find to the /// introduction node with the blinded route. - next_blinding_override: Option, + pub(super) next_blinding_override: Option, } /// Similar to [`ForwardTlvs`], but these TLVs are for the final node. @@ -128,7 +128,7 @@ pub(crate) struct ReceiveTlvs { /// If `path_id` is `Some`, it is used to identify the blinded route that this onion message is /// sending to. This is useful for receivers to check that said blinded route is being used in /// the right context. - path_id: Option<[u8; 32]>, + pub(super) path_id: Option<[u8; 32]>, } impl Writeable for ForwardTlvs { diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index e2d7b51a9b5..f4cb57f28df 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -10,10 +10,14 @@ //! LDK sends, receives, and forwards onion messages via the [`OnionMessenger`]. See its docs for //! more information. -use bitcoin::secp256k1::{self, PublicKey, Secp256k1}; +use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey}; use chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Sign}; use ln::msgs; +use ln::onion_utils; +use super::blinded_route::{BlindedRoute, ForwardTlvs, ReceiveTlvs}; +use super::packet::{BIG_PACKET_HOP_DATA_LEN, ForwardControlTlvs, Packet, Payload, ReceiveControlTlvs, SMALL_PACKET_HOP_DATA_LEN}; +use super::utils; use util::logger::Logger; use core::ops::Deref; @@ -37,6 +41,23 @@ pub struct OnionMessenger // custom_handler: CustomHandler, // handles custom onion messages } +/// The destination of an onion message. +pub enum Destination { + /// We're sending this onion message to a node. + Node(PublicKey), + /// We're sending this onion message to a blinded route. + BlindedRoute(BlindedRoute), +} + +impl Destination { + pub(super) fn num_hops(&self) -> usize { + match self { + Destination::Node(_) => 1, + Destination::BlindedRoute(BlindedRoute { blinded_hops, .. }) => blinded_hops.len(), + } + } +} + impl OnionMessenger where K::Target: KeysInterface, L::Target: Logger, @@ -53,6 +74,36 @@ impl OnionMessenger logger, } } + + /// Send an empty onion message to `destination`, routing it through `intermediate_nodes`. + pub fn send_onion_message(&self, intermediate_nodes: &[PublicKey], destination: Destination) -> Result<(), secp256k1::Error> { + let blinding_secret_bytes = self.keys_manager.get_secure_random_bytes(); + let blinding_secret = SecretKey::from_slice(&blinding_secret_bytes[..]).expect("RNG is busted"); + let (introduction_node_id, blinding_point) = if intermediate_nodes.len() != 0 { + (intermediate_nodes[0], PublicKey::from_secret_key(&self.secp_ctx, &blinding_secret)) + } else { + match destination { + Destination::Node(pk) => (pk, PublicKey::from_secret_key(&self.secp_ctx, &blinding_secret)), + Destination::BlindedRoute(BlindedRoute { introduction_node_id, blinding_point, .. }) => + (introduction_node_id, blinding_point), + } + }; + let (packet_payloads, packet_keys) = packet_payloads_and_keys( + &self.secp_ctx, intermediate_nodes, destination, &blinding_secret)?; + + let prng_seed = self.keys_manager.get_secure_random_bytes(); + let onion_packet = construct_onion_message_packet(packet_payloads, packet_keys, prng_seed); + + let mut pending_per_peer_msgs = self.pending_messages.lock().unwrap(); + let pending_msgs = pending_per_peer_msgs.entry(introduction_node_id).or_insert(Vec::new()); + pending_msgs.push( + msgs::OnionMessage { + blinding_point, + onion_routing_packet: onion_packet, + } + ); + Ok(()) + } } // TODO: parameterize the below Simple* types with OnionMessenger and handle the messages it @@ -69,3 +120,90 @@ pub type SimpleArcOnionMessenger = OnionMessenger = OnionMessenger; + +/// Construct onion packet payloads and keys for sending an onion message along the given +/// `unblinded_path` to the given `destination`. +fn packet_payloads_and_keys( + secp_ctx: &Secp256k1, unblinded_path: &[PublicKey], destination: Destination, session_priv: &SecretKey +) -> Result<(Vec<(Payload, [u8; 32])>, Vec), secp256k1::Error> { + let num_hops = unblinded_path.len() + destination.num_hops(); + let mut payloads = Vec::with_capacity(num_hops); + let mut onion_packet_keys = Vec::with_capacity(num_hops); + + let (mut intro_node_id_blinding_pt, num_blinded_hops) = if let Destination::BlindedRoute(BlindedRoute { + introduction_node_id, blinding_point, blinded_hops }) = &destination { + (Some((*introduction_node_id, *blinding_point)), blinded_hops.len()) } else { (None, 0) }; + let num_unblinded_hops = num_hops - num_blinded_hops; + + let mut unblinded_path_idx = 0; + let mut blinded_path_idx = 0; + let mut prev_control_tlvs_ss = None; + utils::construct_keys_callback(secp_ctx, unblinded_path, Some(destination), session_priv, |_, onion_packet_ss, ephemeral_pubkey, control_tlvs_ss, unblinded_pk_opt, enc_payload_opt| { + if num_unblinded_hops != 0 && unblinded_path_idx < num_unblinded_hops { + if let Some(ss) = prev_control_tlvs_ss.take() { + payloads.push((Payload::Forward(ForwardControlTlvs::Unblinded( + ForwardTlvs { + next_node_id: unblinded_pk_opt.unwrap(), + next_blinding_override: None, + } + )), ss)); + } + prev_control_tlvs_ss = Some(control_tlvs_ss); + unblinded_path_idx += 1; + } else if let Some((intro_node_id, blinding_pt)) = intro_node_id_blinding_pt.take() { + if let Some(control_tlvs_ss) = prev_control_tlvs_ss.take() { + payloads.push((Payload::Forward(ForwardControlTlvs::Unblinded(ForwardTlvs { + next_node_id: intro_node_id, + next_blinding_override: Some(blinding_pt), + })), control_tlvs_ss)); + } + if let Some(encrypted_payload) = enc_payload_opt { + payloads.push((Payload::Forward(ForwardControlTlvs::Blinded(encrypted_payload)), + control_tlvs_ss)); + } else { debug_assert!(false); } + blinded_path_idx += 1; + } else if blinded_path_idx < num_blinded_hops - 1 && enc_payload_opt.is_some() { + payloads.push((Payload::Forward(ForwardControlTlvs::Blinded(enc_payload_opt.unwrap())), + control_tlvs_ss)); + blinded_path_idx += 1; + } else if let Some(encrypted_payload) = enc_payload_opt { + payloads.push((Payload::Receive { + control_tlvs: ReceiveControlTlvs::Blinded(encrypted_payload), + }, control_tlvs_ss)); + } + + let (rho, mu) = onion_utils::gen_rho_mu_from_shared_secret(onion_packet_ss.as_ref()); + onion_packet_keys.push(onion_utils::OnionKeys { + #[cfg(test)] + shared_secret: onion_packet_ss, + #[cfg(test)] + blinding_factor: [0; 32], + ephemeral_pubkey, + rho, + mu, + }); + })?; + + if let Some(control_tlvs_ss) = prev_control_tlvs_ss { + payloads.push((Payload::Receive { + control_tlvs: ReceiveControlTlvs::Unblinded(ReceiveTlvs { path_id: None, }) + }, control_tlvs_ss)); + } + + Ok((payloads, onion_packet_keys)) +} + +fn construct_onion_message_packet(payloads: Vec<(Payload, [u8; 32])>, onion_keys: Vec, prng_seed: [u8; 32]) -> Packet { + // Spec rationale: + // "`len` allows larger messages to be sent than the standard 1300 bytes allowed for an HTLC + // onion, but this should be used sparingly as it is reduces anonymity set, hence the + // recommendation that it either look like an HTLC onion, or if larger, be a fixed size." + let payloads_ser_len = onion_utils::payloads_serialized_length(&payloads); + let hop_data_len = if payloads_ser_len <= SMALL_PACKET_HOP_DATA_LEN { + SMALL_PACKET_HOP_DATA_LEN + } else if payloads_ser_len <= BIG_PACKET_HOP_DATA_LEN { + BIG_PACKET_HOP_DATA_LEN + } else { payloads_ser_len }; + + onion_utils::construct_onion_message_packet::<_, _>(payloads, onion_keys, prng_seed, hop_data_len) +} diff --git a/lightning/src/onion_message/mod.rs b/lightning/src/onion_message/mod.rs index 291030f4e26..9236341fa61 100644 --- a/lightning/src/onion_message/mod.rs +++ b/lightning/src/onion_message/mod.rs @@ -16,5 +16,5 @@ mod utils; // Re-export structs so they can be imported with just the `onion_message::` module prefix. pub use self::blinded_route::{BlindedRoute, BlindedHop}; -pub use self::messenger::{OnionMessenger, SimpleArcOnionMessenger, SimpleRefOnionMessenger}; +pub use self::messenger::{Destination, OnionMessenger, SimpleArcOnionMessenger, SimpleRefOnionMessenger}; pub(crate) use self::packet::Packet; diff --git a/lightning/src/onion_message/packet.rs b/lightning/src/onion_message/packet.rs index 37f178f5521..ed0206a2adb 100644 --- a/lightning/src/onion_message/packet.rs +++ b/lightning/src/onion_message/packet.rs @@ -13,12 +13,19 @@ use bitcoin::secp256k1::PublicKey; use ln::msgs::DecodeError; use ln::onion_utils; +use super::blinded_route::{ForwardTlvs, ReceiveTlvs}; +use util::chacha20poly1305rfc::ChaChaPolyWriteAdapter; use util::ser::{LengthRead, LengthReadable, Readable, Writeable, Writer}; use core::cmp; use io; use prelude::*; +// Per the spec, an onion message packet's `hop_data` field length should be +// SMALL_PACKET_HOP_DATA_LEN if it fits, else BIG_PACKET_HOP_DATA_LEN if it fits. +pub(super) const SMALL_PACKET_HOP_DATA_LEN: usize = 1300; +pub(super) const BIG_PACKET_HOP_DATA_LEN: usize = 32768; + #[derive(Clone, Debug, PartialEq)] pub(crate) struct Packet { version: u8, @@ -80,3 +87,72 @@ impl LengthReadable for Packet { }) } } + +/// Onion message payloads contain "control" TLVs and "data" TLVs. Control TLVs are used to route +/// the onion message from hop to hop and for path verification, whereas data TLVs contain the onion +/// message content itself, such as an invoice request. +pub(super) enum Payload { + /// This payload is for an intermediate hop. + Forward(ForwardControlTlvs), + /// This payload is for the final hop. + Receive { + control_tlvs: ReceiveControlTlvs, + // Coming soon: + // reply_path: Option, + // message: Message, + } +} + +// Coming soon: +// enum Message { +// InvoiceRequest(InvoiceRequest), +// Invoice(Invoice), +// InvoiceError(InvoiceError), +// CustomMessage, +// } + +/// Forward control TLVs in their blinded and unblinded form. +pub(super) enum ForwardControlTlvs { + /// If we're sending to a blinded route, the node that constructed the blinded route has provided + /// this hop's control TLVs, already encrypted into bytes. + Blinded(Vec), + /// If we're constructing an onion message hop through an intermediate unblinded node, we'll need + /// to construct the intermediate hop's control TLVs in their unblinded state to avoid encoding + /// them into an intermediate Vec. See [`super::blinded_route::ForwardTlvs`] for more info. + Unblinded(ForwardTlvs), +} + +/// Receive control TLVs in their blinded and unblinded form. +pub(super) enum ReceiveControlTlvs { + /// See [`ForwardControlTlvs::Blinded`]. + Blinded(Vec), + /// See [`ForwardControlTlvs::Unblinded`] and [`super::blinded_route::ReceiveTlvs`]. + Unblinded(ReceiveTlvs), +} + +// Uses the provided secret to simultaneously encode and encrypt the unblinded control TLVs. +impl Writeable for (Payload, [u8; 32]) { + fn write(&self, w: &mut W) -> Result<(), io::Error> { + match &self.0 { + Payload::Forward(ForwardControlTlvs::Blinded(encrypted_bytes)) | + Payload::Receive { control_tlvs: ReceiveControlTlvs::Blinded(encrypted_bytes)} => { + encode_varint_length_prefixed_tlv!(w, { + (4, encrypted_bytes, vec_type) + }) + }, + Payload::Forward(ForwardControlTlvs::Unblinded(control_tlvs)) => { + let write_adapter = ChaChaPolyWriteAdapter::new(self.1, &control_tlvs); + encode_varint_length_prefixed_tlv!(w, { + (4, write_adapter, required) + }) + }, + Payload::Receive { control_tlvs: ReceiveControlTlvs::Unblinded(control_tlvs)} => { + let write_adapter = ChaChaPolyWriteAdapter::new(self.1, &control_tlvs); + encode_varint_length_prefixed_tlv!(w, { + (4, write_adapter, required) + }) + }, + } + Ok(()) + } +} diff --git a/lightning/src/onion_message/utils.rs b/lightning/src/onion_message/utils.rs index 785a373caa4..9b95183e74b 100644 --- a/lightning/src/onion_message/utils.rs +++ b/lightning/src/onion_message/utils.rs @@ -16,14 +16,16 @@ use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey}; use bitcoin::secp256k1::ecdh::SharedSecret; use ln::onion_utils; +use super::blinded_route::BlindedRoute; +use super::messenger::Destination; use prelude::*; // TODO: DRY with onion_utils::construct_onion_keys_callback #[inline] pub(super) fn construct_keys_callback)>( - secp_ctx: &Secp256k1, unblinded_path: &[PublicKey], + FType: FnMut(PublicKey, SharedSecret, PublicKey, [u8; 32], Option, Option>)>( + secp_ctx: &Secp256k1, unblinded_path: &[PublicKey], destination: Option, session_priv: &SecretKey, mut callback: FType ) -> Result<(), secp256k1::Error> { let mut msg_blinding_point_priv = session_priv.clone(); @@ -32,7 +34,7 @@ pub(super) fn construct_keys_callback { + ($pk: expr, $blinded: expr, $encrypted_payload: expr) => {{ let encrypted_data_ss = SharedSecret::new(&$pk, &msg_blinding_point_priv); let blinded_hop_pk = if $blinded { $pk } else { @@ -49,7 +51,14 @@ pub(super) fn construct_keys_callback { + let (encrypted_data_ss, onion_packet_ss) = build_keys!($pk, $blinded, $encrypted_payload); let msg_blinding_point_blinding_factor = { let mut sha = Sha256::engine(); @@ -73,7 +82,19 @@ pub(super) fn construct_keys_callback { + build_keys!(pk, false, None); + }, + Destination::BlindedRoute(BlindedRoute { blinded_hops, .. }) => { + for hop in blinded_hops { + build_keys_in_loop!(hop.blinded_node_id, true, Some(hop.encrypted_payload)); + } + }, + } } Ok(()) } From bf007ea7632b7b95c280440bc50713e55784e315 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 27 May 2022 18:39:56 -0700 Subject: [PATCH 30/91] Implement receiving and forwarding onion messages This required adapting `onion_utils::decode_next_hop` to work for both payments and onion messages. Currently we just print out the path_id of any onion messages we receive. In the future, these received onion messages will be redirected to their respective handlers: i.e. an invoice_request will go to an InvoiceHandler, custom onion messages will go to a custom handler, etc. --- lightning/src/ln/channelmanager.rs | 4 +- lightning/src/ln/msgs.rs | 10 ++- lightning/src/ln/onion_utils.rs | 88 +++++++++++++++---- lightning/src/onion_message/messenger.rs | 96 +++++++++++++++++++- lightning/src/onion_message/packet.rs | 106 +++++++++++++++++++++-- 5 files changed, 277 insertions(+), 27 deletions(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 36c14242d9e..40218089e3e 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -2159,7 +2159,7 @@ impl ChannelMana } } - let next_hop = match onion_utils::decode_next_hop(shared_secret, &msg.onion_routing_packet.hop_data[..], msg.onion_routing_packet.hmac, msg.payment_hash) { + let next_hop = match onion_utils::decode_next_payment_hop(shared_secret, &msg.onion_routing_packet.hop_data[..], msg.onion_routing_packet.hmac, msg.payment_hash) { Ok(res) => res, Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => { return_malformed_err!(err_msg, err_code); @@ -3058,7 +3058,7 @@ impl ChannelMana let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode); if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id) { let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes(); - let next_hop = match onion_utils::decode_next_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) { + let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) { Ok(res) => res, Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => { let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner(); diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 950d6a35947..01863a9071c 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -42,7 +42,7 @@ use io_extras::read_to_end; use util::events::MessageSendEventsProvider; use util::logger; -use util::ser::{LengthReadable, Readable, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt, Hostname}; +use util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt, Hostname}; use ln::{PaymentPreimage, PaymentHash, PaymentSecret}; @@ -1417,6 +1417,14 @@ impl Writeable for OnionHopData { } } +// ReadableArgs because we need onion_utils::decode_next_hop to accommodate payment packets and +// onion message packets. +impl ReadableArgs<()> for OnionHopData { + fn read(r: &mut R, _arg: ()) -> Result { + ::read(r) + } +} + impl Readable for OnionHopData { fn read(mut r: &mut R) -> Result { use bitcoin::consensus::encode::{Decodable, Error, VarInt}; diff --git a/lightning/src/ln/onion_utils.rs b/lightning/src/ln/onion_utils.rs index ce91d0d04f7..145eb8acbae 100644 --- a/lightning/src/ln/onion_utils.rs +++ b/lightning/src/ln/onion_utils.rs @@ -15,7 +15,7 @@ use routing::gossip::NetworkUpdate; use routing::router::RouteHop; use util::chacha20::{ChaCha20, ChaChaReader}; use util::errors::{self, APIError}; -use util::ser::{Readable, Writeable, LengthCalculatingWriter}; +use util::ser::{Readable, ReadableArgs, Writeable, LengthCalculatingWriter}; use util::logger::Logger; use bitcoin::hashes::{Hash, HashEngine}; @@ -82,7 +82,7 @@ pub(super) fn gen_ammag_from_shared_secret(shared_secret: &[u8]) -> [u8; 32] { Hmac::from_engine(hmac).into_inner() } -pub(super) fn next_hop_packet_pubkey(secp_ctx: &Secp256k1, mut packet_pubkey: PublicKey, packet_shared_secret: &[u8; 32]) -> Result { +pub(crate) fn next_hop_packet_pubkey(secp_ctx: &Secp256k1, mut packet_pubkey: PublicKey, packet_shared_secret: &[u8; 32]) -> Result { let blinding_factor = { let mut sha = Sha256::engine(); sha.input(&packet_pubkey.serialize()[..]); @@ -580,7 +580,50 @@ pub(super) fn process_onion_failure(secp_ctx: & } else { unreachable!(); } } -/// Data decrypted from the onion payload. +/// An input used when decoding an onion packet. +pub(crate) trait DecodeInput { + type Arg; + /// If Some, this is the input when checking the hmac of the onion packet. + fn payment_hash(&self) -> Option<&PaymentHash>; + /// Read argument when decrypting our hop payload. + fn read_arg(self) -> Self::Arg; +} + +impl DecodeInput for PaymentHash { + type Arg = (); + fn payment_hash(&self) -> Option<&PaymentHash> { + Some(self) + } + fn read_arg(self) -> Self::Arg { () } +} + +impl DecodeInput for SharedSecret { + type Arg = SharedSecret; + fn payment_hash(&self) -> Option<&PaymentHash> { + None + } + fn read_arg(self) -> Self::Arg { self } +} + +/// Allows `decode_next_hop` to return the next hop packet bytes for either payments or onion +/// message forwards. +pub(crate) trait NextPacketBytes: AsMut<[u8]> { + fn new(len: usize) -> Self; +} + +impl NextPacketBytes for FixedSizeOnionPacket { + fn new(_len: usize) -> Self { + Self([0 as u8; ONION_DATA_LEN]) + } +} + +impl NextPacketBytes for Vec { + fn new(len: usize) -> Self { + vec![0 as u8; len] + } +} + +/// Data decrypted from a payment's onion payload. pub(crate) enum Hop { /// This onion payload was for us, not for forwarding to a next-hop. Contains information for /// verifying the incoming payment. @@ -592,11 +635,12 @@ pub(crate) enum Hop { /// HMAC of the next hop's onion packet. next_hop_hmac: [u8; 32], /// Bytes of the onion packet we're forwarding. - new_packet_bytes: [u8; 20*65], + new_packet_bytes: [u8; ONION_DATA_LEN], }, } /// Error returned when we fail to decode the onion packet. +#[derive(Debug)] pub(crate) enum OnionDecodeErr { /// The HMAC of the onion packet did not match the hop data. Malformed { @@ -610,11 +654,27 @@ pub(crate) enum OnionDecodeErr { }, } -pub(crate) fn decode_next_hop(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], payment_hash: PaymentHash) -> Result { +pub(crate) fn decode_next_payment_hop(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], payment_hash: PaymentHash) -> Result { + match decode_next_hop(shared_secret, hop_data, hmac_bytes, payment_hash) { + Ok((next_hop_data, None)) => Ok(Hop::Receive(next_hop_data)), + Ok((next_hop_data, Some((next_hop_hmac, FixedSizeOnionPacket(new_packet_bytes))))) => { + Ok(Hop::Forward { + next_hop_data, + next_hop_hmac, + new_packet_bytes + }) + }, + Err(e) => Err(e), + } +} + +pub(crate) fn decode_next_hop, N: NextPacketBytes>(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], decode_input: D) -> Result<(R, Option<([u8; 32], N)>), OnionDecodeErr> { let (rho, mu) = gen_rho_mu_from_shared_secret(&shared_secret); let mut hmac = HmacEngine::::new(&mu); hmac.input(hop_data); - hmac.input(&payment_hash.0[..]); + if let Some(payment_hash) = decode_input.payment_hash() { + hmac.input(&payment_hash.0[..]); + } if !fixed_time_eq(&Hmac::from_engine(hmac).into_inner(), &hmac_bytes) { return Err(OnionDecodeErr::Malformed { err_msg: "HMAC Check failed", @@ -624,7 +684,7 @@ pub(crate) fn decode_next_hop(shared_secret: [u8; 32], hop_data: &[u8], hmac_byt let mut chacha = ChaCha20::new(&rho, &[0u8; 8]); let mut chacha_stream = ChaChaReader { chacha: &mut chacha, read: Cursor::new(&hop_data[..]) }; - match ::read(&mut chacha_stream) { + match R::read(&mut chacha_stream, decode_input.read_arg()) { Err(err) => { let error_code = match err { msgs::DecodeError::UnknownVersion => 0x4000 | 1, // unknown realm byte @@ -662,10 +722,10 @@ pub(crate) fn decode_next_hop(shared_secret: [u8; 32], hop_data: &[u8], hmac_byt chacha_stream.read_exact(&mut next_bytes).unwrap(); assert_ne!(next_bytes[..], [0; 32][..]); } - return Ok(Hop::Receive(msg)); + return Ok((msg, None)); // We are the final destination for this packet } else { - let mut new_packet_bytes = [0; 20*65]; - let read_pos = chacha_stream.read(&mut new_packet_bytes).unwrap(); + let mut new_packet_bytes = N::new(hop_data.len()); + let read_pos = chacha_stream.read(new_packet_bytes.as_mut()).unwrap(); #[cfg(debug_assertions)] { // Check two things: @@ -677,12 +737,8 @@ pub(crate) fn decode_next_hop(shared_secret: [u8; 32], hop_data: &[u8], hmac_byt } // Once we've emptied the set of bytes our peer gave us, encrypt 0 bytes until we // fill the onion hop data we'll forward to our next-hop peer. - chacha_stream.chacha.process_in_place(&mut new_packet_bytes[read_pos..]); - return Ok(Hop::Forward { - next_hop_data: msg, - next_hop_hmac: hmac, - new_packet_bytes, - }) + chacha_stream.chacha.process_in_place(&mut new_packet_bytes.as_mut()[read_pos..]); + return Ok((msg, Some((hmac, new_packet_bytes)))) // This packet needs forwarding } }, } diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index f4cb57f28df..3b18b1186fe 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -10,9 +10,12 @@ //! LDK sends, receives, and forwards onion messages via the [`OnionMessenger`]. See its docs for //! more information. +use bitcoin::hashes::{Hash, HashEngine}; +use bitcoin::hashes::hmac::{Hmac, HmacEngine}; +use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey}; -use chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Sign}; +use chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Recipient, Sign}; use ln::msgs; use ln::onion_utils; use super::blinded_route::{BlindedRoute, ForwardTlvs, ReceiveTlvs}; @@ -104,6 +107,97 @@ impl OnionMessenger ); Ok(()) } + + /// Handle an incoming onion message. Currently, if a message was destined for us we will log, but + /// soon we'll delegate the onion message to a handler that can generate invoices or send + /// payments. + pub fn handle_onion_message(&self, _peer_node_id: &PublicKey, msg: &msgs::OnionMessage) { + let control_tlvs_ss = match self.keys_manager.ecdh(Recipient::Node, &msg.blinding_point, None) { + Ok(ss) => ss, + Err(e) => { + log_error!(self.logger, "Failed to retrieve node secret: {:?}", e); + return + } + }; + let onion_decode_ss = { + let blinding_factor = { + let mut hmac = HmacEngine::::new(b"blinded_node_id"); + hmac.input(control_tlvs_ss.as_ref()); + Hmac::from_engine(hmac).into_inner() + }; + match self.keys_manager.ecdh(Recipient::Node, &msg.onion_routing_packet.public_key, + Some(&blinding_factor)) + { + Ok(ss) => ss.secret_bytes(), + Err(()) => { + log_trace!(self.logger, "Failed to compute onion packet shared secret"); + return + } + } + }; + match onion_utils::decode_next_hop(onion_decode_ss, &msg.onion_routing_packet.hop_data[..], + msg.onion_routing_packet.hmac, control_tlvs_ss) + { + Ok((Payload::Receive { + control_tlvs: ReceiveControlTlvs::Unblinded(ReceiveTlvs { path_id }) + }, None)) => { + log_info!(self.logger, "Received an onion message with path_id: {:02x?}", path_id); + }, + Ok((Payload::Forward(ForwardControlTlvs::Unblinded(ForwardTlvs { + next_node_id, next_blinding_override + })), Some((next_hop_hmac, new_packet_bytes)))) => { + // TODO: we need to check whether `next_node_id` is our node, in which case this is a dummy + // blinded hop and this onion message is destined for us. In this situation, we should keep + // unwrapping the onion layers to get to the final payload. Since we don't have the option + // of creating blinded routes with dummy hops currently, we should be ok to not handle this + // for now. + let new_pubkey = match onion_utils::next_hop_packet_pubkey(&self.secp_ctx, msg.onion_routing_packet.public_key, &onion_decode_ss) { + Ok(pk) => pk, + Err(e) => { + log_trace!(self.logger, "Failed to compute next hop packet pubkey: {}", e); + return + } + }; + let outgoing_packet = Packet { + version: 0, + public_key: new_pubkey, + hop_data: new_packet_bytes, + hmac: next_hop_hmac, + }; + + let mut pending_per_peer_msgs = self.pending_messages.lock().unwrap(); + let pending_msgs = pending_per_peer_msgs.entry(next_node_id).or_insert(Vec::new()); + pending_msgs.push( + msgs::OnionMessage { + blinding_point: match next_blinding_override { + Some(blinding_point) => blinding_point, + None => { + let blinding_factor = { + let mut sha = Sha256::engine(); + sha.input(&msg.blinding_point.serialize()[..]); + sha.input(control_tlvs_ss.as_ref()); + Sha256::from_engine(sha).into_inner() + }; + let mut next_blinding_point = msg.blinding_point; + if let Err(e) = next_blinding_point.mul_assign(&self.secp_ctx, &blinding_factor[..]) { + log_trace!(self.logger, "Failed to compute next blinding point: {}", e); + return + } + next_blinding_point + }, + }, + onion_routing_packet: outgoing_packet, + }, + ); + }, + Err(e) => { + log_trace!(self.logger, "Errored decoding onion message packet: {:?}", e); + }, + _ => { + log_trace!(self.logger, "Received bogus onion message packet, either the sender encoded a final hop as a forwarding hop or vice versa"); + }, + }; + } } // TODO: parameterize the below Simple* types with OnionMessenger and handle the messages it diff --git a/lightning/src/onion_message/packet.rs b/lightning/src/onion_message/packet.rs index ed0206a2adb..a3414d844ed 100644 --- a/lightning/src/onion_message/packet.rs +++ b/lightning/src/onion_message/packet.rs @@ -10,15 +10,16 @@ //! Structs and enums useful for constructing and reading an onion message packet. use bitcoin::secp256k1::PublicKey; +use bitcoin::secp256k1::ecdh::SharedSecret; use ln::msgs::DecodeError; use ln::onion_utils; use super::blinded_route::{ForwardTlvs, ReceiveTlvs}; -use util::chacha20poly1305rfc::ChaChaPolyWriteAdapter; -use util::ser::{LengthRead, LengthReadable, Readable, Writeable, Writer}; +use util::chacha20poly1305rfc::{ChaChaPolyReadAdapter, ChaChaPolyWriteAdapter}; +use util::ser::{FixedLengthReader, LengthRead, LengthReadable, LengthReadableArgs, Readable, ReadableArgs, Writeable, Writer}; use core::cmp; -use io; +use io::{self, Read}; use prelude::*; // Per the spec, an onion message packet's `hop_data` field length should be @@ -28,14 +29,14 @@ pub(super) const BIG_PACKET_HOP_DATA_LEN: usize = 32768; #[derive(Clone, Debug, PartialEq)] pub(crate) struct Packet { - version: u8, - public_key: PublicKey, + pub(super) version: u8, + pub(super) public_key: PublicKey, // Unlike the onion packets used for payments, onion message packets can have payloads greater // than 1300 bytes. // TODO: if 1300 ends up being the most common size, optimize this to be: // enum { ThirteenHundred([u8; 1300]), VarLen(Vec) } - hop_data: Vec, - hmac: [u8; 32], + pub(super) hop_data: Vec, + pub(super) hmac: [u8; 32], } impl onion_utils::Packet for Packet { @@ -156,3 +157,94 @@ impl Writeable for (Payload, [u8; 32]) { Ok(()) } } + +// Uses the provided secret to simultaneously decode and decrypt the control TLVs. +impl ReadableArgs for Payload { + fn read(mut r: &mut R, encrypted_tlvs_ss: SharedSecret) -> Result { + use bitcoin::consensus::encode::{Decodable, Error, VarInt}; + let v: VarInt = Decodable::consensus_decode(&mut r) + .map_err(|e| match e { + Error::Io(ioe) => DecodeError::from(ioe), + _ => DecodeError::InvalidValue + })?; + + let mut rd = FixedLengthReader::new(r, v.0); + // TODO: support reply paths + let mut _reply_path_bytes: Option> = Some(Vec::new()); + let mut read_adapter: Option> = None; + let rho = onion_utils::gen_rho_from_shared_secret(&encrypted_tlvs_ss.secret_bytes()); + decode_tlv_stream!(&mut rd, { + (2, _reply_path_bytes, vec_type), + (4, read_adapter, (option: LengthReadableArgs, rho)) + }); + rd.eat_remaining().map_err(|_| DecodeError::ShortRead)?; + + match read_adapter { + None => return Err(DecodeError::InvalidValue), + Some(ChaChaPolyReadAdapter { readable: ControlTlvs::Forward(tlvs)}) => { + Ok(Payload::Forward(ForwardControlTlvs::Unblinded(tlvs))) + }, + Some(ChaChaPolyReadAdapter { readable: ControlTlvs::Receive(tlvs)}) => { + Ok(Payload::Receive { control_tlvs: ReceiveControlTlvs::Unblinded(tlvs)}) + }, + } + } +} + +/// When reading a packet off the wire, we don't know a priori whether the packet is to be forwarded +/// or received. Thus we read a ControlTlvs rather than reading a ForwardControlTlvs or +/// ReceiveControlTlvs directly. +pub(super) enum ControlTlvs { + /// This onion message is intended to be forwarded. + Forward(ForwardTlvs), + /// This onion message is intended to be received. + Receive(ReceiveTlvs), +} + +impl Readable for ControlTlvs { + fn read(mut r: &mut R) -> Result { + let mut _padding: Option = None; + let mut _short_channel_id: Option = None; + let mut next_node_id: Option = None; + let mut path_id: Option<[u8; 32]> = None; + let mut next_blinding_override: Option = None; + decode_tlv_stream!(&mut r, { + (1, _padding, option), + (2, _short_channel_id, option), + (4, next_node_id, option), + (6, path_id, option), + (8, next_blinding_override, option), + }); + + let valid_fwd_fmt = next_node_id.is_some() && path_id.is_none(); + let valid_recv_fmt = next_node_id.is_none() && next_blinding_override.is_none(); + + let payload_fmt = if valid_fwd_fmt { + ControlTlvs::Forward(ForwardTlvs { + next_node_id: next_node_id.unwrap(), + next_blinding_override, + }) + } else if valid_recv_fmt { + ControlTlvs::Receive(ReceiveTlvs { + path_id, + }) + } else { + return Err(DecodeError::InvalidValue) + }; + + Ok(payload_fmt) + } +} + +/// Reads padding to the end, ignoring what's read. +pub(crate) struct Padding {} +impl Readable for Padding { + #[inline] + fn read(reader: &mut R) -> Result { + loop { + let mut buf = [0; 8192]; + if reader.read(&mut buf[..])? == 0 { break; } + } + Ok(Self {}) + } +} From b26fb851cd08882e0fec4c1f67b6ba946c3138ae Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 23 Jun 2022 15:45:33 -0400 Subject: [PATCH 31/91] Significantly expand onion message documentation --- lightning/src/onion_message/messenger.rs | 50 +++++++++++++++++++++++- lightning/src/onion_message/mod.rs | 11 ++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 3b18b1186fe..77542872022 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -28,9 +28,56 @@ use sync::{Arc, Mutex}; use prelude::*; /// A sender, receiver and forwarder of onion messages. In upcoming releases, this object will be -/// used to retrieve invoices and fulfill invoice requests from [offers]. +/// used to retrieve invoices and fulfill invoice requests from [offers]. Currently, only sending +/// and receiving empty onion messages is supported. +/// +/// # Example +/// +// Needs to be `ignore` until the `onion_message` module is made public, otherwise this is a test +// failure. +/// ```ignore +/// # extern crate bitcoin; +/// # use bitcoin::hashes::_export::_core::time::Duration; +/// # use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey}; +/// # use lightning::chain::keysinterface::{InMemorySigner, KeysManager, KeysInterface}; +/// # use lightning::onion_message::{BlindedRoute, Destination, OnionMessenger}; +/// # use lightning::util::logger::{Logger, Record}; +/// # use std::sync::Arc; +/// # struct FakeLogger {}; +/// # impl Logger for FakeLogger { +/// # fn log(&self, record: &Record) { unimplemented!() } +/// # } +/// # let seed = [42u8; 32]; +/// # let time = Duration::from_secs(123456); +/// # let keys_manager = KeysManager::new(&seed, time.as_secs(), time.subsec_nanos()); +/// # let logger = Arc::new(FakeLogger {}); +/// # let node_secret = SecretKey::from_slice(&hex::decode("0101010101010101010101010101010101010101010101010101010101010101").unwrap()[..]).unwrap(); +/// # let secp_ctx = Secp256k1::new(); +/// # let hop_node_id1 = PublicKey::from_secret_key(&secp_ctx, &node_secret); +/// # let (hop_node_id2, hop_node_id3, hop_node_id4) = (hop_node_id1, hop_node_id1, +/// hop_node_id1); +/// # let destination_node_id = hop_node_id1; +/// # +/// // Create the onion messenger. This must use the same `keys_manager` as is passed to your +/// // ChannelManager. +/// let onion_messenger = OnionMessenger::new(&keys_manager, logger); +/// +/// // Send an empty onion message to a node id. +/// let intermediate_hops = [hop_node_id1, hop_node_id2]; +/// onion_messenger.send_onion_message(&intermediate_hops, Destination::Node(destination_node_id)); +/// +/// // Create a blinded route to yourself, for someone to send an onion message to. +/// # let your_node_id = hop_node_id1; +/// let hops = [hop_node_id3, hop_node_id4, your_node_id]; +/// let blinded_route = BlindedRoute::new::(&hops, &keys_manager, &secp_ctx).unwrap(); +/// +/// // Send an empty onion message to a blinded route. +/// # let intermediate_hops = [hop_node_id1, hop_node_id2]; +/// onion_messenger.send_onion_message(&intermediate_hops, Destination::BlindedRoute(blinded_route)); +/// ``` /// /// [offers]: +/// [`OnionMessenger`]: crate::onion_message::OnionMessenger pub struct OnionMessenger where K::Target: KeysInterface, L::Target: Logger, @@ -79,6 +126,7 @@ impl OnionMessenger } /// Send an empty onion message to `destination`, routing it through `intermediate_nodes`. + /// See [`OnionMessenger`] for example usage. pub fn send_onion_message(&self, intermediate_nodes: &[PublicKey], destination: Destination) -> Result<(), secp256k1::Error> { let blinding_secret_bytes = self.keys_manager.get_secure_random_bytes(); let blinding_secret = SecretKey::from_slice(&blinding_secret_bytes[..]).expect("RNG is busted"); diff --git a/lightning/src/onion_message/mod.rs b/lightning/src/onion_message/mod.rs index 9236341fa61..a874fbc4335 100644 --- a/lightning/src/onion_message/mod.rs +++ b/lightning/src/onion_message/mod.rs @@ -8,6 +8,17 @@ // licenses. //! Onion Messages: sending, receiving, forwarding, and ancillary utilities live here +//! +//! Onion messages are multi-purpose messages sent between peers over the lightning network. In the +//! near future, they will be used to communicate invoices for [offers], unlocking use cases such as +//! static invoices, refunds and proof of payer. Further, you will be able to accept payments +//! without revealing your node id through the use of [blinded routes]. +//! +//! LDK sends and receives onion messages via the [`OnionMessenger`]. See its documentation for more +//! information on its usage. +//! +//! [offers]: +//! [blinded routes]: crate::onion_message::BlindedRoute mod blinded_route; mod messenger; From eaff561e244943f6383ec9fb26e3f71dc628a104 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Sat, 4 Jun 2022 13:22:20 -0700 Subject: [PATCH 32/91] Add test utilities and integration tests for onion messages --- .../src/onion_message/functional_tests.rs | 107 ++++++++++++++++++ lightning/src/onion_message/messenger.rs | 8 ++ lightning/src/onion_message/mod.rs | 2 + 3 files changed, 117 insertions(+) create mode 100644 lightning/src/onion_message/functional_tests.rs diff --git a/lightning/src/onion_message/functional_tests.rs b/lightning/src/onion_message/functional_tests.rs new file mode 100644 index 00000000000..510852cef83 --- /dev/null +++ b/lightning/src/onion_message/functional_tests.rs @@ -0,0 +1,107 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! Onion message testing and test utilities live here. + +use chain::keysinterface::{KeysInterface, Recipient}; +use super::{BlindedRoute, Destination, OnionMessenger}; +use util::enforcing_trait_impls::EnforcingSigner; +use util::test_utils; + +use bitcoin::network::constants::Network; +use bitcoin::secp256k1::{PublicKey, Secp256k1}; + +use sync::Arc; + +struct MessengerNode { + keys_manager: Arc, + messenger: OnionMessenger, Arc>, + logger: Arc, +} + +impl MessengerNode { + fn get_node_pk(&self) -> PublicKey { + let secp_ctx = Secp256k1::new(); + PublicKey::from_secret_key(&secp_ctx, &self.keys_manager.get_node_secret(Recipient::Node).unwrap()) + } +} + +fn create_nodes(num_messengers: u8) -> Vec { + let mut res = Vec::new(); + for i in 0..num_messengers { + let logger = Arc::new(test_utils::TestLogger::with_id(format!("node {}", i))); + let seed = [i as u8; 32]; + let keys_manager = Arc::new(test_utils::TestKeysInterface::new(&seed, Network::Testnet)); + res.push(MessengerNode { + keys_manager: keys_manager.clone(), + messenger: OnionMessenger::new(keys_manager, logger.clone()), + logger, + }); + } + res +} + +fn pass_along_path(mut path: Vec, expected_path_id: Option<[u8; 32]>) { + let mut prev_node = path.remove(0); + let num_nodes = path.len(); + for (idx, node) in path.into_iter().enumerate() { + let events = prev_node.messenger.release_pending_msgs(); + assert_eq!(events.len(), 1); + let onion_msg = { + let msgs = events.get(&node.get_node_pk()).unwrap(); + assert_eq!(msgs.len(), 1); + msgs[0].clone() + }; + node.messenger.handle_onion_message(&prev_node.get_node_pk(), &onion_msg); + if idx == num_nodes - 1 { + node.logger.assert_log_contains( + "lightning::onion_message::messenger".to_string(), + format!("Received an onion message with path_id: {:02x?}", expected_path_id).to_string(), 1); + } + prev_node = node; + } +} + +#[test] +fn one_hop() { + let nodes = create_nodes(2); + + nodes[0].messenger.send_onion_message(&[], Destination::Node(nodes[1].get_node_pk())).unwrap(); + pass_along_path(nodes, None); +} + +#[test] +fn two_unblinded_hops() { + let nodes = create_nodes(3); + + nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk()], Destination::Node(nodes[2].get_node_pk())).unwrap(); + pass_along_path(nodes, None); +} + +#[test] +fn two_unblinded_two_blinded() { + let nodes = create_nodes(5); + + let secp_ctx = Secp256k1::new(); + let blinded_route = BlindedRoute::new::(&[nodes[3].get_node_pk(), nodes[4].get_node_pk()], &*nodes[4].keys_manager, &secp_ctx).unwrap(); + + nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], Destination::BlindedRoute(blinded_route)).unwrap(); + pass_along_path(nodes, None); +} + +#[test] +fn three_blinded_hops() { + let nodes = create_nodes(4); + + let secp_ctx = Secp256k1::new(); + let blinded_route = BlindedRoute::new::(&[nodes[1].get_node_pk(), nodes[2].get_node_pk(), nodes[3].get_node_pk()], &*nodes[3].keys_manager, &secp_ctx).unwrap(); + + nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap(); + pass_along_path(nodes, None); +} diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 77542872022..930d90ebac3 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -246,6 +246,14 @@ impl OnionMessenger }, }; } + + #[cfg(test)] + pub(super) fn release_pending_msgs(&self) -> HashMap> { + let mut pending_msgs = self.pending_messages.lock().unwrap(); + let mut msgs = HashMap::new(); + core::mem::swap(&mut *pending_msgs, &mut msgs); + msgs + } } // TODO: parameterize the below Simple* types with OnionMessenger and handle the messages it diff --git a/lightning/src/onion_message/mod.rs b/lightning/src/onion_message/mod.rs index a874fbc4335..966b10b60fb 100644 --- a/lightning/src/onion_message/mod.rs +++ b/lightning/src/onion_message/mod.rs @@ -24,6 +24,8 @@ mod blinded_route; mod messenger; mod packet; mod utils; +#[cfg(test)] +mod functional_tests; // Re-export structs so they can be imported with just the `onion_message::` module prefix. pub use self::blinded_route::{BlindedRoute, BlindedHop}; From 6500c99f29d626c78c7848586a0018453f64fe45 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 23 Jun 2022 15:47:25 -0400 Subject: [PATCH 33/91] Add SendError enum for onion messages and error on too-big packets --- .../src/onion_message/functional_tests.rs | 18 ++++++++++-- lightning/src/onion_message/messenger.rs | 28 +++++++++++++++---- lightning/src/onion_message/mod.rs | 2 +- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/lightning/src/onion_message/functional_tests.rs b/lightning/src/onion_message/functional_tests.rs index 510852cef83..560028c5bba 100644 --- a/lightning/src/onion_message/functional_tests.rs +++ b/lightning/src/onion_message/functional_tests.rs @@ -10,12 +10,12 @@ //! Onion message testing and test utilities live here. use chain::keysinterface::{KeysInterface, Recipient}; -use super::{BlindedRoute, Destination, OnionMessenger}; +use super::{BlindedRoute, Destination, OnionMessenger, SendError}; use util::enforcing_trait_impls::EnforcingSigner; use util::test_utils; use bitcoin::network::constants::Network; -use bitcoin::secp256k1::{PublicKey, Secp256k1}; +use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey}; use sync::Arc; @@ -105,3 +105,17 @@ fn three_blinded_hops() { nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap(); pass_along_path(nodes, None); } + +#[test] +fn too_big_packet_error() { + // Make sure we error as expected if a packet is too big to send. + let nodes = create_nodes(1); + + let hop_secret = SecretKey::from_slice(&hex::decode("0101010101010101010101010101010101010101010101010101010101010101").unwrap()[..]).unwrap(); + let secp_ctx = Secp256k1::new(); + let hop_node_id = PublicKey::from_secret_key(&secp_ctx, &hop_secret); + + let hops = [hop_node_id; 400]; + let err = nodes[0].messenger.send_onion_message(&hops, Destination::Node(hop_node_id)).unwrap_err(); + assert_eq!(err, SendError::TooBigPacket); +} diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 930d90ebac3..6f5d9e8a52c 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -108,6 +108,18 @@ impl Destination { } } +/// Errors that may occur when [sending an onion message]. +/// +/// [sending an onion message]: OnionMessenger::send_onion_message +#[derive(Debug, PartialEq)] +pub enum SendError { + /// Errored computing onion message packet keys. + Secp256k1(secp256k1::Error), + /// Because implementations such as Eclair will drop onion messages where the message packet + /// exceeds 32834 bytes, we refuse to send messages where the packet exceeds this size. + TooBigPacket, +} + impl OnionMessenger where K::Target: KeysInterface, L::Target: Logger, @@ -127,7 +139,7 @@ impl OnionMessenger /// Send an empty onion message to `destination`, routing it through `intermediate_nodes`. /// See [`OnionMessenger`] for example usage. - pub fn send_onion_message(&self, intermediate_nodes: &[PublicKey], destination: Destination) -> Result<(), secp256k1::Error> { + pub fn send_onion_message(&self, intermediate_nodes: &[PublicKey], destination: Destination) -> Result<(), SendError> { let blinding_secret_bytes = self.keys_manager.get_secure_random_bytes(); let blinding_secret = SecretKey::from_slice(&blinding_secret_bytes[..]).expect("RNG is busted"); let (introduction_node_id, blinding_point) = if intermediate_nodes.len() != 0 { @@ -140,10 +152,12 @@ impl OnionMessenger } }; let (packet_payloads, packet_keys) = packet_payloads_and_keys( - &self.secp_ctx, intermediate_nodes, destination, &blinding_secret)?; + &self.secp_ctx, intermediate_nodes, destination, &blinding_secret) + .map_err(|e| SendError::Secp256k1(e))?; let prng_seed = self.keys_manager.get_secure_random_bytes(); - let onion_packet = construct_onion_message_packet(packet_payloads, packet_keys, prng_seed); + let onion_packet = construct_onion_message_packet( + packet_payloads, packet_keys, prng_seed).map_err(|()| SendError::TooBigPacket)?; let mut pending_per_peer_msgs = self.pending_messages.lock().unwrap(); let pending_msgs = pending_per_peer_msgs.entry(introduction_node_id).or_insert(Vec::new()); @@ -343,7 +357,8 @@ fn packet_payloads_and_keys( Ok((payloads, onion_packet_keys)) } -fn construct_onion_message_packet(payloads: Vec<(Payload, [u8; 32])>, onion_keys: Vec, prng_seed: [u8; 32]) -> Packet { +/// Errors if the serialized payload size exceeds onion_message::BIG_PACKET_HOP_DATA_LEN +fn construct_onion_message_packet(payloads: Vec<(Payload, [u8; 32])>, onion_keys: Vec, prng_seed: [u8; 32]) -> Result { // Spec rationale: // "`len` allows larger messages to be sent than the standard 1300 bytes allowed for an HTLC // onion, but this should be used sparingly as it is reduces anonymity set, hence the @@ -353,7 +368,8 @@ fn construct_onion_message_packet(payloads: Vec<(Payload, [u8; 32])>, onion_keys SMALL_PACKET_HOP_DATA_LEN } else if payloads_ser_len <= BIG_PACKET_HOP_DATA_LEN { BIG_PACKET_HOP_DATA_LEN - } else { payloads_ser_len }; + } else { return Err(()) }; - onion_utils::construct_onion_message_packet::<_, _>(payloads, onion_keys, prng_seed, hop_data_len) + Ok(onion_utils::construct_onion_message_packet::<_, _>( + payloads, onion_keys, prng_seed, hop_data_len)) } diff --git a/lightning/src/onion_message/mod.rs b/lightning/src/onion_message/mod.rs index 966b10b60fb..2e23b76ada3 100644 --- a/lightning/src/onion_message/mod.rs +++ b/lightning/src/onion_message/mod.rs @@ -29,5 +29,5 @@ mod functional_tests; // Re-export structs so they can be imported with just the `onion_message::` module prefix. pub use self::blinded_route::{BlindedRoute, BlindedHop}; -pub use self::messenger::{Destination, OnionMessenger, SimpleArcOnionMessenger, SimpleRefOnionMessenger}; +pub use self::messenger::{Destination, OnionMessenger, SendError, SimpleArcOnionMessenger, SimpleRefOnionMessenger}; pub(crate) use self::packet::Packet; From 39397d4e14ed87e62cf8f313de749b7b5785561e Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 23 Jun 2022 15:51:43 -0400 Subject: [PATCH 34/91] Error when attempting to send an OM to a blinded route with 0 hops --- .../src/onion_message/functional_tests.rs | 21 +++++++++++++++++++ lightning/src/onion_message/messenger.rs | 8 +++++++ 2 files changed, 29 insertions(+) diff --git a/lightning/src/onion_message/functional_tests.rs b/lightning/src/onion_message/functional_tests.rs index 560028c5bba..695064e467c 100644 --- a/lightning/src/onion_message/functional_tests.rs +++ b/lightning/src/onion_message/functional_tests.rs @@ -119,3 +119,24 @@ fn too_big_packet_error() { let err = nodes[0].messenger.send_onion_message(&hops, Destination::Node(hop_node_id)).unwrap_err(); assert_eq!(err, SendError::TooBigPacket); } + +#[test] +fn invalid_blinded_route_error() { + // Make sure we error as expected if a provided blinded route has 0 or 1 hops. + let mut nodes = create_nodes(3); + let (node1, node2, node3) = (nodes.remove(0), nodes.remove(0), nodes.remove(0)); + + // 0 hops + let secp_ctx = Secp256k1::new(); + let mut blinded_route = BlindedRoute::new::(&[node2.get_node_pk(), node3.get_node_pk()], &*node3.keys_manager, &secp_ctx).unwrap(); + blinded_route.blinded_hops.clear(); + let err = node1.messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap_err(); + assert_eq!(err, SendError::TooFewBlindedHops); + + // 1 hop + let mut blinded_route = BlindedRoute::new::(&[node2.get_node_pk(), node3.get_node_pk()], &*node3.keys_manager, &secp_ctx).unwrap(); + blinded_route.blinded_hops.remove(0); + assert_eq!(blinded_route.blinded_hops.len(), 1); + let err = node1.messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap_err(); + assert_eq!(err, SendError::TooFewBlindedHops); +} diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 6f5d9e8a52c..7eba3cdd254 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -118,6 +118,9 @@ pub enum SendError { /// Because implementations such as Eclair will drop onion messages where the message packet /// exceeds 32834 bytes, we refuse to send messages where the packet exceeds this size. TooBigPacket, + /// The provided [`Destination`] was an invalid [`BlindedRoute`], due to having fewer than two + /// blinded hops. + TooFewBlindedHops, } impl OnionMessenger @@ -140,6 +143,11 @@ impl OnionMessenger /// Send an empty onion message to `destination`, routing it through `intermediate_nodes`. /// See [`OnionMessenger`] for example usage. pub fn send_onion_message(&self, intermediate_nodes: &[PublicKey], destination: Destination) -> Result<(), SendError> { + if let Destination::BlindedRoute(BlindedRoute { ref blinded_hops, .. }) = destination { + if blinded_hops.len() < 2 { + return Err(SendError::TooFewBlindedHops); + } + } let blinding_secret_bytes = self.keys_manager.get_secure_random_bytes(); let blinding_secret = SecretKey::from_slice(&blinding_secret_bytes[..]).expect("RNG is busted"); let (introduction_node_id, blinding_point) = if intermediate_nodes.len() != 0 { From 17ec697f8f09d608f6a3d2703ed4f4409773b4bf Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Tue, 12 Jul 2022 16:04:42 -0400 Subject: [PATCH 35/91] Fix possible incomplete read bug on onion packet decode Pre-existing to this PR, we were reading next packet bytes with io::Read::read, which is not guaranteed to read all the bytes we need, only guaranteed to read *some* bytes. We fix this to be read_exact, which is guaranteed to read all the next hop packet bytes. --- lightning/src/ln/onion_utils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lightning/src/ln/onion_utils.rs b/lightning/src/ln/onion_utils.rs index 145eb8acbae..f81c619d735 100644 --- a/lightning/src/ln/onion_utils.rs +++ b/lightning/src/ln/onion_utils.rs @@ -725,7 +725,8 @@ pub(crate) fn decode_next_hop, N: NextPa return Ok((msg, None)); // We are the final destination for this packet } else { let mut new_packet_bytes = N::new(hop_data.len()); - let read_pos = chacha_stream.read(new_packet_bytes.as_mut()).unwrap(); + let read_pos = hop_data.len() - chacha_stream.read.position() as usize; + chacha_stream.read_exact(&mut new_packet_bytes.as_mut()[..read_pos]).unwrap(); #[cfg(debug_assertions)] { // Check two things: From fc771d3b2042cfab0f60b2344089e789b461b6e7 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Fri, 5 Aug 2022 03:20:53 +0000 Subject: [PATCH 36/91] [C Bindings] Expose channel and nodes list in `ReadOnlyNetworkGraph` --- lightning/src/routing/gossip.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 37bf1ea5a6f..1757d137300 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -1796,6 +1796,12 @@ impl ReadOnlyNetworkGraph<'_> { self.channels.get(&short_channel_id) } + #[cfg(c_bindings)] // Non-bindings users should use `channels` + /// Returns the list of channels in the graph + pub fn list_channels(&self) -> Vec { + self.channels.keys().map(|c| *c).collect() + } + /// Returns all known nodes' public keys along with announced node info. /// /// (C-not exported) because we have no mapping for `BTreeMap`s @@ -1808,6 +1814,12 @@ impl ReadOnlyNetworkGraph<'_> { self.nodes.get(node_id) } + #[cfg(c_bindings)] // Non-bindings users should use `nodes` + /// Returns the list of nodes in the graph + pub fn list_nodes(&self) -> Vec { + self.nodes.keys().map(|n| *n).collect() + } + /// Get network addresses by node id. /// Returns None if the requested node is completely unknown, /// or if node announcement for the node was never received. From dcef41d17b822c7d6da36239101cef96df011f61 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 5 Aug 2022 22:18:27 -0400 Subject: [PATCH 37/91] Minor msgs::OnionHopData cleanups --- lightning/src/ln/msgs.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 05a336c18ef..9886bfeea5e 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -1417,14 +1417,6 @@ impl Writeable for OnionHopData { } } -// ReadableArgs because we need onion_utils::decode_next_hop to accommodate payment packets and -// onion message packets. -impl ReadableArgs<()> for OnionHopData { - fn read(r: &mut R, _arg: ()) -> Result { - ::read(r) - } -} - impl Readable for OnionHopData { fn read(mut r: &mut R) -> Result { use bitcoin::consensus::encode::{Decodable, Error, VarInt}; @@ -1441,12 +1433,12 @@ impl Readable for OnionHopData { let mut short_id: Option = None; let mut payment_data: Option = None; let mut keysend_preimage: Option = None; - // The TLV type is chosen to be compatible with lnd and c-lightning. decode_tlv_stream!(&mut rd, { (2, amt, required), (4, cltv_value, required), (6, short_id, option), (8, payment_data, option), + // See https://github.com/lightning/blips/blob/master/blip-0003.md (5482373484, keysend_preimage, option) }); rd.eat_remaining().map_err(|_| DecodeError::ShortRead)?; @@ -1488,6 +1480,14 @@ impl Readable for OnionHopData { } } +// ReadableArgs because we need onion_utils::decode_next_hop to accommodate payment packets and +// onion message packets. +impl ReadableArgs<()> for OnionHopData { + fn read(r: &mut R, _arg: ()) -> Result { + ::read(r) + } +} + impl Writeable for Ping { fn write(&self, w: &mut W) -> Result<(), io::Error> { self.ponglen.write(w)?; From c242003dd3c430a8395af1e48bd1e5c1f5ed94ae Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Sun, 7 Aug 2022 13:49:10 -0400 Subject: [PATCH 38/91] Fix CI to error on doc links to private items Somehow we weren't doing this. --- lightning-background-processor/src/lib.rs | 3 +++ lightning-block-sync/src/lib.rs | 3 +++ lightning-invoice/src/lib.rs | 5 ++++- lightning-net-tokio/src/lib.rs | 4 +++- lightning-persister/src/lib.rs | 3 +++ lightning-rapid-gossip-sync/src/lib.rs | 5 ++++- lightning/src/lib.rs | 3 +++ 7 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lightning-background-processor/src/lib.rs b/lightning-background-processor/src/lib.rs index 484439b3907..3088954c8f9 100644 --- a/lightning-background-processor/src/lib.rs +++ b/lightning-background-processor/src/lib.rs @@ -2,7 +2,10 @@ //! running properly, and (2) either can or should be run in the background. See docs for //! [`BackgroundProcessor`] for more details on the nitty-gritty. +// Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. #![deny(broken_intra_doc_links)] +#![deny(private_intra_doc_links)] + #![deny(missing_docs)] #![deny(unsafe_code)] diff --git a/lightning-block-sync/src/lib.rs b/lightning-block-sync/src/lib.rs index 321dd57e471..823cb5eb554 100644 --- a/lightning-block-sync/src/lib.rs +++ b/lightning-block-sync/src/lib.rs @@ -13,7 +13,10 @@ //! Both features support either blocking I/O using `std::net::TcpStream` or, with feature `tokio`, //! non-blocking I/O using `tokio::net::TcpStream` from inside a Tokio runtime. +// Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. #![deny(broken_intra_doc_links)] +#![deny(private_intra_doc_links)] + #![deny(missing_docs)] #![deny(unsafe_code)] diff --git a/lightning-invoice/src/lib.rs b/lightning-invoice/src/lib.rs index bad024c66c5..c7d7a4042f7 100644 --- a/lightning-invoice/src/lib.rs +++ b/lightning-invoice/src/lib.rs @@ -1,9 +1,12 @@ +// Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. +#![deny(broken_intra_doc_links)] +#![deny(private_intra_doc_links)] + #![deny(missing_docs)] #![deny(non_upper_case_globals)] #![deny(non_camel_case_types)] #![deny(non_snake_case)] #![deny(unused_mut)] -#![deny(broken_intra_doc_links)] #![cfg_attr(docsrs, feature(doc_auto_cfg))] diff --git a/lightning-net-tokio/src/lib.rs b/lightning-net-tokio/src/lib.rs index 645a7434e45..f980addefcf 100644 --- a/lightning-net-tokio/src/lib.rs +++ b/lightning-net-tokio/src/lib.rs @@ -66,9 +66,11 @@ //! } //! ``` +// Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. #![deny(broken_intra_doc_links)] -#![deny(missing_docs)] +#![deny(private_intra_doc_links)] +#![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_auto_cfg))] use bitcoin::secp256k1::PublicKey; diff --git a/lightning-persister/src/lib.rs b/lightning-persister/src/lib.rs index 3e32791711e..a63f11ca1b1 100644 --- a/lightning-persister/src/lib.rs +++ b/lightning-persister/src/lib.rs @@ -1,6 +1,9 @@ //! Utilities that handle persisting Rust-Lightning data to disk via standard filesystem APIs. +// Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. #![deny(broken_intra_doc_links)] +#![deny(private_intra_doc_links)] + #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_auto_cfg))] diff --git a/lightning-rapid-gossip-sync/src/lib.rs b/lightning-rapid-gossip-sync/src/lib.rs index 6e9280f86a3..70758e1fe07 100644 --- a/lightning-rapid-gossip-sync/src/lib.rs +++ b/lightning-rapid-gossip-sync/src/lib.rs @@ -1,6 +1,9 @@ +// Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. +#![deny(broken_intra_doc_links)] +#![deny(private_intra_doc_links)] + #![deny(missing_docs)] #![deny(unsafe_code)] -#![deny(broken_intra_doc_links)] #![deny(non_upper_case_globals)] #![deny(non_camel_case_types)] #![deny(non_snake_case)] diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index 2e6b3ab3c0a..45072710735 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -39,7 +39,10 @@ #![cfg_attr(not(any(test, fuzzing, feature = "_test_utils")), deny(missing_docs))] #![cfg_attr(not(any(test, fuzzing, feature = "_test_utils")), forbid(unsafe_code))] + +// Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. #![deny(broken_intra_doc_links)] +#![deny(private_intra_doc_links)] // In general, rust is absolutely horrid at supporting users doing things like, // for example, compiling Rust code for real environments. Disable useless lints From 353b1e7e4661acbba74a32631352daaace5358d3 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sun, 7 Aug 2022 19:02:33 +0000 Subject: [PATCH 39/91] Update libfuzzer-sys to new upstream inclusion method Dunno why they changed it, but the old "depend directly on git" thing that cargo-fuzz used forever is now deprecated that that repo is archived, they've now moved to another repo and publish properly on crates.io. --- fuzz/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 66dabcfe4be..5569e96cab0 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -23,7 +23,7 @@ lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync" } bitcoin = { version = "0.28.1", features = ["secp-lowmemory"] } hex = "0.3" honggfuzz = { version = "0.5", optional = true } -libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer-sys.git", optional = true } +libfuzzer-sys = { version = "0.4", optional = true } [build-dependencies] cc = "1.0" From 4a1ee5f9a984c9b0c0892025d624ade734337b1a Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Tue, 9 Aug 2022 21:20:56 +0000 Subject: [PATCH 40/91] Use util methods in `Peer` to decide when to forward This consolidates our various checks on peer buffer space into the `Peer` impl itself, making the thresholds at which we stop taking various actions on a peer more readable as a whole. This commit was primarily authored by `Valentine Wallace ` with some amendments by `Matt Corallo `. --- lightning/src/ln/peer_handler.rs | 52 +++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index d34fdfeb52a..6c9a608b624 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -378,6 +378,31 @@ impl Peer { InitSyncTracker::NodesSyncing(pk) => pk < node_id, } } + + /// Returns the number of gossip messages we can fit in this peer's buffer. + fn gossip_buffer_slots_available(&self) -> usize { + OUTBOUND_BUFFER_LIMIT_READ_PAUSE.saturating_sub(self.pending_outbound_buffer.len()) + } + + /// Returns whether we should be reading bytes from this peer, based on whether its outbound + /// buffer still has space and we don't need to pause reads to get some writes out. + fn should_read(&self) -> bool { + self.pending_outbound_buffer.len() < OUTBOUND_BUFFER_LIMIT_READ_PAUSE + } + + fn should_backfill_gossip(&self) -> bool { + self.pending_outbound_buffer.len() < OUTBOUND_BUFFER_LIMIT_READ_PAUSE && + self.msgs_sent_since_pong < BUFFER_DRAIN_MSGS_PER_TICK + } + + /// Returns whether this peer's buffer is full and we should drop gossip messages. + fn buffer_full_drop_gossip(&self) -> bool { + if self.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP + || self.msgs_sent_since_pong > BUFFER_DRAIN_MSGS_PER_TICK * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO { + return false + } + true + } } /// SimpleArcPeerManager is useful when you need a PeerManager with a static lifetime, e.g. @@ -710,11 +735,11 @@ impl P fn do_attempt_write_data(&self, descriptor: &mut Descriptor, peer: &mut Peer) { while !peer.awaiting_write_event { - if peer.pending_outbound_buffer.len() < OUTBOUND_BUFFER_LIMIT_READ_PAUSE && peer.msgs_sent_since_pong < BUFFER_DRAIN_MSGS_PER_TICK { + if peer.should_backfill_gossip() { match peer.sync_status { InitSyncTracker::NoSyncRequested => {}, InitSyncTracker::ChannelsSyncing(c) if c < 0xffff_ffff_ffff_ffff => { - let steps = ((OUTBOUND_BUFFER_LIMIT_READ_PAUSE - peer.pending_outbound_buffer.len() + 2) / 3) as u8; + let steps = ((peer.gossip_buffer_slots_available() + 2) / 3) as u8; let all_messages = self.message_handler.route_handler.get_next_channel_announcements(c, steps); for &(ref announce, ref update_a_option, ref update_b_option) in all_messages.iter() { self.enqueue_message(peer, announce); @@ -731,7 +756,7 @@ impl P } }, InitSyncTracker::ChannelsSyncing(c) if c == 0xffff_ffff_ffff_ffff => { - let steps = (OUTBOUND_BUFFER_LIMIT_READ_PAUSE - peer.pending_outbound_buffer.len()) as u8; + let steps = peer.gossip_buffer_slots_available() as u8; let all_messages = self.message_handler.route_handler.get_next_node_announcements(None, steps); for msg in all_messages.iter() { self.enqueue_message(peer, msg); @@ -743,7 +768,7 @@ impl P }, InitSyncTracker::ChannelsSyncing(_) => unreachable!(), InitSyncTracker::NodesSyncing(key) => { - let steps = (OUTBOUND_BUFFER_LIMIT_READ_PAUSE - peer.pending_outbound_buffer.len()) as u8; + let steps = peer.gossip_buffer_slots_available() as u8; let all_messages = self.message_handler.route_handler.get_next_node_announcements(Some(&key), steps); for msg in all_messages.iter() { self.enqueue_message(peer, msg); @@ -765,11 +790,10 @@ impl P Some(buff) => buff, }; - let should_be_reading = peer.pending_outbound_buffer.len() < OUTBOUND_BUFFER_LIMIT_READ_PAUSE; let pending = &next_buff[peer.pending_outbound_buffer_first_msg_offset..]; - let data_sent = descriptor.send_data(pending, should_be_reading); + let data_sent = descriptor.send_data(pending, peer.should_read()); peer.pending_outbound_buffer_first_msg_offset += data_sent; - if peer.pending_outbound_buffer_first_msg_offset == next_buff.len() { true } else { false } + peer.pending_outbound_buffer_first_msg_offset == next_buff.len() } { peer.pending_outbound_buffer_first_msg_offset = 0; peer.pending_outbound_buffer.pop_front(); @@ -1045,7 +1069,7 @@ impl P } } } - pause_read = peer.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_READ_PAUSE; + pause_read = !peer.should_read(); if let Some(message) = msg_to_handle { match self.handle_message(&peer_mutex, peer_lock, message) { @@ -1308,9 +1332,7 @@ impl P !peer.should_forward_channel_announcement(msg.contents.short_channel_id) { continue } - if peer.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP - || peer.msgs_sent_since_pong > BUFFER_DRAIN_MSGS_PER_TICK * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO - { + if peer.buffer_full_drop_gossip() { log_gossip!(self.logger, "Skipping broadcast message to {:?} as its outbound buffer is full", peer.their_node_id); continue; } @@ -1334,9 +1356,7 @@ impl P !peer.should_forward_node_announcement(msg.contents.node_id) { continue } - if peer.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP - || peer.msgs_sent_since_pong > BUFFER_DRAIN_MSGS_PER_TICK * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO - { + if peer.buffer_full_drop_gossip() { log_gossip!(self.logger, "Skipping broadcast message to {:?} as its outbound buffer is full", peer.their_node_id); continue; } @@ -1359,9 +1379,7 @@ impl P !peer.should_forward_channel_announcement(msg.contents.short_channel_id) { continue } - if peer.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP - || peer.msgs_sent_since_pong > BUFFER_DRAIN_MSGS_PER_TICK * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO - { + if peer.buffer_full_drop_gossip() { log_gossip!(self.logger, "Skipping broadcast message to {:?} as its outbound buffer is full", peer.their_node_id); continue; } From 4f6da92c4e1f42259fbc01aee1812a2fdd9d8638 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 9 Aug 2022 21:45:19 +0000 Subject: [PATCH 41/91] Clarify comment on BUFFER_DRAIN_MSGS_PER_TICK. --- lightning/src/ln/peer_handler.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 6c9a608b624..165d607f199 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -323,6 +323,10 @@ const MAX_BUFFER_DRAIN_TICK_INTERVALS_PER_PEER: i8 = 4; /// tick. Once we have sent this many messages since the last ping, we send a ping right away to /// ensures we don't just fill up our send buffer and leave the peer with too many messages to /// process before the next ping. +/// +/// Note that we continue responding to other messages even after we've sent this many messages, so +/// it's more of a general guideline used for gossip backfill (and gossip forwarding, times +/// [`FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO`]) than a hard limit. const BUFFER_DRAIN_MSGS_PER_TICK: usize = 32; struct Peer { From 7e05623befc111422d1d4230f4e51629566a2ae6 Mon Sep 17 00:00:00 2001 From: Devrandom Date: Tue, 9 Aug 2022 17:39:51 +0200 Subject: [PATCH 42/91] Update bitcoin crate to 0.29.0 --- fuzz/Cargo.toml | 2 +- fuzz/src/chanmon_consistency.rs | 12 +- fuzz/src/full_stack.rs | 14 ++- fuzz/src/process_network_graph.rs | 3 +- lightning-background-processor/Cargo.toml | 2 +- lightning-background-processor/src/lib.rs | 7 +- lightning-block-sync/Cargo.toml | 2 +- lightning-block-sync/src/convert.rs | 3 +- lightning-block-sync/src/test_utils.rs | 4 +- lightning-invoice/Cargo.toml | 4 +- lightning-invoice/fuzz/Cargo.toml | 2 +- lightning-net-tokio/Cargo.toml | 2 +- lightning-persister/Cargo.toml | 2 +- lightning-persister/src/lib.rs | 5 +- lightning-rapid-gossip-sync/Cargo.toml | 2 +- lightning/Cargo.toml | 4 +- lightning/src/chain/chainmonitor.rs | 7 +- lightning/src/chain/channelmonitor.rs | 38 +++---- lightning/src/chain/keysinterface.rs | 18 +-- lightning/src/chain/onchaintx.rs | 5 +- lightning/src/chain/package.rs | 8 +- lightning/src/ln/chan_utils.rs | 44 ++++--- lightning/src/ln/chanmon_update_fail_tests.rs | 11 +- lightning/src/ln/channel.rs | 3 +- lightning/src/ln/channelmanager.rs | 10 +- lightning/src/ln/functional_test_utils.rs | 19 ++-- lightning/src/ln/functional_tests.rs | 107 +++++++++--------- lightning/src/ln/monitor_tests.rs | 2 +- lightning/src/ln/onion_utils.rs | 8 +- lightning/src/ln/payment_tests.rs | 5 +- lightning/src/ln/reorg_tests.rs | 8 +- lightning/src/ln/script.rs | 2 +- lightning/src/onion_message/messenger.rs | 12 +- lightning/src/onion_message/utils.rs | 10 +- lightning/src/util/events.rs | 4 +- lightning/src/util/macro_logger.rs | 2 +- lightning/src/util/ser.rs | 2 +- lightning/src/util/test_utils.rs | 8 +- lightning/src/util/transaction_utils.rs | 17 ++- 39 files changed, 221 insertions(+), 199 deletions(-) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 5569e96cab0..be37fb83cdd 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -20,7 +20,7 @@ stdin_fuzz = [] afl = { version = "0.4", optional = true } lightning = { path = "../lightning", features = ["regex"] } lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync" } -bitcoin = { version = "0.28.1", features = ["secp-lowmemory"] } +bitcoin = { version = "0.29.0", features = ["secp-lowmemory"] } hex = "0.3" honggfuzz = { version = "0.5", optional = true } libfuzzer-sys = { version = "0.4", optional = true } diff --git a/fuzz/src/chanmon_consistency.rs b/fuzz/src/chanmon_consistency.rs index c11e1baf617..9b3f1cd365b 100644 --- a/fuzz/src/chanmon_consistency.rs +++ b/fuzz/src/chanmon_consistency.rs @@ -18,11 +18,13 @@ //! send-side handling is correct, other peers. We consider it a failure if any action results in a //! channel being force-closed. +use bitcoin::TxMerkleNode; use bitcoin::blockdata::block::BlockHeader; use bitcoin::blockdata::constants::genesis_block; use bitcoin::blockdata::transaction::{Transaction, TxOut}; use bitcoin::blockdata::script::{Builder, Script}; use bitcoin::blockdata::opcodes; +use bitcoin::blockdata::locktime::PackedLockTime; use bitcoin::network::constants::Network; use bitcoin::hashes::Hash as TraitImport; @@ -53,7 +55,7 @@ use lightning::routing::router::{Route, RouteHop}; use utils::test_logger::{self, Output}; use utils::test_persister::TestPersister; -use bitcoin::secp256k1::{PublicKey,SecretKey}; +use bitcoin::secp256k1::{PublicKey, SecretKey, Scalar}; use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; use bitcoin::secp256k1::Secp256k1; @@ -169,7 +171,7 @@ impl KeysInterface for KeyProvider { fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { let mut node_secret = self.get_node_secret(recipient)?; if let Some(tweak) = tweak { - node_secret.mul_assign(tweak).map_err(|_| ())?; + node_secret = node_secret.mul_tweak(&Scalar::from_be_bytes(*tweak).unwrap()).unwrap(); } Ok(SharedSecret::new(other_key, &node_secret)) } @@ -447,7 +449,7 @@ pub fn do_test(data: &[u8], underlying_out: Out) { let events = $source.get_and_clear_pending_events(); assert_eq!(events.len(), 1); if let events::Event::FundingGenerationReady { ref temporary_channel_id, ref channel_value_satoshis, ref output_script, .. } = events[0] { - let tx = Transaction { version: $chan_id, lock_time: 0, input: Vec::new(), output: vec![TxOut { + let tx = Transaction { version: $chan_id, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: vec![TxOut { value: *channel_value_satoshis, script_pubkey: output_script.clone(), }]}; funding_output = OutPoint { txid: tx.txid(), index: 0 }; @@ -481,11 +483,11 @@ pub fn do_test(data: &[u8], underlying_out: Out) { macro_rules! confirm_txn { ($node: expr) => { { let chain_hash = genesis_block(Network::Bitcoin).block_hash(); - let mut header = BlockHeader { version: 0x20000000, prev_blockhash: chain_hash, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let mut header = BlockHeader { version: 0x20000000, prev_blockhash: chain_hash, merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; let txdata: Vec<_> = channel_txn.iter().enumerate().map(|(i, tx)| (i + 1, tx)).collect(); $node.transactions_confirmed(&header, &txdata, 1); for _ in 2..100 { - header = BlockHeader { version: 0x20000000, prev_blockhash: header.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + header = BlockHeader { version: 0x20000000, prev_blockhash: header.block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; } $node.best_block_updated(&header, 99); } } diff --git a/fuzz/src/full_stack.rs b/fuzz/src/full_stack.rs index f0f8054624d..b65d2a34a95 100644 --- a/fuzz/src/full_stack.rs +++ b/fuzz/src/full_stack.rs @@ -13,13 +13,15 @@ //! or payments to send/ways to handle events generated. //! This test has been very useful, though due to its complexity good starting inputs are critical. +use bitcoin::TxMerkleNode; use bitcoin::blockdata::block::BlockHeader; +use bitcoin::blockdata::constants::genesis_block; use bitcoin::blockdata::transaction::{Transaction, TxOut}; use bitcoin::blockdata::script::{Builder, Script}; use bitcoin::blockdata::opcodes; +use bitcoin::blockdata::locktime::PackedLockTime; use bitcoin::consensus::encode::deserialize; use bitcoin::network::constants::Network; -use bitcoin::blockdata::constants::genesis_block; use bitcoin::hashes::Hash as TraitImport; use bitcoin::hashes::HashEngine as TraitImportEngine; @@ -50,7 +52,7 @@ use lightning::util::ser::ReadableArgs; use utils::test_logger; use utils::test_persister::TestPersister; -use bitcoin::secp256k1::{PublicKey,SecretKey}; +use bitcoin::secp256k1::{PublicKey, SecretKey, Scalar}; use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; use bitcoin::secp256k1::Secp256k1; @@ -213,7 +215,7 @@ impl<'a> MoneyLossDetector<'a> { } self.blocks_connected += 1; - let header = BlockHeader { version: 0x20000000, prev_blockhash: self.header_hashes[self.height].0, merkle_root: Default::default(), time: self.blocks_connected, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: self.header_hashes[self.height].0, merkle_root: TxMerkleNode::all_zeros(), time: self.blocks_connected, bits: 42, nonce: 42 }; self.height += 1; self.manager.transactions_confirmed(&header, &txdata, self.height as u32); self.manager.best_block_updated(&header, self.height as u32); @@ -230,7 +232,7 @@ impl<'a> MoneyLossDetector<'a> { fn disconnect_block(&mut self) { if self.height > 0 && (self.max_height < 6 || self.height >= self.max_height - 6) { - let header = BlockHeader { version: 0x20000000, prev_blockhash: self.header_hashes[self.height - 1].0, merkle_root: Default::default(), time: self.header_hashes[self.height].1, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: self.header_hashes[self.height - 1].0, merkle_root: TxMerkleNode::all_zeros(), time: self.header_hashes[self.height].1, bits: 42, nonce: 42 }; self.manager.block_disconnected(&header, self.height as u32); self.monitor.block_disconnected(&header, self.height as u32); self.height -= 1; @@ -273,7 +275,7 @@ impl KeysInterface for KeyProvider { fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { let mut node_secret = self.get_node_secret(recipient)?; if let Some(tweak) = tweak { - node_secret.mul_assign(tweak).map_err(|_| ())?; + node_secret = node_secret.mul_tweak(&Scalar::from_be_bytes(*tweak).unwrap()).unwrap(); } Ok(SharedSecret::new(other_key, &node_secret)) } @@ -564,7 +566,7 @@ pub fn do_test(data: &[u8], logger: &Arc) { }, 10 => { 'outer_loop: for funding_generation in pending_funding_generation.drain(..) { - let mut tx = Transaction { version: 0, lock_time: 0, input: Vec::new(), output: vec![TxOut { + let mut tx = Transaction { version: 0, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: vec![TxOut { value: funding_generation.2, script_pubkey: funding_generation.3, }] }; let funding_output = 'search_loop: loop { diff --git a/fuzz/src/process_network_graph.rs b/fuzz/src/process_network_graph.rs index d649710526a..3c8e37175d3 100644 --- a/fuzz/src/process_network_graph.rs +++ b/fuzz/src/process_network_graph.rs @@ -1,11 +1,12 @@ // Imports that need to be added manually use lightning_rapid_gossip_sync::RapidGossipSync; +use bitcoin::hashes::Hash as TraitImport; use utils::test_logger; /// Actual fuzz test, method signature and name are fixed fn do_test(data: &[u8], out: Out) { - let block_hash = bitcoin::BlockHash::default(); + let block_hash = bitcoin::BlockHash::all_zeros(); let logger = test_logger::TestLogger::new("".to_owned(), out); let network_graph = lightning::routing::gossip::NetworkGraph::new(block_hash, &logger); let rapid_sync = RapidGossipSync::new(&network_graph); diff --git a/lightning-background-processor/Cargo.toml b/lightning-background-processor/Cargo.toml index 2df83f2b75a..ef07a3c9df5 100644 --- a/lightning-background-processor/Cargo.toml +++ b/lightning-background-processor/Cargo.toml @@ -14,7 +14,7 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [dependencies] -bitcoin = "0.28.1" +bitcoin = "0.29.0" lightning = { version = "0.0.110", path = "../lightning", features = ["std"] } lightning-rapid-gossip-sync = { version = "0.0.110", path = "../lightning-rapid-gossip-sync" } diff --git a/lightning-background-processor/src/lib.rs b/lightning-background-processor/src/lib.rs index 484439b3907..d13c1311e29 100644 --- a/lightning-background-processor/src/lib.rs +++ b/lightning-background-processor/src/lib.rs @@ -488,6 +488,7 @@ impl Drop for BackgroundProcessor { mod tests { use bitcoin::blockdata::block::BlockHeader; use bitcoin::blockdata::constants::genesis_block; + use bitcoin::blockdata::locktime::PackedLockTime; use bitcoin::blockdata::transaction::{Transaction, TxOut}; use bitcoin::network::constants::Network; use lightning::chain::{BestBlock, Confirm, chainmonitor}; @@ -513,6 +514,8 @@ mod tests { use std::sync::{Arc, Mutex}; use std::sync::mpsc::SyncSender; use std::time::Duration; + use bitcoin::hashes::Hash; + use bitcoin::TxMerkleNode; use lightning::routing::scoring::{FixedPenaltyScorer}; use lightning_rapid_gossip_sync::RapidGossipSync; use super::{BackgroundProcessor, GossipSync, FRESHNESS_TIMER}; @@ -700,7 +703,7 @@ mod tests { assert_eq!(channel_value_satoshis, $channel_value); assert_eq!(user_channel_id, 42); - let tx = Transaction { version: 1 as i32, lock_time: 0, input: Vec::new(), output: vec![TxOut { + let tx = Transaction { version: 1 as i32, lock_time: PackedLockTime(0), input: Vec::new(), output: vec![TxOut { value: channel_value_satoshis, script_pubkey: output_script.clone(), }]}; (temporary_channel_id, tx) @@ -722,7 +725,7 @@ mod tests { for i in 1..=depth { let prev_blockhash = node.best_block.block_hash(); let height = node.best_block.height() + 1; - let header = BlockHeader { version: 0x20000000, prev_blockhash, merkle_root: Default::default(), time: height, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash, merkle_root: TxMerkleNode::all_zeros(), time: height, bits: 42, nonce: 42 }; let txdata = vec![(0, tx)]; node.best_block = BestBlock::new(header.block_hash(), height); match i { diff --git a/lightning-block-sync/Cargo.toml b/lightning-block-sync/Cargo.toml index c6650208e26..27fa32149f1 100644 --- a/lightning-block-sync/Cargo.toml +++ b/lightning-block-sync/Cargo.toml @@ -18,7 +18,7 @@ rest-client = [ "serde", "serde_json", "chunked_transfer" ] rpc-client = [ "serde", "serde_json", "chunked_transfer" ] [dependencies] -bitcoin = "0.28.1" +bitcoin = "0.29.0" lightning = { version = "0.0.110", path = "../lightning" } futures = { version = "0.3" } tokio = { version = "1.0", features = [ "io-util", "net", "time" ], optional = true } diff --git a/lightning-block-sync/src/convert.rs b/lightning-block-sync/src/convert.rs index 8023c837519..ed28833b7b3 100644 --- a/lightning-block-sync/src/convert.rs +++ b/lightning-block-sync/src/convert.rs @@ -15,6 +15,7 @@ use serde_json; use std::convert::From; use std::convert::TryFrom; use std::convert::TryInto; +use bitcoin::hashes::Hash; /// Conversion from `std::io::Error` into `BlockSourceError`. impl From for BlockSourceError { @@ -57,7 +58,7 @@ impl TryInto for JsonResponse { // Add an empty previousblockhash for the genesis block. if let None = header.get("previousblockhash") { - let hash: BlockHash = Default::default(); + let hash: BlockHash = BlockHash::all_zeros(); header.as_object_mut().unwrap().insert("previousblockhash".to_string(), serde_json::json!(hash.to_hex())); } diff --git a/lightning-block-sync/src/test_utils.rs b/lightning-block-sync/src/test_utils.rs index baaab456b5a..0c402deb329 100644 --- a/lightning-block-sync/src/test_utils.rs +++ b/lightning-block-sync/src/test_utils.rs @@ -7,7 +7,7 @@ use bitcoin::hash_types::BlockHash; use bitcoin::network::constants::Network; use bitcoin::util::uint::Uint256; use bitcoin::util::hash::bitcoin_merkle_root; -use bitcoin::Transaction; +use bitcoin::{PackedLockTime, Transaction}; use lightning::chain; @@ -45,7 +45,7 @@ impl Blockchain { // but that's OK because those tests don't trigger the check. let coinbase = Transaction { version: 0, - lock_time: 0, + lock_time: PackedLockTime::ZERO, input: vec![], output: vec![] }; diff --git a/lightning-invoice/Cargo.toml b/lightning-invoice/Cargo.toml index cc0e68cfc42..b7b8f7f0146 100644 --- a/lightning-invoice/Cargo.toml +++ b/lightning-invoice/Cargo.toml @@ -19,9 +19,9 @@ no-std = ["hashbrown", "lightning/no-std", "core2/alloc"] std = ["bitcoin_hashes/std", "num-traits/std", "lightning/std", "bech32/std"] [dependencies] -bech32 = { version = "0.8", default-features = false } +bech32 = { version = "0.9.0", default-features = false } lightning = { version = "0.0.110", path = "../lightning", default-features = false } -secp256k1 = { version = "0.22", default-features = false, features = ["recovery", "alloc"] } +secp256k1 = { version = "0.24.0", default-features = false, features = ["recovery", "alloc"] } num-traits = { version = "0.2.8", default-features = false } bitcoin_hashes = { version = "0.10", default-features = false } hashbrown = { version = "0.11", optional = true } diff --git a/lightning-invoice/fuzz/Cargo.toml b/lightning-invoice/fuzz/Cargo.toml index d741864aea2..833606c1ad8 100644 --- a/lightning-invoice/fuzz/Cargo.toml +++ b/lightning-invoice/fuzz/Cargo.toml @@ -16,7 +16,7 @@ honggfuzz = { version = "0.5", optional = true } afl = { version = "0.4", optional = true } lightning-invoice = { path = ".." } lightning = { path = "../../lightning", features = ["regex"] } -bech32 = "0.8" +bech32 = "0.9.0" # Prevent this from interfering with workspaces [workspace] diff --git a/lightning-net-tokio/Cargo.toml b/lightning-net-tokio/Cargo.toml index dd7770cbe8c..a3e0cbb4586 100644 --- a/lightning-net-tokio/Cargo.toml +++ b/lightning-net-tokio/Cargo.toml @@ -15,7 +15,7 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [dependencies] -bitcoin = "0.28.1" +bitcoin = "0.29.0" lightning = { version = "0.0.110", path = "../lightning" } tokio = { version = "1.0", features = [ "io-util", "macros", "rt", "sync", "net", "time" ] } diff --git a/lightning-persister/Cargo.toml b/lightning-persister/Cargo.toml index 7de0ddc153c..728743c8a4d 100644 --- a/lightning-persister/Cargo.toml +++ b/lightning-persister/Cargo.toml @@ -16,7 +16,7 @@ rustdoc-args = ["--cfg", "docsrs"] _bench_unstable = ["lightning/_bench_unstable"] [dependencies] -bitcoin = "0.28.1" +bitcoin = "0.29.0" lightning = { version = "0.0.110", path = "../lightning" } libc = "0.2" diff --git a/lightning-persister/src/lib.rs b/lightning-persister/src/lib.rs index 3e32791711e..a8792666b0f 100644 --- a/lightning-persister/src/lib.rs +++ b/lightning-persister/src/lib.rs @@ -134,7 +134,7 @@ mod tests { use crate::FilesystemPersister; use bitcoin::blockdata::block::{Block, BlockHeader}; use bitcoin::hashes::hex::FromHex; - use bitcoin::Txid; + use bitcoin::{Txid, TxMerkleNode}; use lightning::chain::ChannelMonitorUpdateErr; use lightning::chain::chainmonitor::Persist; use lightning::chain::transaction::OutPoint; @@ -144,6 +144,7 @@ mod tests { use lightning::util::events::{ClosureReason, MessageSendEventsProvider}; use lightning::util::test_utils; use std::fs; + use bitcoin::hashes::Hash; #[cfg(target_os = "windows")] use { lightning::get_event_msg, @@ -221,7 +222,7 @@ mod tests { let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap(); assert_eq!(node_txn.len(), 1); - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[0].clone()]}); check_closed_broadcast!(nodes[1], true); check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed); diff --git a/lightning-rapid-gossip-sync/Cargo.toml b/lightning-rapid-gossip-sync/Cargo.toml index 39518bbbebd..b8bc8437bca 100644 --- a/lightning-rapid-gossip-sync/Cargo.toml +++ b/lightning-rapid-gossip-sync/Cargo.toml @@ -14,7 +14,7 @@ _bench_unstable = [] [dependencies] lightning = { version = "0.0.110", path = "../lightning" } -bitcoin = { version = "0.28.1", default-features = false } +bitcoin = { version = "0.29.0", default-features = false } [dev-dependencies] lightning = { version = "0.0.110", path = "../lightning", features = ["_test_utils"] } diff --git a/lightning/Cargo.toml b/lightning/Cargo.toml index 7980a9b1dc8..fde54661420 100644 --- a/lightning/Cargo.toml +++ b/lightning/Cargo.toml @@ -38,7 +38,7 @@ grind_signatures = [] default = ["std", "grind_signatures"] [dependencies] -bitcoin = { version = "0.28.1", default-features = false, features = ["secp-recovery"] } +bitcoin = { version = "0.29.0", default-features = false, features = ["secp-recovery"] } hashbrown = { version = "0.11", optional = true } hex = { version = "0.4", optional = true } @@ -52,6 +52,6 @@ hex = "0.4" regex = "1.5.6" [dev-dependencies.bitcoin] -version = "0.28.1" +version = "0.29.0" default-features = false features = ["bitcoinconsensus", "secp-recovery"] diff --git a/lightning/src/chain/chainmonitor.rs b/lightning/src/chain/chainmonitor.rs index 5c4ede0b161..5296a33df79 100644 --- a/lightning/src/chain/chainmonitor.rs +++ b/lightning/src/chain/chainmonitor.rs @@ -733,7 +733,8 @@ impl even #[cfg(test)] mod tests { - use bitcoin::BlockHeader; + use bitcoin::{BlockHeader, TxMerkleNode}; + use bitcoin::hashes::Hash; use ::{check_added_monitors, check_closed_broadcast, check_closed_event}; use ::{expect_payment_sent, expect_payment_claimed, expect_payment_sent_without_paths, expect_payment_path_successful, get_event_msg}; use ::{get_htlc_update_msgs, get_local_commitment_txn, get_revoke_commit_msgs, get_route_and_payment_hash, unwrap_send_err}; @@ -900,7 +901,7 @@ mod tests { let new_header = BlockHeader { version: 2, time: 0, bits: 0, nonce: 0, prev_blockhash: nodes[0].best_block_info().0, - merkle_root: Default::default() }; + merkle_root: TxMerkleNode::all_zeros() }; nodes[0].chain_monitor.chain_monitor.transactions_confirmed(&new_header, &[(0, &remote_txn[0]), (1, &remote_txn[1])], nodes[0].best_block_info().1 + 1); assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty()); @@ -926,7 +927,7 @@ mod tests { let latest_header = BlockHeader { version: 2, time: 0, bits: 0, nonce: 0, prev_blockhash: nodes[0].best_block_info().0, - merkle_root: Default::default() }; + merkle_root: TxMerkleNode::all_zeros() }; nodes[0].chain_monitor.chain_monitor.best_block_updated(&latest_header, nodes[0].best_block_info().1 + LATENCY_GRACE_PERIOD_BLOCKS); } else { let persistences = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clone(); diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 855263fe53b..4bb08724c17 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -404,7 +404,7 @@ impl Writeable for OnchainEventEntry { impl MaybeReadable for OnchainEventEntry { fn read(reader: &mut R) -> Result, DecodeError> { - let mut txid = Default::default(); + let mut txid = Txid::all_zeros(); let mut height = 0; let mut event = None; read_tlv_fields!(reader, { @@ -1756,12 +1756,12 @@ macro_rules! fail_unbroadcast_htlcs { #[cfg(test)] pub fn deliberately_bogus_accepted_htlc_witness_program() -> Vec { - let mut ret = [opcodes::all::OP_NOP.into_u8(); 136]; - ret[131] = opcodes::all::OP_DROP.into_u8(); - ret[132] = opcodes::all::OP_DROP.into_u8(); - ret[133] = opcodes::all::OP_DROP.into_u8(); - ret[134] = opcodes::all::OP_DROP.into_u8(); - ret[135] = opcodes::OP_TRUE.into_u8(); + let mut ret = [opcodes::all::OP_NOP.to_u8(); 136]; + ret[131] = opcodes::all::OP_DROP.to_u8(); + ret[132] = opcodes::all::OP_DROP.to_u8(); + ret[133] = opcodes::all::OP_DROP.to_u8(); + ret[134] = opcodes::all::OP_DROP.to_u8(); + ret[135] = opcodes::OP_TRUE.to_u8(); Vec::from(&ret[..]) } @@ -2110,7 +2110,7 @@ impl ChannelMonitorImpl { }; } - let commitment_number = 0xffffffffffff - ((((tx.input[0].sequence as u64 & 0xffffff) << 3*8) | (tx.lock_time as u64 & 0xffffff)) ^ self.commitment_transaction_number_obscure_factor); + let commitment_number = 0xffffffffffff - ((((tx.input[0].sequence.0 as u64 & 0xffffff) << 3*8) | (tx.lock_time.0 as u64 & 0xffffff)) ^ self.commitment_transaction_number_obscure_factor); if commitment_number >= self.get_min_seen_secret() { let secret = self.get_secret(commitment_number).unwrap(); let per_commitment_key = ignore_error!(SecretKey::from_slice(&secret)); @@ -2495,7 +2495,7 @@ impl ChannelMonitorImpl { log_info!(logger, "Channel {} closed by funding output spend in txid {}.", log_bytes!(self.funding_info.0.to_channel_id()), tx.txid()); self.funding_spend_seen = true; - if (tx.input[0].sequence >> 8*3) as u8 == 0x80 && (tx.lock_time >> 8*3) as u8 == 0x20 { + if (tx.input[0].sequence.0 >> 8*3) as u8 == 0x80 && (tx.lock_time.0 >> 8*3) as u8 == 0x20 { let (mut new_outpoints, new_outputs) = self.check_spend_counterparty_transaction(&tx, height, &logger); if !new_outputs.1.is_empty() { watch_outputs.push(new_outputs); @@ -3469,7 +3469,7 @@ mod tests { use util::ser::{ReadableArgs, Writeable}; use sync::{Arc, Mutex}; use io; - use bitcoin::Witness; + use bitcoin::{PackedLockTime, Sequence, TxMerkleNode, Witness}; use prelude::*; fn do_test_funding_spend_refuses_updates(use_local_txn: bool) { @@ -3513,7 +3513,7 @@ mod tests { let new_header = BlockHeader { version: 2, time: 0, bits: 0, nonce: 0, prev_blockhash: nodes[0].best_block_info().0, - merkle_root: Default::default() }; + merkle_root: TxMerkleNode::all_zeros() }; let conf_height = nodes[0].best_block_info().1 + 1; nodes[1].chain_monitor.chain_monitor.transactions_confirmed(&new_header, &[(0, broadcast_tx)], conf_height); @@ -3573,7 +3573,7 @@ mod tests { let fee_estimator = TestFeeEstimator { sat_per_kw: Mutex::new(253) }; let dummy_key = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()); - let dummy_tx = Transaction { version: 0, lock_time: 0, input: Vec::new(), output: Vec::new() }; + let dummy_tx = Transaction { version: 0, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: Vec::new() }; let mut preimages = Vec::new(); { @@ -3639,7 +3639,7 @@ mod tests { delayed_payment_basepoint: PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[47; 32]).unwrap()), htlc_basepoint: PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[48; 32]).unwrap()) }; - let funding_outpoint = OutPoint { txid: Default::default(), index: u16::max_value() }; + let funding_outpoint = OutPoint { txid: Txid::all_zeros(), index: u16::max_value() }; let channel_parameters = ChannelTransactionParameters { holder_pubkeys: keys.holder_channel_pubkeys.clone(), holder_selected_contest_delay: 66, @@ -3753,7 +3753,7 @@ mod tests { // Justice tx with 1 to_holder, 2 revoked offered HTLCs, 1 revoked received HTLCs for &opt_anchors in [false, true].iter() { - let mut claim_tx = Transaction { version: 0, lock_time: 0, input: Vec::new(), output: Vec::new() }; + let mut claim_tx = Transaction { version: 0, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: Vec::new() }; let mut sum_actual_sigs = 0; for i in 0..4 { claim_tx.input.push(TxIn { @@ -3762,7 +3762,7 @@ mod tests { vout: i, }, script_sig: Script::new(), - sequence: 0xfffffffd, + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, witness: Witness::new(), }); } @@ -3785,7 +3785,7 @@ mod tests { // Claim tx with 1 offered HTLCs, 3 received HTLCs for &opt_anchors in [false, true].iter() { - let mut claim_tx = Transaction { version: 0, lock_time: 0, input: Vec::new(), output: Vec::new() }; + let mut claim_tx = Transaction { version: 0, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: Vec::new() }; let mut sum_actual_sigs = 0; for i in 0..4 { claim_tx.input.push(TxIn { @@ -3794,7 +3794,7 @@ mod tests { vout: i, }, script_sig: Script::new(), - sequence: 0xfffffffd, + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, witness: Witness::new(), }); } @@ -3817,7 +3817,7 @@ mod tests { // Justice tx with 1 revoked HTLC-Success tx output for &opt_anchors in [false, true].iter() { - let mut claim_tx = Transaction { version: 0, lock_time: 0, input: Vec::new(), output: Vec::new() }; + let mut claim_tx = Transaction { version: 0, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: Vec::new() }; let mut sum_actual_sigs = 0; claim_tx.input.push(TxIn { previous_output: BitcoinOutPoint { @@ -3825,7 +3825,7 @@ mod tests { vout: 0, }, script_sig: Script::new(), - sequence: 0xfffffffd, + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, witness: Witness::new(), }); claim_tx.output.push(TxOut { diff --git a/lightning/src/chain/keysinterface.rs b/lightning/src/chain/keysinterface.rs index 9a3baea8bb4..000c497d8b7 100644 --- a/lightning/src/chain/keysinterface.rs +++ b/lightning/src/chain/keysinterface.rs @@ -25,11 +25,11 @@ use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::hashes::sha256d::Hash as Sha256dHash; use bitcoin::hash_types::WPubkeyHash; -use bitcoin::secp256k1::{SecretKey, PublicKey}; +use bitcoin::secp256k1::{SecretKey, PublicKey, Scalar}; use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature, Signing}; use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; -use bitcoin::{secp256k1, Witness}; +use bitcoin::{PackedLockTime, secp256k1, Sequence, Witness}; use util::{byte_utils, transaction_utils}; use util::crypto::{hkdf_extract_expand_twice, sign}; @@ -626,7 +626,7 @@ impl InMemorySigner { if spend_tx.input.len() <= input_idx { return Err(()); } if !spend_tx.input[input_idx].script_sig.is_empty() { return Err(()); } if spend_tx.input[input_idx].previous_output != descriptor.outpoint.into_bitcoin_outpoint() { return Err(()); } - if spend_tx.input[input_idx].sequence != descriptor.to_self_delay as u32 { return Err(()); } + if spend_tx.input[input_idx].sequence.0 != descriptor.to_self_delay as u32 { return Err(()); } let delayed_payment_key = chan_utils::derive_private_key(&secp_ctx, &descriptor.per_commitment_point, &self.delayed_payment_base_key) .expect("We constructed the payment_base_key, so we can only fail here if the RNG is busted."); @@ -1022,7 +1022,7 @@ impl KeysManager { input.push(TxIn { previous_output: descriptor.outpoint.into_bitcoin_outpoint(), script_sig: Script::new(), - sequence: 0, + sequence: Sequence::ZERO, witness: Witness::new(), }); witness_weight += StaticPaymentOutputDescriptor::MAX_WITNESS_LENGTH; @@ -1033,7 +1033,7 @@ impl KeysManager { input.push(TxIn { previous_output: descriptor.outpoint.into_bitcoin_outpoint(), script_sig: Script::new(), - sequence: descriptor.to_self_delay as u32, + sequence: Sequence(descriptor.to_self_delay as u32), witness: Witness::new(), }); witness_weight += DelayedPaymentOutputDescriptor::MAX_WITNESS_LENGTH; @@ -1044,7 +1044,7 @@ impl KeysManager { input.push(TxIn { previous_output: outpoint.into_bitcoin_outpoint(), script_sig: Script::new(), - sequence: 0, + sequence: Sequence::ZERO, witness: Witness::new(), }); witness_weight += 1 + 73 + 34; @@ -1056,7 +1056,7 @@ impl KeysManager { } let mut spend_tx = Transaction { version: 2, - lock_time: 0, + lock_time: PackedLockTime(0), input, output: outputs, }; @@ -1143,7 +1143,7 @@ impl KeysInterface for KeysManager { fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { let mut node_secret = self.get_node_secret(recipient)?; if let Some(tweak) = tweak { - node_secret.mul_assign(tweak).map_err(|_| ())?; + node_secret = node_secret.mul_tweak(&Scalar::from_be_bytes(*tweak).unwrap()).map_err(|_| ())?; } Ok(SharedSecret::new(other_key, &node_secret)) } @@ -1235,7 +1235,7 @@ impl KeysInterface for PhantomKeysManager { fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { let mut node_secret = self.get_node_secret(recipient)?; if let Some(tweak) = tweak { - node_secret.mul_assign(tweak).map_err(|_| ())?; + node_secret = node_secret.mul_tweak(&Scalar::from_be_bytes(*tweak).unwrap()).map_err(|_| ())?; } Ok(SharedSecret::new(other_key, &node_secret)) } diff --git a/lightning/src/chain/onchaintx.rs b/lightning/src/chain/onchaintx.rs index ac01cacaa86..8f62c43c44e 100644 --- a/lightning/src/chain/onchaintx.rs +++ b/lightning/src/chain/onchaintx.rs @@ -38,6 +38,7 @@ use alloc::collections::BTreeMap; use core::cmp; use core::ops::Deref; use core::mem::replace; +use bitcoin::hashes::Hash; const MAX_ALLOC_SIZE: usize = 64*1024; @@ -92,7 +93,7 @@ impl Writeable for OnchainEventEntry { impl MaybeReadable for OnchainEventEntry { fn read(reader: &mut R) -> Result, DecodeError> { - let mut txid = Default::default(); + let mut txid = Txid::all_zeros(); let mut height = 0; let mut event = None; read_tlv_fields!(reader, { @@ -389,7 +390,7 @@ impl OnchainTxHandler { if cached_request.is_malleable() { let predicted_weight = cached_request.package_weight(&self.destination_script, self.channel_transaction_parameters.opt_anchors.is_some()); if let Some((output_value, new_feerate)) = - cached_request.compute_package_output(predicted_weight, self.destination_script.dust_value().as_sat(), fee_estimator, logger) { + cached_request.compute_package_output(predicted_weight, self.destination_script.dust_value().to_sat(), fee_estimator, logger) { assert!(new_feerate != 0); let transaction = cached_request.finalize_package(self, output_value, self.destination_script.clone(), logger).unwrap(); diff --git a/lightning/src/chain/package.rs b/lightning/src/chain/package.rs index 30530303e59..c945d8909da 100644 --- a/lightning/src/chain/package.rs +++ b/lightning/src/chain/package.rs @@ -36,7 +36,7 @@ use prelude::*; use core::cmp; use core::mem; use core::ops::Deref; -use bitcoin::Witness; +use bitcoin::{PackedLockTime, Sequence, Witness}; use super::chaininterface::LowerBoundedFeeEstimator; @@ -393,7 +393,7 @@ impl PackageSolvingData { if let Ok(chan_keys) = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint) { let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&outp.htlc, onchain_handler.opt_anchors(), &chan_keys.broadcaster_htlc_key, &chan_keys.countersignatory_htlc_key, &chan_keys.revocation_key); - bumped_tx.lock_time = outp.htlc.cltv_expiry; // Right now we don't aggregate time-locked transaction, if we do we should set lock_time before to avoid breaking hash computation + bumped_tx.lock_time = PackedLockTime(outp.htlc.cltv_expiry); // Right now we don't aggregate time-locked transaction, if we do we should set lock_time before to avoid breaking hash computation if let Ok(sig) = onchain_handler.signer.sign_counterparty_htlc_transaction(&bumped_tx, i, &outp.htlc.amount_msat / 1000, &outp.per_commitment_point, &outp.htlc, &onchain_handler.secp_ctx) { let mut ser_sig = sig.serialize_der().to_vec(); ser_sig.push(EcdsaSighashType::All as u8); @@ -615,7 +615,7 @@ impl PackageTemplate { PackageMalleability::Malleable => { let mut bumped_tx = Transaction { version: 2, - lock_time: 0, + lock_time: PackedLockTime::ZERO, input: vec![], output: vec![TxOut { script_pubkey: destination_script, @@ -626,7 +626,7 @@ impl PackageTemplate { bumped_tx.input.push(TxIn { previous_output: *outpoint, script_sig: Script::new(), - sequence: 0xfffffffd, + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, witness: Witness::new(), }); } diff --git a/lightning/src/ln/chan_utils.rs b/lightning/src/ln/chan_utils.rs index 41d1eff856a..d53863289bc 100644 --- a/lightning/src/ln/chan_utils.rs +++ b/lightning/src/ln/chan_utils.rs @@ -26,10 +26,10 @@ use util::ser::{Readable, Writeable, Writer}; use util::{byte_utils, transaction_utils}; use bitcoin::hash_types::WPubkeyHash; -use bitcoin::secp256k1::{SecretKey, PublicKey}; +use bitcoin::secp256k1::{SecretKey, PublicKey, Scalar}; use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature, Message}; use bitcoin::secp256k1::Error as SecpError; -use bitcoin::{secp256k1, Witness}; +use bitcoin::{PackedLockTime, secp256k1, Sequence, Witness}; use io; use prelude::*; @@ -101,7 +101,7 @@ pub fn build_closing_transaction(to_holder_value_sat: u64, to_counterparty_value ins.push(TxIn { previous_output: funding_outpoint, script_sig: Script::new(), - sequence: 0xffffffff, + sequence: Sequence::MAX, witness: Witness::new(), }); ins @@ -132,7 +132,7 @@ pub fn build_closing_transaction(to_holder_value_sat: u64, to_counterparty_value Transaction { version: 2, - lock_time: 0, + lock_time: PackedLockTime::ZERO, input: txins, output: outputs, } @@ -264,9 +264,7 @@ pub fn derive_private_key(secp_ctx: &Secp256k1, per_co sha.input(&PublicKey::from_secret_key(&secp_ctx, &base_secret).serialize()); let res = Sha256::from_engine(sha).into_inner(); - let mut key = base_secret.clone(); - key.add_assign(&res)?; - Ok(key) + base_secret.clone().add_tweak(&Scalar::from_be_bytes(res).unwrap()) } /// Derives a per-commitment-transaction public key (eg an htlc key or a delayed_payment key) @@ -313,12 +311,9 @@ pub fn derive_private_revocation_key(secp_ctx: &Secp256k1 Sha256::from_engine(sha).into_inner() }; - let mut countersignatory_contrib = countersignatory_revocation_base_secret.clone(); - countersignatory_contrib.mul_assign(&rev_append_commit_hash_key)?; - let mut broadcaster_contrib = per_commitment_secret.clone(); - broadcaster_contrib.mul_assign(&commit_append_rev_hash_key)?; - countersignatory_contrib.add_assign(&broadcaster_contrib[..])?; - Ok(countersignatory_contrib) + let countersignatory_contrib = countersignatory_revocation_base_secret.clone().mul_tweak(&Scalar::from_be_bytes(rev_append_commit_hash_key).unwrap())?; + let broadcaster_contrib = per_commitment_secret.clone().mul_tweak(&Scalar::from_be_bytes(commit_append_rev_hash_key).unwrap())?; + countersignatory_contrib.add_tweak(&Scalar::from_be_bytes(broadcaster_contrib.secret_bytes()).unwrap()) } /// Derives a per-commitment-transaction revocation public key from its constituent parts. This is @@ -348,10 +343,8 @@ pub fn derive_public_revocation_key(secp_ctx: &Secp2 Sha256::from_engine(sha).into_inner() }; - let mut countersignatory_contrib = countersignatory_revocation_base_point.clone(); - countersignatory_contrib.mul_assign(&secp_ctx, &rev_append_commit_hash_key)?; - let mut broadcaster_contrib = per_commitment_point.clone(); - broadcaster_contrib.mul_assign(&secp_ctx, &commit_append_rev_hash_key)?; + let countersignatory_contrib = countersignatory_revocation_base_point.clone().mul_tweak(&secp_ctx, &Scalar::from_be_bytes(rev_append_commit_hash_key).unwrap())?; + let broadcaster_contrib = per_commitment_point.clone().mul_tweak(&secp_ctx, &Scalar::from_be_bytes(commit_append_rev_hash_key).unwrap())?; countersignatory_contrib.combine(&broadcaster_contrib) } @@ -614,7 +607,7 @@ pub fn build_htlc_transaction(commitment_txid: &Txid, feerate_per_kw: u32, conte vout: htlc.transaction_output_index.expect("Can't build an HTLC transaction for a dust output"), }, script_sig: Script::new(), - sequence: if opt_anchors { 1 } else { 0 }, + sequence: Sequence(if opt_anchors { 1 } else { 0 }), witness: Witness::new(), }); @@ -633,7 +626,7 @@ pub fn build_htlc_transaction(commitment_txid: &Txid, feerate_per_kw: u32, conte Transaction { version: 2, - lock_time: if htlc.offered { htlc.cltv_expiry } else { 0 }, + lock_time: PackedLockTime(if htlc.offered { htlc.cltv_expiry } else { 0 }), input: txins, output: txouts, } @@ -863,7 +856,7 @@ impl HolderCommitmentTransaction { holder_selected_contest_delay: 0, is_outbound_from_holder: false, counterparty_parameters: Some(CounterpartyChannelTransactionParameters { pubkeys: channel_pubkeys.clone(), selected_contest_delay: 0 }), - funding_outpoint: Some(chain::transaction::OutPoint { txid: Default::default(), index: 0 }), + funding_outpoint: Some(chain::transaction::OutPoint { txid: Txid::all_zeros(), index: 0 }), opt_anchors: None }; let mut htlcs_with_aux: Vec<(_, ())> = Vec::new(); @@ -1167,7 +1160,7 @@ impl CommitmentTransaction { fn make_transaction(obscured_commitment_transaction_number: u64, txins: Vec, outputs: Vec) -> Transaction { Transaction { version: 2, - lock_time: ((0x20 as u32) << 8 * 3) | ((obscured_commitment_transaction_number & 0xffffffu64) as u32), + lock_time: PackedLockTime(((0x20 as u32) << 8 * 3) | ((obscured_commitment_transaction_number & 0xffffffu64) as u32)), input: txins, output: outputs, } @@ -1291,8 +1284,8 @@ impl CommitmentTransaction { ins.push(TxIn { previous_output: channel_parameters.funding_outpoint(), script_sig: Script::new(), - sequence: ((0x80 as u32) << 8 * 3) - | ((obscured_commitment_transaction_number >> 3 * 8) as u32), + sequence: Sequence(((0x80 as u32) << 8 * 3) + | ((obscured_commitment_transaction_number >> 3 * 8) as u32)), witness: Witness::new(), }); ins @@ -1508,7 +1501,8 @@ mod tests { use bitcoin::secp256k1::{PublicKey, SecretKey, Secp256k1}; use util::test_utils; use chain::keysinterface::{KeysInterface, BaseSign}; - use bitcoin::Network; + use bitcoin::{Network, Txid}; + use bitcoin::hashes::Hash; use ln::PaymentHash; use bitcoin::hashes::hex::ToHex; @@ -1533,7 +1527,7 @@ mod tests { holder_selected_contest_delay: 0, is_outbound_from_holder: false, counterparty_parameters: Some(CounterpartyChannelTransactionParameters { pubkeys: counterparty_pubkeys.clone(), selected_contest_delay: 0 }), - funding_outpoint: Some(chain::transaction::OutPoint { txid: Default::default(), index: 0 }), + funding_outpoint: Some(chain::transaction::OutPoint { txid: Txid::all_zeros(), index: 0 }), opt_anchors: None }; diff --git a/lightning/src/ln/chanmon_update_fail_tests.rs b/lightning/src/ln/chanmon_update_fail_tests.rs index 02ac7b48137..f5977abd4fc 100644 --- a/lightning/src/ln/chanmon_update_fail_tests.rs +++ b/lightning/src/ln/chanmon_update_fail_tests.rs @@ -36,6 +36,8 @@ use ln::functional_test_utils::*; use util::test_utils; use io; +use bitcoin::hashes::Hash; +use bitcoin::TxMerkleNode; use prelude::*; use sync::{Arc, Mutex}; @@ -116,7 +118,14 @@ fn test_monitor_and_persister_update_fail() { assert!(chain_mon.watch_channel(outpoint, new_monitor).is_ok()); chain_mon }; - let header = BlockHeader { version: 0x20000000, prev_blockhash: Default::default(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { + version: 0x20000000, + prev_blockhash: BlockHash::all_zeros(), + merkle_root: TxMerkleNode::all_zeros(), + time: 42, + bits: 42, + nonce: 42 + }; chain_mon.chain_monitor.block_connected(&Block { header, txdata: vec![] }, 200); // Set the persister's return value to be a TemporaryFailure. diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index fdd073f632a..7e978d9407c 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -6612,6 +6612,7 @@ mod tests { use bitcoin::hashes::Hash; use bitcoin::hash_types::WPubkeyHash; use bitcoin::bech32::u5; + use bitcoin::PackedLockTime; use bitcoin::util::address::WitnessVersion; use prelude::*; @@ -6872,7 +6873,7 @@ mod tests { // Node A --> Node B: funding created let output_script = node_a_chan.get_funding_redeemscript(); - let tx = Transaction { version: 1, lock_time: 0, input: Vec::new(), output: vec![TxOut { + let tx = Transaction { version: 1, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: vec![TxOut { value: 10000000, script_pubkey: output_script.clone(), }]}; let funding_outpoint = OutPoint{ txid: tx.txid(), index: 0 }; diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 692d46eda90..e8830ab5ffe 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -32,7 +32,7 @@ use bitcoin::hash_types::{BlockHash, Txid}; use bitcoin::secp256k1::{SecretKey,PublicKey}; use bitcoin::secp256k1::Secp256k1; use bitcoin::secp256k1::ecdh::SharedSecret; -use bitcoin::secp256k1; +use bitcoin::{LockTime, secp256k1, Sequence}; use chain; use chain::{Confirm, ChannelMonitorUpdateErr, Watch, BestBlock}; @@ -2897,7 +2897,7 @@ impl ChannelMana // constituting our Lightning node might not have perfect sync about their blockchain views. Thus, if // the wallet module is in advance on the LDK view, allow one more block of headroom. // TODO: updated if/when https://github.com/rust-bitcoin/rust-bitcoin/pull/994 landed and rust-bitcoin bumped. - if !funding_transaction.input.iter().all(|input| input.sequence == 0xffffffff) && funding_transaction.lock_time < 500_000_000 && funding_transaction.lock_time > height + 2 { + if !funding_transaction.input.iter().all(|input| input.sequence == Sequence::MAX) && LockTime::from(funding_transaction.lock_time).is_block_height() && funding_transaction.lock_time.0 > height + 2 { return Err(APIError::APIMisuseError { err: "Funding transaction absolute timelock is non-final".to_owned() }); @@ -7974,7 +7974,7 @@ pub mod bench { use bitcoin::hashes::Hash; use bitcoin::hashes::sha256::Hash as Sha256; - use bitcoin::{Block, BlockHeader, Transaction, TxOut}; + use bitcoin::{Block, BlockHeader, PackedLockTime, Transaction, TxMerkleNode, TxOut}; use sync::{Arc, Mutex}; @@ -8036,7 +8036,7 @@ pub mod bench { let tx; if let Event::FundingGenerationReady { temporary_channel_id, output_script, .. } = get_event!(node_a_holder, Event::FundingGenerationReady) { - tx = Transaction { version: 2, lock_time: 0, input: Vec::new(), output: vec![TxOut { + tx = Transaction { version: 2, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: vec![TxOut { value: 8_000_000, script_pubkey: output_script, }]}; node_a.funding_transaction_generated(&temporary_channel_id, &node_b.get_our_node_id(), tx.clone()).unwrap(); @@ -8048,7 +8048,7 @@ pub mod bench { assert_eq!(&tx_broadcaster.txn_broadcasted.lock().unwrap()[..], &[tx.clone()]); let block = Block { - header: BlockHeader { version: 0x20000000, prev_blockhash: genesis_hash, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }, + header: BlockHeader { version: 0x20000000, prev_blockhash: genesis_hash, merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }, txdata: vec![tx], }; Listen::block_connected(&node_a, &block, 1); diff --git a/lightning/src/ln/functional_test_utils.rs b/lightning/src/ln/functional_test_utils.rs index 54d199a26f8..4c91d4f7b1c 100644 --- a/lightning/src/ln/functional_test_utils.rs +++ b/lightning/src/ln/functional_test_utils.rs @@ -47,6 +47,7 @@ use alloc::rc::Rc; use sync::{Arc, Mutex}; use core::mem; use core::iter::repeat; +use bitcoin::{PackedLockTime, TxMerkleNode}; pub const CHAN_CONFIRM_DEPTH: u32 = 10; @@ -77,11 +78,11 @@ pub fn confirm_transaction_at<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, tx: &T connect_blocks(node, conf_height - first_connect_height); } let mut block = Block { - header: BlockHeader { version: 0x20000000, prev_blockhash: node.best_block_hash(), merkle_root: Default::default(), time: conf_height, bits: 42, nonce: 42 }, + header: BlockHeader { version: 0x20000000, prev_blockhash: node.best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: conf_height, bits: 42, nonce: 42 }, txdata: Vec::new(), }; for _ in 0..*node.network_chan_count.borrow() { // Make sure we don't end up with channels at the same short id by offsetting by chan_count - block.txdata.push(Transaction { version: 0, lock_time: 0, input: Vec::new(), output: Vec::new() }); + block.txdata.push(Transaction { version: 0, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: Vec::new() }); } block.txdata.push(tx.clone()); connect_block(node, &block); @@ -148,7 +149,7 @@ pub fn connect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, depth: u32) -> let height = node.best_block_info().1 + 1; let mut block = Block { - header: BlockHeader { version: 0x2000000, prev_blockhash: node.best_block_hash(), merkle_root: Default::default(), time: height, bits: 42, nonce: 42 }, + header: BlockHeader { version: 0x2000000, prev_blockhash: node.best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: height, bits: 42, nonce: 42 }, txdata: vec![], }; assert!(depth >= 1); @@ -156,7 +157,7 @@ pub fn connect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, depth: u32) -> let prev_blockhash = block.header.block_hash(); do_connect_block(node, block, skip_intermediaries); block = Block { - header: BlockHeader { version: 0x20000000, prev_blockhash, merkle_root: Default::default(), time: height + i, bits: 42, nonce: 42 }, + header: BlockHeader { version: 0x20000000, prev_blockhash, merkle_root: TxMerkleNode::all_zeros(), time: height + i, bits: 42, nonce: 42 }, txdata: vec![], }; } @@ -619,7 +620,7 @@ pub fn create_funding_transaction<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, expected_ assert_eq!(*channel_value_satoshis, expected_chan_value); assert_eq!(user_channel_id, expected_user_chan_id); - let tx = Transaction { version: chan_id as i32, lock_time: 0, input: Vec::new(), output: vec![TxOut { + let tx = Transaction { version: chan_id as i32, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: vec![TxOut { value: *channel_value_satoshis, script_pubkey: output_script.clone(), }]}; let funding_outpoint = OutPoint { txid: tx.txid(), index: 0 }; @@ -893,11 +894,11 @@ macro_rules! check_spends { { $( for outp in $spends_txn.output.iter() { - assert!(outp.value >= outp.script_pubkey.dust_value().as_sat(), "Input tx output didn't meet dust limit"); + assert!(outp.value >= outp.script_pubkey.dust_value().to_sat(), "Input tx output didn't meet dust limit"); } )* for outp in $tx.output.iter() { - assert!(outp.value >= outp.script_pubkey.dust_value().as_sat(), "Spending tx output didn't meet dust limit"); + assert!(outp.value >= outp.script_pubkey.dust_value().to_sat(), "Spending tx output didn't meet dust limit"); } let get_output = |out_point: &bitcoin::blockdata::transaction::OutPoint| { $( @@ -2125,9 +2126,9 @@ pub fn test_txn_broadcast<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, chan: &(msgs::Cha if tx.input.len() == 1 && tx.input[0].previous_output.txid == res[0].txid() { check_spends!(tx, res[0]); if has_htlc_tx == HTLCType::TIMEOUT { - assert!(tx.lock_time != 0); + assert!(tx.lock_time.0 != 0); } else { - assert!(tx.lock_time == 0); + assert!(tx.lock_time.0 == 0); } res.push(tx.clone()); false diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index d7be8966250..c41d5402ff3 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -42,7 +42,7 @@ use bitcoin::blockdata::script::{Builder, Script}; use bitcoin::blockdata::opcodes; use bitcoin::blockdata::constants::genesis_block; use bitcoin::network::constants::Network; -use bitcoin::{Transaction, TxIn, TxOut, Witness}; +use bitcoin::{PackedLockTime, Sequence, Transaction, TxIn, TxMerkleNode, TxOut, Witness}; use bitcoin::OutPoint as BitcoinOutPoint; use bitcoin::secp256k1::Secp256k1; @@ -55,6 +55,7 @@ use prelude::*; use alloc::collections::BTreeSet; use core::default::Default; use core::iter::repeat; +use bitcoin::hashes::Hash; use sync::{Arc, Mutex}; use ln::functional_test_utils::*; @@ -503,7 +504,7 @@ fn do_test_sanity_on_in_flight_opens(steps: u8) { if steps & 0b1000_0000 != 0{ let block = Block { - header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }, + header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }, txdata: vec![], }; connect_block(&nodes[0], &block); @@ -2717,11 +2718,11 @@ fn test_htlc_on_chain_success() { assert_eq!(node_txn[1].input[0].witness.clone().last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT); assert!(node_txn[0].output[0].script_pubkey.is_v0_p2wsh()); // revokeable output assert!(node_txn[1].output[0].script_pubkey.is_v0_p2wsh()); // revokeable output - assert_eq!(node_txn[0].lock_time, 0); - assert_eq!(node_txn[1].lock_time, 0); + assert_eq!(node_txn[0].lock_time.0, 0); + assert_eq!(node_txn[1].lock_time.0, 0); // Verify that B's ChannelManager is able to extract preimage from HTLC Success tx and pass it backward - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42}; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42}; connect_block(&nodes[1], &Block { header, txdata: node_txn}); connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires { @@ -2791,8 +2792,8 @@ fn test_htlc_on_chain_success() { // Node[0]: ChannelManager: 3 (commtiemtn tx, 2*HTLC-Timeout tx), ChannelMonitor: 2 HTLC-timeout check_spends!(node_txn[1], $commitment_tx); check_spends!(node_txn[2], $commitment_tx); - assert_ne!(node_txn[1].lock_time, 0); - assert_ne!(node_txn[2].lock_time, 0); + assert_ne!(node_txn[1].lock_time.0, 0); + assert_ne!(node_txn[2].lock_time.0, 0); if $htlc_offered { assert_eq!(node_txn[1].input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT); assert_eq!(node_txn[2].input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT); @@ -2841,7 +2842,7 @@ fn test_htlc_on_chain_success() { assert_eq!(commitment_spend.input.len(), 2); assert_eq!(commitment_spend.input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT); assert_eq!(commitment_spend.input[1].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT); - assert_eq!(commitment_spend.lock_time, 0); + assert_eq!(commitment_spend.lock_time.0, 0); assert!(commitment_spend.output[0].script_pubkey.is_v0_p2wpkh()); // direct payment check_spends!(node_txn[3], chan_1.3); assert_eq!(node_txn[3].input[0].witness.clone().last().unwrap().len(), 71); @@ -2851,7 +2852,7 @@ fn test_htlc_on_chain_success() { // we already checked the same situation with A. // Verify that A's ChannelManager is able to extract preimage from preimage tx and generate PaymentSent - let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42}; + let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42}; connect_block(&nodes[0], &Block { header, txdata: vec![node_a_commitment_tx[0].clone(), commitment_spend.clone()] }); connect_blocks(&nodes[0], TEST_FINAL_CLTV + MIN_CLTV_EXPIRY_DELTA as u32 - 1); // Confirm blocks until the HTLC expires check_closed_broadcast!(nodes[0], true); @@ -3408,7 +3409,7 @@ fn test_htlc_ignore_latest_remote_commitment() { assert_eq!(node_txn.len(), 3); assert_eq!(node_txn[0], node_txn[1]); - let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[1].clone()]}); check_closed_broadcast!(nodes[1], true); check_added_monitors!(nodes[1], 1); @@ -3491,7 +3492,7 @@ fn test_force_close_fail_back() { assert_eq!(node_txn.len(), 1); assert_eq!(node_txn[0].input.len(), 1); assert_eq!(node_txn[0].input[0].previous_output.txid, tx.txid()); - assert_eq!(node_txn[0].lock_time, 0); // Must be an HTLC-Success + assert_eq!(node_txn[0].lock_time.0, 0); // Must be an HTLC-Success assert_eq!(node_txn[0].input[0].witness.len(), 5); // Must be an HTLC-Success check_spends!(node_txn[0], tx); @@ -4259,7 +4260,7 @@ fn do_test_htlc_timeout(send_partial_mpp: bool) { }; let mut block = Block { - header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }, + header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }, txdata: vec![], }; connect_block(&nodes[0], &block); @@ -4804,7 +4805,7 @@ fn test_claim_sizeable_push_msat() { assert_eq!(spend_txn.len(), 1); assert_eq!(spend_txn[0].input.len(), 1); check_spends!(spend_txn[0], node_txn[0]); - assert_eq!(spend_txn[0].input[0].sequence, BREAKDOWN_TIMEOUT as u32); + assert_eq!(spend_txn[0].input[0].sequence.0, BREAKDOWN_TIMEOUT as u32); } #[test] @@ -5034,10 +5035,10 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_timeout_tx() { assert_eq!(revoked_htlc_txn[1].input.len(), 1); assert_eq!(revoked_htlc_txn[1].input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT); check_spends!(revoked_htlc_txn[1], revoked_local_txn[0]); - assert_ne!(revoked_htlc_txn[1].lock_time, 0); // HTLC-Timeout + assert_ne!(revoked_htlc_txn[1].lock_time.0, 0); // HTLC-Timeout // B will generate justice tx from A's revoked commitment/HTLC tx - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[1], &Block { header, txdata: vec![revoked_local_txn[0].clone(), revoked_htlc_txn[1].clone()] }); check_closed_broadcast!(nodes[1], true); check_added_monitors!(nodes[1], 1); @@ -5111,7 +5112,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_success_tx() { assert_eq!(revoked_local_txn[0].output[unspent_local_txn_output].script_pubkey.len(), 2 + 20); // P2WPKH // A will generate justice tx from B's revoked commitment/HTLC tx - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header, txdata: vec![revoked_local_txn[0].clone(), revoked_htlc_txn[0].clone()] }); check_closed_broadcast!(nodes[0], true); check_added_monitors!(nodes[0], 1); @@ -5208,10 +5209,10 @@ fn test_onchain_to_onchain_claim() { assert_eq!(c_txn[1].input[0].witness.clone().last().unwrap().len(), 71); assert_eq!(c_txn[2].input[0].witness.clone().last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT); assert!(c_txn[0].output[0].script_pubkey.is_v0_p2wsh()); // revokeable output - assert_eq!(c_txn[0].lock_time, 0); // Success tx + assert_eq!(c_txn[0].lock_time.0, 0); // Success tx // So we broadcast C's commitment tx and HTLC-Success on B's chain, we should successfully be able to extract preimage and update downstream monitor - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42}; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42}; connect_block(&nodes[1], &Block { header, txdata: vec![c_txn[1].clone(), c_txn[2].clone()]}); check_added_monitors!(nodes[1], 1); let events = nodes[1].node.get_and_clear_pending_events(); @@ -5269,7 +5270,7 @@ fn test_onchain_to_onchain_claim() { check_spends!(b_txn[0], commitment_tx[0]); assert_eq!(b_txn[0].input[0].witness.clone().last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT); assert!(b_txn[0].output[0].script_pubkey.is_v0_p2wpkh()); // direct payment - assert_eq!(b_txn[0].lock_time, 0); // Success tx + assert_eq!(b_txn[0].lock_time.0, 0); // Success tx check_closed_broadcast!(nodes[1], true); check_added_monitors!(nodes[1], 1); @@ -5477,7 +5478,7 @@ fn test_dynamic_spendable_outputs_local_htlc_success_tx() { assert_eq!(spend_txn.len(), 1); assert_eq!(spend_txn[0].input.len(), 1); check_spends!(spend_txn[0], node_tx); - assert_eq!(spend_txn[0].input[0].sequence, BREAKDOWN_TIMEOUT as u32); + assert_eq!(spend_txn[0].input[0].sequence.0, BREAKDOWN_TIMEOUT as u32); } fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, announce_latest: bool) { @@ -5825,11 +5826,11 @@ fn test_dynamic_spendable_outputs_local_htlc_timeout_tx() { check_spends!(spend_txn[0], local_txn[0]); assert_eq!(spend_txn[1].input.len(), 1); check_spends!(spend_txn[1], htlc_timeout); - assert_eq!(spend_txn[1].input[0].sequence, BREAKDOWN_TIMEOUT as u32); + assert_eq!(spend_txn[1].input[0].sequence.0, BREAKDOWN_TIMEOUT as u32); assert_eq!(spend_txn[2].input.len(), 2); check_spends!(spend_txn[2], local_txn[0], htlc_timeout); - assert!(spend_txn[2].input[0].sequence == BREAKDOWN_TIMEOUT as u32 || - spend_txn[2].input[1].sequence == BREAKDOWN_TIMEOUT as u32); + assert!(spend_txn[2].input[0].sequence.0 == BREAKDOWN_TIMEOUT as u32 || + spend_txn[2].input[1].sequence.0 == BREAKDOWN_TIMEOUT as u32); } #[test] @@ -5908,11 +5909,11 @@ fn test_key_derivation_params() { check_spends!(spend_txn[0], local_txn_1[0]); assert_eq!(spend_txn[1].input.len(), 1); check_spends!(spend_txn[1], htlc_timeout); - assert_eq!(spend_txn[1].input[0].sequence, BREAKDOWN_TIMEOUT as u32); + assert_eq!(spend_txn[1].input[0].sequence.0, BREAKDOWN_TIMEOUT as u32); assert_eq!(spend_txn[2].input.len(), 2); check_spends!(spend_txn[2], local_txn_1[0], htlc_timeout); - assert!(spend_txn[2].input[0].sequence == BREAKDOWN_TIMEOUT as u32 || - spend_txn[2].input[1].sequence == BREAKDOWN_TIMEOUT as u32); + assert!(spend_txn[2].input[0].sequence.0 == BREAKDOWN_TIMEOUT as u32 || + spend_txn[2].input[1].sequence.0 == BREAKDOWN_TIMEOUT as u32); } #[test] @@ -5971,7 +5972,7 @@ fn do_htlc_claim_local_commitment_only(use_dust: bool) { let starting_block = nodes[1].best_block_info(); let mut block = Block { - header: BlockHeader { version: 0x20000000, prev_blockhash: starting_block.0, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }, + header: BlockHeader { version: 0x20000000, prev_blockhash: starting_block.0, merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }, txdata: vec![], }; for _ in starting_block.1 + 1..TEST_FINAL_CLTV - CLTV_CLAIM_BUFFER + starting_block.1 + 2 { @@ -6002,7 +6003,7 @@ fn do_htlc_claim_current_remote_commitment_only(use_dust: bool) { // to "time out" the HTLC. let starting_block = nodes[1].best_block_info(); - let mut header = BlockHeader { version: 0x20000000, prev_blockhash: starting_block.0, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let mut header = BlockHeader { version: 0x20000000, prev_blockhash: starting_block.0, merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; for _ in starting_block.1 + 1..TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + starting_block.1 + 2 { connect_block(&nodes[0], &Block { header, txdata: Vec::new()}); @@ -6049,7 +6050,7 @@ fn do_htlc_claim_previous_remote_commitment_only(use_dust: bool, check_revoke_no let starting_block = nodes[1].best_block_info(); let mut block = Block { - header: BlockHeader { version: 0x20000000, prev_blockhash: starting_block.0, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }, + header: BlockHeader { version: 0x20000000, prev_blockhash: starting_block.0, merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }, txdata: vec![], }; for _ in starting_block.1 + 1..TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + CHAN_CONFIRM_DEPTH + 2 { @@ -7335,7 +7336,7 @@ fn do_test_sweep_outbound_htlc_failure_update(revoked: bool, local: bool) { if !revoked { assert_eq!(timeout_tx[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT); } else { - assert_eq!(timeout_tx[0].lock_time, 0); + assert_eq!(timeout_tx[0].lock_time.0, 0); } // We fail non-dust-HTLC 2 by broadcast of local timeout/revocation-claim tx mine_transaction(&nodes[0], &timeout_tx[0]); @@ -7755,7 +7756,7 @@ fn test_bump_penalty_txn_on_revoked_commitment() { // Actually revoke tx by claiming a HTLC claim_payment(&nodes[0], &vec!(&nodes[1])[..], payment_preimage); - let header = BlockHeader { version: 0x20000000, prev_blockhash: header_114, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: header_114, merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[1], &Block { header, txdata: vec![revoked_txn[0].clone()] }); check_added_monitors!(nodes[1], 1); @@ -7855,7 +7856,7 @@ fn test_bump_penalty_txn_on_revoked_htlcs() { // Revoke local commitment tx claim_payment(&nodes[0], &vec!(&nodes[1])[..], payment_preimage); - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; // B will generate both revoked HTLC-timeout/HTLC-preimage txn from revoked commitment tx connect_block(&nodes[1], &Block { header, txdata: vec![revoked_local_txn[0].clone()] }); check_closed_broadcast!(nodes[1], true); @@ -7878,9 +7879,9 @@ fn test_bump_penalty_txn_on_revoked_htlcs() { // Broadcast set of revoked txn on A let hash_128 = connect_blocks(&nodes[0], 40); - let header_11 = BlockHeader { version: 0x20000000, prev_blockhash: hash_128, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header_11 = BlockHeader { version: 0x20000000, prev_blockhash: hash_128, merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header: header_11, txdata: vec![revoked_local_txn[0].clone()] }); - let header_129 = BlockHeader { version: 0x20000000, prev_blockhash: header_11.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header_129 = BlockHeader { version: 0x20000000, prev_blockhash: header_11.block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header: header_129, txdata: vec![revoked_htlc_txn[0].clone(), revoked_htlc_txn[2].clone()] }); let events = nodes[0].node.get_and_clear_pending_events(); expect_pending_htlcs_forwardable_from_events!(nodes[0], events[0..1], true); @@ -7937,9 +7938,9 @@ fn test_bump_penalty_txn_on_revoked_htlcs() { } // Connect one more block to see if bumped penalty are issued for HTLC txn - let header_130 = BlockHeader { version: 0x20000000, prev_blockhash: header_129.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header_130 = BlockHeader { version: 0x20000000, prev_blockhash: header_129.block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header: header_130, txdata: penalty_txn }); - let header_131 = BlockHeader { version: 0x20000000, prev_blockhash: header_130.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header_131 = BlockHeader { version: 0x20000000, prev_blockhash: header_130.block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header: header_131, txdata: Vec::new() }); { let mut node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap(); @@ -7978,7 +7979,7 @@ fn test_bump_penalty_txn_on_revoked_htlcs() { txn }; // Broadcast claim txn and confirm blocks to avoid further bumps on this outputs - let header_145 = BlockHeader { version: 0x20000000, prev_blockhash: header_144, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header_145 = BlockHeader { version: 0x20000000, prev_blockhash: header_144, merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header: header_145, txdata: node_txn }); connect_blocks(&nodes[0], 20); { @@ -8193,7 +8194,7 @@ fn test_bump_txn_sanitize_tracking_maps() { node_txn.clear(); penalty_txn }; - let header_130 = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header_130 = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header: header_130, txdata: penalty_txn }); connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1); { @@ -8669,7 +8670,7 @@ fn test_secret_timeout() { header: BlockHeader { version: 0x2000000, prev_blockhash: node_1_blocks.last().unwrap().0.block_hash(), - merkle_root: Default::default(), + merkle_root: TxMerkleNode::all_zeros(), time: node_1_blocks.len() as u32 + 7200, bits: 42, nonce: 42 }, txdata: vec![], } @@ -8814,7 +8815,7 @@ fn test_update_err_monitor_lockdown() { assert!(watchtower.watch_channel(outpoint, new_monitor).is_ok()); watchtower }; - let header = BlockHeader { version: 0x20000000, prev_blockhash: Default::default(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; let block = Block { header, txdata: vec![] }; // Make the tx_broadcaster aware of enough blocks that it doesn't think we're violating // transaction lock time requirements here. @@ -8878,7 +8879,7 @@ fn test_concurrent_monitor_claim() { assert!(watchtower.watch_channel(outpoint, new_monitor).is_ok()); watchtower }; - let header = BlockHeader { version: 0x20000000, prev_blockhash: Default::default(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; let block = Block { header, txdata: vec![] }; // Make the tx_broadcaster aware of enough blocks that it doesn't think we're violating // transaction lock time requirements here. @@ -8907,7 +8908,7 @@ fn test_concurrent_monitor_claim() { assert!(watchtower.watch_channel(outpoint, new_monitor).is_ok()); watchtower }; - let header = BlockHeader { version: 0x20000000, prev_blockhash: Default::default(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; watchtower_bob.chain_monitor.block_connected(&Block { header, txdata: vec![] }, CHAN_CONFIRM_DEPTH + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS); // Route another payment to generate another update with still previous HTLC pending @@ -8932,7 +8933,7 @@ fn test_concurrent_monitor_claim() { check_added_monitors!(nodes[0], 1); //// Provide one more block to watchtower Bob, expect broadcast of commitment and HTLC-Timeout - let header = BlockHeader { version: 0x20000000, prev_blockhash: Default::default(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; watchtower_bob.chain_monitor.block_connected(&Block { header, txdata: vec![] }, CHAN_CONFIRM_DEPTH + 1 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS); // Watchtower Bob should have broadcast a commitment/HTLC-timeout @@ -8945,7 +8946,7 @@ fn test_concurrent_monitor_claim() { }; // We confirm Bob's state Y on Alice, she should broadcast a HTLC-timeout - let header = BlockHeader { version: 0x20000000, prev_blockhash: Default::default(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; watchtower_alice.chain_monitor.block_connected(&Block { header, txdata: vec![bob_state_y.clone()] }, CHAN_CONFIRM_DEPTH + 2 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS); { let htlc_txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcasted.lock().unwrap(); @@ -9020,7 +9021,7 @@ fn test_htlc_no_detection() { check_spends!(local_txn[0], chan_1.3); // Timeout HTLC on A's chain and so it can generate a HTLC-Timeout tx - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header, txdata: vec![local_txn[0].clone()] }); // We deliberately connect the local tx twice as this should provoke a failure calling // this test before #653 fix. @@ -9038,7 +9039,7 @@ fn test_htlc_no_detection() { node_txn[1].clone() }; - let header_201 = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 }; + let header_201 = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }; connect_block(&nodes[0], &Block { header: header_201, txdata: vec![htlc_timeout.clone()] }); connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1); expect_payment_failed!(nodes[0], our_payment_hash, true); @@ -9099,7 +9100,7 @@ fn do_test_onchain_htlc_settlement_after_close(broadcast_alice: bool, go_onchain true => alice_txn.clone(), false => get_local_commitment_txn!(nodes[1], chan_ab.2) }; - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42}; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42}; connect_block(&nodes[1], &Block { header, txdata: vec![txn_to_broadcast[0].clone()]}); let mut bob_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); if broadcast_alice { @@ -9182,7 +9183,7 @@ fn do_test_onchain_htlc_settlement_after_close(broadcast_alice: bool, go_onchain let mut txn_to_broadcast = alice_txn.clone(); if !broadcast_alice { txn_to_broadcast = get_local_commitment_txn!(nodes[1], chan_ab.2); } if !go_onchain_before_fulfill { - let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42}; + let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42}; connect_block(&nodes[1], &Block { header, txdata: vec![txn_to_broadcast[0].clone()]}); // If Bob was the one to force-close, he will have already passed these checks earlier. if broadcast_alice { @@ -9514,14 +9515,14 @@ fn test_invalid_funding_tx() { // long the ChannelMonitor will try to read 32 bytes from the second-to-last element, panicing // as its not 32 bytes long. let mut spend_tx = Transaction { - version: 2i32, lock_time: 0, + version: 2i32, lock_time: PackedLockTime::ZERO, input: tx.output.iter().enumerate().map(|(idx, _)| TxIn { previous_output: BitcoinOutPoint { txid: tx.txid(), vout: idx as u32, }, script_sig: Script::new(), - sequence: 0xfffffffd, + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, witness: Witness::from_vec(channelmonitor::deliberately_bogus_accepted_htlc_witness()) }).collect(), output: vec![TxOut { @@ -10432,12 +10433,12 @@ fn test_non_final_funding_tx() { let chan_id = *nodes[0].network_chan_count.borrow(); let events = nodes[0].node.get_and_clear_pending_events(); - let input = TxIn { previous_output: BitcoinOutPoint::null(), script_sig: bitcoin::Script::new(), sequence: 0x1, witness: Witness::from_vec(vec!(vec!(1))) }; + let input = TxIn { previous_output: BitcoinOutPoint::null(), script_sig: bitcoin::Script::new(), sequence: Sequence(1), witness: Witness::from_vec(vec!(vec!(1))) }; assert_eq!(events.len(), 1); let mut tx = match events[0] { Event::FundingGenerationReady { ref channel_value_satoshis, ref output_script, .. } => { // Timelock the transaction _beyond_ the best client height + 2. - Transaction { version: chan_id as i32, lock_time: best_height + 3, input: vec![input], output: vec![TxOut { + Transaction { version: chan_id as i32, lock_time: PackedLockTime(best_height + 3), input: vec![input], output: vec![TxOut { value: *channel_value_satoshis, script_pubkey: output_script.clone(), }]} }, @@ -10452,7 +10453,7 @@ fn test_non_final_funding_tx() { } // However, transaction should be accepted if it's in a +2 headroom from best block. - tx.lock_time -= 1; + tx.lock_time = PackedLockTime(tx.lock_time.0 - 1); assert!(nodes[0].node.funding_transaction_generated(&temp_channel_id, &nodes[1].node.get_our_node_id(), tx.clone()).is_ok()); get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()); } diff --git a/lightning/src/ln/monitor_tests.rs b/lightning/src/ln/monitor_tests.rs index 4f36b9a8881..9be2059a705 100644 --- a/lightning/src/ln/monitor_tests.rs +++ b/lightning/src/ln/monitor_tests.rs @@ -663,7 +663,7 @@ fn test_balances_on_local_commitment_htlcs() { claimable_height: htlc_cltv_timeout, }]), sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); - assert_eq!(as_txn[1].lock_time, nodes[0].best_block_info().1 + 1); // as_txn[1] can be included in the next block + assert_eq!(as_txn[1].lock_time.0, nodes[0].best_block_info().1 + 1); // as_txn[1] can be included in the next block // Now confirm nodes[0]'s HTLC-Timeout transaction, which changes the claimable balance to an // "awaiting confirmations" one. diff --git a/lightning/src/ln/onion_utils.rs b/lightning/src/ln/onion_utils.rs index f81c619d735..3795ad5ee77 100644 --- a/lightning/src/ln/onion_utils.rs +++ b/lightning/src/ln/onion_utils.rs @@ -23,7 +23,7 @@ use bitcoin::hashes::cmp::fixed_time_eq; use bitcoin::hashes::hmac::{Hmac, HmacEngine}; use bitcoin::hashes::sha256::Hash as Sha256; -use bitcoin::secp256k1::{SecretKey,PublicKey}; +use bitcoin::secp256k1::{SecretKey, PublicKey, Scalar}; use bitcoin::secp256k1::Secp256k1; use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1; @@ -82,7 +82,7 @@ pub(super) fn gen_ammag_from_shared_secret(shared_secret: &[u8]) -> [u8; 32] { Hmac::from_engine(hmac).into_inner() } -pub(crate) fn next_hop_packet_pubkey(secp_ctx: &Secp256k1, mut packet_pubkey: PublicKey, packet_shared_secret: &[u8; 32]) -> Result { +pub(crate) fn next_hop_packet_pubkey(secp_ctx: &Secp256k1, packet_pubkey: PublicKey, packet_shared_secret: &[u8; 32]) -> Result { let blinding_factor = { let mut sha = Sha256::engine(); sha.input(&packet_pubkey.serialize()[..]); @@ -90,7 +90,7 @@ pub(crate) fn next_hop_packet_pubkey bo if script.is_p2pkh() || script.is_p2sh() || script.is_v0_p2wpkh() || script.is_v0_p2wsh() { true } else if features.supports_shutdown_anysegwit() { - script.is_witness_program() && script.as_bytes()[0] != SEGWIT_V0.into_u8() + script.is_witness_program() && script.as_bytes()[0] != SEGWIT_V0.to_u8() } else { false } diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 7eba3cdd254..2b2b1134577 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -13,7 +13,7 @@ use bitcoin::hashes::{Hash, HashEngine}; use bitcoin::hashes::hmac::{Hmac, HmacEngine}; use bitcoin::hashes::sha256::Hash as Sha256; -use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey}; +use bitcoin::secp256k1::{self, PublicKey, Scalar, Secp256k1, SecretKey}; use chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Recipient, Sign}; use ln::msgs; @@ -249,11 +249,13 @@ impl OnionMessenger Sha256::from_engine(sha).into_inner() }; let mut next_blinding_point = msg.blinding_point; - if let Err(e) = next_blinding_point.mul_assign(&self.secp_ctx, &blinding_factor[..]) { - log_trace!(self.logger, "Failed to compute next blinding point: {}", e); - return + match next_blinding_point.mul_tweak(&self.secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap()) { + Ok(bp) => bp, + Err(e) => { + log_trace!(self.logger, "Failed to compute next blinding point: {}", e); + return + } } - next_blinding_point }, }, onion_routing_packet: outgoing_packet, diff --git a/lightning/src/onion_message/utils.rs b/lightning/src/onion_message/utils.rs index 9b95183e74b..52cadf6c9db 100644 --- a/lightning/src/onion_message/utils.rs +++ b/lightning/src/onion_message/utils.rs @@ -12,7 +12,7 @@ use bitcoin::hashes::{Hash, HashEngine}; use bitcoin::hashes::hmac::{Hmac, HmacEngine}; use bitcoin::hashes::sha256::Hash as Sha256; -use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey}; +use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey, Scalar}; use bitcoin::secp256k1::ecdh::SharedSecret; use ln::onion_utils; @@ -43,9 +43,7 @@ pub(super) fn construct_keys_callback { let f = || { let mut channel_id = [0; 32]; - let mut transaction = Transaction{ version: 2, lock_time: 0, input: Vec::new(), output: Vec::new() }; + let mut transaction = Transaction{ version: 2, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: Vec::new() }; read_tlv_fields!(reader, { (0, channel_id, required), (2, transaction, required), diff --git a/lightning/src/util/macro_logger.rs b/lightning/src/util/macro_logger.rs index eadfdbdfc67..63496b28362 100644 --- a/lightning/src/util/macro_logger.rs +++ b/lightning/src/util/macro_logger.rs @@ -91,7 +91,7 @@ impl<'a> core::fmt::Display for DebugTx<'a> { fn fmt(&self, f: &mut core::fmt::Formatter) -> Result<(), core::fmt::Error> { if self.0.input.len() >= 1 && self.0.input.iter().any(|i| !i.witness.is_empty()) { if self.0.input.len() == 1 && self.0.input[0].witness.last().unwrap().len() == 71 && - (self.0.input[0].sequence >> 8*3) as u8 == 0x80 { + (self.0.input[0].sequence.0 >> 8*3) as u8 == 0x80 { write!(f, "commitment tx ")?; } else if self.0.input.len() == 1 && self.0.input[0].witness.last().unwrap().len() == 71 { write!(f, "closing tx ")?; diff --git a/lightning/src/util/ser.rs b/lightning/src/util/ser.rs index ecf85839a5a..bec915bae4e 100644 --- a/lightning/src/util/ser.rs +++ b/lightning/src/util/ser.rs @@ -860,7 +860,7 @@ macro_rules! impl_consensus_ser { ($bitcoin_type: ty) => { impl Writeable for $bitcoin_type { fn write(&self, writer: &mut W) -> Result<(), io::Error> { - match self.consensus_encode(WriterWriteAdaptor(writer)) { + match self.consensus_encode(&mut WriterWriteAdaptor(writer)) { Ok(_) => Ok(()), Err(e) => Err(e), } diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index 2f80fb73a41..4dcbc3e5b38 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -51,6 +51,7 @@ use chain::keysinterface::{InMemorySigner, Recipient, KeyMaterial}; #[cfg(feature = "std")] use std::time::{SystemTime, UNIX_EPOCH}; +use bitcoin::Sequence; pub struct TestVecWriter(pub Vec); impl Writer for TestVecWriter { @@ -241,10 +242,11 @@ impl TestBroadcaster { impl chaininterface::BroadcasterInterface for TestBroadcaster { fn broadcast_transaction(&self, tx: &Transaction) { - assert!(tx.lock_time < 1_500_000_000); - if tx.lock_time > self.blocks.lock().unwrap().len() as u32 + 1 && tx.lock_time < 500_000_000 { + let lock_time = tx.lock_time.0; + assert!(lock_time < 1_500_000_000); + if lock_time > self.blocks.lock().unwrap().len() as u32 + 1 && lock_time < 500_000_000 { for inp in tx.input.iter() { - if inp.sequence != 0xffffffff { + if inp.sequence != Sequence::MAX { panic!("We should never broadcast a transaction before its locktime ({})!", tx.lock_time); } } diff --git a/lightning/src/util/transaction_utils.rs b/lightning/src/util/transaction_utils.rs index 12768543783..028a08345ed 100644 --- a/lightning/src/util/transaction_utils.rs +++ b/lightning/src/util/transaction_utils.rs @@ -56,7 +56,7 @@ pub(crate) fn maybe_add_change_output(tx: &mut Transaction, input_value: u64, wi weight_with_change += (VarInt(tx.output.len() as u64 + 1).len() - VarInt(tx.output.len() as u64).len()) as i64 * 4; // When calculating weight, add two for the flag bytes let change_value: i64 = (input_value - output_value) as i64 - weight_with_change * feerate_sat_per_1000_weight as i64 / 1000; - if change_value >= dust_value.as_sat() as i64 { + if change_value >= dust_value.to_sat() as i64 { change_output.value = change_value as u64; tx.output.push(change_output); Ok(weight_with_change as usize) @@ -75,9 +75,8 @@ mod tests { use bitcoin::blockdata::script::{Script, Builder}; use bitcoin::hash_types::{PubkeyHash, Txid}; - use bitcoin::hashes::sha256d::Hash as Sha256dHash; use bitcoin::hashes::Hash; - use bitcoin::Witness; + use bitcoin::{PackedLockTime, Sequence, Witness}; use hex::decode; @@ -215,7 +214,7 @@ mod tests { #[test] fn test_tx_value_overrun() { // If we have a bogus input amount or outputs valued more than inputs, we should fail - let mut tx = Transaction { version: 2, lock_time: 0, input: Vec::new(), output: vec![TxOut { + let mut tx = Transaction { version: 2, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: vec![TxOut { script_pubkey: Script::new(), value: 1000 }] }; assert!(maybe_add_change_output(&mut tx, 21_000_000_0000_0001, 0, 253, Script::new()).is_err()); @@ -226,10 +225,10 @@ mod tests { #[test] fn test_tx_change_edge() { // Check that we never add dust outputs - let mut tx = Transaction { version: 2, lock_time: 0, input: Vec::new(), output: Vec::new() }; + let mut tx = Transaction { version: 2, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: Vec::new() }; let orig_wtxid = tx.wtxid(); let output_spk = Script::new_p2pkh(&PubkeyHash::hash(&[0; 0])); - assert_eq!(output_spk.dust_value().as_sat(), 546); + assert_eq!(output_spk.dust_value().to_sat(), 546); // 9 sats isn't enough to pay fee on a dummy transaction... assert_eq!(tx.weight() as u64, 40); // ie 10 vbytes assert!(maybe_add_change_output(&mut tx, 9, 0, 250, output_spk.clone()).is_err()); @@ -260,8 +259,8 @@ mod tests { #[test] fn test_tx_extra_outputs() { // Check that we correctly handle existing outputs - let mut tx = Transaction { version: 2, lock_time: 0, input: vec![TxIn { - previous_output: OutPoint::new(Txid::from_hash(Sha256dHash::default()), 0), script_sig: Script::new(), witness: Witness::new(), sequence: 0, + let mut tx = Transaction { version: 2, lock_time: PackedLockTime::ZERO, input: vec![TxIn { + previous_output: OutPoint::new(Txid::all_zeros(), 0), script_sig: Script::new(), witness: Witness::new(), sequence: Sequence::ZERO, }], output: vec![TxOut { script_pubkey: Builder::new().push_int(1).into_script(), value: 1000 }] }; @@ -269,7 +268,7 @@ mod tests { let orig_weight = tx.weight(); assert_eq!(orig_weight / 4, 61); - assert_eq!(Builder::new().push_int(2).into_script().dust_value().as_sat(), 474); + assert_eq!(Builder::new().push_int(2).into_script().dust_value().to_sat(), 474); // Input value of the output value + fee - 1 should fail: assert!(maybe_add_change_output(&mut tx, 1000 + 61 + 100 - 1, 400, 250, Builder::new().push_int(2).into_script()).is_err()); From 11166aa83623c7e80015d50dbfc9fc8529f969a2 Mon Sep 17 00:00:00 2001 From: Devrandom Date: Wed, 10 Aug 2022 18:04:59 +0200 Subject: [PATCH 43/91] Modify ecdh to take Scalar --- fuzz/src/chanmon_consistency.rs | 4 ++-- fuzz/src/full_stack.rs | 4 ++-- lightning/src/chain/keysinterface.rs | 10 +++++----- lightning/src/ln/channel.rs | 4 ++-- lightning/src/onion_message/messenger.rs | 2 +- lightning/src/util/test_utils.rs | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/fuzz/src/chanmon_consistency.rs b/fuzz/src/chanmon_consistency.rs index 9b3f1cd365b..372bed60493 100644 --- a/fuzz/src/chanmon_consistency.rs +++ b/fuzz/src/chanmon_consistency.rs @@ -168,10 +168,10 @@ impl KeysInterface for KeyProvider { Ok(SecretKey::from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, self.node_id]).unwrap()) } - fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&Scalar>) -> Result { let mut node_secret = self.get_node_secret(recipient)?; if let Some(tweak) = tweak { - node_secret = node_secret.mul_tweak(&Scalar::from_be_bytes(*tweak).unwrap()).unwrap(); + node_secret = node_secret.mul_tweak(tweak).unwrap(); } Ok(SharedSecret::new(other_key, &node_secret)) } diff --git a/fuzz/src/full_stack.rs b/fuzz/src/full_stack.rs index b65d2a34a95..c1d797ea579 100644 --- a/fuzz/src/full_stack.rs +++ b/fuzz/src/full_stack.rs @@ -272,10 +272,10 @@ impl KeysInterface for KeyProvider { Ok(self.node_secret.clone()) } - fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&Scalar>) -> Result { let mut node_secret = self.get_node_secret(recipient)?; if let Some(tweak) = tweak { - node_secret = node_secret.mul_tweak(&Scalar::from_be_bytes(*tweak).unwrap()).unwrap(); + node_secret = node_secret.mul_tweak(tweak).unwrap(); } Ok(SharedSecret::new(other_key, &node_secret)) } diff --git a/lightning/src/chain/keysinterface.rs b/lightning/src/chain/keysinterface.rs index 000c497d8b7..73b8a1b9822 100644 --- a/lightning/src/chain/keysinterface.rs +++ b/lightning/src/chain/keysinterface.rs @@ -410,7 +410,7 @@ pub trait KeysInterface { /// secret, though this is less efficient. /// /// [`node secret`]: Self::get_node_secret - fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result; + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&Scalar>) -> Result; /// Get a script pubkey which we send funds to when claiming on-chain contestable outputs. /// /// This method should return a different value each time it is called, to avoid linking @@ -1140,10 +1140,10 @@ impl KeysInterface for KeysManager { } } - fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&Scalar>) -> Result { let mut node_secret = self.get_node_secret(recipient)?; if let Some(tweak) = tweak { - node_secret = node_secret.mul_tweak(&Scalar::from_be_bytes(*tweak).unwrap()).map_err(|_| ())?; + node_secret = node_secret.mul_tweak(tweak).map_err(|_| ())?; } Ok(SharedSecret::new(other_key, &node_secret)) } @@ -1232,10 +1232,10 @@ impl KeysInterface for PhantomKeysManager { } } - fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&Scalar>) -> Result { let mut node_secret = self.get_node_secret(recipient)?; if let Some(tweak) = tweak { - node_secret = node_secret.mul_tweak(&Scalar::from_be_bytes(*tweak).unwrap()).map_err(|_| ())?; + node_secret = node_secret.mul_tweak(tweak).map_err(|_| ())?; } Ok(SharedSecret::new(other_key, &node_secret)) } diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index 7e978d9407c..624f4d6b688 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -6603,7 +6603,7 @@ mod tests { use util::errors::APIError; use util::test_utils; use util::test_utils::OnGetShutdownScriptpubkey; - use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature}; + use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature, Scalar}; use bitcoin::secp256k1::ffi::Signature as FFISignature; use bitcoin::secp256k1::{SecretKey,PublicKey}; use bitcoin::secp256k1::ecdh::SharedSecret; @@ -6648,7 +6648,7 @@ mod tests { type Signer = InMemorySigner; fn get_node_secret(&self, _recipient: Recipient) -> Result { panic!(); } - fn ecdh(&self, _recipient: Recipient, _other_key: &PublicKey, _tweak: Option<&[u8; 32]>) -> Result { panic!(); } + fn ecdh(&self, _recipient: Recipient, _other_key: &PublicKey, _tweak: Option<&Scalar>) -> Result { panic!(); } fn get_inbound_payment_key_material(&self) -> KeyMaterial { panic!(); } fn get_destination_script(&self) -> Script { let secp_ctx = Secp256k1::signing_only(); diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 2b2b1134577..04424896197 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -196,7 +196,7 @@ impl OnionMessenger Hmac::from_engine(hmac).into_inner() }; match self.keys_manager.ecdh(Recipient::Node, &msg.onion_routing_packet.public_key, - Some(&blinding_factor)) + Some(&Scalar::from_be_bytes(blinding_factor).unwrap())) { Ok(ss) => ss.secret_bytes(), Err(()) => { diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index 4dcbc3e5b38..4dcbde4236a 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -34,7 +34,7 @@ use bitcoin::blockdata::block::Block; use bitcoin::network::constants::Network; use bitcoin::hash_types::{BlockHash, Txid}; -use bitcoin::secp256k1::{SecretKey, PublicKey, Secp256k1, ecdsa::Signature}; +use bitcoin::secp256k1::{SecretKey, PublicKey, Secp256k1, ecdsa::Signature, Scalar}; use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; @@ -75,7 +75,7 @@ impl keysinterface::KeysInterface for OnlyReadsKeysInterface { type Signer = EnforcingSigner; fn get_node_secret(&self, _recipient: Recipient) -> Result { unreachable!(); } - fn ecdh(&self, _recipient: Recipient, _other_key: &PublicKey, _tweak: Option<&[u8; 32]>) -> Result { unreachable!(); } + fn ecdh(&self, _recipient: Recipient, _other_key: &PublicKey, _tweak: Option<&Scalar>) -> Result { unreachable!(); } fn get_inbound_payment_key_material(&self) -> KeyMaterial { unreachable!(); } fn get_destination_script(&self) -> Script { unreachable!(); } fn get_shutdown_scriptpubkey(&self) -> ShutdownScript { unreachable!(); } @@ -602,7 +602,7 @@ impl keysinterface::KeysInterface for TestKeysInterface { fn get_node_secret(&self, recipient: Recipient) -> Result { self.backing.get_node_secret(recipient) } - fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&[u8; 32]>) -> Result { + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&Scalar>) -> Result { self.backing.ecdh(recipient, other_key, tweak) } fn get_inbound_payment_key_material(&self) -> keysinterface::KeyMaterial { From a23681fde10b08d7a838c27ffb851915ccf784ad Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 5 Aug 2022 22:19:50 -0400 Subject: [PATCH 44/91] Fix bug in onion payment payload decode Prior to this change, we could have failed to decode a valid payload of size >253. This is because we were decoding the length (a BigSize, big-endian) as a VarInt (little-endian). Found in #1652. --- lightning/src/ln/msgs.rs | 53 +++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 9886bfeea5e..4db15257189 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -42,7 +42,7 @@ use io_extras::read_to_end; use util::events::MessageSendEventsProvider; use util::logger; -use util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt, Hostname}; +use util::ser::{BigSize, LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt, Hostname}; use ln::{PaymentPreimage, PaymentHash, PaymentSecret}; @@ -1418,16 +1418,11 @@ impl Writeable for OnionHopData { } impl Readable for OnionHopData { - fn read(mut r: &mut R) -> Result { - use bitcoin::consensus::encode::{Decodable, Error, VarInt}; - let v: VarInt = Decodable::consensus_decode(&mut r) - .map_err(|e| match e { - Error::Io(ioe) => DecodeError::from(ioe), - _ => DecodeError::InvalidValue - })?; + fn read(r: &mut R) -> Result { + let b: BigSize = Readable::read(r)?; const LEGACY_ONION_HOP_FLAG: u64 = 0; - let (format, amt, cltv_value) = if v.0 != LEGACY_ONION_HOP_FLAG { - let mut rd = FixedLengthReader::new(r, v.0); + let (format, amt, cltv_value) = if b.0 != LEGACY_ONION_HOP_FLAG { + let mut rd = FixedLengthReader::new(r, b.0); let mut amt = HighZeroBytesDroppedVarInt(0u64); let mut cltv_value = HighZeroBytesDroppedVarInt(0u32); let mut short_id: Option = None; @@ -1913,7 +1908,7 @@ mod tests { use bitcoin::secp256k1::{PublicKey,SecretKey}; use bitcoin::secp256k1::{Secp256k1, Message}; - use io::Cursor; + use io::{self, Cursor}; use prelude::*; use core::convert::TryFrom; @@ -2824,4 +2819,40 @@ mod tests { assert_eq!(gossip_timestamp_filter.first_timestamp, 1590000000); assert_eq!(gossip_timestamp_filter.timestamp_range, 0xffff_ffff); } + + #[test] + fn decode_onion_hop_data_len_as_bigsize() { + // Tests that we can decode an onion payload that is >253 bytes. + // Previously, receiving a payload of this size could've caused us to fail to decode a valid + // payload, because we were decoding the length (a BigSize, big-endian) as a VarInt + // (little-endian). + + // Encode a test onion payload with a big custom TLV such that it's >253 bytes, forcing the + // payload length to be encoded over multiple bytes rather than a single u8. + let big_payload = encode_big_payload().unwrap(); + let mut rd = Cursor::new(&big_payload[..]); + ::read(&mut rd).unwrap(); + } + // see above test, needs to be a separate method for use of the serialization macros. + fn encode_big_payload() -> Result, io::Error> { + use util::ser::HighZeroBytesDroppedVarInt; + let payload = msgs::OnionHopData { + format: OnionHopDataFormat::NonFinalNode { + short_channel_id: 0xdeadbeef1bad1dea, + }, + amt_to_forward: 1000, + outgoing_cltv_value: 0xffffffff, + }; + let mut encoded_payload = Vec::new(); + let test_bytes = vec![42u8; 1000]; + if let OnionHopDataFormat::NonFinalNode { short_channel_id } = payload.format { + encode_varint_length_prefixed_tlv!(&mut encoded_payload, { + (1, test_bytes, vec_type), + (2, HighZeroBytesDroppedVarInt(payload.amt_to_forward), required), + (4, HighZeroBytesDroppedVarInt(payload.outgoing_cltv_value), required), + (6, short_channel_id, required) + }); + } + Ok(encoded_payload) + } } From dfbebbf4c31135791dd402a4fc64abc7e0b8e158 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Sat, 6 Aug 2022 23:06:19 -0400 Subject: [PATCH 45/91] Rename HighZeroBytesDroppedVarInt to HighZeroBytesDroppedBigSize As observed by @wpaulino, this struct encodes its bytes as big-endian, therefore it's a BigSize, not a VarInt. --- lightning/src/ln/msgs.rs | 24 ++++++++++++------------ lightning/src/util/ser.rs | 10 +++++----- lightning/src/util/ser_macros.rs | 10 +++++----- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 4db15257189..95c7e61b674 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -42,7 +42,7 @@ use io_extras::read_to_end; use util::events::MessageSendEventsProvider; use util::logger; -use util::ser::{BigSize, LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt, Hostname}; +use util::ser::{BigSize, LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname}; use ln::{PaymentPreimage, PaymentHash, PaymentSecret}; @@ -1375,14 +1375,14 @@ impl Writeable for OnionMessage { impl Writeable for FinalOnionHopData { fn write(&self, w: &mut W) -> Result<(), io::Error> { self.payment_secret.0.write(w)?; - HighZeroBytesDroppedVarInt(self.total_msat).write(w) + HighZeroBytesDroppedBigSize(self.total_msat).write(w) } } impl Readable for FinalOnionHopData { fn read(r: &mut R) -> Result { let secret: [u8; 32] = Readable::read(r)?; - let amt: HighZeroBytesDroppedVarInt = Readable::read(r)?; + let amt: HighZeroBytesDroppedBigSize = Readable::read(r)?; Ok(Self { payment_secret: PaymentSecret(secret), total_msat: amt.0 }) } } @@ -1399,15 +1399,15 @@ impl Writeable for OnionHopData { }, OnionHopDataFormat::NonFinalNode { short_channel_id } => { encode_varint_length_prefixed_tlv!(w, { - (2, HighZeroBytesDroppedVarInt(self.amt_to_forward), required), - (4, HighZeroBytesDroppedVarInt(self.outgoing_cltv_value), required), + (2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required), + (4, HighZeroBytesDroppedBigSize(self.outgoing_cltv_value), required), (6, short_channel_id, required) }); }, OnionHopDataFormat::FinalNode { ref payment_data, ref keysend_preimage } => { encode_varint_length_prefixed_tlv!(w, { - (2, HighZeroBytesDroppedVarInt(self.amt_to_forward), required), - (4, HighZeroBytesDroppedVarInt(self.outgoing_cltv_value), required), + (2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required), + (4, HighZeroBytesDroppedBigSize(self.outgoing_cltv_value), required), (8, payment_data, option), (5482373484, keysend_preimage, option) }); @@ -1423,8 +1423,8 @@ impl Readable for OnionHopData { const LEGACY_ONION_HOP_FLAG: u64 = 0; let (format, amt, cltv_value) = if b.0 != LEGACY_ONION_HOP_FLAG { let mut rd = FixedLengthReader::new(r, b.0); - let mut amt = HighZeroBytesDroppedVarInt(0u64); - let mut cltv_value = HighZeroBytesDroppedVarInt(0u32); + let mut amt = HighZeroBytesDroppedBigSize(0u64); + let mut cltv_value = HighZeroBytesDroppedBigSize(0u32); let mut short_id: Option = None; let mut payment_data: Option = None; let mut keysend_preimage: Option = None; @@ -2835,7 +2835,7 @@ mod tests { } // see above test, needs to be a separate method for use of the serialization macros. fn encode_big_payload() -> Result, io::Error> { - use util::ser::HighZeroBytesDroppedVarInt; + use util::ser::HighZeroBytesDroppedBigSize; let payload = msgs::OnionHopData { format: OnionHopDataFormat::NonFinalNode { short_channel_id: 0xdeadbeef1bad1dea, @@ -2848,8 +2848,8 @@ mod tests { if let OnionHopDataFormat::NonFinalNode { short_channel_id } = payload.format { encode_varint_length_prefixed_tlv!(&mut encoded_payload, { (1, test_bytes, vec_type), - (2, HighZeroBytesDroppedVarInt(payload.amt_to_forward), required), - (4, HighZeroBytesDroppedVarInt(payload.outgoing_cltv_value), required), + (2, HighZeroBytesDroppedBigSize(payload.amt_to_forward), required), + (4, HighZeroBytesDroppedBigSize(payload.outgoing_cltv_value), required), (6, short_channel_id, required) }); } diff --git a/lightning/src/util/ser.rs b/lightning/src/util/ser.rs index ecf85839a5a..333c552d20f 100644 --- a/lightning/src/util/ser.rs +++ b/lightning/src/util/ser.rs @@ -400,7 +400,7 @@ impl Readable for BigSize { /// variable-length integer which is simply truncated by skipping high zero bytes. This type /// encapsulates such integers implementing Readable/Writeable for them. #[cfg_attr(test, derive(PartialEq, Debug))] -pub(crate) struct HighZeroBytesDroppedVarInt(pub T); +pub(crate) struct HighZeroBytesDroppedBigSize(pub T); macro_rules! impl_writeable_primitive { ($val_type:ty, $len: expr) => { @@ -410,7 +410,7 @@ macro_rules! impl_writeable_primitive { writer.write_all(&self.to_be_bytes()) } } - impl Writeable for HighZeroBytesDroppedVarInt<$val_type> { + impl Writeable for HighZeroBytesDroppedBigSize<$val_type> { #[inline] fn write(&self, writer: &mut W) -> Result<(), io::Error> { // Skip any full leading 0 bytes when writing (in BE): @@ -425,9 +425,9 @@ macro_rules! impl_writeable_primitive { Ok(<$val_type>::from_be_bytes(buf)) } } - impl Readable for HighZeroBytesDroppedVarInt<$val_type> { + impl Readable for HighZeroBytesDroppedBigSize<$val_type> { #[inline] - fn read(reader: &mut R) -> Result, DecodeError> { + fn read(reader: &mut R) -> Result, DecodeError> { // We need to accept short reads (read_len == 0) as "EOF" and handle them as simply // the high bytes being dropped. To do so, we start reading into the middle of buf // and then convert the appropriate number of bytes with extra high bytes out of @@ -443,7 +443,7 @@ macro_rules! impl_writeable_primitive { let first_byte = $len - ($len - total_read_len); let mut bytes = [0; $len]; bytes.copy_from_slice(&buf[first_byte..first_byte + $len]); - Ok(HighZeroBytesDroppedVarInt(<$val_type>::from_be_bytes(bytes))) + Ok(HighZeroBytesDroppedBigSize(<$val_type>::from_be_bytes(bytes))) } else { // If the encoding had extra zero bytes, return a failure even though we know // what they meant (as the TLV test vectors require this) diff --git a/lightning/src/util/ser_macros.rs b/lightning/src/util/ser_macros.rs index 351c2a1f5e2..94990fcb8a1 100644 --- a/lightning/src/util/ser_macros.rs +++ b/lightning/src/util/ser_macros.rs @@ -563,7 +563,7 @@ mod tests { use io::{self, Cursor}; use prelude::*; use ln::msgs::DecodeError; - use util::ser::{Writeable, HighZeroBytesDroppedVarInt, VecWriter}; + use util::ser::{Writeable, HighZeroBytesDroppedBigSize, VecWriter}; use bitcoin::secp256k1::PublicKey; // The BOLT TLV test cases don't include any tests which use our "required-value" logic since @@ -632,9 +632,9 @@ mod tests { } // BOLT TLV test cases - fn tlv_reader_n1(s: &[u8]) -> Result<(Option>, Option, Option<(PublicKey, u64, u64)>, Option), DecodeError> { + fn tlv_reader_n1(s: &[u8]) -> Result<(Option>, Option, Option<(PublicKey, u64, u64)>, Option), DecodeError> { let mut s = Cursor::new(s); - let mut tlv1: Option> = None; + let mut tlv1: Option> = None; let mut tlv2: Option = None; let mut tlv3: Option<(PublicKey, u64, u64)> = None; let mut tlv4: Option = None; @@ -765,11 +765,11 @@ mod tests { assert_eq!(stream.0, ::hex::decode("06fd00ff02abcd").unwrap()); stream.0.clear(); - encode_varint_length_prefixed_tlv!(&mut stream, {(0, 1u64, required), (42, None::, option), (0xff, HighZeroBytesDroppedVarInt(0u64), required)}); + encode_varint_length_prefixed_tlv!(&mut stream, {(0, 1u64, required), (42, None::, option), (0xff, HighZeroBytesDroppedBigSize(0u64), required)}); assert_eq!(stream.0, ::hex::decode("0e00080000000000000001fd00ff00").unwrap()); stream.0.clear(); - encode_varint_length_prefixed_tlv!(&mut stream, {(0, Some(1u64), option), (0xff, HighZeroBytesDroppedVarInt(0u64), required)}); + encode_varint_length_prefixed_tlv!(&mut stream, {(0, Some(1u64), option), (0xff, HighZeroBytesDroppedBigSize(0u64), required)}); assert_eq!(stream.0, ::hex::decode("0e00080000000000000001fd00ff00").unwrap()); Ok(()) From 81b7b03d4ff7424e61b073b81cf4edf627c8694b Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 5 Aug 2022 13:15:57 -0400 Subject: [PATCH 46/91] Fix fuzzer-found underflow --- lightning/src/onion_message/packet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightning/src/onion_message/packet.rs b/lightning/src/onion_message/packet.rs index a3414d844ed..d4ba28c843b 100644 --- a/lightning/src/onion_message/packet.rs +++ b/lightning/src/onion_message/packet.rs @@ -69,7 +69,7 @@ impl LengthReadable for Packet { let public_key = Readable::read(r)?; let mut hop_data = Vec::new(); - let hop_data_len = r.total_bytes() as usize - 66; // 1 (version) + 33 (pubkey) + 32 (HMAC) = 66 + let hop_data_len = r.total_bytes().saturating_sub(66) as usize; // 1 (version) + 33 (pubkey) + 32 (HMAC) = 66 let mut read_idx = 0; while read_idx < hop_data_len { let mut read_buffer = [0; READ_BUFFER_SIZE]; From 2f4457fd7e188587cdba04ce1651ecf2db0c9fbf Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 11 Aug 2022 14:27:45 +0200 Subject: [PATCH 47/91] Drop return value from `Filter::register_output` This commit removes the return value from `Filter::register_output` as creating a suitable value almost always entails blocking operations (e.g., lookups via network request), which however conflicts with the requirement that user calls should avoid blocking calls at all cost. Removing the return value also rendered quite a bit of test code for dependent transaction handling superfluous, which is therefore also removed with this commit. --- lightning/src/chain/chainmonitor.rs | 161 +++++++++------------------- lightning/src/chain/mod.rs | 17 ++- lightning/src/util/test_utils.rs | 69 +----------- 3 files changed, 59 insertions(+), 188 deletions(-) diff --git a/lightning/src/chain/chainmonitor.rs b/lightning/src/chain/chainmonitor.rs index 5c4ede0b161..2b3326472f9 100644 --- a/lightning/src/chain/chainmonitor.rs +++ b/lightning/src/chain/chainmonitor.rs @@ -263,82 +263,67 @@ where C::Target: chain::Filter, where FN: Fn(&ChannelMonitor, &TransactionData) -> Vec { - let mut dependent_txdata = Vec::new(); - { - let monitor_states = self.monitors.write().unwrap(); - if let Some(height) = best_height { - // If the best block height is being updated, update highest_chain_height under the - // monitors write lock. - let old_height = self.highest_chain_height.load(Ordering::Acquire); - let new_height = height as usize; - if new_height > old_height { - self.highest_chain_height.store(new_height, Ordering::Release); - } + let monitor_states = self.monitors.write().unwrap(); + if let Some(height) = best_height { + // If the best block height is being updated, update highest_chain_height under the + // monitors write lock. + let old_height = self.highest_chain_height.load(Ordering::Acquire); + let new_height = height as usize; + if new_height > old_height { + self.highest_chain_height.store(new_height, Ordering::Release); } + } - for (funding_outpoint, monitor_state) in monitor_states.iter() { - let monitor = &monitor_state.monitor; - let mut txn_outputs; - { - txn_outputs = process(monitor, txdata); - let update_id = MonitorUpdateId { - contents: UpdateOrigin::ChainSync(self.sync_persistence_id.get_increment()), - }; - let mut pending_monitor_updates = monitor_state.pending_monitor_updates.lock().unwrap(); - if let Some(height) = best_height { - if !monitor_state.has_pending_chainsync_updates(&pending_monitor_updates) { - // If there are not ChainSync persists awaiting completion, go ahead and - // set last_chain_persist_height here - we wouldn't want the first - // TemporaryFailure to always immediately be considered "overly delayed". - monitor_state.last_chain_persist_height.store(height as usize, Ordering::Release); - } + for (funding_outpoint, monitor_state) in monitor_states.iter() { + let monitor = &monitor_state.monitor; + let mut txn_outputs; + { + txn_outputs = process(monitor, txdata); + let update_id = MonitorUpdateId { + contents: UpdateOrigin::ChainSync(self.sync_persistence_id.get_increment()), + }; + let mut pending_monitor_updates = monitor_state.pending_monitor_updates.lock().unwrap(); + if let Some(height) = best_height { + if !monitor_state.has_pending_chainsync_updates(&pending_monitor_updates) { + // If there are not ChainSync persists awaiting completion, go ahead and + // set last_chain_persist_height here - we wouldn't want the first + // TemporaryFailure to always immediately be considered "overly delayed". + monitor_state.last_chain_persist_height.store(height as usize, Ordering::Release); } + } - log_trace!(self.logger, "Syncing Channel Monitor for channel {}", log_funding_info!(monitor)); - match self.persister.update_persisted_channel(*funding_outpoint, &None, monitor, update_id) { - Ok(()) => - log_trace!(self.logger, "Finished syncing Channel Monitor for channel {}", log_funding_info!(monitor)), - Err(ChannelMonitorUpdateErr::PermanentFailure) => { - monitor_state.channel_perm_failed.store(true, Ordering::Release); - self.pending_monitor_events.lock().unwrap().push((*funding_outpoint, vec![MonitorEvent::UpdateFailed(*funding_outpoint)], monitor.get_counterparty_node_id())); - }, - Err(ChannelMonitorUpdateErr::TemporaryFailure) => { - log_debug!(self.logger, "Channel Monitor sync for channel {} in progress, holding events until completion!", log_funding_info!(monitor)); - pending_monitor_updates.push(update_id); - }, - } + log_trace!(self.logger, "Syncing Channel Monitor for channel {}", log_funding_info!(monitor)); + match self.persister.update_persisted_channel(*funding_outpoint, &None, monitor, update_id) { + Ok(()) => + log_trace!(self.logger, "Finished syncing Channel Monitor for channel {}", log_funding_info!(monitor)), + Err(ChannelMonitorUpdateErr::PermanentFailure) => { + monitor_state.channel_perm_failed.store(true, Ordering::Release); + self.pending_monitor_events.lock().unwrap().push((*funding_outpoint, vec![MonitorEvent::UpdateFailed(*funding_outpoint)], monitor.get_counterparty_node_id())); + }, + Err(ChannelMonitorUpdateErr::TemporaryFailure) => { + log_debug!(self.logger, "Channel Monitor sync for channel {} in progress, holding events until completion!", log_funding_info!(monitor)); + pending_monitor_updates.push(update_id); + }, } + } - // Register any new outputs with the chain source for filtering, storing any dependent - // transactions from within the block that previously had not been included in txdata. - if let Some(ref chain_source) = self.chain_source { - let block_hash = header.block_hash(); - for (txid, mut outputs) in txn_outputs.drain(..) { - for (idx, output) in outputs.drain(..) { - // Register any new outputs with the chain source for filtering and recurse - // if it indicates that there are dependent transactions within the block - // that had not been previously included in txdata. - let output = WatchedOutput { - block_hash: Some(block_hash), - outpoint: OutPoint { txid, index: idx as u16 }, - script_pubkey: output.script_pubkey, - }; - if let Some(tx) = chain_source.register_output(output) { - dependent_txdata.push(tx); - } - } + // Register any new outputs with the chain source for filtering, storing any dependent + // transactions from within the block that previously had not been included in txdata. + if let Some(ref chain_source) = self.chain_source { + let block_hash = header.block_hash(); + for (txid, mut outputs) in txn_outputs.drain(..) { + for (idx, output) in outputs.drain(..) { + // Register any new outputs with the chain source for filtering + let output = WatchedOutput { + block_hash: Some(block_hash), + outpoint: OutPoint { txid, index: idx as u16 }, + script_pubkey: output.script_pubkey, + }; + chain_source.register_output(output) } } } } - - // Recursively call for any dependent transactions that were identified by the chain source. - if !dependent_txdata.is_empty() { - dependent_txdata.sort_unstable_by_key(|(index, _tx)| *index); - dependent_txdata.dedup_by_key(|(index, _tx)| *index); - let txdata: Vec<_> = dependent_txdata.iter().map(|(index, tx)| (*index, tx)).collect(); - self.process_chain_data(header, None, &txdata, process); // We skip the best height the second go-around - } } /// Creates a new `ChainMonitor` used to watch on-chain activity pertaining to channels. @@ -745,50 +730,6 @@ mod tests { use ln::msgs::ChannelMessageHandler; use util::errors::APIError; use util::events::{ClosureReason, MessageSendEvent, MessageSendEventsProvider}; - use util::test_utils::{OnRegisterOutput, TxOutReference}; - - /// Tests that in-block dependent transactions are processed by `block_connected` when not - /// included in `txdata` but returned by [`chain::Filter::register_output`]. For instance, - /// a (non-anchor) commitment transaction's HTLC output may be spent in the same block as the - /// commitment transaction itself. An Electrum client may filter the commitment transaction but - /// needs to return the HTLC transaction so it can be processed. - #[test] - fn connect_block_checks_dependent_transactions() { - let chanmon_cfgs = create_chanmon_cfgs(2); - let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - let channel = create_announced_chan_between_nodes( - &nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - - // Send a payment, saving nodes[0]'s revoked commitment and HTLC-Timeout transactions. - let (commitment_tx, htlc_tx) = { - let payment_preimage = route_payment(&nodes[0], &vec!(&nodes[1])[..], 5_000_000).0; - let mut txn = get_local_commitment_txn!(nodes[0], channel.2); - claim_payment(&nodes[0], &vec!(&nodes[1])[..], payment_preimage); - - assert_eq!(txn.len(), 2); - (txn.remove(0), txn.remove(0)) - }; - - // Set expectations on nodes[1]'s chain source to return dependent transactions. - let htlc_output = TxOutReference(commitment_tx.clone(), 0); - let to_local_output = TxOutReference(commitment_tx.clone(), 1); - let htlc_timeout_output = TxOutReference(htlc_tx.clone(), 0); - nodes[1].chain_source - .expect(OnRegisterOutput { with: htlc_output, returns: Some((1, htlc_tx)) }) - .expect(OnRegisterOutput { with: to_local_output, returns: None }) - .expect(OnRegisterOutput { with: htlc_timeout_output, returns: None }); - - // Notify nodes[1] that nodes[0]'s revoked commitment transaction was mined. The chain - // source should return the dependent HTLC transaction when the HTLC output is registered. - mine_transaction(&nodes[1], &commitment_tx); - - // Clean up so uninteresting assertions don't fail. - check_added_monitors!(nodes[1], 1); - nodes[1].node.get_and_clear_pending_msg_events(); - nodes[1].node.get_and_clear_pending_events(); - } #[test] fn test_async_ooo_offchain_updates() { diff --git a/lightning/src/chain/mod.rs b/lightning/src/chain/mod.rs index a0eb17c6be0..170f543e1dc 100644 --- a/lightning/src/chain/mod.rs +++ b/lightning/src/chain/mod.rs @@ -12,7 +12,7 @@ use bitcoin::blockdata::block::{Block, BlockHeader}; use bitcoin::blockdata::constants::genesis_block; use bitcoin::blockdata::script::Script; -use bitcoin::blockdata::transaction::{Transaction, TxOut}; +use bitcoin::blockdata::transaction::TxOut; use bitcoin::hash_types::{BlockHash, Txid}; use bitcoin::network::constants::Network; use bitcoin::secp256k1::PublicKey; @@ -333,21 +333,18 @@ pub trait Filter { /// Registers interest in spends of a transaction output. /// - /// Optionally, when `output.block_hash` is set, should return any transaction spending the - /// output that is found in the corresponding block along with its index. - /// - /// This return value is useful for Electrum clients in order to supply in-block descendant - /// transactions which otherwise were not included. This is not necessary for other clients if - /// such descendant transactions were already included (e.g., when a BIP 157 client provides the - /// full block). - fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)>; + /// Note that this method might be called during processing of a new block. You therefore need + /// to ensure that also dependent output spents within an already connected block are correctly + /// handled, e.g., by re-scanning the block in question whenever new outputs have been + /// registered mid-processing. + fn register_output(&self, output: WatchedOutput); } /// A transaction output watched by a [`ChannelMonitor`] for spends on-chain. /// /// Used to convey to a [`Filter`] such an output with a given spending condition. Any transaction /// spending the output must be given to [`ChannelMonitor::block_connected`] either directly or via -/// the return value of [`Filter::register_output`]. +/// [`Confirm::transactions_confirmed`]. /// /// If `block_hash` is `Some`, this indicates the output was created in the corresponding block and /// may have been spent there. See [`Filter::register_output`] for details. diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index 2f80fb73a41..c45585c65fa 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -728,7 +728,6 @@ pub struct TestChainSource { pub utxo_ret: Mutex>, pub watched_txn: Mutex>, pub watched_outputs: Mutex>, - expectations: Mutex>>, } impl TestChainSource { @@ -739,17 +738,8 @@ impl TestChainSource { utxo_ret: Mutex::new(Ok(TxOut { value: u64::max_value(), script_pubkey })), watched_txn: Mutex::new(HashSet::new()), watched_outputs: Mutex::new(HashSet::new()), - expectations: Mutex::new(None), } } - - /// Sets an expectation that [`chain::Filter::register_output`] is called. - pub fn expect(&self, expectation: OnRegisterOutput) -> &Self { - self.expectations.lock().unwrap() - .get_or_insert_with(|| VecDeque::new()) - .push_back(expectation); - self - } } impl chain::Access for TestChainSource { @@ -767,24 +757,8 @@ impl chain::Filter for TestChainSource { self.watched_txn.lock().unwrap().insert((*txid, script_pubkey.clone())); } - fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)> { - let dependent_tx = match &mut *self.expectations.lock().unwrap() { - None => None, - Some(expectations) => match expectations.pop_front() { - None => { - panic!("Unexpected register_output: {:?}", - (output.outpoint, output.script_pubkey)); - }, - Some(expectation) => { - assert_eq!(output.outpoint, expectation.outpoint()); - assert_eq!(&output.script_pubkey, expectation.script_pubkey()); - expectation.returns - }, - }, - }; - + fn register_output(&self, output: WatchedOutput) { self.watched_outputs.lock().unwrap().insert((output.outpoint, output.script_pubkey)); - dependent_tx } } @@ -793,47 +767,6 @@ impl Drop for TestChainSource { if panicking() { return; } - - if let Some(expectations) = &*self.expectations.lock().unwrap() { - if !expectations.is_empty() { - panic!("Unsatisfied expectations: {:?}", expectations); - } - } - } -} - -/// An expectation that [`chain::Filter::register_output`] was called with a transaction output and -/// returns an optional dependent transaction that spends the output in the same block. -pub struct OnRegisterOutput { - /// The transaction output to register. - pub with: TxOutReference, - - /// A dependent transaction spending the output along with its position in the block. - pub returns: Option<(usize, Transaction)>, -} - -/// A transaction output as identified by an index into a transaction's output list. -pub struct TxOutReference(pub Transaction, pub usize); - -impl OnRegisterOutput { - fn outpoint(&self) -> OutPoint { - let txid = self.with.0.txid(); - let index = self.with.1 as u16; - OutPoint { txid, index } - } - - fn script_pubkey(&self) -> &Script { - let index = self.with.1; - &self.with.0.output[index].script_pubkey - } -} - -impl core::fmt::Debug for OnRegisterOutput { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("OnRegisterOutput") - .field("outpoint", &self.outpoint()) - .field("script_pubkey", self.script_pubkey()) - .finish() } } From c562f3338b5a13caaed437c8e307b2947f98f2b9 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 11 Aug 2022 14:33:22 +0200 Subject: [PATCH 48/91] Clarify 'should' vs 'will' in `get_relevant_txids` --- lightning/src/chain/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lightning/src/chain/mod.rs b/lightning/src/chain/mod.rs index 170f543e1dc..42508569575 100644 --- a/lightning/src/chain/mod.rs +++ b/lightning/src/chain/mod.rs @@ -151,15 +151,15 @@ pub trait Confirm { /// in the event of a chain reorganization, it must not be called with a `header` that is no /// longer in the chain as of the last call to [`best_block_updated`]. /// - /// [chain order]: Confirm#Order + /// [chain order]: Confirm#order /// [`best_block_updated`]: Self::best_block_updated fn transactions_confirmed(&self, header: &BlockHeader, txdata: &TransactionData, height: u32); /// Processes a transaction that is no longer confirmed as result of a chain reorganization. /// /// Should be called for any transaction returned by [`get_relevant_txids`] if it has been - /// reorganized out of the best chain. Once called, the given transaction should not be returned - /// by [`get_relevant_txids`] unless it has been reconfirmed via [`transactions_confirmed`]. + /// reorganized out of the best chain. Once called, the given transaction will not be returned + /// by [`get_relevant_txids`], unless it has been reconfirmed via [`transactions_confirmed`]. /// /// [`get_relevant_txids`]: Self::get_relevant_txids /// [`transactions_confirmed`]: Self::transactions_confirmed @@ -173,9 +173,9 @@ pub trait Confirm { /// Returns transactions that should be monitored for reorganization out of the chain. /// - /// Should include any transactions passed to [`transactions_confirmed`] that have insufficient - /// confirmations to be safe from a chain reorganization. Should not include any transactions - /// passed to [`transaction_unconfirmed`] unless later reconfirmed. + /// Will include any transactions passed to [`transactions_confirmed`] that have insufficient + /// confirmations to be safe from a chain reorganization. Will not include any transactions + /// passed to [`transaction_unconfirmed`], unless later reconfirmed. /// /// May be called to determine the subset of transactions that must still be monitored for /// reorganization. Will be idempotent between calls but may change as a result of calls to the From 7717fa23a8cccdd31afa086f5c2487e9ba90bbfc Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 9 Aug 2022 21:26:16 +0000 Subject: [PATCH 49/91] Backfill gossip without buffering directly in LDK Instead of backfilling gossip by buffering (up to) ten messages at a time, only buffer one message at a time, as the peers' outbound socket buffer drains. This moves the outbound backfill messages out of `PeerHandler` and into the operating system buffer, where it arguably belongs. Not buffering causes us to walk the gossip B-Trees somewhat more often, but avoids allocating vecs for the responses. While its probably (without having benchmarked it) a net performance loss, it simplifies buffer tracking and leaves us with more room to play with the buffer sizing constants as we add onion message forwarding which is an important win. Note that because we change how often we check if we're out of messages to send before pinging, we slightly change how many messages are exchanged at once, impacting the `test_do_attempt_write_data` constants. --- lightning-net-tokio/src/lib.rs | 4 +- lightning/src/ln/functional_test_utils.rs | 12 ++-- lightning/src/ln/msgs.rs | 16 ++--- lightning/src/ln/peer_handler.rs | 65 +++++++++----------- lightning/src/routing/gossip.rs | 73 ++++++++++------------- lightning/src/util/test_utils.rs | 23 +++---- 6 files changed, 84 insertions(+), 109 deletions(-) diff --git a/lightning-net-tokio/src/lib.rs b/lightning-net-tokio/src/lib.rs index 645a7434e45..1f00fcb3ca0 100644 --- a/lightning-net-tokio/src/lib.rs +++ b/lightning-net-tokio/src/lib.rs @@ -562,8 +562,8 @@ mod tests { fn handle_node_announcement(&self, _msg: &NodeAnnouncement) -> Result { Ok(false) } fn handle_channel_announcement(&self, _msg: &ChannelAnnouncement) -> Result { Ok(false) } fn handle_channel_update(&self, _msg: &ChannelUpdate) -> Result { Ok(false) } - fn get_next_channel_announcements(&self, _starting_point: u64, _batch_amount: u8) -> Vec<(ChannelAnnouncement, Option, Option)> { Vec::new() } - fn get_next_node_announcements(&self, _starting_point: Option<&PublicKey>, _batch_amount: u8) -> Vec { Vec::new() } + fn get_next_channel_announcement(&self, _starting_point: u64) -> Option<(ChannelAnnouncement, Option, Option)> { None } + fn get_next_node_announcement(&self, _starting_point: Option<&PublicKey>) -> Option { None } fn peer_connected(&self, _their_node_id: &PublicKey, _init_msg: &Init) { } fn handle_reply_channel_range(&self, _their_node_id: &PublicKey, _msg: ReplyChannelRange) -> Result<(), LightningError> { Ok(()) } fn handle_reply_short_channel_ids_end(&self, _their_node_id: &PublicKey, _msg: ReplyShortChannelIdsEnd) -> Result<(), LightningError> { Ok(()) } diff --git a/lightning/src/ln/functional_test_utils.rs b/lightning/src/ln/functional_test_utils.rs index 54d199a26f8..aa2ad2177c4 100644 --- a/lightning/src/ln/functional_test_utils.rs +++ b/lightning/src/ln/functional_test_utils.rs @@ -318,20 +318,20 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> { ); let mut chan_progress = 0; loop { - let orig_announcements = self.gossip_sync.get_next_channel_announcements(chan_progress, 255); - let deserialized_announcements = gossip_sync.get_next_channel_announcements(chan_progress, 255); + let orig_announcements = self.gossip_sync.get_next_channel_announcement(chan_progress); + let deserialized_announcements = gossip_sync.get_next_channel_announcement(chan_progress); assert!(orig_announcements == deserialized_announcements); - chan_progress = match orig_announcements.last() { + chan_progress = match orig_announcements { Some(announcement) => announcement.0.contents.short_channel_id + 1, None => break, }; } let mut node_progress = None; loop { - let orig_announcements = self.gossip_sync.get_next_node_announcements(node_progress.as_ref(), 255); - let deserialized_announcements = gossip_sync.get_next_node_announcements(node_progress.as_ref(), 255); + let orig_announcements = self.gossip_sync.get_next_node_announcement(node_progress.as_ref()); + let deserialized_announcements = gossip_sync.get_next_node_announcement(node_progress.as_ref()); assert!(orig_announcements == deserialized_announcements); - node_progress = match orig_announcements.last() { + node_progress = match orig_announcements { Some(announcement) => Some(announcement.contents.node_id), None => break, }; diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 05a336c18ef..975b40c9c21 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -915,15 +915,15 @@ pub trait RoutingMessageHandler : MessageSendEventsProvider { /// Handle an incoming channel_update message, returning true if it should be forwarded on, /// false or returning an Err otherwise. fn handle_channel_update(&self, msg: &ChannelUpdate) -> Result; - /// Gets a subset of the channel announcements and updates required to dump our routing table - /// to a remote node, starting at the short_channel_id indicated by starting_point and - /// including the batch_amount entries immediately higher in numerical value than starting_point. - fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(ChannelAnnouncement, Option, Option)>; - /// Gets a subset of the node announcements required to dump our routing table to a remote node, - /// starting at the node *after* the provided publickey and including batch_amount entries - /// immediately higher (as defined by ::cmp) than starting_point. + /// Gets channel announcements and updates required to dump our routing table to a remote node, + /// starting at the short_channel_id indicated by starting_point and including announcements + /// for a single channel. + fn get_next_channel_announcement(&self, starting_point: u64) -> Option<(ChannelAnnouncement, Option, Option)>; + /// Gets a node announcement required to dump our routing table to a remote node, starting at + /// the node *after* the provided pubkey and including up to one announcement immediately + /// higher (as defined by ::cmp) than starting_point. /// If None is provided for starting_point, we start at the first node. - fn get_next_node_announcements(&self, starting_point: Option<&PublicKey>, batch_amount: u8) -> Vec; + fn get_next_node_announcement(&self, starting_point: Option<&PublicKey>) -> Option; /// Called when a connection is established with a peer. This can be used to /// perform routing table synchronization using a strategy defined by the /// implementor. diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 165d607f199..277eab58943 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -67,9 +67,9 @@ impl RoutingMessageHandler for IgnoringMessageHandler { fn handle_node_announcement(&self, _msg: &msgs::NodeAnnouncement) -> Result { Ok(false) } fn handle_channel_announcement(&self, _msg: &msgs::ChannelAnnouncement) -> Result { Ok(false) } fn handle_channel_update(&self, _msg: &msgs::ChannelUpdate) -> Result { Ok(false) } - fn get_next_channel_announcements(&self, _starting_point: u64, _batch_amount: u8) -> - Vec<(msgs::ChannelAnnouncement, Option, Option)> { Vec::new() } - fn get_next_node_announcements(&self, _starting_point: Option<&PublicKey>, _batch_amount: u8) -> Vec { Vec::new() } + fn get_next_channel_announcement(&self, _starting_point: u64) -> + Option<(msgs::ChannelAnnouncement, Option, Option)> { None } + fn get_next_node_announcement(&self, _starting_point: Option<&PublicKey>) -> Option { None } fn peer_connected(&self, _their_node_id: &PublicKey, _init: &msgs::Init) {} fn handle_reply_channel_range(&self, _their_node_id: &PublicKey, _msg: msgs::ReplyChannelRange) -> Result<(), LightningError> { Ok(()) } fn handle_reply_short_channel_ids_end(&self, _their_node_id: &PublicKey, _msg: msgs::ReplyShortChannelIdsEnd) -> Result<(), LightningError> { Ok(()) } @@ -383,19 +383,17 @@ impl Peer { } } - /// Returns the number of gossip messages we can fit in this peer's buffer. - fn gossip_buffer_slots_available(&self) -> usize { - OUTBOUND_BUFFER_LIMIT_READ_PAUSE.saturating_sub(self.pending_outbound_buffer.len()) - } - /// Returns whether we should be reading bytes from this peer, based on whether its outbound /// buffer still has space and we don't need to pause reads to get some writes out. fn should_read(&self) -> bool { self.pending_outbound_buffer.len() < OUTBOUND_BUFFER_LIMIT_READ_PAUSE } - fn should_backfill_gossip(&self) -> bool { - self.pending_outbound_buffer.len() < OUTBOUND_BUFFER_LIMIT_READ_PAUSE && + /// Determines if we should push additional gossip messages onto a peer's outbound buffer for + /// backfilling gossip data to the peer. This is checked every time the peer's buffer may have + /// been drained. + fn should_buffer_gossip_backfill(&self) -> bool { + self.pending_outbound_buffer.is_empty() && self.msgs_sent_since_pong < BUFFER_DRAIN_MSGS_PER_TICK } @@ -739,46 +737,39 @@ impl P fn do_attempt_write_data(&self, descriptor: &mut Descriptor, peer: &mut Peer) { while !peer.awaiting_write_event { - if peer.should_backfill_gossip() { + if peer.should_buffer_gossip_backfill() { match peer.sync_status { InitSyncTracker::NoSyncRequested => {}, InitSyncTracker::ChannelsSyncing(c) if c < 0xffff_ffff_ffff_ffff => { - let steps = ((peer.gossip_buffer_slots_available() + 2) / 3) as u8; - let all_messages = self.message_handler.route_handler.get_next_channel_announcements(c, steps); - for &(ref announce, ref update_a_option, ref update_b_option) in all_messages.iter() { - self.enqueue_message(peer, announce); - if let &Some(ref update_a) = update_a_option { - self.enqueue_message(peer, update_a); + if let Some((announce, update_a_option, update_b_option)) = + self.message_handler.route_handler.get_next_channel_announcement(c) + { + self.enqueue_message(peer, &announce); + if let Some(update_a) = update_a_option { + self.enqueue_message(peer, &update_a); } - if let &Some(ref update_b) = update_b_option { - self.enqueue_message(peer, update_b); + if let Some(update_b) = update_b_option { + self.enqueue_message(peer, &update_b); } peer.sync_status = InitSyncTracker::ChannelsSyncing(announce.contents.short_channel_id + 1); - } - if all_messages.is_empty() || all_messages.len() != steps as usize { + } else { peer.sync_status = InitSyncTracker::ChannelsSyncing(0xffff_ffff_ffff_ffff); } }, InitSyncTracker::ChannelsSyncing(c) if c == 0xffff_ffff_ffff_ffff => { - let steps = peer.gossip_buffer_slots_available() as u8; - let all_messages = self.message_handler.route_handler.get_next_node_announcements(None, steps); - for msg in all_messages.iter() { - self.enqueue_message(peer, msg); + if let Some(msg) = self.message_handler.route_handler.get_next_node_announcement(None) { + self.enqueue_message(peer, &msg); peer.sync_status = InitSyncTracker::NodesSyncing(msg.contents.node_id); - } - if all_messages.is_empty() || all_messages.len() != steps as usize { + } else { peer.sync_status = InitSyncTracker::NoSyncRequested; } }, InitSyncTracker::ChannelsSyncing(_) => unreachable!(), InitSyncTracker::NodesSyncing(key) => { - let steps = peer.gossip_buffer_slots_available() as u8; - let all_messages = self.message_handler.route_handler.get_next_node_announcements(Some(&key), steps); - for msg in all_messages.iter() { - self.enqueue_message(peer, msg); + if let Some(msg) = self.message_handler.route_handler.get_next_node_announcement(Some(&key)) { + self.enqueue_message(peer, &msg); peer.sync_status = InitSyncTracker::NodesSyncing(msg.contents.node_id); - } - if all_messages.is_empty() || all_messages.len() != steps as usize { + } else { peer.sync_status = InitSyncTracker::NoSyncRequested; } }, @@ -2082,10 +2073,10 @@ mod tests { // Check that each peer has received the expected number of channel updates and channel // announcements. - assert_eq!(cfgs[0].routing_handler.chan_upds_recvd.load(Ordering::Acquire), 100); - assert_eq!(cfgs[0].routing_handler.chan_anns_recvd.load(Ordering::Acquire), 50); - assert_eq!(cfgs[1].routing_handler.chan_upds_recvd.load(Ordering::Acquire), 100); - assert_eq!(cfgs[1].routing_handler.chan_anns_recvd.load(Ordering::Acquire), 50); + assert_eq!(cfgs[0].routing_handler.chan_upds_recvd.load(Ordering::Acquire), 108); + assert_eq!(cfgs[0].routing_handler.chan_anns_recvd.load(Ordering::Acquire), 54); + assert_eq!(cfgs[1].routing_handler.chan_upds_recvd.load(Ordering::Acquire), 108); + assert_eq!(cfgs[1].routing_handler.chan_anns_recvd.load(Ordering::Acquire), 54); } #[test] diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 1757d137300..11a302891c1 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -318,11 +318,10 @@ where C::Target: chain::Access, L::Target: Logger Ok(msg.contents.excess_data.len() <= MAX_EXCESS_BYTES_FOR_RELAY) } - fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(ChannelAnnouncement, Option, Option)> { - let mut result = Vec::with_capacity(batch_amount as usize); + fn get_next_channel_announcement(&self, starting_point: u64) -> Option<(ChannelAnnouncement, Option, Option)> { let channels = self.network_graph.channels.read().unwrap(); let mut iter = channels.range(starting_point..); - while result.len() < batch_amount as usize { + loop { if let Some((_, ref chan)) = iter.next() { if chan.announcement_message.is_some() { let chan_announcement = chan.announcement_message.clone().unwrap(); @@ -334,20 +333,18 @@ where C::Target: chain::Access, L::Target: Logger if let Some(two_to_one) = chan.two_to_one.as_ref() { two_to_one_announcement = two_to_one.last_update_message.clone(); } - result.push((chan_announcement, one_to_two_announcement, two_to_one_announcement)); + return Some((chan_announcement, one_to_two_announcement, two_to_one_announcement)); } else { // TODO: We may end up sending un-announced channel_updates if we are sending // initial sync data while receiving announce/updates for this channel. } } else { - return result; + return None; } } - result } - fn get_next_node_announcements(&self, starting_point: Option<&PublicKey>, batch_amount: u8) -> Vec { - let mut result = Vec::with_capacity(batch_amount as usize); + fn get_next_node_announcement(&self, starting_point: Option<&PublicKey>) -> Option { let nodes = self.network_graph.nodes.read().unwrap(); let mut iter = if let Some(pubkey) = starting_point { let mut iter = nodes.range(NodeId::from_pubkey(pubkey)..); @@ -356,18 +353,17 @@ where C::Target: chain::Access, L::Target: Logger } else { nodes.range::(..) }; - while result.len() < batch_amount as usize { + loop { if let Some((_, ref node)) = iter.next() { if let Some(node_info) = node.announcement_info.as_ref() { - if node_info.announcement_message.is_some() { - result.push(node_info.announcement_message.clone().unwrap()); + if let Some(msg) = node_info.announcement_message.clone() { + return Some(msg); } } } else { - return result; + return None; } } - result } /// Initiates a stateless sync of routing gossip information with a peer @@ -2412,8 +2408,8 @@ mod tests { let node_2_privkey = &SecretKey::from_slice(&[41; 32]).unwrap(); // Channels were not announced yet. - let channels_with_announcements = gossip_sync.get_next_channel_announcements(0, 1); - assert_eq!(channels_with_announcements.len(), 0); + let channels_with_announcements = gossip_sync.get_next_channel_announcement(0); + assert!(channels_with_announcements.is_none()); let short_channel_id; { @@ -2427,17 +2423,15 @@ mod tests { } // Contains initial channel announcement now. - let channels_with_announcements = gossip_sync.get_next_channel_announcements(short_channel_id, 1); - assert_eq!(channels_with_announcements.len(), 1); - if let Some(channel_announcements) = channels_with_announcements.first() { - let &(_, ref update_1, ref update_2) = channel_announcements; + let channels_with_announcements = gossip_sync.get_next_channel_announcement(short_channel_id); + if let Some(channel_announcements) = channels_with_announcements { + let (_, ref update_1, ref update_2) = channel_announcements; assert_eq!(update_1, &None); assert_eq!(update_2, &None); } else { panic!(); } - { // Valid channel update let valid_channel_update = get_signed_channel_update(|unsigned_channel_update| { @@ -2450,10 +2444,9 @@ mod tests { } // Now contains an initial announcement and an update. - let channels_with_announcements = gossip_sync.get_next_channel_announcements(short_channel_id, 1); - assert_eq!(channels_with_announcements.len(), 1); - if let Some(channel_announcements) = channels_with_announcements.first() { - let &(_, ref update_1, ref update_2) = channel_announcements; + let channels_with_announcements = gossip_sync.get_next_channel_announcement(short_channel_id); + if let Some(channel_announcements) = channels_with_announcements { + let (_, ref update_1, ref update_2) = channel_announcements; assert_ne!(update_1, &None); assert_eq!(update_2, &None); } else { @@ -2473,10 +2466,9 @@ mod tests { } // Test that announcements with excess data won't be returned - let channels_with_announcements = gossip_sync.get_next_channel_announcements(short_channel_id, 1); - assert_eq!(channels_with_announcements.len(), 1); - if let Some(channel_announcements) = channels_with_announcements.first() { - let &(_, ref update_1, ref update_2) = channel_announcements; + let channels_with_announcements = gossip_sync.get_next_channel_announcement(short_channel_id); + if let Some(channel_announcements) = channels_with_announcements { + let (_, ref update_1, ref update_2) = channel_announcements; assert_eq!(update_1, &None); assert_eq!(update_2, &None); } else { @@ -2484,8 +2476,8 @@ mod tests { } // Further starting point have no channels after it - let channels_with_announcements = gossip_sync.get_next_channel_announcements(short_channel_id + 1000, 1); - assert_eq!(channels_with_announcements.len(), 0); + let channels_with_announcements = gossip_sync.get_next_channel_announcement(short_channel_id + 1000); + assert!(channels_with_announcements.is_none()); } #[test] @@ -2497,8 +2489,8 @@ mod tests { let node_id_1 = PublicKey::from_secret_key(&secp_ctx, node_1_privkey); // No nodes yet. - let next_announcements = gossip_sync.get_next_node_announcements(None, 10); - assert_eq!(next_announcements.len(), 0); + let next_announcements = gossip_sync.get_next_node_announcement(None); + assert!(next_announcements.is_none()); { // Announce a channel to add 2 nodes @@ -2509,10 +2501,9 @@ mod tests { }; } - // Nodes were never announced - let next_announcements = gossip_sync.get_next_node_announcements(None, 3); - assert_eq!(next_announcements.len(), 0); + let next_announcements = gossip_sync.get_next_node_announcement(None); + assert!(next_announcements.is_none()); { let valid_announcement = get_signed_node_announcement(|_| {}, node_1_privkey, &secp_ctx); @@ -2528,12 +2519,12 @@ mod tests { }; } - let next_announcements = gossip_sync.get_next_node_announcements(None, 3); - assert_eq!(next_announcements.len(), 2); + let next_announcements = gossip_sync.get_next_node_announcement(None); + assert!(next_announcements.is_some()); // Skip the first node. - let next_announcements = gossip_sync.get_next_node_announcements(Some(&node_id_1), 2); - assert_eq!(next_announcements.len(), 1); + let next_announcements = gossip_sync.get_next_node_announcement(Some(&node_id_1)); + assert!(next_announcements.is_some()); { // Later announcement which should not be relayed (excess data) prevent us from sharing a node @@ -2547,8 +2538,8 @@ mod tests { }; } - let next_announcements = gossip_sync.get_next_node_announcements(Some(&node_id_1), 2); - assert_eq!(next_announcements.len(), 0); + let next_announcements = gossip_sync.get_next_node_announcement(Some(&node_id_1)); + assert!(next_announcements.is_none()); } #[test] diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index 2f80fb73a41..fe009cc16b8 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -45,7 +45,7 @@ use prelude::*; use core::time::Duration; use sync::{Mutex, Arc}; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use core::{cmp, mem}; +use core::mem; use bitcoin::bech32::u5; use chain::keysinterface::{InMemorySigner, Recipient, KeyMaterial}; @@ -445,23 +445,16 @@ impl msgs::RoutingMessageHandler for TestRoutingMessageHandler { self.chan_upds_recvd.fetch_add(1, Ordering::AcqRel); Err(msgs::LightningError { err: "".to_owned(), action: msgs::ErrorAction::IgnoreError }) } - fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(msgs::ChannelAnnouncement, Option, Option)> { - let mut chan_anns = Vec::new(); - const TOTAL_UPDS: u64 = 50; - let end: u64 = cmp::min(starting_point + batch_amount as u64, TOTAL_UPDS); - for i in starting_point..end { - let chan_upd_1 = get_dummy_channel_update(i); - let chan_upd_2 = get_dummy_channel_update(i); - let chan_ann = get_dummy_channel_announcement(i); + fn get_next_channel_announcement(&self, starting_point: u64) -> Option<(msgs::ChannelAnnouncement, Option, Option)> { + let chan_upd_1 = get_dummy_channel_update(starting_point); + let chan_upd_2 = get_dummy_channel_update(starting_point); + let chan_ann = get_dummy_channel_announcement(starting_point); - chan_anns.push((chan_ann, Some(chan_upd_1), Some(chan_upd_2))); - } - - chan_anns + Some((chan_ann, Some(chan_upd_1), Some(chan_upd_2))) } - fn get_next_node_announcements(&self, _starting_point: Option<&PublicKey>, _batch_amount: u8) -> Vec { - Vec::new() + fn get_next_node_announcement(&self, _starting_point: Option<&PublicKey>) -> Option { + None } fn peer_connected(&self, their_node_id: &PublicKey, init_msg: &msgs::Init) { From 9115f66dbc3824b2d655eb5440d122bdd1282eff Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Fri, 13 May 2022 05:11:14 +0000 Subject: [PATCH 50/91] Store the full event transaction in `OnchainEvent` structs When we see a transaction which generates some `OnchainEvent`, its useful to have the full transaction around for later analysis. Specifically, it lets us check the list of outputs which were spent in the transaction, allowing us to look up, e.g. which HTLC outpoint was spent in a transaction. This will be used in a few commits to do exactly that - figure out which HTLC a given `OnchainEvent` corresponds with. --- lightning/src/chain/channelmonitor.rs | 28 +++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 4bb08724c17..5cd031143be 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -315,6 +315,7 @@ struct OnchainEventEntry { txid: Txid, height: u32, event: OnchainEvent, + transaction: Option, // Added as optional, but always filled in, in LDK 0.0.110 } impl OnchainEventEntry { @@ -395,6 +396,7 @@ impl Writeable for OnchainEventEntry { fn write(&self, writer: &mut W) -> Result<(), io::Error> { write_tlv_fields!(writer, { (0, self.txid, required), + (1, self.transaction, option), (2, self.height, required), (4, self.event, required), }); @@ -405,15 +407,17 @@ impl Writeable for OnchainEventEntry { impl MaybeReadable for OnchainEventEntry { fn read(reader: &mut R) -> Result, DecodeError> { let mut txid = Txid::all_zeros(); + let mut transaction = None; let mut height = 0; let mut event = None; read_tlv_fields!(reader, { (0, txid, required), + (1, transaction, option), (2, height, required), (4, event, ignorable), }); if let Some(ev) = event { - Ok(Some(Self { txid, height, event: ev })) + Ok(Some(Self { txid, transaction, height, event: ev })) } else { Ok(None) } @@ -1683,8 +1687,10 @@ impl ChannelMonitor { /// as long as we examine both the current counterparty commitment transaction and, if it hasn't /// been revoked yet, the previous one, we we will never "forget" to resolve an HTLC. macro_rules! fail_unbroadcast_htlcs { - ($self: expr, $commitment_tx_type: expr, $commitment_txid_confirmed: expr, + ($self: expr, $commitment_tx_type: expr, $commitment_txid_confirmed: expr, $commitment_tx_confirmed: expr, $commitment_tx_conf_height: expr, $confirmed_htlcs_list: expr, $logger: expr) => { { + debug_assert_eq!($commitment_tx_confirmed.txid(), $commitment_txid_confirmed); + macro_rules! check_htlc_fails { ($txid: expr, $commitment_tx: expr) => { if let Some(ref latest_outpoints) = $self.counterparty_claimable_outpoints.get($txid) { @@ -1724,6 +1730,7 @@ macro_rules! fail_unbroadcast_htlcs { }); let entry = OnchainEventEntry { txid: $commitment_txid_confirmed, + transaction: Some($commitment_tx_confirmed.clone()), height: $commitment_tx_conf_height, event: OnchainEvent::HTLCUpdate { source: (**source).clone(), @@ -2155,13 +2162,13 @@ impl ChannelMonitorImpl { self.counterparty_commitment_txn_on_chain.insert(commitment_txid, commitment_number); if let Some(per_commitment_data) = per_commitment_option { - fail_unbroadcast_htlcs!(self, "revoked_counterparty", commitment_txid, height, + fail_unbroadcast_htlcs!(self, "revoked_counterparty", commitment_txid, tx, height, per_commitment_data.iter().map(|(htlc, htlc_source)| (htlc, htlc_source.as_ref().map(|htlc_source| htlc_source.as_ref())) ), logger); } else { debug_assert!(false, "We should have per-commitment option for any recognized old commitment txn"); - fail_unbroadcast_htlcs!(self, "revoked counterparty", commitment_txid, height, + fail_unbroadcast_htlcs!(self, "revoked counterparty", commitment_txid, tx, height, [].iter().map(|reference| *reference), logger); } } @@ -2179,7 +2186,7 @@ impl ChannelMonitorImpl { self.counterparty_commitment_txn_on_chain.insert(commitment_txid, commitment_number); log_info!(logger, "Got broadcast of non-revoked counterparty commitment transaction {}", commitment_txid); - fail_unbroadcast_htlcs!(self, "counterparty", commitment_txid, height, + fail_unbroadcast_htlcs!(self, "counterparty", commitment_txid, tx, height, per_commitment_data.iter().map(|(htlc, htlc_source)| (htlc, htlc_source.as_ref().map(|htlc_source| htlc_source.as_ref())) ), logger); @@ -2338,7 +2345,7 @@ impl ChannelMonitorImpl { let res = self.get_broadcasted_holder_claims(&self.current_holder_commitment_tx, height); let mut to_watch = self.get_broadcasted_holder_watch_outputs(&self.current_holder_commitment_tx, tx); append_onchain_update!(res, to_watch); - fail_unbroadcast_htlcs!(self, "latest holder", commitment_txid, height, + fail_unbroadcast_htlcs!(self, "latest holder", commitment_txid, tx, height, self.current_holder_commitment_tx.htlc_outputs.iter() .map(|(htlc, _, htlc_source)| (htlc, htlc_source.as_ref())), logger); } else if let &Some(ref holder_tx) = &self.prev_holder_signed_commitment_tx { @@ -2348,7 +2355,7 @@ impl ChannelMonitorImpl { let res = self.get_broadcasted_holder_claims(holder_tx, height); let mut to_watch = self.get_broadcasted_holder_watch_outputs(holder_tx, tx); append_onchain_update!(res, to_watch); - fail_unbroadcast_htlcs!(self, "previous holder", commitment_txid, height, + fail_unbroadcast_htlcs!(self, "previous holder", commitment_txid, tx, height, holder_tx.htlc_outputs.iter().map(|(htlc, _, htlc_source)| (htlc, htlc_source.as_ref())), logger); } @@ -2514,6 +2521,7 @@ impl ChannelMonitorImpl { let txid = tx.txid(); self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry { txid, + transaction: Some((*tx).clone()), height: height, event: OnchainEvent::FundingSpendConfirmation { on_local_output_csv: balance_spendable_csv, @@ -2932,7 +2940,7 @@ impl ChannelMonitorImpl { let outbound_htlc = $holder_tx == htlc_output.offered; if !outbound_htlc || revocation_sig_claim { self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry { - txid: tx.txid(), height, + txid: tx.txid(), height, transaction: Some(tx.clone()), event: OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx: input.previous_output.vout, preimage: if accepted_preimage_claim || offered_preimage_claim { @@ -2984,6 +2992,7 @@ impl ChannelMonitorImpl { self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry { txid: tx.txid(), height, + transaction: Some(tx.clone()), event: OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx: input.previous_output.vout, preimage: Some(payment_preimage), @@ -3004,6 +3013,7 @@ impl ChannelMonitorImpl { } else { false }) { self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry { txid: tx.txid(), + transaction: Some(tx.clone()), height, event: OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx: input.previous_output.vout, @@ -3030,6 +3040,7 @@ impl ChannelMonitorImpl { }); let entry = OnchainEventEntry { txid: tx.txid(), + transaction: Some(tx.clone()), height, event: OnchainEvent::HTLCUpdate { source, payment_hash, @@ -3103,6 +3114,7 @@ impl ChannelMonitorImpl { if let Some(spendable_output) = spendable_output { let entry = OnchainEventEntry { txid: tx.txid(), + transaction: Some(tx.clone()), height: height, event: OnchainEvent::MaturingOutput { descriptor: spendable_output.clone() }, }; From 0ec54ecd977732a83947668e3e6432c7b9dc0295 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sat, 30 Apr 2022 20:29:31 +0000 Subject: [PATCH 51/91] Track counterparty payout info in counterparty commitment txn When handling a revoked counterparty commitment transaction which was broadcast on-chain, we occasionally need to look up which output (and its value) was to the counterparty (the `to_self` output). This will allow us to generate `Balance`s for the user for the revoked output. --- lightning/src/chain/channelmonitor.rs | 95 +++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 5cd031143be..fa4ab601121 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -54,6 +54,7 @@ use util::events::Event; use prelude::*; use core::{cmp, mem}; use io::{self, Error}; +use core::convert::TryInto; use core::ops::Deref; use sync::Mutex; @@ -345,6 +346,11 @@ impl OnchainEventEntry { } } +/// The (output index, sats value) for the counterparty's output in a commitment transaction. +/// +/// This was added as an `Option` in 0.0.110. +type CommitmentTxCounterpartyOutputInfo = Option<(u32, u64)>; + /// Upon discovering of some classes of onchain tx by ChannelMonitor, we may have to take actions on it /// once they mature to enough confirmations (ANTI_REORG_DELAY) #[derive(PartialEq)] @@ -372,6 +378,9 @@ enum OnchainEvent { /// The CSV delay for the output of the funding spend transaction (implying it is a local /// commitment transaction, and this is the delay on the to_self output). on_local_output_csv: Option, + /// If the funding spend transaction was a known remote commitment transaction, we track + /// the output index and amount of the counterparty's `to_self` output here. + commitment_tx_to_counterparty_output: CommitmentTxCounterpartyOutputInfo, }, /// A spend of a commitment transaction HTLC output, set in the cases where *no* `HTLCUpdate` /// is constructed. This is used when @@ -436,6 +445,7 @@ impl_writeable_tlv_based_enum_upgradable!(OnchainEvent, }, (3, FundingSpendConfirmation) => { (0, on_local_output_csv, option), + (1, commitment_tx_to_counterparty_output, option), }, (5, HTLCSpendConfirmation) => { (0, commitment_tx_output_idx, required), @@ -714,6 +724,7 @@ pub(crate) struct ChannelMonitorImpl { funding_spend_seen: bool, funding_spend_confirmed: Option, + confirmed_commitment_tx_counterparty_output: CommitmentTxCounterpartyOutputInfo, /// The set of HTLCs which have been either claimed or failed on chain and have reached /// the requisite confirmations on the claim/fail transaction (either ANTI_REORG_DELAY or the /// spending CSV for revocable outputs). @@ -783,6 +794,7 @@ impl PartialEq for ChannelMonitorImpl { self.holder_tx_signed != other.holder_tx_signed || self.funding_spend_seen != other.funding_spend_seen || self.funding_spend_confirmed != other.funding_spend_confirmed || + self.confirmed_commitment_tx_counterparty_output != other.confirmed_commitment_tx_counterparty_output || self.htlcs_resolved_on_chain != other.htlcs_resolved_on_chain { false @@ -962,6 +974,7 @@ impl Writeable for ChannelMonitorImpl { (5, self.pending_monitor_events, vec_type), (7, self.funding_spend_seen, required), (9, self.counterparty_node_id, option), + (11, self.confirmed_commitment_tx_counterparty_output, option), }); Ok(()) @@ -1068,6 +1081,7 @@ impl ChannelMonitor { holder_tx_signed: false, funding_spend_seen: false, funding_spend_confirmed: None, + confirmed_commitment_tx_counterparty_output: None, htlcs_resolved_on_chain: Vec::new(), best_block, @@ -1918,7 +1932,7 @@ impl ChannelMonitorImpl { // First check if a counterparty commitment transaction has been broadcasted: macro_rules! claim_htlcs { ($commitment_number: expr, $txid: expr) => { - let htlc_claim_reqs = self.get_counterparty_htlc_output_claim_reqs($commitment_number, $txid, None); + let (htlc_claim_reqs, _) = self.get_counterparty_output_claim_info($commitment_number, $txid, None); self.onchain_tx_handler.update_claims_view(&Vec::new(), htlc_claim_reqs, self.best_block.height(), self.best_block.height(), broadcaster, fee_estimator, logger); } } @@ -2097,13 +2111,18 @@ impl ChannelMonitorImpl { /// data in counterparty_claimable_outpoints. Will directly claim any HTLC outputs which expire at a /// height > height + CLTV_SHARED_CLAIM_BUFFER. In any case, will install monitoring for /// HTLC-Success/HTLC-Timeout transactions. - /// Return updates for HTLC pending in the channel and failed automatically by the broadcast of - /// revoked counterparty commitment tx - fn check_spend_counterparty_transaction(&mut self, tx: &Transaction, height: u32, logger: &L) -> (Vec, TransactionOutputs) where L::Target: Logger { + /// + /// Returns packages to claim the revoked output(s), as well as additional outputs to watch and + /// general information about the output that is to the counterparty in the commitment + /// transaction. + fn check_spend_counterparty_transaction(&mut self, tx: &Transaction, height: u32, logger: &L) + -> (Vec, TransactionOutputs, CommitmentTxCounterpartyOutputInfo) + where L::Target: Logger { // Most secp and related errors trying to create keys means we have no hope of constructing // a spend transaction...so we return no transactions to broadcast let mut claimable_outpoints = Vec::new(); let mut watch_outputs = Vec::new(); + let mut to_counterparty_output_info = None; let commitment_txid = tx.txid(); //TODO: This is gonna be a performance bottleneck for watchtowers! let per_commitment_option = self.counterparty_claimable_outpoints.get(&commitment_txid); @@ -2112,7 +2131,7 @@ impl ChannelMonitorImpl { ( $thing : expr ) => { match $thing { Ok(a) => a, - Err(_) => return (claimable_outpoints, (commitment_txid, watch_outputs)) + Err(_) => return (claimable_outpoints, (commitment_txid, watch_outputs), to_counterparty_output_info) } }; } @@ -2134,6 +2153,8 @@ impl ChannelMonitorImpl { let revk_outp = RevokedOutput::build(per_commitment_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key, self.counterparty_commitment_params.counterparty_htlc_base_key, per_commitment_key, outp.value, self.counterparty_commitment_params.on_counterparty_tx_csv); let justice_package = PackageTemplate::build_package(commitment_txid, idx as u32, PackageSolvingData::RevokedOutput(revk_outp), height + self.counterparty_commitment_params.on_counterparty_tx_csv as u32, true, height); claimable_outpoints.push(justice_package); + to_counterparty_output_info = + Some((idx.try_into().expect("Txn can't have more than 2^32 outputs"), outp.value)); } } @@ -2143,7 +2164,9 @@ impl ChannelMonitorImpl { if let Some(transaction_output_index) = htlc.transaction_output_index { if transaction_output_index as usize >= tx.output.len() || tx.output[transaction_output_index as usize].value != htlc.amount_msat / 1000 { - return (claimable_outpoints, (commitment_txid, watch_outputs)); // Corrupted per_commitment_data, fuck this user + // per_commitment_data is corrupt or our commitment signing key leaked! + return (claimable_outpoints, (commitment_txid, watch_outputs), + to_counterparty_output_info); } let revk_htlc_outp = RevokedHTLCOutput::build(per_commitment_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key, self.counterparty_commitment_params.counterparty_htlc_base_key, per_commitment_key, htlc.amount_msat / 1000, htlc.clone(), self.onchain_tx_handler.channel_transaction_parameters.opt_anchors.is_some()); let justice_package = PackageTemplate::build_package(commitment_txid, transaction_output_index, PackageSolvingData::RevokedHTLCOutput(revk_htlc_outp), htlc.cltv_expiry, true, height); @@ -2191,17 +2214,22 @@ impl ChannelMonitorImpl { (htlc, htlc_source.as_ref().map(|htlc_source| htlc_source.as_ref())) ), logger); - let htlc_claim_reqs = self.get_counterparty_htlc_output_claim_reqs(commitment_number, commitment_txid, Some(tx)); + let (htlc_claim_reqs, counterparty_output_info) = + self.get_counterparty_output_claim_info(commitment_number, commitment_txid, Some(tx)); + to_counterparty_output_info = counterparty_output_info; for req in htlc_claim_reqs { claimable_outpoints.push(req); } } - (claimable_outpoints, (commitment_txid, watch_outputs)) + (claimable_outpoints, (commitment_txid, watch_outputs), to_counterparty_output_info) } - fn get_counterparty_htlc_output_claim_reqs(&self, commitment_number: u64, commitment_txid: Txid, tx: Option<&Transaction>) -> Vec { + /// Returns the HTLC claim package templates and the counterparty output info + fn get_counterparty_output_claim_info(&self, commitment_number: u64, commitment_txid: Txid, tx: Option<&Transaction>) + -> (Vec, CommitmentTxCounterpartyOutputInfo) { let mut claimable_outpoints = Vec::new(); + let mut to_counterparty_output_info: CommitmentTxCounterpartyOutputInfo = None; if let Some(htlc_outputs) = self.counterparty_claimable_outpoints.get(&commitment_txid) { if let Some(per_commitment_points) = self.their_cur_per_commitment_points { let per_commitment_point_option = @@ -2215,12 +2243,43 @@ impl ChannelMonitorImpl { if per_commitment_points.0 == commitment_number + 1 { Some(point) } else { None } } else { None }; if let Some(per_commitment_point) = per_commitment_point_option { + if let Some(transaction) = tx { + let revokeable_p2wsh_opt = + if let Ok(revocation_pubkey) = chan_utils::derive_public_revocation_key( + &self.secp_ctx, &per_commitment_point, &self.holder_revocation_basepoint) + { + if let Ok(delayed_key) = chan_utils::derive_public_key(&self.secp_ctx, + &per_commitment_point, + &self.counterparty_commitment_params.counterparty_delayed_payment_base_key) + { + Some(chan_utils::get_revokeable_redeemscript(&revocation_pubkey, + self.counterparty_commitment_params.on_counterparty_tx_csv, + &delayed_key).to_v0_p2wsh()) + } else { + debug_assert!(false, "Failed to derive a delayed payment key for a commitment state we accepted"); + None + } + } else { + debug_assert!(false, "Failed to derive a revocation pubkey key for a commitment state we accepted"); + None + }; + if let Some(revokeable_p2wsh) = revokeable_p2wsh_opt { + for (idx, outp) in transaction.output.iter().enumerate() { + if outp.script_pubkey == revokeable_p2wsh { + to_counterparty_output_info = + Some((idx.try_into().expect("Can't have > 2^32 outputs"), outp.value)); + } + } + } + } + for (_, &(ref htlc, _)) in htlc_outputs.iter().enumerate() { if let Some(transaction_output_index) = htlc.transaction_output_index { if let Some(transaction) = tx { if transaction_output_index as usize >= transaction.output.len() || transaction.output[transaction_output_index as usize].value != htlc.amount_msat / 1000 { - return claimable_outpoints; // Corrupted per_commitment_data, fuck this user + // per_commitment_data is corrupt or our commitment signing key leaked! + return (claimable_outpoints, to_counterparty_output_info); } } let preimage = if htlc.offered { if let Some(p) = self.payment_preimages.get(&htlc.payment_hash) { Some(*p) } else { None } } else { None }; @@ -2247,7 +2306,7 @@ impl ChannelMonitorImpl { } } } - claimable_outpoints + (claimable_outpoints, to_counterparty_output_info) } /// Attempts to claim a counterparty HTLC-Success/HTLC-Timeout's outputs using the revocation key @@ -2502,14 +2561,19 @@ impl ChannelMonitorImpl { log_info!(logger, "Channel {} closed by funding output spend in txid {}.", log_bytes!(self.funding_info.0.to_channel_id()), tx.txid()); self.funding_spend_seen = true; + let mut commitment_tx_to_counterparty_output = None; if (tx.input[0].sequence.0 >> 8*3) as u8 == 0x80 && (tx.lock_time.0 >> 8*3) as u8 == 0x20 { - let (mut new_outpoints, new_outputs) = self.check_spend_counterparty_transaction(&tx, height, &logger); + let (mut new_outpoints, new_outputs, counterparty_output_idx_sats) = + self.check_spend_counterparty_transaction(&tx, height, &logger); + commitment_tx_to_counterparty_output = counterparty_output_idx_sats; if !new_outputs.1.is_empty() { watch_outputs.push(new_outputs); } claimable_outpoints.append(&mut new_outpoints); if new_outpoints.is_empty() { if let Some((mut new_outpoints, new_outputs)) = self.check_spend_holder_transaction(&tx, height, &logger) { + debug_assert!(commitment_tx_to_counterparty_output.is_none(), + "A commitment transaction matched as both a counterparty and local commitment tx?"); if !new_outputs.1.is_empty() { watch_outputs.push(new_outputs); } @@ -2525,6 +2589,7 @@ impl ChannelMonitorImpl { height: height, event: OnchainEvent::FundingSpendConfirmation { on_local_output_csv: balance_spendable_csv, + commitment_tx_to_counterparty_output, }, }); } else { @@ -2661,8 +2726,9 @@ impl ChannelMonitorImpl { OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } => { self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC { commitment_tx_output_idx, payment_preimage: preimage }); }, - OnchainEvent::FundingSpendConfirmation { .. } => { + OnchainEvent::FundingSpendConfirmation { commitment_tx_to_counterparty_output, .. } => { self.funding_spend_confirmed = Some(entry.txid); + self.confirmed_commitment_tx_counterparty_output = commitment_tx_to_counterparty_output; }, } } @@ -3375,12 +3441,14 @@ impl<'a, Signer: Sign, K: KeysInterface> ReadableArgs<&'a K> let mut htlcs_resolved_on_chain = Some(Vec::new()); let mut funding_spend_seen = Some(false); let mut counterparty_node_id = None; + let mut confirmed_commitment_tx_counterparty_output = None; read_tlv_fields!(reader, { (1, funding_spend_confirmed, option), (3, htlcs_resolved_on_chain, vec_type), (5, pending_monitor_events, vec_type), (7, funding_spend_seen, option), (9, counterparty_node_id, option), + (11, confirmed_commitment_tx_counterparty_output, option), }); let mut secp_ctx = Secp256k1::new(); @@ -3431,6 +3499,7 @@ impl<'a, Signer: Sign, K: KeysInterface> ReadableArgs<&'a K> holder_tx_signed, funding_spend_seen: funding_spend_seen.unwrap(), funding_spend_confirmed, + confirmed_commitment_tx_counterparty_output, htlcs_resolved_on_chain: htlcs_resolved_on_chain.unwrap(), best_block, From 66ced68ec652ff84b654bd50919a70f866418089 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 14 Jul 2022 21:23:39 +0000 Subject: [PATCH 52/91] Clean up nesting in `get_counterparty_output_claim_info` --- lightning/src/chain/channelmonitor.rs | 148 ++++++++++++++------------ 1 file changed, 78 insertions(+), 70 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index fa4ab601121..4d109294cf0 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -2230,82 +2230,90 @@ impl ChannelMonitorImpl { -> (Vec, CommitmentTxCounterpartyOutputInfo) { let mut claimable_outpoints = Vec::new(); let mut to_counterparty_output_info: CommitmentTxCounterpartyOutputInfo = None; - if let Some(htlc_outputs) = self.counterparty_claimable_outpoints.get(&commitment_txid) { - if let Some(per_commitment_points) = self.their_cur_per_commitment_points { - let per_commitment_point_option = - // If the counterparty commitment tx is the latest valid state, use their latest - // per-commitment point - if per_commitment_points.0 == commitment_number { Some(&per_commitment_points.1) } - else if let Some(point) = per_commitment_points.2.as_ref() { - // If counterparty commitment tx is the state previous to the latest valid state, use - // their previous per-commitment point (non-atomicity of revocation means it's valid for - // them to temporarily have two valid commitment txns from our viewpoint) - if per_commitment_points.0 == commitment_number + 1 { Some(point) } else { None } - } else { None }; - if let Some(per_commitment_point) = per_commitment_point_option { - if let Some(transaction) = tx { - let revokeable_p2wsh_opt = - if let Ok(revocation_pubkey) = chan_utils::derive_public_revocation_key( - &self.secp_ctx, &per_commitment_point, &self.holder_revocation_basepoint) - { - if let Ok(delayed_key) = chan_utils::derive_public_key(&self.secp_ctx, - &per_commitment_point, - &self.counterparty_commitment_params.counterparty_delayed_payment_base_key) - { - Some(chan_utils::get_revokeable_redeemscript(&revocation_pubkey, - self.counterparty_commitment_params.on_counterparty_tx_csv, - &delayed_key).to_v0_p2wsh()) - } else { - debug_assert!(false, "Failed to derive a delayed payment key for a commitment state we accepted"); - None - } - } else { - debug_assert!(false, "Failed to derive a revocation pubkey key for a commitment state we accepted"); - None - }; - if let Some(revokeable_p2wsh) = revokeable_p2wsh_opt { - for (idx, outp) in transaction.output.iter().enumerate() { - if outp.script_pubkey == revokeable_p2wsh { - to_counterparty_output_info = - Some((idx.try_into().expect("Can't have > 2^32 outputs"), outp.value)); - } - } - } + + let htlc_outputs = match self.counterparty_claimable_outpoints.get(&commitment_txid) { + Some(outputs) => outputs, + None => return (claimable_outpoints, to_counterparty_output_info), + }; + let per_commitment_points = match self.their_cur_per_commitment_points { + Some(points) => points, + None => return (claimable_outpoints, to_counterparty_output_info), + }; + + let per_commitment_point = + // If the counterparty commitment tx is the latest valid state, use their latest + // per-commitment point + if per_commitment_points.0 == commitment_number { &per_commitment_points.1 } + else if let Some(point) = per_commitment_points.2.as_ref() { + // If counterparty commitment tx is the state previous to the latest valid state, use + // their previous per-commitment point (non-atomicity of revocation means it's valid for + // them to temporarily have two valid commitment txns from our viewpoint) + if per_commitment_points.0 == commitment_number + 1 { + point + } else { return (claimable_outpoints, to_counterparty_output_info); } + } else { return (claimable_outpoints, to_counterparty_output_info); }; + + if let Some(transaction) = tx { + let revokeable_p2wsh_opt = + if let Ok(revocation_pubkey) = chan_utils::derive_public_revocation_key( + &self.secp_ctx, &per_commitment_point, &self.holder_revocation_basepoint) + { + if let Ok(delayed_key) = chan_utils::derive_public_key(&self.secp_ctx, + &per_commitment_point, + &self.counterparty_commitment_params.counterparty_delayed_payment_base_key) + { + Some(chan_utils::get_revokeable_redeemscript(&revocation_pubkey, + self.counterparty_commitment_params.on_counterparty_tx_csv, + &delayed_key).to_v0_p2wsh()) + } else { + debug_assert!(false, "Failed to derive a delayed payment key for a commitment state we accepted"); + None + } + } else { + debug_assert!(false, "Failed to derive a revocation pubkey key for a commitment state we accepted"); + None + }; + if let Some(revokeable_p2wsh) = revokeable_p2wsh_opt { + for (idx, outp) in transaction.output.iter().enumerate() { + if outp.script_pubkey == revokeable_p2wsh { + to_counterparty_output_info = + Some((idx.try_into().expect("Can't have > 2^32 outputs"), outp.value)); } + } + } + } - for (_, &(ref htlc, _)) in htlc_outputs.iter().enumerate() { - if let Some(transaction_output_index) = htlc.transaction_output_index { - if let Some(transaction) = tx { - if transaction_output_index as usize >= transaction.output.len() || - transaction.output[transaction_output_index as usize].value != htlc.amount_msat / 1000 { - // per_commitment_data is corrupt or our commitment signing key leaked! - return (claimable_outpoints, to_counterparty_output_info); - } - } - let preimage = if htlc.offered { if let Some(p) = self.payment_preimages.get(&htlc.payment_hash) { Some(*p) } else { None } } else { None }; - if preimage.is_some() || !htlc.offered { - let counterparty_htlc_outp = if htlc.offered { - PackageSolvingData::CounterpartyOfferedHTLCOutput( - CounterpartyOfferedHTLCOutput::build(*per_commitment_point, - self.counterparty_commitment_params.counterparty_delayed_payment_base_key, - self.counterparty_commitment_params.counterparty_htlc_base_key, - preimage.unwrap(), htlc.clone())) - } else { - PackageSolvingData::CounterpartyReceivedHTLCOutput( - CounterpartyReceivedHTLCOutput::build(*per_commitment_point, - self.counterparty_commitment_params.counterparty_delayed_payment_base_key, - self.counterparty_commitment_params.counterparty_htlc_base_key, - htlc.clone())) - }; - let aggregation = if !htlc.offered { false } else { true }; - let counterparty_package = PackageTemplate::build_package(commitment_txid, transaction_output_index, counterparty_htlc_outp, htlc.cltv_expiry,aggregation, 0); - claimable_outpoints.push(counterparty_package); - } + for (_, &(ref htlc, _)) in htlc_outputs.iter().enumerate() { + if let Some(transaction_output_index) = htlc.transaction_output_index { + if let Some(transaction) = tx { + if transaction_output_index as usize >= transaction.output.len() || + transaction.output[transaction_output_index as usize].value != htlc.amount_msat / 1000 { + // per_commitment_data is corrupt or our commitment signing key leaked! + return (claimable_outpoints, to_counterparty_output_info); } - } + } + let preimage = if htlc.offered { if let Some(p) = self.payment_preimages.get(&htlc.payment_hash) { Some(*p) } else { None } } else { None }; + if preimage.is_some() || !htlc.offered { + let counterparty_htlc_outp = if htlc.offered { + PackageSolvingData::CounterpartyOfferedHTLCOutput( + CounterpartyOfferedHTLCOutput::build(*per_commitment_point, + self.counterparty_commitment_params.counterparty_delayed_payment_base_key, + self.counterparty_commitment_params.counterparty_htlc_base_key, + preimage.unwrap(), htlc.clone())) + } else { + PackageSolvingData::CounterpartyReceivedHTLCOutput( + CounterpartyReceivedHTLCOutput::build(*per_commitment_point, + self.counterparty_commitment_params.counterparty_delayed_payment_base_key, + self.counterparty_commitment_params.counterparty_htlc_base_key, + htlc.clone())) + }; + let aggregation = if !htlc.offered { false } else { true }; + let counterparty_package = PackageTemplate::build_package(commitment_txid, transaction_output_index, counterparty_htlc_outp, htlc.cltv_expiry,aggregation, 0); + claimable_outpoints.push(counterparty_package); } } } + (claimable_outpoints, to_counterparty_output_info) } From e76ac333304c4080f56f7931ec555d3a1f8269af Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 17 May 2022 23:57:52 +0000 Subject: [PATCH 53/91] Fix off-by-one in test_onchain_htlc_claim_reorg_remote_commitment The test intended to disconnect a transaction previously connected but didn't disconnect enough blocks to do so, leading to it confirming two conflicting transactions. In the next few commits this will become an assertion failure. --- lightning/src/ln/reorg_tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lightning/src/ln/reorg_tests.rs b/lightning/src/ln/reorg_tests.rs index cbf920f587d..e4b916c9345 100644 --- a/lightning/src/ln/reorg_tests.rs +++ b/lightning/src/ln/reorg_tests.rs @@ -130,7 +130,8 @@ fn do_test_onchain_htlc_reorg(local_commitment: bool, claim: bool) { assert_eq!(nodes[1].node.get_and_clear_pending_events().len(), 0); if claim { - disconnect_blocks(&nodes[1], ANTI_REORG_DELAY - 2); + // Disconnect Node 1's HTLC-Timeout which was connected above + disconnect_blocks(&nodes[1], ANTI_REORG_DELAY - 1); let block = Block { header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 }, From 9469ce0f3877843997f382da2e04e603335e37d7 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 19 May 2022 01:50:37 +0000 Subject: [PATCH 54/91] Track HTLC-Success/HTLC-Timeout claims of revoked outputs When a counterparty broadcasts a revoked commitment transaction, followed immediately by HTLC-Success/-Timeout spends thereof, we'd like to have an `onchain_events_awaiting_threshold_conf` entry for them. This does so using the `HTLCSpendConfirmation` entry, giving it (slightly) new meaning. Because all existing uses of `HTLCSpendConfirmation` already check if the relevant commitment transaction is revoked first, this should be trivially backwards compatible. We will ultimately figure out if something is being spent via the `OnchainTxHandler`, but to do so we need to look up the output via the HTLC transaction txid, which this allows us to do. --- lightning/src/chain/channelmonitor.rs | 44 ++++++++++++--------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 4d109294cf0..0d1c458bc2d 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -369,6 +369,8 @@ enum OnchainEvent { /// transaction which appeared on chain. commitment_tx_output_idx: Option, }, + /// An output waiting on [`ANTI_REORG_DELAY`] confirmations before we hand the user the + /// [`SpendableOutputDescriptor`]. MaturingOutput { descriptor: SpendableOutputDescriptor, }, @@ -390,6 +392,9 @@ enum OnchainEvent { /// * an inbound HTLC is claimed by us (with a preimage). /// * a revoked-state HTLC transaction was broadcasted, which was claimed by the revocation /// signature. + /// * a revoked-state HTLC transaction was broadcasted, which was claimed by an + /// HTLC-Success/HTLC-Failure transaction (and is still claimable with a revocation + /// signature). HTLCSpendConfirmation { commitment_tx_output_idx: u32, /// If the claim was made by either party with a preimage, this is filled in @@ -2965,7 +2970,7 @@ impl ChannelMonitorImpl { log_error!(logger, "Input spending {} ({}:{}) in {} resolves {} HTLC with payment hash {} with {}!", $tx_info, input.previous_output.txid, input.previous_output.vout, tx.txid(), if outbound_htlc { "outbound" } else { "inbound" }, log_bytes!($htlc.payment_hash.0), - if revocation_sig_claim { "revocation sig" } else { "preimage claim after we'd passed the HTLC resolution back" }); + if revocation_sig_claim { "revocation sig" } else { "preimage claim after we'd passed the HTLC resolution back. We can likely claim the HTLC output with a revocation claim" }); } else { log_info!(logger, "Input spending {} ({}:{}) in {} resolves {} HTLC with payment hash {} with {}", $tx_info, input.previous_output.txid, input.previous_output.vout, tx.txid(), @@ -3012,29 +3017,20 @@ impl ChannelMonitorImpl { if payment_data.is_none() { log_claim!($tx_info, $holder_tx, htlc_output, false); let outbound_htlc = $holder_tx == htlc_output.offered; - if !outbound_htlc || revocation_sig_claim { - self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry { - txid: tx.txid(), height, transaction: Some(tx.clone()), - event: OnchainEvent::HTLCSpendConfirmation { - commitment_tx_output_idx: input.previous_output.vout, - preimage: if accepted_preimage_claim || offered_preimage_claim { - Some(payment_preimage) } else { None }, - // If this is a payment to us (!outbound_htlc, above), - // wait for the CSV delay before dropping the HTLC from - // claimable balance if the claim was an HTLC-Success - // transaction. - on_to_local_output_csv: if accepted_preimage_claim { - Some(self.on_holder_tx_csv) } else { None }, - }, - }); - } else { - // Outbound claims should always have payment_data, unless - // we've already failed the HTLC as the commitment transaction - // which was broadcasted was revoked. In that case, we should - // spend the HTLC output here immediately, and expose that fact - // as a Balance, something which we do not yet do. - // TODO: Track the above as claimable! - } + self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry { + txid: tx.txid(), height, transaction: Some(tx.clone()), + event: OnchainEvent::HTLCSpendConfirmation { + commitment_tx_output_idx: input.previous_output.vout, + preimage: if accepted_preimage_claim || offered_preimage_claim { + Some(payment_preimage) } else { None }, + // If this is a payment to us (ie !outbound_htlc), wait for + // the CSV delay before dropping the HTLC from claimable + // balance if the claim was an HTLC-Success transaction (ie + // accepted_preimage_claim). + on_to_local_output_csv: if accepted_preimage_claim && !outbound_htlc { + Some(self.on_holder_tx_csv) } else { None }, + }, + }); continue 'outer_loop; } } From 04d8f5e3f11c156a862940856002971dcb7ac358 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sat, 21 May 2022 01:11:52 +0000 Subject: [PATCH 55/91] Track the txid that resolves HTLCs even after resolution completes We need this information when we look up if we still need to spend a revoked output from an HTLC-Success/HTLC-Timeout transaction for balance calculation. --- lightning/src/chain/channelmonitor.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 0d1c458bc2d..377657d6bca 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -588,12 +588,17 @@ pub enum Balance { #[derive(PartialEq)] struct IrrevocablyResolvedHTLC { commitment_tx_output_idx: u32, + /// The txid of the transaction which resolved the HTLC, this may be a commitment (if the HTLC + /// was not present in the confirmed commitment transaction), HTLC-Success, or HTLC-Timeout + /// transaction. + resolving_txid: Option, // Added as optional, but always filled in, in 0.0.110 /// Only set if the HTLC claim was ours using a payment preimage payment_preimage: Option, } impl_writeable_tlv_based!(IrrevocablyResolvedHTLC, { (0, commitment_tx_output_idx, required), + (1, resolving_txid, option), (2, payment_preimage, option), }); @@ -2727,7 +2732,10 @@ impl ChannelMonitorImpl { htlc_value_satoshis, })); if let Some(idx) = commitment_tx_output_idx { - self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC { commitment_tx_output_idx: idx, payment_preimage: None }); + self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC { + commitment_tx_output_idx: idx, resolving_txid: Some(entry.txid), + payment_preimage: None, + }); } }, OnchainEvent::MaturingOutput { descriptor } => { @@ -2737,7 +2745,10 @@ impl ChannelMonitorImpl { }); }, OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } => { - self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC { commitment_tx_output_idx, payment_preimage: preimage }); + self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC { + commitment_tx_output_idx, resolving_txid: Some(entry.txid), + payment_preimage: preimage, + }); }, OnchainEvent::FundingSpendConfirmation { commitment_tx_to_counterparty_output, .. } => { self.funding_spend_confirmed = Some(entry.txid); From a17794c3800c94fd23e767c1ee3ffd411d7ace81 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 17 May 2022 20:45:17 +0000 Subject: [PATCH 56/91] Scan `onchain_events_awaiting_threshold_conf` once in balance calc Instead of a series of different `onchain_events_awaiting_threshold_conf.iter()...` calls to scan for HTLC status in balance calculation, pull them all out into one `for ... { match ... }` to do it once and simplify the code somewhat. --- lightning/src/chain/channelmonitor.rs | 115 +++++++++++++++----------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 377657d6bca..f08895718f1 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1449,67 +1449,82 @@ impl ChannelMonitor { ($holder_commitment: expr, $htlc_iter: expr) => { for htlc in $htlc_iter { if let Some(htlc_commitment_tx_output_idx) = htlc.transaction_output_index { - if let Some(conf_thresh) = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { - if let OnchainEvent::MaturingOutput { descriptor: SpendableOutputDescriptor::DelayedPaymentOutput(descriptor) } = &event.event { - if descriptor.outpoint.index as u32 == htlc_commitment_tx_output_idx { Some(event.confirmation_threshold()) } else { None } - } else { None } - }) { + let mut htlc_update_pending = None; + let mut htlc_spend_pending = None; + let mut delayed_output_pending = None; + for event in us.onchain_events_awaiting_threshold_conf.iter() { + match event.event { + OnchainEvent::HTLCUpdate { commitment_tx_output_idx, htlc_value_satoshis, .. } + if commitment_tx_output_idx == Some(htlc_commitment_tx_output_idx) => { + debug_assert!(htlc_update_pending.is_none()); + htlc_update_pending = + Some((htlc_value_satoshis.unwrap(), event.confirmation_threshold())); + }, + OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } + if commitment_tx_output_idx == htlc_commitment_tx_output_idx => { + debug_assert!(htlc_spend_pending.is_none()); + htlc_spend_pending = Some((event.confirmation_threshold(), preimage.is_some())); + }, + OnchainEvent::MaturingOutput { + descriptor: SpendableOutputDescriptor::DelayedPaymentOutput(ref descriptor) } + if descriptor.outpoint.index as u32 == htlc_commitment_tx_output_idx => { + debug_assert!(delayed_output_pending.is_none()); + delayed_output_pending = Some(event.confirmation_threshold()); + }, + _ => {}, + } + } + let htlc_resolved = us.htlcs_resolved_on_chain.iter() + .find(|v| v.commitment_tx_output_idx == htlc_commitment_tx_output_idx); + debug_assert!(htlc_update_pending.is_some() as u8 + htlc_spend_pending.is_some() as u8 + htlc_resolved.is_some() as u8 <= 1); + + if let Some(conf_thresh) = delayed_output_pending { debug_assert!($holder_commitment); res.push(Balance::ClaimableAwaitingConfirmations { claimable_amount_satoshis: htlc.amount_msat / 1000, confirmation_height: conf_thresh, }); - } else if us.htlcs_resolved_on_chain.iter().any(|v| v.commitment_tx_output_idx == htlc_commitment_tx_output_idx) { + } else if htlc_resolved.is_some() { // Funding transaction spends should be fully confirmed by the time any // HTLC transactions are resolved, unless we're talking about a holder // commitment tx, whose resolution is delayed until the CSV timeout is // reached, even though HTLCs may be resolved after only // ANTI_REORG_DELAY confirmations. debug_assert!($holder_commitment || us.funding_spend_confirmed.is_some()); - } else if htlc.offered == $holder_commitment { - // If the payment was outbound, check if there's an HTLCUpdate - // indicating we have spent this HTLC with a timeout, claiming it back - // and awaiting confirmations on it. - let htlc_update_pending = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { - if let OnchainEvent::HTLCUpdate { commitment_tx_output_idx: Some(commitment_tx_output_idx), .. } = event.event { - if commitment_tx_output_idx == htlc_commitment_tx_output_idx { - Some(event.confirmation_threshold()) } else { None } - } else { None } - }); - if let Some(conf_thresh) = htlc_update_pending { - res.push(Balance::ClaimableAwaitingConfirmations { - claimable_amount_satoshis: htlc.amount_msat / 1000, - confirmation_height: conf_thresh, - }); - } else { - res.push(Balance::MaybeClaimableHTLCAwaitingTimeout { - claimable_amount_satoshis: htlc.amount_msat / 1000, - claimable_height: htlc.cltv_expiry, - }); - } - } else if us.payment_preimages.get(&htlc.payment_hash).is_some() { - // Otherwise (the payment was inbound), only expose it as claimable if - // we know the preimage. - // Note that if there is a pending claim, but it did not use the - // preimage, we lost funds to our counterparty! We will then continue - // to show it as ContentiousClaimable until ANTI_REORG_DELAY. - let htlc_spend_pending = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { - if let OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } = event.event { - if commitment_tx_output_idx == htlc_commitment_tx_output_idx { - Some((event.confirmation_threshold(), preimage.is_some())) - } else { None } - } else { None } - }); - if let Some((conf_thresh, true)) = htlc_spend_pending { - res.push(Balance::ClaimableAwaitingConfirmations { - claimable_amount_satoshis: htlc.amount_msat / 1000, - confirmation_height: conf_thresh, - }); - } else { - res.push(Balance::ContentiousClaimable { - claimable_amount_satoshis: htlc.amount_msat / 1000, - timeout_height: htlc.cltv_expiry, - }); + } else { + if htlc.offered == $holder_commitment { + // If the payment was outbound, check if there's an HTLCUpdate + // indicating we have spent this HTLC with a timeout, claiming it back + // and awaiting confirmations on it. + if let Some((value, conf_thresh)) = htlc_update_pending { + res.push(Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: value, + confirmation_height: conf_thresh, + }); + } else { + res.push(Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: htlc.amount_msat / 1000, + claimable_height: htlc.cltv_expiry, + }); + } + } else if us.payment_preimages.get(&htlc.payment_hash).is_some() { + // Otherwise (the payment was inbound), only expose it as claimable if + // we know the preimage. + // Note that if there is a pending claim, but it did not use the + // preimage, we lost funds to our counterparty! We will then continue + // to show it as ContentiousClaimable until ANTI_REORG_DELAY. + debug_assert!(htlc_update_pending.is_none()); + if let Some((conf_thresh, true)) = htlc_spend_pending { + res.push(Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: htlc.amount_msat / 1000, + confirmation_height: conf_thresh, + }); + } else { + res.push(Balance::ContentiousClaimable { + claimable_amount_satoshis: htlc.amount_msat / 1000, + timeout_height: htlc.cltv_expiry, + }); + } } } } From f712eb41572d98af2e234bb05013728138f3f4f5 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Wed, 13 Jul 2022 01:20:09 +0000 Subject: [PATCH 57/91] Simplify claimable balance trivially with consistent accessors --- lightning/src/chain/channelmonitor.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index f08895718f1..863ce4759e6 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1457,8 +1457,8 @@ impl ChannelMonitor { OnchainEvent::HTLCUpdate { commitment_tx_output_idx, htlc_value_satoshis, .. } if commitment_tx_output_idx == Some(htlc_commitment_tx_output_idx) => { debug_assert!(htlc_update_pending.is_none()); - htlc_update_pending = - Some((htlc_value_satoshis.unwrap(), event.confirmation_threshold())); + debug_assert_eq!(htlc_value_satoshis.unwrap(), htlc.amount_msat / 1000); + htlc_update_pending = Some(event.confirmation_threshold()); }, OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } if commitment_tx_output_idx == htlc_commitment_tx_output_idx => { @@ -1496,9 +1496,9 @@ impl ChannelMonitor { // If the payment was outbound, check if there's an HTLCUpdate // indicating we have spent this HTLC with a timeout, claiming it back // and awaiting confirmations on it. - if let Some((value, conf_thresh)) = htlc_update_pending { + if let Some(conf_thresh) = htlc_update_pending { res.push(Balance::ClaimableAwaitingConfirmations { - claimable_amount_satoshis: value, + claimable_amount_satoshis: htlc.amount_msat / 1000, confirmation_height: conf_thresh, }); } else { From 0ee88d029b9024b7e284b02d645746dc7123bc52 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 15 Aug 2022 18:08:13 +0000 Subject: [PATCH 58/91] Replace manual iteration with for loops in outbound gossip sync --- lightning/src/routing/gossip.rs | 55 ++++++++++++++------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 11a302891c1..925cd4cd4d3 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -41,7 +41,7 @@ use core::{cmp, fmt}; use sync::{RwLock, RwLockReadGuard}; use core::sync::atomic::{AtomicUsize, Ordering}; use sync::Mutex; -use core::ops::Deref; +use core::ops::{Bound, Deref}; use bitcoin::hashes::hex::ToHex; #[cfg(feature = "std")] @@ -320,50 +320,41 @@ where C::Target: chain::Access, L::Target: Logger fn get_next_channel_announcement(&self, starting_point: u64) -> Option<(ChannelAnnouncement, Option, Option)> { let channels = self.network_graph.channels.read().unwrap(); - let mut iter = channels.range(starting_point..); - loop { - if let Some((_, ref chan)) = iter.next() { - if chan.announcement_message.is_some() { - let chan_announcement = chan.announcement_message.clone().unwrap(); - let mut one_to_two_announcement: Option = None; - let mut two_to_one_announcement: Option = None; - if let Some(one_to_two) = chan.one_to_two.as_ref() { - one_to_two_announcement = one_to_two.last_update_message.clone(); - } - if let Some(two_to_one) = chan.two_to_one.as_ref() { - two_to_one_announcement = two_to_one.last_update_message.clone(); - } - return Some((chan_announcement, one_to_two_announcement, two_to_one_announcement)); - } else { - // TODO: We may end up sending un-announced channel_updates if we are sending - // initial sync data while receiving announce/updates for this channel. + for (_, ref chan) in channels.range(starting_point..) { + if chan.announcement_message.is_some() { + let chan_announcement = chan.announcement_message.clone().unwrap(); + let mut one_to_two_announcement: Option = None; + let mut two_to_one_announcement: Option = None; + if let Some(one_to_two) = chan.one_to_two.as_ref() { + one_to_two_announcement = one_to_two.last_update_message.clone(); } + if let Some(two_to_one) = chan.two_to_one.as_ref() { + two_to_one_announcement = two_to_one.last_update_message.clone(); + } + return Some((chan_announcement, one_to_two_announcement, two_to_one_announcement)); } else { - return None; + // TODO: We may end up sending un-announced channel_updates if we are sending + // initial sync data while receiving announce/updates for this channel. } } + None } fn get_next_node_announcement(&self, starting_point: Option<&PublicKey>) -> Option { let nodes = self.network_graph.nodes.read().unwrap(); - let mut iter = if let Some(pubkey) = starting_point { - let mut iter = nodes.range(NodeId::from_pubkey(pubkey)..); - iter.next(); - iter + let iter = if let Some(pubkey) = starting_point { + nodes.range((Bound::Excluded(NodeId::from_pubkey(pubkey)), Bound::Unbounded)) } else { - nodes.range::(..) + nodes.range(..) }; - loop { - if let Some((_, ref node)) = iter.next() { - if let Some(node_info) = node.announcement_info.as_ref() { - if let Some(msg) = node_info.announcement_message.clone() { - return Some(msg); - } + for (_, ref node) in iter { + if let Some(node_info) = node.announcement_info.as_ref() { + if let Some(msg) = node_info.announcement_message.clone() { + return Some(msg); } - } else { - return None; } } + None } /// Initiates a stateless sync of routing gossip information with a peer From 8ffaacb3c3d1979fdf7c986b4e7f671926e9e6b5 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 15 Aug 2022 19:38:45 +0000 Subject: [PATCH 59/91] Drop uneccessary `if {...; bool}` pattern in PeerManager --- lightning/src/ln/peer_handler.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 277eab58943..623aa969dfd 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -779,17 +779,15 @@ impl P self.maybe_send_extra_ping(peer); } - if { - let next_buff = match peer.pending_outbound_buffer.front() { - None => return, - Some(buff) => buff, - }; + let next_buff = match peer.pending_outbound_buffer.front() { + None => return, + Some(buff) => buff, + }; - let pending = &next_buff[peer.pending_outbound_buffer_first_msg_offset..]; - let data_sent = descriptor.send_data(pending, peer.should_read()); - peer.pending_outbound_buffer_first_msg_offset += data_sent; - peer.pending_outbound_buffer_first_msg_offset == next_buff.len() - } { + let pending = &next_buff[peer.pending_outbound_buffer_first_msg_offset..]; + let data_sent = descriptor.send_data(pending, peer.should_read()); + peer.pending_outbound_buffer_first_msg_offset += data_sent; + if peer.pending_outbound_buffer_first_msg_offset == next_buff.len() { peer.pending_outbound_buffer_first_msg_offset = 0; peer.pending_outbound_buffer.pop_front(); } else { From 5a8ede09fb3c8bbcd8694d94c12dac9ea7485537 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 24 May 2022 23:57:56 +0000 Subject: [PATCH 60/91] Expose counterparty-revoked-outputs in `get_claimable_balance` This uses the various new tracking added in the prior commits to expose a new `Balance` type - `CounterpartyRevokedOutputClaimable`. Some nontrivial work is required, however, as we now have to track HTLC outputs as spendable in a transaction that comes *after* an HTLC-Success/HTLC-Timeout transaction, which we previously didn't need to do. Thus, we have to check if an `onchain_events_awaiting_threshold_conf` event spends a commitment transaction's HTLC output while walking events. Further, because we now need to track HTLC outputs after the HTLC-Success/HTLC-Timeout confirms, and because we have to track the counterparty's `to_self` output as a contentious output which could be claimed by either party, we have to examine the `OnchainTxHandler`'s set of outputs to spend when determining if certain outputs are still spendable. Two new tests are added which test various different transaction formats, and hopefully provide good test coverage of the various revoked output paths. --- lightning/src/chain/channelmonitor.rs | 137 ++++- lightning/src/chain/onchaintx.rs | 4 + lightning/src/ln/functional_test_utils.rs | 19 +- lightning/src/ln/monitor_tests.rs | 658 ++++++++++++++++++++++ 4 files changed, 799 insertions(+), 19 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 863ce4759e6..08e5fedd8d3 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -22,6 +22,7 @@ use bitcoin::blockdata::block::BlockHeader; use bitcoin::blockdata::transaction::{TxOut,Transaction}; +use bitcoin::blockdata::transaction::OutPoint as BitcoinOutPoint; use bitcoin::blockdata::script::{Script, Builder}; use bitcoin::blockdata::opcodes; @@ -382,6 +383,9 @@ enum OnchainEvent { on_local_output_csv: Option, /// If the funding spend transaction was a known remote commitment transaction, we track /// the output index and amount of the counterparty's `to_self` output here. + /// + /// This allows us to generate a [`Balance::CounterpartyRevokedOutputClaimable`] for the + /// counterparty output. commitment_tx_to_counterparty_output: CommitmentTxCounterpartyOutputInfo, }, /// A spend of a commitment transaction HTLC output, set in the cases where *no* `HTLCUpdate` @@ -582,6 +586,18 @@ pub enum Balance { /// done so. claimable_height: u32, }, + /// The channel has been closed, and our counterparty broadcasted a revoked commitment + /// transaction. + /// + /// Thus, we're able to claim all outputs in the commitment transaction, one of which has the + /// following amount. + CounterpartyRevokedOutputClaimable { + /// The amount, in satoshis, of the output which we can claim. + /// + /// Note that for outputs from HTLC balances this may be excluding some on-chain fees that + /// were already spent. + claimable_amount_satoshis: u64, + }, } /// An HTLC which has been irrevocably resolved on-chain, and has reached ANTI_REORG_DELAY. @@ -1421,9 +1437,9 @@ impl ChannelMonitor { /// balance, or until our counterparty has claimed the balance and accrued several /// confirmations on the claim transaction. /// - /// Note that the balances available when you or your counterparty have broadcasted revoked - /// state(s) may not be fully captured here. - // TODO, fix that ^ + /// Note that for `ChannelMonitors` which track a channel which went on-chain with versions of + /// LDK prior to 0.0.108, balances may not be fully captured if our counterparty broadcasted + /// a revoked state. /// /// See [`Balance`] for additional details on the types of claimable balances which /// may be returned here and their meanings. @@ -1432,9 +1448,13 @@ impl ChannelMonitor { let us = self.inner.lock().unwrap(); let mut confirmed_txid = us.funding_spend_confirmed; + let mut confirmed_counterparty_output = us.confirmed_commitment_tx_counterparty_output; let mut pending_commitment_tx_conf_thresh = None; let funding_spend_pending = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { - if let OnchainEvent::FundingSpendConfirmation { .. } = event.event { + if let OnchainEvent::FundingSpendConfirmation { commitment_tx_to_counterparty_output, .. } = + event.event + { + confirmed_counterparty_output = commitment_tx_to_counterparty_output; Some((event.txid, event.confirmation_threshold())) } else { None } }); @@ -1446,9 +1466,10 @@ impl ChannelMonitor { } macro_rules! walk_htlcs { - ($holder_commitment: expr, $htlc_iter: expr) => { + ($holder_commitment: expr, $counterparty_revoked_commitment: expr, $htlc_iter: expr) => { for htlc in $htlc_iter { if let Some(htlc_commitment_tx_output_idx) = htlc.transaction_output_index { + let mut htlc_spend_txid_opt = None; let mut htlc_update_pending = None; let mut htlc_spend_pending = None; let mut delayed_output_pending = None; @@ -1456,12 +1477,16 @@ impl ChannelMonitor { match event.event { OnchainEvent::HTLCUpdate { commitment_tx_output_idx, htlc_value_satoshis, .. } if commitment_tx_output_idx == Some(htlc_commitment_tx_output_idx) => { + debug_assert!(htlc_spend_txid_opt.is_none()); + htlc_spend_txid_opt = event.transaction.as_ref().map(|tx| tx.txid()); debug_assert!(htlc_update_pending.is_none()); debug_assert_eq!(htlc_value_satoshis.unwrap(), htlc.amount_msat / 1000); htlc_update_pending = Some(event.confirmation_threshold()); }, OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } if commitment_tx_output_idx == htlc_commitment_tx_output_idx => { + debug_assert!(htlc_spend_txid_opt.is_none()); + htlc_spend_txid_opt = event.transaction.as_ref().map(|tx| tx.txid()); debug_assert!(htlc_spend_pending.is_none()); htlc_spend_pending = Some((event.confirmation_threshold(), preimage.is_some())); }, @@ -1475,22 +1500,69 @@ impl ChannelMonitor { } } let htlc_resolved = us.htlcs_resolved_on_chain.iter() - .find(|v| v.commitment_tx_output_idx == htlc_commitment_tx_output_idx); + .find(|v| if v.commitment_tx_output_idx == htlc_commitment_tx_output_idx { + debug_assert!(htlc_spend_txid_opt.is_none()); + htlc_spend_txid_opt = v.resolving_txid; + true + } else { false }); debug_assert!(htlc_update_pending.is_some() as u8 + htlc_spend_pending.is_some() as u8 + htlc_resolved.is_some() as u8 <= 1); + let htlc_output_to_spend = + if let Some(txid) = htlc_spend_txid_opt { + debug_assert!( + us.onchain_tx_handler.channel_transaction_parameters.opt_anchors.is_none(), + "This code needs updating for anchors"); + BitcoinOutPoint::new(txid, 0) + } else { + BitcoinOutPoint::new(confirmed_txid.unwrap(), htlc_commitment_tx_output_idx) + }; + let htlc_output_needs_spending = us.onchain_tx_handler.is_output_spend_pending(&htlc_output_to_spend); + if let Some(conf_thresh) = delayed_output_pending { debug_assert!($holder_commitment); res.push(Balance::ClaimableAwaitingConfirmations { claimable_amount_satoshis: htlc.amount_msat / 1000, confirmation_height: conf_thresh, }); - } else if htlc_resolved.is_some() { + } else if htlc_resolved.is_some() && !htlc_output_needs_spending { // Funding transaction spends should be fully confirmed by the time any // HTLC transactions are resolved, unless we're talking about a holder // commitment tx, whose resolution is delayed until the CSV timeout is // reached, even though HTLCs may be resolved after only // ANTI_REORG_DELAY confirmations. debug_assert!($holder_commitment || us.funding_spend_confirmed.is_some()); + } else if $counterparty_revoked_commitment { + let htlc_output_claim_pending = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { + if let OnchainEvent::MaturingOutput { + descriptor: SpendableOutputDescriptor::StaticOutput { .. } + } = &event.event { + if event.transaction.as_ref().map(|tx| tx.input.iter().any(|inp| { + if let Some(htlc_spend_txid) = htlc_spend_txid_opt { + Some(tx.txid()) == htlc_spend_txid_opt || + inp.previous_output.txid == htlc_spend_txid + } else { + Some(inp.previous_output.txid) == confirmed_txid && + inp.previous_output.vout == htlc_commitment_tx_output_idx + } + })).unwrap_or(false) { + Some(()) + } else { None } + } else { None } + }); + if htlc_output_claim_pending.is_some() { + // We already push `Balance`s onto the `res` list for every + // `StaticOutput` in a `MaturingOutput` in the revoked + // counterparty commitment transaction case generally, so don't + // need to do so again here. + } else { + debug_assert!(htlc_update_pending.is_none(), + "HTLCUpdate OnchainEvents should never appear for preimage claims"); + debug_assert!(!htlc.offered || htlc_spend_pending.is_none() || !htlc_spend_pending.unwrap().1, + "We don't (currently) generate preimage claims against revoked outputs, where did you get one?!"); + res.push(Balance::CounterpartyRevokedOutputClaimable { + claimable_amount_satoshis: htlc.amount_msat / 1000, + }); + } } else { if htlc.offered == $holder_commitment { // If the payment was outbound, check if there's an HTLCUpdate @@ -1534,8 +1606,8 @@ impl ChannelMonitor { if let Some(txid) = confirmed_txid { let mut found_commitment_tx = false; - if Some(txid) == us.current_counterparty_commitment_txid || Some(txid) == us.prev_counterparty_commitment_txid { - walk_htlcs!(false, us.counterparty_claimable_outpoints.get(&txid).unwrap().iter().map(|(a, _)| a)); + if let Some(counterparty_tx_htlcs) = us.counterparty_claimable_outpoints.get(&txid) { + // First look for the to_remote output back to us. if let Some(conf_thresh) = pending_commitment_tx_conf_thresh { if let Some(value) = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { if let OnchainEvent::MaturingOutput { @@ -1554,9 +1626,50 @@ impl ChannelMonitor { // confirmation with the same height or have never met our dust amount. } } + if Some(txid) == us.current_counterparty_commitment_txid || Some(txid) == us.prev_counterparty_commitment_txid { + walk_htlcs!(false, false, counterparty_tx_htlcs.iter().map(|(a, _)| a)); + } else { + walk_htlcs!(false, true, counterparty_tx_htlcs.iter().map(|(a, _)| a)); + // The counterparty broadcasted a revoked state! + // Look for any StaticOutputs first, generating claimable balances for those. + // If any match the confirmed counterparty revoked to_self output, skip + // generating a CounterpartyRevokedOutputClaimable. + let mut spent_counterparty_output = false; + for event in us.onchain_events_awaiting_threshold_conf.iter() { + if let OnchainEvent::MaturingOutput { + descriptor: SpendableOutputDescriptor::StaticOutput { output, .. } + } = &event.event { + res.push(Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: output.value, + confirmation_height: event.confirmation_threshold(), + }); + if let Some(confirmed_to_self_idx) = confirmed_counterparty_output.map(|(idx, _)| idx) { + if event.transaction.as_ref().map(|tx| + tx.input.iter().any(|inp| inp.previous_output.vout == confirmed_to_self_idx) + ).unwrap_or(false) { + spent_counterparty_output = true; + } + } + } + } + + if spent_counterparty_output { + } else if let Some((confirmed_to_self_idx, amt)) = confirmed_counterparty_output { + let output_spendable = us.onchain_tx_handler + .is_output_spend_pending(&BitcoinOutPoint::new(txid, confirmed_to_self_idx)); + if output_spendable { + res.push(Balance::CounterpartyRevokedOutputClaimable { + claimable_amount_satoshis: amt, + }); + } + } else { + // Counterparty output is missing, either it was broadcasted on a + // previous version of LDK or the counterparty hadn't met dust. + } + } found_commitment_tx = true; } else if txid == us.current_holder_commitment_tx.txid { - walk_htlcs!(true, us.current_holder_commitment_tx.htlc_outputs.iter().map(|(a, _, _)| a)); + walk_htlcs!(true, false, us.current_holder_commitment_tx.htlc_outputs.iter().map(|(a, _, _)| a)); if let Some(conf_thresh) = pending_commitment_tx_conf_thresh { res.push(Balance::ClaimableAwaitingConfirmations { claimable_amount_satoshis: us.current_holder_commitment_tx.to_self_value_sat, @@ -1566,7 +1679,7 @@ impl ChannelMonitor { found_commitment_tx = true; } else if let Some(prev_commitment) = &us.prev_holder_signed_commitment_tx { if txid == prev_commitment.txid { - walk_htlcs!(true, prev_commitment.htlc_outputs.iter().map(|(a, _, _)| a)); + walk_htlcs!(true, false, prev_commitment.htlc_outputs.iter().map(|(a, _, _)| a)); if let Some(conf_thresh) = pending_commitment_tx_conf_thresh { res.push(Balance::ClaimableAwaitingConfirmations { claimable_amount_satoshis: prev_commitment.to_self_value_sat, @@ -1587,8 +1700,6 @@ impl ChannelMonitor { }); } } - // TODO: Add logic to provide claimable balances for counterparty broadcasting revoked - // outputs. } else { let mut claimable_inbound_htlc_value_sat = 0; for (htlc, _, _) in us.current_holder_commitment_tx.htlc_outputs.iter() { diff --git a/lightning/src/chain/onchaintx.rs b/lightning/src/chain/onchaintx.rs index 8f62c43c44e..0f2edff5ed7 100644 --- a/lightning/src/chain/onchaintx.rs +++ b/lightning/src/chain/onchaintx.rs @@ -691,6 +691,10 @@ impl OnchainTxHandler { } } + pub(crate) fn is_output_spend_pending(&self, outpoint: &BitcoinOutPoint) -> bool { + self.claimable_outpoints.get(outpoint).is_some() + } + pub(crate) fn get_relevant_txids(&self) -> Vec { let mut txids: Vec = self.onchain_events_awaiting_threshold_conf .iter() diff --git a/lightning/src/ln/functional_test_utils.rs b/lightning/src/ln/functional_test_utils.rs index 4c91d4f7b1c..85077683fbb 100644 --- a/lightning/src/ln/functional_test_utils.rs +++ b/lightning/src/ln/functional_test_utils.rs @@ -1534,13 +1534,11 @@ macro_rules! expect_payment_failed { }; } -pub fn expect_payment_failed_conditions<'a, 'b, 'c, 'd, 'e>( - node: &'a Node<'b, 'c, 'd>, expected_payment_hash: PaymentHash, expected_rejected_by_dest: bool, - conditions: PaymentFailedConditions<'e> +pub fn expect_payment_failed_conditions_event<'a, 'b, 'c, 'd, 'e>( + node: &'a Node<'b, 'c, 'd>, payment_failed_event: Event, expected_payment_hash: PaymentHash, + expected_rejected_by_dest: bool, conditions: PaymentFailedConditions<'e> ) { - let mut events = node.node.get_and_clear_pending_events(); - assert_eq!(events.len(), 1); - let expected_payment_id = match events.pop().unwrap() { + let expected_payment_id = match payment_failed_event { Event::PaymentPathFailed { payment_hash, rejected_by_dest, path, retry, payment_id, network_update, short_channel_id, #[cfg(test)] error_code, @@ -1603,6 +1601,15 @@ pub fn expect_payment_failed_conditions<'a, 'b, 'c, 'd, 'e>( } } +pub fn expect_payment_failed_conditions<'a, 'b, 'c, 'd, 'e>( + node: &'a Node<'b, 'c, 'd>, expected_payment_hash: PaymentHash, expected_rejected_by_dest: bool, + conditions: PaymentFailedConditions<'e> +) { + let mut events = node.node.get_and_clear_pending_events(); + assert_eq!(events.len(), 1); + expect_payment_failed_conditions_event(node, events.pop().unwrap(), expected_payment_hash, expected_rejected_by_dest, conditions); +} + pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_paths: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) -> PaymentId { let payment_id = origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap(); check_added_monitors!(origin_node, expected_paths.len()); diff --git a/lightning/src/ln/monitor_tests.rs b/lightning/src/ln/monitor_tests.rs index 9be2059a705..67ea07f2abd 100644 --- a/lightning/src/ln/monitor_tests.rs +++ b/lightning/src/ln/monitor_tests.rs @@ -11,6 +11,7 @@ use chain::channelmonitor::{ANTI_REORG_DELAY, Balance}; use chain::transaction::OutPoint; +use chain::chaininterface::LowerBoundedFeeEstimator; use ln::channel; use ln::channelmanager::BREAKDOWN_TIMEOUT; use ln::features::InitFeatures; @@ -228,6 +229,17 @@ fn sorted_vec(mut v: Vec) -> Vec { v } +/// Asserts that `a` and `b` are close, but maybe off by up to 5. +/// This is useful when checking fees and weights on transactions as things may vary by a few based +/// on signature size and signature size estimation being non-exact. +fn fuzzy_assert_eq>(a: V, b: V) { + let a_u64 = a.try_into().map_err(|_| ()).unwrap(); + let b_u64 = b.try_into().map_err(|_| ()).unwrap(); + eprintln!("Checking {} and {} for fuzzy equality", a_u64, b_u64); + assert!(a_u64 >= b_u64 - 5); + assert!(b_u64 >= a_u64 - 5); +} + fn do_test_claim_value_force_close(prev_commitment_tx: bool) { // Tests `get_claimable_balances` with an HTLC across a force-close. // We build a channel with an HTLC pending, then force close the channel and check that the @@ -734,3 +746,649 @@ fn test_balances_on_local_commitment_htlcs() { assert!(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty()); test_spendable_output(&nodes[0], &as_txn[1]); } + +fn sorted_vec_with_additions(v_orig: &Vec, extra_ts: &[&T]) -> Vec { + let mut v = v_orig.clone(); + for t in extra_ts { + v.push((*t).clone()); + } + v.sort_unstable(); + v +} + +fn do_test_revoked_counterparty_commitment_balances(confirm_htlc_spend_first: bool) { + // Tests `get_claimable_balances` for revoked counterparty commitment transactions. + let mut chanmon_cfgs = create_chanmon_cfgs(2); + // We broadcast a second-to-latest commitment transaction, without providing the revocation + // secret to the counterparty. However, because we always immediately take the revocation + // secret from the keys_manager, we would panic at broadcast as we're trying to sign a + // transaction which, from the point of view of our keys_manager, is revoked. + chanmon_cfgs[1].keys_manager.disable_revocation_policy_check = true; + let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + + let (_, _, chan_id, funding_tx) = + create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 100_000_000, InitFeatures::known(), InitFeatures::known()); + let funding_outpoint = OutPoint { txid: funding_tx.txid(), index: 0 }; + assert_eq!(funding_outpoint.to_channel_id(), chan_id); + + // We create five HTLCs for B to claim against A's revoked commitment transaction: + // + // (1) one for which A is the originator and B knows the preimage + // (2) one for which B is the originator where the HTLC has since timed-out + // (3) one for which B is the originator but where the HTLC has not yet timed-out + // (4) one dust HTLC which is lost in the channel closure + // (5) one that actually isn't in the revoked commitment transaction at all, but was added in + // later commitment transaction updates + // + // Though they could all be claimed in a single claim transaction, due to CLTV timeouts they + // are all currently claimed in separate transactions, which helps us test as we can claim + // HTLCs individually. + + let (claimed_payment_preimage, claimed_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 3_000_000); + let timeout_payment_hash = route_payment(&nodes[1], &[&nodes[0]], 4_000_000).1; + let dust_payment_hash = route_payment(&nodes[1], &[&nodes[0]], 3_000).1; + + let htlc_cltv_timeout = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1; // Note ChannelManager adds one to CLTV timeouts for safety + + connect_blocks(&nodes[0], 10); + connect_blocks(&nodes[1], 10); + + let live_htlc_cltv_timeout = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1; // Note ChannelManager adds one to CLTV timeouts for safety + let live_payment_hash = route_payment(&nodes[1], &[&nodes[0]], 5_000_000).1; + + // Get the latest commitment transaction from A and then update the fee to revoke it + let as_revoked_txn = get_local_commitment_txn!(nodes[0], chan_id); + let opt_anchors = get_opt_anchors!(nodes[0], chan_id); + + let chan_feerate = get_feerate!(nodes[0], chan_id) as u64; + + let missing_htlc_cltv_timeout = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1; // Note ChannelManager adds one to CLTV timeouts for safety + let missing_htlc_payment_hash = route_payment(&nodes[1], &[&nodes[0]], 2_000_000).1; + + nodes[1].node.claim_funds(claimed_payment_preimage); + expect_payment_claimed!(nodes[1], claimed_payment_hash, 3_000_000); + check_added_monitors!(nodes[1], 1); + let _b_htlc_msgs = get_htlc_update_msgs!(&nodes[1], nodes[0].node.get_our_node_id()); + + connect_blocks(&nodes[0], htlc_cltv_timeout + 1 - 10); + check_closed_broadcast!(nodes[0], true); + check_added_monitors!(nodes[0], 1); + + let mut events = nodes[0].node.get_and_clear_pending_events(); + assert_eq!(events.len(), 6); + let mut failed_payments: HashSet<_> = + [timeout_payment_hash, dust_payment_hash, live_payment_hash, missing_htlc_payment_hash] + .iter().map(|a| *a).collect(); + events.retain(|ev| { + match ev { + Event::HTLCHandlingFailed { failed_next_destination: HTLCDestination::NextHopChannel { node_id, channel_id }, .. } => { + assert_eq!(*channel_id, chan_id); + assert_eq!(*node_id, Some(nodes[1].node.get_our_node_id())); + false + }, + Event::HTLCHandlingFailed { failed_next_destination: HTLCDestination::FailedPayment { payment_hash }, .. } => { + assert!(failed_payments.remove(payment_hash)); + false + }, + _ => true, + } + }); + assert!(failed_payments.is_empty()); + if let Event::PendingHTLCsForwardable { .. } = events[0] {} else { panic!(); } + match &events[1] { + Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {}, + _ => panic!(), + } + + connect_blocks(&nodes[1], htlc_cltv_timeout + 1 - 10); + check_closed_broadcast!(nodes[1], true); + check_added_monitors!(nodes[1], 1); + check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed); + + // Prior to channel closure, B considers the preimage HTLC as its own, and otherwise only + // lists the two on-chain timeout-able HTLCs as claimable balances. + assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose { + claimable_amount_satoshis: 100_000 - 5_000 - 4_000 - 3 - 2_000 + 3_000, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 2_000, + claimable_height: missing_htlc_cltv_timeout, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 4_000, + claimable_height: htlc_cltv_timeout, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 5_000, + claimable_height: live_htlc_cltv_timeout, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[1], &as_revoked_txn[0]); + let mut claim_txn: Vec<_> = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().drain(..).filter(|tx| tx.input.iter().any(|inp| inp.previous_output.txid == as_revoked_txn[0].txid())).collect(); + // Currently the revoked commitment is claimed in four transactions as the HTLCs all expire + // quite soon. + assert_eq!(claim_txn.len(), 4); + claim_txn.sort_unstable_by_key(|tx| tx.output.iter().map(|output| output.value).sum::()); + + // The following constants were determined experimentally + const BS_TO_SELF_CLAIM_EXP_WEIGHT: usize = 483; + const OUTBOUND_HTLC_CLAIM_EXP_WEIGHT: usize = 571; + const INBOUND_HTLC_CLAIM_EXP_WEIGHT: usize = 578; + + // Check that the weight is close to the expected weight. Note that signature sizes vary + // somewhat so it may not always be exact. + fuzzy_assert_eq(claim_txn[0].weight(), OUTBOUND_HTLC_CLAIM_EXP_WEIGHT); + fuzzy_assert_eq(claim_txn[1].weight(), INBOUND_HTLC_CLAIM_EXP_WEIGHT); + fuzzy_assert_eq(claim_txn[2].weight(), INBOUND_HTLC_CLAIM_EXP_WEIGHT); + fuzzy_assert_eq(claim_txn[3].weight(), BS_TO_SELF_CLAIM_EXP_WEIGHT); + + // The expected balance for the next three checks, with the largest-HTLC and to_self output + // claim balances separated out. + let expected_balance = vec![Balance::ClaimableAwaitingConfirmations { + // to_remote output in A's revoked commitment + claimable_amount_satoshis: 100_000 - 5_000 - 4_000 - 3, + confirmation_height: nodes[1].best_block_info().1 + 5, + }, Balance::CounterpartyRevokedOutputClaimable { + claimable_amount_satoshis: 3_000, + }, Balance::CounterpartyRevokedOutputClaimable { + claimable_amount_satoshis: 4_000, + }]; + + let to_self_unclaimed_balance = Balance::CounterpartyRevokedOutputClaimable { + claimable_amount_satoshis: 1_000_000 - 100_000 - 3_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 3 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + }; + let to_self_claimed_avail_height; + let largest_htlc_unclaimed_balance = Balance::CounterpartyRevokedOutputClaimable { + claimable_amount_satoshis: 5_000, + }; + let largest_htlc_claimed_avail_height; + + // Once the channel has been closed by A, B now considers all of the commitment transactions' + // outputs as `CounterpartyRevokedOutputClaimable`. + assert_eq!(sorted_vec_with_additions(&expected_balance, &[&to_self_unclaimed_balance, &largest_htlc_unclaimed_balance]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + if confirm_htlc_spend_first { + mine_transaction(&nodes[1], &claim_txn[2]); + largest_htlc_claimed_avail_height = nodes[1].best_block_info().1 + 5; + to_self_claimed_avail_height = nodes[1].best_block_info().1 + 6; // will be claimed in the next block + } else { + // Connect the to_self output claim, taking all of A's non-HTLC funds + mine_transaction(&nodes[1], &claim_txn[3]); + to_self_claimed_avail_height = nodes[1].best_block_info().1 + 5; + largest_htlc_claimed_avail_height = nodes[1].best_block_info().1 + 6; // will be claimed in the next block + } + + let largest_htlc_claimed_balance = Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 5_000 - chan_feerate * INBOUND_HTLC_CLAIM_EXP_WEIGHT as u64 / 1000, + confirmation_height: largest_htlc_claimed_avail_height, + }; + let to_self_claimed_balance = Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 1_000_000 - 100_000 - 3_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 3 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000 + - chan_feerate * claim_txn[3].weight() as u64 / 1000, + confirmation_height: to_self_claimed_avail_height, + }; + + if confirm_htlc_spend_first { + assert_eq!(sorted_vec_with_additions(&expected_balance, &[&to_self_unclaimed_balance, &largest_htlc_claimed_balance]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + } else { + assert_eq!(sorted_vec_with_additions(&expected_balance, &[&to_self_claimed_balance, &largest_htlc_unclaimed_balance]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + } + + if confirm_htlc_spend_first { + mine_transaction(&nodes[1], &claim_txn[3]); + } else { + mine_transaction(&nodes[1], &claim_txn[2]); + } + assert_eq!(sorted_vec_with_additions(&expected_balance, &[&to_self_claimed_balance, &largest_htlc_claimed_balance]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + // Finally, connect the last two remaining HTLC spends and check that they move to + // `ClaimableAwaitingConfirmations` + mine_transaction(&nodes[1], &claim_txn[0]); + mine_transaction(&nodes[1], &claim_txn[1]); + + assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + // to_remote output in A's revoked commitment + claimable_amount_satoshis: 100_000 - 5_000 - 4_000 - 3, + confirmation_height: nodes[1].best_block_info().1 + 1, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 1_000_000 - 100_000 - 3_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 3 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000 + - chan_feerate * claim_txn[3].weight() as u64 / 1000, + confirmation_height: to_self_claimed_avail_height, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 3_000 - chan_feerate * OUTBOUND_HTLC_CLAIM_EXP_WEIGHT as u64 / 1000, + confirmation_height: nodes[1].best_block_info().1 + 4, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 4_000 - chan_feerate * INBOUND_HTLC_CLAIM_EXP_WEIGHT as u64 / 1000, + confirmation_height: nodes[1].best_block_info().1 + 5, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 5_000 - chan_feerate * INBOUND_HTLC_CLAIM_EXP_WEIGHT as u64 / 1000, + confirmation_height: largest_htlc_claimed_avail_height, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[1], 1); + test_spendable_output(&nodes[1], &as_revoked_txn[0]); + + let mut payment_failed_events = nodes[1].node.get_and_clear_pending_events(); + expect_payment_failed_conditions_event(&nodes[1], payment_failed_events.pop().unwrap(), + dust_payment_hash, true, PaymentFailedConditions::new()); + expect_payment_failed_conditions_event(&nodes[1], payment_failed_events.pop().unwrap(), + missing_htlc_payment_hash, true, PaymentFailedConditions::new()); + assert!(payment_failed_events.is_empty()); + + connect_blocks(&nodes[1], 1); + test_spendable_output(&nodes[1], &claim_txn[if confirm_htlc_spend_first { 2 } else { 3 }]); + connect_blocks(&nodes[1], 1); + test_spendable_output(&nodes[1], &claim_txn[if confirm_htlc_spend_first { 3 } else { 2 }]); + expect_payment_failed!(nodes[1], live_payment_hash, true); + connect_blocks(&nodes[1], 1); + test_spendable_output(&nodes[1], &claim_txn[0]); + connect_blocks(&nodes[1], 1); + test_spendable_output(&nodes[1], &claim_txn[1]); + expect_payment_failed!(nodes[1], timeout_payment_hash, true); + assert_eq!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances(), Vec::new()); +} + +#[test] +fn test_revoked_counterparty_commitment_balances() { + do_test_revoked_counterparty_commitment_balances(true); + do_test_revoked_counterparty_commitment_balances(false); +} + +#[test] +fn test_revoked_counterparty_htlc_tx_balances() { + // Tests `get_claimable_balances` for revocation spends of HTLC transactions. + let mut chanmon_cfgs = create_chanmon_cfgs(2); + chanmon_cfgs[1].keys_manager.disable_revocation_policy_check = true; + let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + + // Create some initial channels + let (_, _, chan_id, funding_tx) = + create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 11_000_000, InitFeatures::known(), InitFeatures::known()); + let funding_outpoint = OutPoint { txid: funding_tx.txid(), index: 0 }; + assert_eq!(funding_outpoint.to_channel_id(), chan_id); + + let payment_preimage = route_payment(&nodes[0], &[&nodes[1]], 3_000_000).0; + let failed_payment_hash = route_payment(&nodes[1], &[&nodes[0]], 1_000_000).1; + let revoked_local_txn = get_local_commitment_txn!(nodes[1], chan_id); + assert_eq!(revoked_local_txn[0].input.len(), 1); + assert_eq!(revoked_local_txn[0].input[0].previous_output.txid, funding_tx.txid()); + + // The to-be-revoked commitment tx should have two HTLCs and an output for both sides + assert_eq!(revoked_local_txn[0].output.len(), 4); + + claim_payment(&nodes[0], &[&nodes[1]], payment_preimage); + + let chan_feerate = get_feerate!(nodes[0], chan_id) as u64; + let opt_anchors = get_opt_anchors!(nodes[0], chan_id); + + // B will generate an HTLC-Success from its revoked commitment tx + mine_transaction(&nodes[1], &revoked_local_txn[0]); + check_closed_broadcast!(nodes[1], true); + check_added_monitors!(nodes[1], 1); + check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed); + let revoked_htlc_success_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + + assert_eq!(revoked_htlc_success_txn.len(), 2); + assert_eq!(revoked_htlc_success_txn[0].input.len(), 1); + assert_eq!(revoked_htlc_success_txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT); + check_spends!(revoked_htlc_success_txn[0], revoked_local_txn[0]); + check_spends!(revoked_htlc_success_txn[1], funding_tx); + + connect_blocks(&nodes[1], TEST_FINAL_CLTV); + let revoked_htlc_timeout_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + assert_eq!(revoked_htlc_timeout_txn.len(), 1); + check_spends!(revoked_htlc_timeout_txn[0], revoked_local_txn[0]); + assert_ne!(revoked_htlc_success_txn[0].input[0].previous_output, revoked_htlc_timeout_txn[0].input[0].previous_output); + assert_eq!(revoked_htlc_success_txn[0].lock_time.0, 0); + assert_ne!(revoked_htlc_timeout_txn[0].lock_time.0, 0); + + // A will generate justice tx from B's revoked commitment/HTLC tx + mine_transaction(&nodes[0], &revoked_local_txn[0]); + check_closed_broadcast!(nodes[0], true); + check_added_monitors!(nodes[0], 1); + check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed); + let to_remote_conf_height = nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 1; + + let as_commitment_claim_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + assert_eq!(as_commitment_claim_txn.len(), 2); + check_spends!(as_commitment_claim_txn[0], revoked_local_txn[0]); + check_spends!(as_commitment_claim_txn[1], funding_tx); + + // The next two checks have the same balance set for A - even though we confirm a revoked HTLC + // transaction our balance tracking doesn't use the on-chain value so the + // `CounterpartyRevokedOutputClaimable` entry doesn't change. + let as_balances = sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + // to_remote output in B's revoked commitment + claimable_amount_satoshis: 1_000_000 - 11_000 - 3_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + confirmation_height: to_remote_conf_height, + }, Balance::CounterpartyRevokedOutputClaimable { + // to_self output in B's revoked commitment + claimable_amount_satoshis: 10_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 1 + claimable_amount_satoshis: 3_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 2 + claimable_amount_satoshis: 1_000, + }]); + assert_eq!(as_balances, + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[0], &revoked_htlc_success_txn[0]); + let as_htlc_claim_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + assert_eq!(as_htlc_claim_tx.len(), 2); + check_spends!(as_htlc_claim_tx[0], revoked_htlc_success_txn[0]); + check_spends!(as_htlc_claim_tx[1], revoked_local_txn[0]); // A has to generate a new claim for the remaining revoked + // outputs (which no longer includes the spent HTLC output) + + assert_eq!(as_balances, + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + assert_eq!(as_htlc_claim_tx[0].output.len(), 1); + fuzzy_assert_eq(as_htlc_claim_tx[0].output[0].value, + 3_000 - chan_feerate * (revoked_htlc_success_txn[0].weight() + as_htlc_claim_tx[0].weight()) as u64 / 1000); + + mine_transaction(&nodes[0], &as_htlc_claim_tx[0]); + assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + // to_remote output in B's revoked commitment + claimable_amount_satoshis: 1_000_000 - 11_000 - 3_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + confirmation_height: to_remote_conf_height, + }, Balance::CounterpartyRevokedOutputClaimable { + // to_self output in B's revoked commitment + claimable_amount_satoshis: 10_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 2 + claimable_amount_satoshis: 1_000, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: as_htlc_claim_tx[0].output[0].value, + confirmation_height: nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 1, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[0], ANTI_REORG_DELAY - 3); + test_spendable_output(&nodes[0], &revoked_local_txn[0]); + assert_eq!(sorted_vec(vec![Balance::CounterpartyRevokedOutputClaimable { + // to_self output to B + claimable_amount_satoshis: 10_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 2 + claimable_amount_satoshis: 1_000, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: as_htlc_claim_tx[0].output[0].value, + confirmation_height: nodes[0].best_block_info().1 + 2, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[0], 2); + test_spendable_output(&nodes[0], &as_htlc_claim_tx[0]); + assert_eq!(sorted_vec(vec![Balance::CounterpartyRevokedOutputClaimable { + // to_self output in B's revoked commitment + claimable_amount_satoshis: 10_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 2 + claimable_amount_satoshis: 1_000, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[0], revoked_htlc_timeout_txn[0].lock_time.0 - nodes[0].best_block_info().1); + expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(&nodes[0], + [HTLCDestination::FailedPayment { payment_hash: failed_payment_hash }]); + // As time goes on A may split its revocation claim transaction into multiple. + let as_fewer_input_rbf = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + for tx in as_fewer_input_rbf.iter() { + check_spends!(tx, revoked_local_txn[0]); + } + + // Connect a number of additional blocks to ensure we don't forget the HTLC output needs + // claiming. + connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1); + let as_fewer_input_rbf = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + for tx in as_fewer_input_rbf.iter() { + check_spends!(tx, revoked_local_txn[0]); + } + + mine_transaction(&nodes[0], &revoked_htlc_timeout_txn[0]); + let as_second_htlc_claim_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + assert_eq!(as_second_htlc_claim_tx.len(), 2); + + check_spends!(as_second_htlc_claim_tx[0], revoked_htlc_timeout_txn[0]); + check_spends!(as_second_htlc_claim_tx[1], revoked_local_txn[0]); + + // Connect blocks to finalize the HTLC resolution with the HTLC-Timeout transaction. In a + // previous iteration of the revoked balance handling this would result in us "forgetting" that + // the revoked HTLC output still needed to be claimed. + connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1); + assert_eq!(sorted_vec(vec![Balance::CounterpartyRevokedOutputClaimable { + // to_self output in B's revoked commitment + claimable_amount_satoshis: 10_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 2 + claimable_amount_satoshis: 1_000, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[0], &as_second_htlc_claim_tx[0]); + assert_eq!(sorted_vec(vec![Balance::CounterpartyRevokedOutputClaimable { + // to_self output in B's revoked commitment + claimable_amount_satoshis: 10_000, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: as_second_htlc_claim_tx[0].output[0].value, + confirmation_height: nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 1, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[0], &as_second_htlc_claim_tx[1]); + assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + // to_self output in B's revoked commitment + claimable_amount_satoshis: as_second_htlc_claim_tx[1].output[0].value, + confirmation_height: nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 1, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: as_second_htlc_claim_tx[0].output[0].value, + confirmation_height: nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 2, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2); + test_spendable_output(&nodes[0], &as_second_htlc_claim_tx[0]); + connect_blocks(&nodes[0], 1); + test_spendable_output(&nodes[0], &as_second_htlc_claim_tx[1]); + + assert_eq!(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances(), Vec::new()); +} + +#[test] +fn test_revoked_counterparty_aggregated_claims() { + // Tests `get_claimable_balances` for revoked counterparty commitment transactions when + // claiming with an aggregated claim transaction. + let mut chanmon_cfgs = create_chanmon_cfgs(2); + // We broadcast a second-to-latest commitment transaction, without providing the revocation + // secret to the counterparty. However, because we always immediately take the revocation + // secret from the keys_manager, we would panic at broadcast as we're trying to sign a + // transaction which, from the point of view of our keys_manager, is revoked. + chanmon_cfgs[1].keys_manager.disable_revocation_policy_check = true; + let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + + let (_, _, chan_id, funding_tx) = + create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 100_000_000, InitFeatures::known(), InitFeatures::known()); + let funding_outpoint = OutPoint { txid: funding_tx.txid(), index: 0 }; + assert_eq!(funding_outpoint.to_channel_id(), chan_id); + + // We create two HTLCs, one which we will give A the preimage to to generate an HTLC-Success + // transaction, and one which we will not, allowing B to claim the HTLC output in an aggregated + // revocation-claim transaction. + + let (claimed_payment_preimage, claimed_payment_hash, ..) = route_payment(&nodes[1], &[&nodes[0]], 3_000_000); + let revoked_payment_hash = route_payment(&nodes[1], &[&nodes[0]], 4_000_000).1; + + let htlc_cltv_timeout = nodes[1].best_block_info().1 + TEST_FINAL_CLTV + 1; // Note ChannelManager adds one to CLTV timeouts for safety + + // Cheat by giving A's ChannelMonitor the preimage to the to-be-claimed HTLC so that we have an + // HTLC-claim transaction on the to-be-revoked state. + get_monitor!(nodes[0], chan_id).provide_payment_preimage(&claimed_payment_hash, &claimed_payment_preimage, + &node_cfgs[0].tx_broadcaster, &LowerBoundedFeeEstimator::new(node_cfgs[0].fee_estimator), &nodes[0].logger); + + // Now get the latest commitment transaction from A and then update the fee to revoke it + let as_revoked_txn = get_local_commitment_txn!(nodes[0], chan_id); + + assert_eq!(as_revoked_txn.len(), 2); + check_spends!(as_revoked_txn[0], funding_tx); + check_spends!(as_revoked_txn[1], as_revoked_txn[0]); // The HTLC-Claim transaction + + let opt_anchors = get_opt_anchors!(nodes[0], chan_id); + let chan_feerate = get_feerate!(nodes[0], chan_id) as u64; + + { + let mut feerate = chanmon_cfgs[0].fee_estimator.sat_per_kw.lock().unwrap(); + *feerate += 1; + } + nodes[0].node.timer_tick_occurred(); + check_added_monitors!(nodes[0], 1); + + let fee_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id()); + nodes[1].node.handle_update_fee(&nodes[0].node.get_our_node_id(), &fee_update.update_fee.unwrap()); + commitment_signed_dance!(nodes[1], nodes[0], fee_update.commitment_signed, false); + + nodes[0].node.claim_funds(claimed_payment_preimage); + expect_payment_claimed!(nodes[0], claimed_payment_hash, 3_000_000); + check_added_monitors!(nodes[0], 1); + let _a_htlc_msgs = get_htlc_update_msgs!(&nodes[0], nodes[1].node.get_our_node_id()); + + assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose { + claimable_amount_satoshis: 100_000 - 4_000 - 3_000, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 4_000, + claimable_height: htlc_cltv_timeout, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 3_000, + claimable_height: htlc_cltv_timeout, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[1], &as_revoked_txn[0]); + check_closed_broadcast!(nodes[1], true); + check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed); + check_added_monitors!(nodes[1], 1); + + let mut claim_txn: Vec<_> = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().drain(..).filter(|tx| tx.input.iter().any(|inp| inp.previous_output.txid == as_revoked_txn[0].txid())).collect(); + // Currently the revoked commitment outputs are all claimed in one aggregated transaction + assert_eq!(claim_txn.len(), 1); + assert_eq!(claim_txn[0].input.len(), 3); + check_spends!(claim_txn[0], as_revoked_txn[0]); + + let to_remote_maturity = nodes[1].best_block_info().1 + ANTI_REORG_DELAY - 1; + + assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + // to_remote output in A's revoked commitment + claimable_amount_satoshis: 100_000 - 4_000 - 3_000, + confirmation_height: to_remote_maturity, + }, Balance::CounterpartyRevokedOutputClaimable { + // to_self output in A's revoked commitment + claimable_amount_satoshis: 1_000_000 - 100_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 1 + claimable_amount_satoshis: 4_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 2 + claimable_amount_satoshis: 3_000, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + // Confirm A's HTLC-Success tranasction which presumably raced B's claim, causing B to create a + // new claim. + mine_transaction(&nodes[1], &as_revoked_txn[1]); + expect_payment_sent!(nodes[1], claimed_payment_preimage); + let mut claim_txn_2: Vec<_> = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); + claim_txn_2.sort_unstable_by_key(|tx| if tx.input.iter().any(|inp| inp.previous_output.txid == as_revoked_txn[0].txid()) { 0 } else { 1 }); + // Once B sees the HTLC-Success transaction it splits its claim transaction into two, though in + // theory it could re-aggregate the claims as well. + assert_eq!(claim_txn_2.len(), 2); + assert_eq!(claim_txn_2[0].input.len(), 2); + check_spends!(claim_txn_2[0], as_revoked_txn[0]); + assert_eq!(claim_txn_2[1].input.len(), 1); + check_spends!(claim_txn_2[1], as_revoked_txn[1]); + + assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + // to_remote output in A's revoked commitment + claimable_amount_satoshis: 100_000 - 4_000 - 3_000, + confirmation_height: to_remote_maturity, + }, Balance::CounterpartyRevokedOutputClaimable { + // to_self output in A's revoked commitment + claimable_amount_satoshis: 1_000_000 - 100_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 1 + claimable_amount_satoshis: 4_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 2 + // The amount here is a bit of a misnomer, really its been reduced by the HTLC + // transaction fee, but the claimable amount is always a bit of an overshoot for HTLCs + // anyway, so its not a big change. + claimable_amount_satoshis: 3_000, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[1], 5); + test_spendable_output(&nodes[1], &as_revoked_txn[0]); + + assert_eq!(sorted_vec(vec![Balance::CounterpartyRevokedOutputClaimable { + // to_self output in A's revoked commitment + claimable_amount_satoshis: 1_000_000 - 100_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 1 + claimable_amount_satoshis: 4_000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 2 + // The amount here is a bit of a misnomer, really its been reduced by the HTLC + // transaction fee, but the claimable amount is always a bit of an overshoot for HTLCs + // anyway, so its not a big change. + claimable_amount_satoshis: 3_000, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[1], &claim_txn_2[1]); + let htlc_2_claim_maturity = nodes[1].best_block_info().1 + ANTI_REORG_DELAY - 1; + + assert_eq!(sorted_vec(vec![Balance::CounterpartyRevokedOutputClaimable { + // to_self output in A's revoked commitment + claimable_amount_satoshis: 1_000_000 - 100_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 1 + claimable_amount_satoshis: 4_000, + }, Balance::ClaimableAwaitingConfirmations { // HTLC 2 + claimable_amount_satoshis: claim_txn_2[1].output[0].value, + confirmation_height: htlc_2_claim_maturity, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[1], 5); + test_spendable_output(&nodes[1], &claim_txn_2[1]); + + assert_eq!(sorted_vec(vec![Balance::CounterpartyRevokedOutputClaimable { + // to_self output in A's revoked commitment + claimable_amount_satoshis: 1_000_000 - 100_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + }, Balance::CounterpartyRevokedOutputClaimable { // HTLC 1 + claimable_amount_satoshis: 4_000, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[1], &claim_txn_2[0]); + let rest_claim_maturity = nodes[1].best_block_info().1 + ANTI_REORG_DELAY - 1; + + assert_eq!(vec![Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: claim_txn_2[0].output[0].value, + confirmation_height: rest_claim_maturity, + }], + nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances()); + + assert!(nodes[1].node.get_and_clear_pending_events().is_empty()); // We shouldn't fail the payment until we spend the output + + connect_blocks(&nodes[1], 5); + expect_payment_failed!(nodes[1], revoked_payment_hash, true); + test_spendable_output(&nodes[1], &claim_txn_2[0]); + assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty()); +} From d8651e3dd57673792344d6aff65eb79fc9f2340b Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sat, 16 Jul 2022 20:41:45 +0000 Subject: [PATCH 61/91] Move per-HTLC logic out of get_claimable_balances into a helper Val suggested this as an obvious cleanup to separate per_HTLC logic from the total commitment transaction logic, separating the large function into two. --- lightning/src/chain/channelmonitor.rs | 275 ++++++++++++++------------ 1 file changed, 146 insertions(+), 129 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 08e5fedd8d3..500f2b521dd 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1427,7 +1427,150 @@ impl ChannelMonitor { pub fn current_best_block(&self) -> BestBlock { self.inner.lock().unwrap().best_block.clone() } +} + +impl ChannelMonitorImpl { + /// Helper for get_claimable_balances which does the work for an individual HTLC, generating up + /// to one `Balance` for the HTLC. + fn get_htlc_balance(&self, htlc: &HTLCOutputInCommitment, holder_commitment: bool, + counterparty_revoked_commitment: bool, confirmed_txid: Option) + -> Option { + let htlc_commitment_tx_output_idx = + if let Some(v) = htlc.transaction_output_index { v } else { return None; }; + + let mut htlc_spend_txid_opt = None; + let mut holder_timeout_spend_pending = None; + let mut htlc_spend_pending = None; + let mut holder_delayed_output_pending = None; + for event in self.onchain_events_awaiting_threshold_conf.iter() { + match event.event { + OnchainEvent::HTLCUpdate { commitment_tx_output_idx, htlc_value_satoshis, .. } + if commitment_tx_output_idx == Some(htlc_commitment_tx_output_idx) => { + debug_assert!(htlc_spend_txid_opt.is_none()); + htlc_spend_txid_opt = event.transaction.as_ref().map(|tx| tx.txid()); + debug_assert!(holder_timeout_spend_pending.is_none()); + debug_assert_eq!(htlc_value_satoshis.unwrap(), htlc.amount_msat / 1000); + holder_timeout_spend_pending = Some(event.confirmation_threshold()); + }, + OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } + if commitment_tx_output_idx == htlc_commitment_tx_output_idx => { + debug_assert!(htlc_spend_txid_opt.is_none()); + htlc_spend_txid_opt = event.transaction.as_ref().map(|tx| tx.txid()); + debug_assert!(htlc_spend_pending.is_none()); + htlc_spend_pending = Some((event.confirmation_threshold(), preimage.is_some())); + }, + OnchainEvent::MaturingOutput { + descriptor: SpendableOutputDescriptor::DelayedPaymentOutput(ref descriptor) } + if descriptor.outpoint.index as u32 == htlc_commitment_tx_output_idx => { + debug_assert!(holder_delayed_output_pending.is_none()); + holder_delayed_output_pending = Some(event.confirmation_threshold()); + }, + _ => {}, + } + } + let htlc_resolved = self.htlcs_resolved_on_chain.iter() + .find(|v| if v.commitment_tx_output_idx == htlc_commitment_tx_output_idx { + debug_assert!(htlc_spend_txid_opt.is_none()); + htlc_spend_txid_opt = v.resolving_txid; + true + } else { false }); + debug_assert!(holder_timeout_spend_pending.is_some() as u8 + htlc_spend_pending.is_some() as u8 + htlc_resolved.is_some() as u8 <= 1); + + let htlc_output_to_spend = + if let Some(txid) = htlc_spend_txid_opt { + debug_assert!( + self.onchain_tx_handler.channel_transaction_parameters.opt_anchors.is_none(), + "This code needs updating for anchors"); + BitcoinOutPoint::new(txid, 0) + } else { + BitcoinOutPoint::new(confirmed_txid.unwrap(), htlc_commitment_tx_output_idx) + }; + let htlc_output_spend_pending = self.onchain_tx_handler.is_output_spend_pending(&htlc_output_to_spend); + if let Some(conf_thresh) = holder_delayed_output_pending { + debug_assert!(holder_commitment); + return Some(Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: htlc.amount_msat / 1000, + confirmation_height: conf_thresh, + }); + } else if htlc_resolved.is_some() && !htlc_output_spend_pending { + // Funding transaction spends should be fully confirmed by the time any + // HTLC transactions are resolved, unless we're talking about a holder + // commitment tx, whose resolution is delayed until the CSV timeout is + // reached, even though HTLCs may be resolved after only + // ANTI_REORG_DELAY confirmations. + debug_assert!(holder_commitment || self.funding_spend_confirmed.is_some()); + } else if counterparty_revoked_commitment { + let htlc_output_claim_pending = self.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { + if let OnchainEvent::MaturingOutput { + descriptor: SpendableOutputDescriptor::StaticOutput { .. } + } = &event.event { + if event.transaction.as_ref().map(|tx| tx.input.iter().any(|inp| { + if let Some(htlc_spend_txid) = htlc_spend_txid_opt { + Some(tx.txid()) == htlc_spend_txid_opt || + inp.previous_output.txid == htlc_spend_txid + } else { + Some(inp.previous_output.txid) == confirmed_txid && + inp.previous_output.vout == htlc_commitment_tx_output_idx + } + })).unwrap_or(false) { + Some(()) + } else { None } + } else { None } + }); + if htlc_output_claim_pending.is_some() { + // We already push `Balance`s onto the `res` list for every + // `StaticOutput` in a `MaturingOutput` in the revoked + // counterparty commitment transaction case generally, so don't + // need to do so again here. + } else { + debug_assert!(holder_timeout_spend_pending.is_none(), + "HTLCUpdate OnchainEvents should never appear for preimage claims"); + debug_assert!(!htlc.offered || htlc_spend_pending.is_none() || !htlc_spend_pending.unwrap().1, + "We don't (currently) generate preimage claims against revoked outputs, where did you get one?!"); + return Some(Balance::CounterpartyRevokedOutputClaimable { + claimable_amount_satoshis: htlc.amount_msat / 1000, + }); + } + } else if htlc.offered == holder_commitment { + // If the payment was outbound, check if there's an HTLCUpdate + // indicating we have spent this HTLC with a timeout, claiming it back + // and awaiting confirmations on it. + if let Some(conf_thresh) = holder_timeout_spend_pending { + return Some(Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: htlc.amount_msat / 1000, + confirmation_height: conf_thresh, + }); + } else { + return Some(Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: htlc.amount_msat / 1000, + claimable_height: htlc.cltv_expiry, + }); + } + } else if self.payment_preimages.get(&htlc.payment_hash).is_some() { + // Otherwise (the payment was inbound), only expose it as claimable if + // we know the preimage. + // Note that if there is a pending claim, but it did not use the + // preimage, we lost funds to our counterparty! We will then continue + // to show it as ContentiousClaimable until ANTI_REORG_DELAY. + debug_assert!(holder_timeout_spend_pending.is_none()); + if let Some((conf_thresh, true)) = htlc_spend_pending { + return Some(Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: htlc.amount_msat / 1000, + confirmation_height: conf_thresh, + }); + } else { + return Some(Balance::ContentiousClaimable { + claimable_amount_satoshis: htlc.amount_msat / 1000, + timeout_height: htlc.cltv_expiry, + }); + } + } + None + } +} + +impl ChannelMonitor { /// Gets the balances in this channel which are either claimable by us if we were to /// force-close the channel now or which are claimable on-chain (possibly awaiting /// confirmation). @@ -1468,136 +1611,10 @@ impl ChannelMonitor { macro_rules! walk_htlcs { ($holder_commitment: expr, $counterparty_revoked_commitment: expr, $htlc_iter: expr) => { for htlc in $htlc_iter { - if let Some(htlc_commitment_tx_output_idx) = htlc.transaction_output_index { - let mut htlc_spend_txid_opt = None; - let mut htlc_update_pending = None; - let mut htlc_spend_pending = None; - let mut delayed_output_pending = None; - for event in us.onchain_events_awaiting_threshold_conf.iter() { - match event.event { - OnchainEvent::HTLCUpdate { commitment_tx_output_idx, htlc_value_satoshis, .. } - if commitment_tx_output_idx == Some(htlc_commitment_tx_output_idx) => { - debug_assert!(htlc_spend_txid_opt.is_none()); - htlc_spend_txid_opt = event.transaction.as_ref().map(|tx| tx.txid()); - debug_assert!(htlc_update_pending.is_none()); - debug_assert_eq!(htlc_value_satoshis.unwrap(), htlc.amount_msat / 1000); - htlc_update_pending = Some(event.confirmation_threshold()); - }, - OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } - if commitment_tx_output_idx == htlc_commitment_tx_output_idx => { - debug_assert!(htlc_spend_txid_opt.is_none()); - htlc_spend_txid_opt = event.transaction.as_ref().map(|tx| tx.txid()); - debug_assert!(htlc_spend_pending.is_none()); - htlc_spend_pending = Some((event.confirmation_threshold(), preimage.is_some())); - }, - OnchainEvent::MaturingOutput { - descriptor: SpendableOutputDescriptor::DelayedPaymentOutput(ref descriptor) } - if descriptor.outpoint.index as u32 == htlc_commitment_tx_output_idx => { - debug_assert!(delayed_output_pending.is_none()); - delayed_output_pending = Some(event.confirmation_threshold()); - }, - _ => {}, - } - } - let htlc_resolved = us.htlcs_resolved_on_chain.iter() - .find(|v| if v.commitment_tx_output_idx == htlc_commitment_tx_output_idx { - debug_assert!(htlc_spend_txid_opt.is_none()); - htlc_spend_txid_opt = v.resolving_txid; - true - } else { false }); - debug_assert!(htlc_update_pending.is_some() as u8 + htlc_spend_pending.is_some() as u8 + htlc_resolved.is_some() as u8 <= 1); - - let htlc_output_to_spend = - if let Some(txid) = htlc_spend_txid_opt { - debug_assert!( - us.onchain_tx_handler.channel_transaction_parameters.opt_anchors.is_none(), - "This code needs updating for anchors"); - BitcoinOutPoint::new(txid, 0) - } else { - BitcoinOutPoint::new(confirmed_txid.unwrap(), htlc_commitment_tx_output_idx) - }; - let htlc_output_needs_spending = us.onchain_tx_handler.is_output_spend_pending(&htlc_output_to_spend); + if htlc.transaction_output_index.is_some() { - if let Some(conf_thresh) = delayed_output_pending { - debug_assert!($holder_commitment); - res.push(Balance::ClaimableAwaitingConfirmations { - claimable_amount_satoshis: htlc.amount_msat / 1000, - confirmation_height: conf_thresh, - }); - } else if htlc_resolved.is_some() && !htlc_output_needs_spending { - // Funding transaction spends should be fully confirmed by the time any - // HTLC transactions are resolved, unless we're talking about a holder - // commitment tx, whose resolution is delayed until the CSV timeout is - // reached, even though HTLCs may be resolved after only - // ANTI_REORG_DELAY confirmations. - debug_assert!($holder_commitment || us.funding_spend_confirmed.is_some()); - } else if $counterparty_revoked_commitment { - let htlc_output_claim_pending = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { - if let OnchainEvent::MaturingOutput { - descriptor: SpendableOutputDescriptor::StaticOutput { .. } - } = &event.event { - if event.transaction.as_ref().map(|tx| tx.input.iter().any(|inp| { - if let Some(htlc_spend_txid) = htlc_spend_txid_opt { - Some(tx.txid()) == htlc_spend_txid_opt || - inp.previous_output.txid == htlc_spend_txid - } else { - Some(inp.previous_output.txid) == confirmed_txid && - inp.previous_output.vout == htlc_commitment_tx_output_idx - } - })).unwrap_or(false) { - Some(()) - } else { None } - } else { None } - }); - if htlc_output_claim_pending.is_some() { - // We already push `Balance`s onto the `res` list for every - // `StaticOutput` in a `MaturingOutput` in the revoked - // counterparty commitment transaction case generally, so don't - // need to do so again here. - } else { - debug_assert!(htlc_update_pending.is_none(), - "HTLCUpdate OnchainEvents should never appear for preimage claims"); - debug_assert!(!htlc.offered || htlc_spend_pending.is_none() || !htlc_spend_pending.unwrap().1, - "We don't (currently) generate preimage claims against revoked outputs, where did you get one?!"); - res.push(Balance::CounterpartyRevokedOutputClaimable { - claimable_amount_satoshis: htlc.amount_msat / 1000, - }); - } - } else { - if htlc.offered == $holder_commitment { - // If the payment was outbound, check if there's an HTLCUpdate - // indicating we have spent this HTLC with a timeout, claiming it back - // and awaiting confirmations on it. - if let Some(conf_thresh) = htlc_update_pending { - res.push(Balance::ClaimableAwaitingConfirmations { - claimable_amount_satoshis: htlc.amount_msat / 1000, - confirmation_height: conf_thresh, - }); - } else { - res.push(Balance::MaybeClaimableHTLCAwaitingTimeout { - claimable_amount_satoshis: htlc.amount_msat / 1000, - claimable_height: htlc.cltv_expiry, - }); - } - } else if us.payment_preimages.get(&htlc.payment_hash).is_some() { - // Otherwise (the payment was inbound), only expose it as claimable if - // we know the preimage. - // Note that if there is a pending claim, but it did not use the - // preimage, we lost funds to our counterparty! We will then continue - // to show it as ContentiousClaimable until ANTI_REORG_DELAY. - debug_assert!(htlc_update_pending.is_none()); - if let Some((conf_thresh, true)) = htlc_spend_pending { - res.push(Balance::ClaimableAwaitingConfirmations { - claimable_amount_satoshis: htlc.amount_msat / 1000, - confirmation_height: conf_thresh, - }); - } else { - res.push(Balance::ContentiousClaimable { - claimable_amount_satoshis: htlc.amount_msat / 1000, - timeout_height: htlc.cltv_expiry, - }); - } - } + if let Some(bal) = us.get_htlc_balance(htlc, $holder_commitment, $counterparty_revoked_commitment, confirmed_txid) { + res.push(bal); } } } From 8424f3f0546144eaaf4f9d2f518e94677804c510 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Sun, 24 Jul 2022 14:47:31 -0400 Subject: [PATCH 62/91] Fuzz test onion messages Also update the fuzz ChaCha20Poly1305 to not mark as finished after a single encrypt_in_place. This is because more bytes may still need to be encrypted, causing us to panic at the assertion that finished == false when we go to encrypt more. Also fix unused_mut warning in messenger + add log on OM forward for testing --- fuzz/src/bin/gen_target.sh | 1 + fuzz/src/bin/onion_message_target.rs | 113 ++++++++++++++++ fuzz/src/lib.rs | 1 + fuzz/src/onion_message.rs | 152 ++++++++++++++++++++++ fuzz/targets.h | 1 + lightning/src/lib.rs | 3 + lightning/src/onion_message/messenger.rs | 3 +- lightning/src/util/chacha20poly1305rfc.rs | 2 +- 8 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 fuzz/src/bin/onion_message_target.rs create mode 100644 fuzz/src/onion_message.rs diff --git a/fuzz/src/bin/gen_target.sh b/fuzz/src/bin/gen_target.sh index c0daa5a3a0f..95e65695eb8 100755 --- a/fuzz/src/bin/gen_target.sh +++ b/fuzz/src/bin/gen_target.sh @@ -9,6 +9,7 @@ GEN_TEST() { GEN_TEST chanmon_deser GEN_TEST chanmon_consistency GEN_TEST full_stack +GEN_TEST onion_message GEN_TEST peer_crypt GEN_TEST process_network_graph GEN_TEST router diff --git a/fuzz/src/bin/onion_message_target.rs b/fuzz/src/bin/onion_message_target.rs new file mode 100644 index 00000000000..e9bcf590da1 --- /dev/null +++ b/fuzz/src/bin/onion_message_target.rs @@ -0,0 +1,113 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +// This file is auto-generated by gen_target.sh based on target_template.txt +// To modify it, modify target_template.txt and run gen_target.sh instead. + +#![cfg_attr(feature = "libfuzzer_fuzz", no_main)] + +#[cfg(not(fuzzing))] +compile_error!("Fuzz targets need cfg=fuzzing"); + +extern crate lightning_fuzz; +use lightning_fuzz::onion_message::*; + +#[cfg(feature = "afl")] +#[macro_use] extern crate afl; +#[cfg(feature = "afl")] +fn main() { + fuzz!(|data| { + onion_message_run(data.as_ptr(), data.len()); + }); +} + +#[cfg(feature = "honggfuzz")] +#[macro_use] extern crate honggfuzz; +#[cfg(feature = "honggfuzz")] +fn main() { + loop { + fuzz!(|data| { + onion_message_run(data.as_ptr(), data.len()); + }); + } +} + +#[cfg(feature = "libfuzzer_fuzz")] +#[macro_use] extern crate libfuzzer_sys; +#[cfg(feature = "libfuzzer_fuzz")] +fuzz_target!(|data: &[u8]| { + onion_message_run(data.as_ptr(), data.len()); +}); + +#[cfg(feature = "stdin_fuzz")] +fn main() { + use std::io::Read; + + let mut data = Vec::with_capacity(8192); + std::io::stdin().read_to_end(&mut data).unwrap(); + onion_message_run(data.as_ptr(), data.len()); +} + +#[test] +fn run_test_cases() { + use std::fs; + use std::io::Read; + use lightning_fuzz::utils::test_logger::StringBuffer; + + use std::sync::{atomic, Arc}; + { + let data: Vec = vec![0]; + onion_message_run(data.as_ptr(), data.len()); + } + let mut threads = Vec::new(); + let threads_running = Arc::new(atomic::AtomicUsize::new(0)); + if let Ok(tests) = fs::read_dir("test_cases/onion_message") { + for test in tests { + let mut data: Vec = Vec::new(); + let path = test.unwrap().path(); + fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap(); + threads_running.fetch_add(1, atomic::Ordering::AcqRel); + + let thread_count_ref = Arc::clone(&threads_running); + let main_thread_ref = std::thread::current(); + threads.push((path.file_name().unwrap().to_str().unwrap().to_string(), + std::thread::spawn(move || { + let string_logger = StringBuffer::new(); + + let panic_logger = string_logger.clone(); + let res = if ::std::panic::catch_unwind(move || { + onion_message_test(&data, panic_logger); + }).is_err() { + Some(string_logger.into_string()) + } else { None }; + thread_count_ref.fetch_sub(1, atomic::Ordering::AcqRel); + main_thread_ref.unpark(); + res + }) + )); + while threads_running.load(atomic::Ordering::Acquire) > 32 { + std::thread::park(); + } + } + } + let mut failed_outputs = Vec::new(); + for (test, thread) in threads.drain(..) { + if let Some(output) = thread.join().unwrap() { + println!("\nOutput of {}:\n{}\n", test, output); + failed_outputs.push(test); + } + } + if !failed_outputs.is_empty() { + println!("Test cases which failed: "); + for case in failed_outputs { + println!("{}", case); + } + panic!(); + } +} diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 5e158aee36f..2238a9702a9 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -17,6 +17,7 @@ pub mod utils; pub mod chanmon_deser; pub mod chanmon_consistency; pub mod full_stack; +pub mod onion_message; pub mod peer_crypt; pub mod process_network_graph; pub mod router; diff --git a/fuzz/src/onion_message.rs b/fuzz/src/onion_message.rs new file mode 100644 index 00000000000..7ab2bd63a9f --- /dev/null +++ b/fuzz/src/onion_message.rs @@ -0,0 +1,152 @@ +// Imports that need to be added manually +use bitcoin::bech32::u5; +use bitcoin::blockdata::script::Script; +use bitcoin::secp256k1::{PublicKey, Scalar, SecretKey}; +use bitcoin::secp256k1::ecdh::SharedSecret; +use bitcoin::secp256k1::ecdsa::RecoverableSignature; + +use lightning::chain::keysinterface::{Recipient, KeyMaterial, KeysInterface}; +use lightning::ln::msgs::{self, DecodeError}; +use lightning::ln::script::ShutdownScript; +use lightning::util::enforcing_trait_impls::EnforcingSigner; +use lightning::util::logger::Logger; +use lightning::util::ser::{Readable, Writer}; +use lightning::onion_message::OnionMessenger; + +use utils::test_logger; + +use std::io::Cursor; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[inline] +/// Actual fuzz test, method signature and name are fixed +pub fn do_test(data: &[u8], logger: &L) { + if let Ok(msg) = ::read(&mut Cursor::new(data)) { + let mut secret_bytes = [0; 32]; + secret_bytes[31] = 2; + let secret = SecretKey::from_slice(&secret_bytes).unwrap(); + let keys_manager = KeyProvider { + node_secret: secret, + counter: AtomicU64::new(0), + }; + let onion_messenger = OnionMessenger::new(&keys_manager, logger); + let mut pk = [2; 33]; pk[1] = 0xff; + let peer_node_id_not_used = PublicKey::from_slice(&pk).unwrap(); + onion_messenger.handle_onion_message(&peer_node_id_not_used, &msg); + } +} + +/// Method that needs to be added manually, {name}_test +pub fn onion_message_test(data: &[u8], out: Out) { + let logger = test_logger::TestLogger::new("".to_owned(), out); + do_test(data, &logger); +} + +/// Method that needs to be added manually, {name}_run +#[no_mangle] +pub extern "C" fn onion_message_run(data: *const u8, datalen: usize) { + let logger = test_logger::TestLogger::new("".to_owned(), test_logger::DevNull {}); + do_test(unsafe { std::slice::from_raw_parts(data, datalen) }, &logger); +} + +pub struct VecWriter(pub Vec); +impl Writer for VecWriter { + fn write_all(&mut self, buf: &[u8]) -> Result<(), ::std::io::Error> { + self.0.extend_from_slice(buf); + Ok(()) + } +} +struct KeyProvider { + node_secret: SecretKey, + counter: AtomicU64, +} +impl KeysInterface for KeyProvider { + type Signer = EnforcingSigner; + + fn get_node_secret(&self, _recipient: Recipient) -> Result { + Ok(self.node_secret.clone()) + } + + fn ecdh(&self, recipient: Recipient, other_key: &PublicKey, tweak: Option<&Scalar>) -> Result { + let mut node_secret = self.get_node_secret(recipient)?; + if let Some(tweak) = tweak { + node_secret = node_secret.mul_tweak(tweak).map_err(|_| ())?; + } + Ok(SharedSecret::new(other_key, &node_secret)) + } + + fn get_inbound_payment_key_material(&self) -> KeyMaterial { unreachable!() } + + fn get_destination_script(&self) -> Script { unreachable!() } + + fn get_shutdown_scriptpubkey(&self) -> ShutdownScript { unreachable!() } + + fn get_channel_signer(&self, _inbound: bool, _channel_value_satoshis: u64) -> EnforcingSigner { + unreachable!() + } + + fn get_secure_random_bytes(&self) -> [u8; 32] { + let ctr = self.counter.fetch_add(1, Ordering::Relaxed); + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (ctr >> 8*7) as u8, (ctr >> 8*6) as u8, (ctr >> 8*5) as u8, (ctr >> 8*4) as u8, (ctr >> 8*3) as u8, (ctr >> 8*2) as u8, (ctr >> 8*1) as u8, 14, (ctr >> 8*0) as u8] + } + + fn read_chan_signer(&self, _data: &[u8]) -> Result { unreachable!() } + + fn sign_invoice(&self, _hrp_bytes: &[u8], _invoice_data: &[u5], _recipient: Recipient) -> Result { + unreachable!() + } +} + +#[cfg(test)] +mod tests { + use lightning::util::logger::{Logger, Record}; + use std::collections::HashMap; + use std::sync::Mutex; + + struct TrackingLogger { + /// (module, message) -> count + pub lines: Mutex>, + } + impl Logger for TrackingLogger { + fn log(&self, record: &Record) { + *self.lines.lock().unwrap().entry((record.module_path.to_string(), format!("{}", record.args))).or_insert(0) += 1; + println!("{:<5} [{} : {}, {}] {}", record.level.to_string(), record.module_path, record.file, record.line, record.args); + } + } + + #[test] + fn test_no_onion_message_breakage() { + let one_hop_om = "020000000000000000000000000000000000000000000000000000000000000e01055600020000000000000000000000000000000000000000000000000000000000000e01120410950000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009300000000000000000000000000000000000000000000000000000000000000"; + let logger = TrackingLogger { lines: Mutex::new(HashMap::new()) }; + super::do_test(&::hex::decode(one_hop_om).unwrap(), &logger); + { + let log_entries = logger.lines.lock().unwrap(); + assert_eq!(log_entries.get(&("lightning::onion_message::messenger".to_string(), "Received an onion message with path_id: None".to_string())), Some(&1)); + } + + let two_unblinded_hops_om = "020000000000000000000000000000000000000000000000000000000000000e01055600020000000000000000000000000000000000000000000000000000000000000e0135043304210200000000000000000000000000000000000000000000000000000000000000039500000000000000000000000000000058000000000000000000000000000000000000000000000000000000000000001204105e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b300000000000000000000000000000000000000000000000000000000000000"; + let logger = TrackingLogger { lines: Mutex::new(HashMap::new()) }; + super::do_test(&::hex::decode(two_unblinded_hops_om).unwrap(), &logger); + { + let log_entries = logger.lines.lock().unwrap(); + assert_eq!(log_entries.get(&("lightning::onion_message::messenger".to_string(), "Forwarding an onion message to peer 020000000000000000000000000000000000000000000000000000000000000003".to_string())), Some(&1)); + } + + let two_unblinded_two_blinded_om = "020000000000000000000000000000000000000000000000000000000000000e01055600020000000000000000000000000000000000000000000000000000000000000e01350433042102000000000000000000000000000000000000000000000000000000000000000395000000000000000000000000000000530000000000000000000000000000000000000000000000000000000000000058045604210200000000000000000000000000000000000000000000000000000000000000040821020000000000000000000000000000000000000000000000000000000000000e015e0000000000000000000000000000006b0000000000000000000000000000000000000000000000000000000000000035043304210200000000000000000000000000000000000000000000000000000000000000054b000000000000000000000000000000e800000000000000000000000000000000000000000000000000000000000000120410ee00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b300000000000000000000000000000000000000000000000000000000000000"; + let logger = TrackingLogger { lines: Mutex::new(HashMap::new()) }; + super::do_test(&::hex::decode(two_unblinded_two_blinded_om).unwrap(), &logger); + { + let log_entries = logger.lines.lock().unwrap(); + assert_eq!(log_entries.get(&("lightning::onion_message::messenger".to_string(), "Forwarding an onion message to peer 020000000000000000000000000000000000000000000000000000000000000003".to_string())), Some(&1)); + } + + let three_blinded_om = "020000000000000000000000000000000000000000000000000000000000000e01055600020000000000000000000000000000000000000000000000000000000000000e013504330421020000000000000000000000000000000000000000000000000000000000000003950000000000000000000000000000007f0000000000000000000000000000000000000000000000000000000000000035043304210200000000000000000000000000000000000000000000000000000000000000045e0000000000000000000000000000004c000000000000000000000000000000000000000000000000000000000000001204104a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b300000000000000000000000000000000000000000000000000000000000000"; + let logger = TrackingLogger { lines: Mutex::new(HashMap::new()) }; + super::do_test(&::hex::decode(three_blinded_om).unwrap(), &logger); + { + let log_entries = logger.lines.lock().unwrap(); + assert_eq!(log_entries.get(&("lightning::onion_message::messenger".to_string(), "Forwarding an onion message to peer 020000000000000000000000000000000000000000000000000000000000000003".to_string())), Some(&1)); + } + } +} diff --git a/fuzz/targets.h b/fuzz/targets.h index 7958a6f614d..cff3f9bdbb5 100644 --- a/fuzz/targets.h +++ b/fuzz/targets.h @@ -2,6 +2,7 @@ void chanmon_deser_run(const unsigned char* data, size_t data_len); void chanmon_consistency_run(const unsigned char* data, size_t data_len); void full_stack_run(const unsigned char* data, size_t data_len); +void onion_message_run(const unsigned char* data, size_t data_len); void peer_crypt_run(const unsigned char* data, size_t data_len); void process_network_graph_run(const unsigned char* data, size_t data_len); void router_run(const unsigned char* data, size_t data_len); diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index 45072710735..199a3cbee4b 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -79,6 +79,9 @@ pub mod util; pub mod chain; pub mod ln; pub mod routing; +#[cfg(fuzzing)] +pub mod onion_message; +#[cfg(not(fuzzing))] #[allow(unused)] mod onion_message; // To be exposed after sending/receiving OMs is supported in PeerManager. diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 04424896197..c264cbc387f 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -248,7 +248,7 @@ impl OnionMessenger sha.input(control_tlvs_ss.as_ref()); Sha256::from_engine(sha).into_inner() }; - let mut next_blinding_point = msg.blinding_point; + let next_blinding_point = msg.blinding_point; match next_blinding_point.mul_tweak(&self.secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap()) { Ok(bp) => bp, Err(e) => { @@ -261,6 +261,7 @@ impl OnionMessenger onion_routing_packet: outgoing_packet, }, ); + log_trace!(self.logger, "Forwarding an onion message to peer {}", next_node_id); }, Err(e) => { log_trace!(self.logger, "Errored decoding onion message packet: {:?}", e); diff --git a/lightning/src/util/chacha20poly1305rfc.rs b/lightning/src/util/chacha20poly1305rfc.rs index befe5d19f5f..1dbd91e65e0 100644 --- a/lightning/src/util/chacha20poly1305rfc.rs +++ b/lightning/src/util/chacha20poly1305rfc.rs @@ -286,10 +286,10 @@ mod fuzzy_chachapoly { pub(super) fn encrypt_in_place(&mut self, _input_output: &mut [u8]) { assert!(self.finished == false); - self.finished = true; } pub(super) fn finish_and_get_tag(&mut self, out_tag: &mut [u8]) { + assert!(self.finished == false); out_tag.copy_from_slice(&self.tag); self.finished = true; } From 6265da0d39104fe311ccd582abed6a3b3fcc4d6c Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sat, 13 Aug 2022 17:29:06 +0000 Subject: [PATCH 63/91] Correct the on-chain script checked in gossip verification The `bitcoin_key_1` and `bitcoin_key_2` fields in `channel_announcement` messages are sorted according to node_ids rather than the keys themselves, however the on-chain funding script is sorted according to the bitcoin keys themselves. Thus, with some probability, we end up checking that the on-chain script matches the wrong script and rejecting the channel announcement. The correct solution is to use our existing channel funding script generation function which ensure we always match what we generate. This was found in testing the Java bindings, where a test checks that retunring the generated funding script in `chain::Access` results in the constructed channel ending up in our network graph. --- lightning/src/routing/gossip.rs | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 1757d137300..7bdcd439285 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -16,13 +16,12 @@ use bitcoin::secp256k1; use bitcoin::hashes::sha256d::Hash as Sha256dHash; use bitcoin::hashes::Hash; -use bitcoin::blockdata::script::Builder; use bitcoin::blockdata::transaction::TxOut; -use bitcoin::blockdata::opcodes; use bitcoin::hash_types::BlockHash; use chain; use chain::Access; +use ln::chan_utils::make_funding_redeemscript; use ln::features::{ChannelFeatures, NodeFeatures}; use ln::msgs::{DecodeError, ErrorAction, Init, LightningError, RoutingMessageHandler, NetAddress, MAX_VALUE_MSAT}; use ln::msgs::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement, GossipTimestampFilter}; @@ -1455,11 +1454,8 @@ impl NetworkGraph where L::Target: Logger { &Some(ref chain_access) => { match chain_access.get_utxo(&msg.chain_hash, msg.short_channel_id) { Ok(TxOut { value, script_pubkey }) => { - let expected_script = Builder::new().push_opcode(opcodes::all::OP_PUSHNUM_2) - .push_slice(&msg.bitcoin_key_1.serialize()) - .push_slice(&msg.bitcoin_key_2.serialize()) - .push_opcode(opcodes::all::OP_PUSHNUM_2) - .push_opcode(opcodes::all::OP_CHECKMULTISIG).into_script().to_v0_p2wsh(); + let expected_script = + make_funding_redeemscript(&msg.bitcoin_key_1, &msg.bitcoin_key_2).to_v0_p2wsh(); if script_pubkey != expected_script { return Err(LightningError{err: format!("Channel announcement key ({}) didn't match on-chain script ({})", script_pubkey.to_hex(), expected_script.to_hex()), action: ErrorAction::IgnoreError}); } @@ -1836,6 +1832,7 @@ impl ReadOnlyNetworkGraph<'_> { #[cfg(test)] mod tests { use chain; + use ln::chan_utils::make_funding_redeemscript; use ln::PaymentHash; use ln::features::{ChannelFeatures, InitFeatures, NodeFeatures}; use routing::gossip::{P2PGossipSync, NetworkGraph, NetworkUpdate, NodeAlias, MAX_EXCESS_BYTES_FOR_RELAY, NodeId, RoutingFees, ChannelUpdateInfo, ChannelInfo, NodeAnnouncementInfo, NodeInfo}; @@ -1853,9 +1850,8 @@ mod tests { use bitcoin::hashes::Hash; use bitcoin::network::constants::Network; use bitcoin::blockdata::constants::genesis_block; - use bitcoin::blockdata::script::{Builder, Script}; + use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::TxOut; - use bitcoin::blockdata::opcodes; use hex; @@ -1945,14 +1941,10 @@ mod tests { } fn get_channel_script(secp_ctx: &Secp256k1) -> Script { - let node_1_btckey = &SecretKey::from_slice(&[40; 32]).unwrap(); - let node_2_btckey = &SecretKey::from_slice(&[39; 32]).unwrap(); - Builder::new().push_opcode(opcodes::all::OP_PUSHNUM_2) - .push_slice(&PublicKey::from_secret_key(&secp_ctx, node_1_btckey).serialize()) - .push_slice(&PublicKey::from_secret_key(&secp_ctx, node_2_btckey).serialize()) - .push_opcode(opcodes::all::OP_PUSHNUM_2) - .push_opcode(opcodes::all::OP_CHECKMULTISIG).into_script() - .to_v0_p2wsh() + let node_1_btckey = SecretKey::from_slice(&[40; 32]).unwrap(); + let node_2_btckey = SecretKey::from_slice(&[39; 32]).unwrap(); + make_funding_redeemscript(&PublicKey::from_secret_key(secp_ctx, &node_1_btckey), + &PublicKey::from_secret_key(secp_ctx, &node_2_btckey)).to_v0_p2wsh() } fn get_signed_channel_update(f: F, node_key: &SecretKey, secp_ctx: &Secp256k1) -> ChannelUpdate { From 36e9725d9a9e9f71b31f8193daccfc96bb94090a Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 15 Aug 2022 19:30:32 +0000 Subject: [PATCH 64/91] Provide guidance on ChannelMonitorUpdate serialized size Users need to make decisions about storage sizing and we need to have advice on the maximum size of various things users need to store. ChannelMonitorUpdates are likely the worst case of this, they're usually at max a few KB, but can get up to a few hundred KB for commitment transactions that have 400+ HTLCs pending. We had one user report an update (likely) going over 400 KiB, which isn't immediately obvious to me is practical, but its within a few multiples of trivially-reachable sizes, so its likely that did occur. To be on the safe side, we simply recommend users ensure they can support "upwards of 1 MiB" here. --- lightning/src/chain/channelmonitor.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 855263fe53b..4225bdee632 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -57,8 +57,13 @@ use io::{self, Error}; use core::ops::Deref; use sync::Mutex; -/// An update generated by the underlying Channel itself which contains some new information the -/// ChannelMonitor should be made aware of. +/// An update generated by the underlying channel itself which contains some new information the +/// [`ChannelMonitor`] should be made aware of. +/// +/// Because this represents only a small number of updates to the underlying state, it is generally +/// much smaller than a full [`ChannelMonitor`]. However, for large single commitment transaction +/// updates (e.g. ones during which there are hundreds of HTLCs pending on the commitment +/// transaction), a single update may reach upwards of 1 MiB in serialized size. #[cfg_attr(any(test, fuzzing, feature = "_test_utils"), derive(PartialEq))] #[derive(Clone)] #[must_use] From 1cc2bc5145ac352fa610454fa081810ff8eb6bb9 Mon Sep 17 00:00:00 2001 From: ok300 <106775972+ok300@users.noreply.github.com> Date: Fri, 19 Aug 2022 16:40:19 +0200 Subject: [PATCH 65/91] Bump bitcoin_hashes to v0.11 --- lightning-invoice/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightning-invoice/Cargo.toml b/lightning-invoice/Cargo.toml index b7b8f7f0146..cd67bcd36dc 100644 --- a/lightning-invoice/Cargo.toml +++ b/lightning-invoice/Cargo.toml @@ -23,7 +23,7 @@ bech32 = { version = "0.9.0", default-features = false } lightning = { version = "0.0.110", path = "../lightning", default-features = false } secp256k1 = { version = "0.24.0", default-features = false, features = ["recovery", "alloc"] } num-traits = { version = "0.2.8", default-features = false } -bitcoin_hashes = { version = "0.10", default-features = false } +bitcoin_hashes = { version = "0.11", default-features = false } hashbrown = { version = "0.11", optional = true } core2 = { version = "0.3.0", default-features = false, optional = true } serde = { version = "1.0.118", optional = true } From e263d904a17cd798a130a2d4cfe8497b79117a58 Mon Sep 17 00:00:00 2001 From: NicolaLS Date: Mon, 27 Jun 2022 14:41:46 +0200 Subject: [PATCH 66/91] derive Hash for Invoice --- lightning-invoice/src/lib.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lightning-invoice/src/lib.rs b/lightning-invoice/src/lib.rs index c7d7a4042f7..4fc58271ed7 100644 --- a/lightning-invoice/src/lib.rs +++ b/lightning-invoice/src/lib.rs @@ -239,7 +239,7 @@ pub struct InvoiceBuilder(&str)` -#[derive(Eq, PartialEq, Debug, Clone)] +#[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct Invoice { signed_invoice: SignedRawInvoice, } @@ -263,7 +263,7 @@ pub enum InvoiceDescription<'f> { /// /// # Invariants /// The hash has to be either from the deserialized invoice or from the serialized `raw_invoice`. -#[derive(Eq, PartialEq, Debug, Clone)] +#[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct SignedRawInvoice { /// The rawInvoice that the signature belongs to raw_invoice: RawInvoice, @@ -286,7 +286,7 @@ pub struct SignedRawInvoice { /// De- and encoding should not lead to information loss but may lead to different hashes. /// /// For methods without docs see the corresponding methods in `Invoice`. -#[derive(Eq, PartialEq, Debug, Clone)] +#[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct RawInvoice { /// human readable part pub hrp: RawHrp, @@ -298,7 +298,7 @@ pub struct RawInvoice { /// Data of the `RawInvoice` that is encoded in the human readable part /// /// (C-not exported) As we don't yet support Option -#[derive(Eq, PartialEq, Debug, Clone)] +#[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct RawHrp { /// The currency deferred from the 3rd and 4th character of the bech32 transaction pub currency: Currency, @@ -311,7 +311,7 @@ pub struct RawHrp { } /// Data of the `RawInvoice` that is encoded in the data part -#[derive(Eq, PartialEq, Debug, Clone)] +#[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct RawDataPart { /// generation time of the invoice pub timestamp: PositiveTimestamp, @@ -326,11 +326,11 @@ pub struct RawDataPart { /// /// The Unix timestamp representing the stored time has to be positive and no greater than /// [`MAX_TIMESTAMP`]. -#[derive(Eq, PartialEq, Debug, Clone)] +#[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct PositiveTimestamp(Duration); /// SI prefixes for the human readable part -#[derive(Eq, PartialEq, Debug, Clone, Copy)] +#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] pub enum SiPrefix { /// 10^-3 Milli, @@ -456,7 +456,7 @@ pub enum Fallback { } /// Recoverable signature -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Hash, Eq, PartialEq)] pub struct InvoiceSignature(pub RecoverableSignature); /// Private routing information From f53492eff29506c2582e199a06f17ffd10304aaa Mon Sep 17 00:00:00 2001 From: Arik Sosman Date: Fri, 19 Aug 2022 10:07:55 -0700 Subject: [PATCH 67/91] Fix script order in gossip key mismatch error. --- lightning/src/routing/gossip.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 483a7bab6d9..b083f402141 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -1444,7 +1444,7 @@ impl NetworkGraph where L::Target: Logger { let expected_script = make_funding_redeemscript(&msg.bitcoin_key_1, &msg.bitcoin_key_2).to_v0_p2wsh(); if script_pubkey != expected_script { - return Err(LightningError{err: format!("Channel announcement key ({}) didn't match on-chain script ({})", script_pubkey.to_hex(), expected_script.to_hex()), action: ErrorAction::IgnoreError}); + return Err(LightningError{err: format!("Channel announcement key ({}) didn't match on-chain script ({})", expected_script.to_hex(), script_pubkey.to_hex()), action: ErrorAction::IgnoreError}); } //TODO: Check if value is worth storing, use it to inform routing, and compare it //to the new HTLC max field in channel_update @@ -2977,7 +2977,7 @@ mod tests { let legacy_chan_update_info_with_none: Vec = hex::decode("2c0004000000170201010402002a060800000000000004d20801000a0d0c00040000000902040000000a0c0100").unwrap(); let read_chan_update_info_res: Result = ::util::ser::Readable::read(&mut legacy_chan_update_info_with_none.as_slice()); assert!(read_chan_update_info_res.is_err()); - + // 2. Test encoding/decoding of ChannelInfo // Check we can encode/decode ChannelInfo without ChannelUpdateInfo fields present. let chan_info_none_updates = ChannelInfo { From 88e562053a80fe4bfc7b602e6b59882e9b519d2f Mon Sep 17 00:00:00 2001 From: Duncan Dean Date: Tue, 23 Aug 2022 09:20:35 +0200 Subject: [PATCH 68/91] Workaround GH Actions issue for 1.45 --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cfb1b9c8474..be52e05776c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,6 +62,8 @@ jobs: - name: Pin tokio to 1.14 for Rust 1.45 if: "matrix.build-net-old-tokio" run: cargo update -p tokio --precise "1.14.0" --verbose + env: + CARGO_NET_GIT_FETCH_WITH_CLI: "true" - name: Build on Rust ${{ matrix.toolchain }} with net-tokio if: "matrix.build-net-tokio && !matrix.coverage" run: cargo build --verbose --color always From 5847abd92db2f5346bdc307ec532df6a3753dec3 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sun, 21 Aug 2022 20:34:22 +0000 Subject: [PATCH 69/91] Avoid querying the chain for outputs for channels we already have If we receive a ChannelAnnouncement message but we already have the channel, there's no reason to do a chain lookup. Instead of immediately calling the user-provided `chain::Access` when handling a ChannelAnnouncement, we first check if we have the corresponding channel in the graph. Note that if we do have the corresponding channel but it was not previously checked against the blockchain, we should still check with the `chain::Access` and update if necessary. --- lightning/src/routing/gossip.rs | 62 ++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index b083f402141..160609216e8 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -1433,6 +1433,39 @@ impl NetworkGraph where L::Target: Logger { return Err(LightningError{err: "Channel announcement node had a channel with itself".to_owned(), action: ErrorAction::IgnoreError}); } + { + let channels = self.channels.read().unwrap(); + + if let Some(chan) = channels.get(&msg.short_channel_id) { + if chan.capacity_sats.is_some() { + // If we'd previously looked up the channel on-chain and checked the script + // against what appears on-chain, ignore the duplicate announcement. + // + // Because a reorg could replace one channel with another at the same SCID, if + // the channel appears to be different, we re-validate. This doesn't expose us + // to any more DoS risk than not, as a peer can always flood us with + // randomly-generated SCID values anyway. + // + // We use the Node IDs rather than the bitcoin_keys to check for "equivalence" + // as we didn't (necessarily) store the bitcoin keys, and we only really care + // if the peers on the channel changed anyway. + if NodeId::from_pubkey(&msg.node_id_1) == chan.node_one && NodeId::from_pubkey(&msg.node_id_2) == chan.node_two { + return Err(LightningError { + err: "Already have chain-validated channel".to_owned(), + action: ErrorAction::IgnoreDuplicateGossip + }); + } + } else if chain_access.is_none() { + // Similarly, if we can't check the chain right now anyway, ignore the + // duplicate announcement without bothering to take the channels write lock. + return Err(LightningError { + err: "Already have non-chain-validated channel".to_owned(), + action: ErrorAction::IgnoreDuplicateGossip + }); + } + } + } + let utxo_value = match &chain_access { &None => { // Tentatively accept, potentially exposing us to DoS attacks @@ -2046,7 +2079,7 @@ mod tests { // drop new one on the floor, since we can't see any changes. match gossip_sync.handle_channel_announcement(&valid_announcement) { Ok(_) => panic!(), - Err(e) => assert_eq!(e.err, "Already have knowledge of channel") + Err(e) => assert_eq!(e.err, "Already have non-chain-validated channel") }; // Test if an associated transaction were not on-chain (or not confirmed). @@ -2080,32 +2113,13 @@ mod tests { }; } - // If we receive announcement for the same channel (but TX is not confirmed), - // drop new one on the floor, since we can't see any changes. - *chain_source.utxo_ret.lock().unwrap() = Err(chain::AccessError::UnknownTx); - match gossip_sync.handle_channel_announcement(&valid_announcement) { - Ok(_) => panic!(), - Err(e) => assert_eq!(e.err, "Channel announced without corresponding UTXO entry") - }; - - // But if it is confirmed, replace the channel + // If we receive announcement for the same channel, once we've validated it against the + // chain, we simply ignore all new (duplicate) announcements. *chain_source.utxo_ret.lock().unwrap() = Ok(TxOut { value: 0, script_pubkey: good_script }); - let valid_announcement = get_signed_channel_announcement(|unsigned_announcement| { - unsigned_announcement.features = ChannelFeatures::empty(); - unsigned_announcement.short_channel_id += 2; - }, node_1_privkey, node_2_privkey, &secp_ctx); match gossip_sync.handle_channel_announcement(&valid_announcement) { - Ok(res) => assert!(res), - _ => panic!() + Ok(_) => panic!(), + Err(e) => assert_eq!(e.err, "Already have chain-validated channel") }; - { - match network_graph.read_only().channels().get(&valid_announcement.contents.short_channel_id) { - Some(channel_entry) => { - assert_eq!(channel_entry.features, ChannelFeatures::empty()); - }, - _ => panic!() - }; - } // Don't relay valid channels with excess data let valid_announcement = get_signed_channel_announcement(|unsigned_announcement| { From dceca5b590dd34370e365307ed8517db5d04440e Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 5 Aug 2022 17:45:43 -0400 Subject: [PATCH 70/91] OM functional tests: update util to take nodes by reference And fix one test to be uniform with the others --- .../src/onion_message/functional_tests.rs | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lightning/src/onion_message/functional_tests.rs b/lightning/src/onion_message/functional_tests.rs index 695064e467c..f3026270f70 100644 --- a/lightning/src/onion_message/functional_tests.rs +++ b/lightning/src/onion_message/functional_tests.rs @@ -47,10 +47,10 @@ fn create_nodes(num_messengers: u8) -> Vec { res } -fn pass_along_path(mut path: Vec, expected_path_id: Option<[u8; 32]>) { - let mut prev_node = path.remove(0); +fn pass_along_path(path: &Vec, expected_path_id: Option<[u8; 32]>) { + let mut prev_node = &path[0]; let num_nodes = path.len(); - for (idx, node) in path.into_iter().enumerate() { + for (idx, node) in path.into_iter().skip(1).enumerate() { let events = prev_node.messenger.release_pending_msgs(); assert_eq!(events.len(), 1); let onion_msg = { @@ -73,7 +73,7 @@ fn one_hop() { let nodes = create_nodes(2); nodes[0].messenger.send_onion_message(&[], Destination::Node(nodes[1].get_node_pk())).unwrap(); - pass_along_path(nodes, None); + pass_along_path(&nodes, None); } #[test] @@ -81,7 +81,7 @@ fn two_unblinded_hops() { let nodes = create_nodes(3); nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk()], Destination::Node(nodes[2].get_node_pk())).unwrap(); - pass_along_path(nodes, None); + pass_along_path(&nodes, None); } #[test] @@ -92,7 +92,7 @@ fn two_unblinded_two_blinded() { let blinded_route = BlindedRoute::new::(&[nodes[3].get_node_pk(), nodes[4].get_node_pk()], &*nodes[4].keys_manager, &secp_ctx).unwrap(); nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], Destination::BlindedRoute(blinded_route)).unwrap(); - pass_along_path(nodes, None); + pass_along_path(&nodes, None); } #[test] @@ -103,7 +103,7 @@ fn three_blinded_hops() { let blinded_route = BlindedRoute::new::(&[nodes[1].get_node_pk(), nodes[2].get_node_pk(), nodes[3].get_node_pk()], &*nodes[3].keys_manager, &secp_ctx).unwrap(); nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap(); - pass_along_path(nodes, None); + pass_along_path(&nodes, None); } #[test] @@ -124,19 +124,18 @@ fn too_big_packet_error() { fn invalid_blinded_route_error() { // Make sure we error as expected if a provided blinded route has 0 or 1 hops. let mut nodes = create_nodes(3); - let (node1, node2, node3) = (nodes.remove(0), nodes.remove(0), nodes.remove(0)); // 0 hops let secp_ctx = Secp256k1::new(); - let mut blinded_route = BlindedRoute::new::(&[node2.get_node_pk(), node3.get_node_pk()], &*node3.keys_manager, &secp_ctx).unwrap(); + let mut blinded_route = BlindedRoute::new::(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap(); blinded_route.blinded_hops.clear(); - let err = node1.messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap_err(); + let err = nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap_err(); assert_eq!(err, SendError::TooFewBlindedHops); // 1 hop - let mut blinded_route = BlindedRoute::new::(&[node2.get_node_pk(), node3.get_node_pk()], &*node3.keys_manager, &secp_ctx).unwrap(); + let mut blinded_route = BlindedRoute::new::(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap(); blinded_route.blinded_hops.remove(0); assert_eq!(blinded_route.blinded_hops.len(), 1); - let err = node1.messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap_err(); + let err = nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap_err(); assert_eq!(err, SendError::TooFewBlindedHops); } From 351349c845b9927ef8a508c1216efe380c41b138 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 5 Aug 2022 17:50:02 -0400 Subject: [PATCH 71/91] Fix bug in onion message payload decode Previously, we were decoding payload lengths as a VarInt. Per the spec, this is wrong -- it should be decoded as a BigSize. This bug also exists in our payment payload decoding, to be fixed separately. Upcoming reply path tests caught this bug because we hadn't encoded a payload greater than 253 before, so we hadn't hit the problem that VarInts are encoded as little-endian whereas BigSizes are encoded as big-endian. --- lightning/src/onion_message/packet.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lightning/src/onion_message/packet.rs b/lightning/src/onion_message/packet.rs index d4ba28c843b..5afe578121b 100644 --- a/lightning/src/onion_message/packet.rs +++ b/lightning/src/onion_message/packet.rs @@ -16,7 +16,7 @@ use ln::msgs::DecodeError; use ln::onion_utils; use super::blinded_route::{ForwardTlvs, ReceiveTlvs}; use util::chacha20poly1305rfc::{ChaChaPolyReadAdapter, ChaChaPolyWriteAdapter}; -use util::ser::{FixedLengthReader, LengthRead, LengthReadable, LengthReadableArgs, Readable, ReadableArgs, Writeable, Writer}; +use util::ser::{BigSize, FixedLengthReader, LengthRead, LengthReadable, LengthReadableArgs, Readable, ReadableArgs, Writeable, Writer}; use core::cmp; use io::{self, Read}; @@ -161,13 +161,7 @@ impl Writeable for (Payload, [u8; 32]) { // Uses the provided secret to simultaneously decode and decrypt the control TLVs. impl ReadableArgs for Payload { fn read(mut r: &mut R, encrypted_tlvs_ss: SharedSecret) -> Result { - use bitcoin::consensus::encode::{Decodable, Error, VarInt}; - let v: VarInt = Decodable::consensus_decode(&mut r) - .map_err(|e| match e { - Error::Io(ioe) => DecodeError::from(ioe), - _ => DecodeError::InvalidValue - })?; - + let v: BigSize = Readable::read(r)?; let mut rd = FixedLengthReader::new(r, v.0); // TODO: support reply paths let mut _reply_path_bytes: Option> = Some(Vec::new()); From 950b7d777a3fb129759932f5a4220ece44e94f41 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 5 Aug 2022 18:03:12 -0400 Subject: [PATCH 72/91] Support sending and receiving reply paths --- fuzz/src/onion_message.rs | 2 +- lightning/src/onion_message/blinded_route.rs | 39 ++++++++++++++++++- .../src/onion_message/functional_tests.rs | 39 +++++++++++++++---- lightning/src/onion_message/messenger.rs | 17 +++++--- lightning/src/onion_message/packet.rs | 27 ++++++++----- 5 files changed, 99 insertions(+), 25 deletions(-) diff --git a/fuzz/src/onion_message.rs b/fuzz/src/onion_message.rs index 7ab2bd63a9f..57603dde110 100644 --- a/fuzz/src/onion_message.rs +++ b/fuzz/src/onion_message.rs @@ -122,7 +122,7 @@ mod tests { super::do_test(&::hex::decode(one_hop_om).unwrap(), &logger); { let log_entries = logger.lines.lock().unwrap(); - assert_eq!(log_entries.get(&("lightning::onion_message::messenger".to_string(), "Received an onion message with path_id: None".to_string())), Some(&1)); + assert_eq!(log_entries.get(&("lightning::onion_message::messenger".to_string(), "Received an onion message with path_id: None and no reply_path".to_string())), Some(&1)); } let two_unblinded_hops_om = "020000000000000000000000000000000000000000000000000000000000000e01055600020000000000000000000000000000000000000000000000000000000000000e0135043304210200000000000000000000000000000000000000000000000000000000000000039500000000000000000000000000000058000000000000000000000000000000000000000000000000000000000000001204105e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b300000000000000000000000000000000000000000000000000000000000000"; diff --git a/lightning/src/onion_message/blinded_route.rs b/lightning/src/onion_message/blinded_route.rs index d18372e3b00..9f1d8db46dd 100644 --- a/lightning/src/onion_message/blinded_route.rs +++ b/lightning/src/onion_message/blinded_route.rs @@ -13,10 +13,10 @@ use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey}; use chain::keysinterface::{KeysInterface, Sign}; use super::utils; +use ln::msgs::DecodeError; use util::chacha20poly1305rfc::ChaChaPolyWriteAdapter; -use util::ser::{VecWriter, Writeable, Writer}; +use util::ser::{Readable, VecWriter, Writeable, Writer}; -use core::iter::FromIterator; use io; use prelude::*; @@ -113,6 +113,41 @@ fn encrypt_payload(payload: P, encrypted_tlvs_ss: [u8; 32]) -> Vec writer.0 } +impl Writeable for BlindedRoute { + fn write(&self, w: &mut W) -> Result<(), io::Error> { + self.introduction_node_id.write(w)?; + self.blinding_point.write(w)?; + (self.blinded_hops.len() as u8).write(w)?; + for hop in &self.blinded_hops { + hop.write(w)?; + } + Ok(()) + } +} + +impl Readable for BlindedRoute { + fn read(r: &mut R) -> Result { + let introduction_node_id = Readable::read(r)?; + let blinding_point = Readable::read(r)?; + let num_hops: u8 = Readable::read(r)?; + if num_hops == 0 { return Err(DecodeError::InvalidValue) } + let mut blinded_hops: Vec = Vec::with_capacity(num_hops.into()); + for _ in 0..num_hops { + blinded_hops.push(Readable::read(r)?); + } + Ok(BlindedRoute { + introduction_node_id, + blinding_point, + blinded_hops, + }) + } +} + +impl_writeable!(BlindedHop, { + blinded_node_id, + encrypted_payload +}); + /// TLVs to encode in an intermediate onion message packet's hop data. When provided in a blinded /// route, they are encoded into [`BlindedHop::encrypted_payload`]. pub(crate) struct ForwardTlvs { diff --git a/lightning/src/onion_message/functional_tests.rs b/lightning/src/onion_message/functional_tests.rs index f3026270f70..ccc834434a1 100644 --- a/lightning/src/onion_message/functional_tests.rs +++ b/lightning/src/onion_message/functional_tests.rs @@ -72,7 +72,7 @@ fn pass_along_path(path: &Vec, expected_path_id: Option<[u8; 32]> fn one_hop() { let nodes = create_nodes(2); - nodes[0].messenger.send_onion_message(&[], Destination::Node(nodes[1].get_node_pk())).unwrap(); + nodes[0].messenger.send_onion_message(&[], Destination::Node(nodes[1].get_node_pk()), None).unwrap(); pass_along_path(&nodes, None); } @@ -80,7 +80,7 @@ fn one_hop() { fn two_unblinded_hops() { let nodes = create_nodes(3); - nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk()], Destination::Node(nodes[2].get_node_pk())).unwrap(); + nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk()], Destination::Node(nodes[2].get_node_pk()), None).unwrap(); pass_along_path(&nodes, None); } @@ -91,7 +91,7 @@ fn two_unblinded_two_blinded() { let secp_ctx = Secp256k1::new(); let blinded_route = BlindedRoute::new::(&[nodes[3].get_node_pk(), nodes[4].get_node_pk()], &*nodes[4].keys_manager, &secp_ctx).unwrap(); - nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], Destination::BlindedRoute(blinded_route)).unwrap(); + nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], Destination::BlindedRoute(blinded_route), None).unwrap(); pass_along_path(&nodes, None); } @@ -102,7 +102,7 @@ fn three_blinded_hops() { let secp_ctx = Secp256k1::new(); let blinded_route = BlindedRoute::new::(&[nodes[1].get_node_pk(), nodes[2].get_node_pk(), nodes[3].get_node_pk()], &*nodes[3].keys_manager, &secp_ctx).unwrap(); - nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap(); + nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route), None).unwrap(); pass_along_path(&nodes, None); } @@ -116,7 +116,7 @@ fn too_big_packet_error() { let hop_node_id = PublicKey::from_secret_key(&secp_ctx, &hop_secret); let hops = [hop_node_id; 400]; - let err = nodes[0].messenger.send_onion_message(&hops, Destination::Node(hop_node_id)).unwrap_err(); + let err = nodes[0].messenger.send_onion_message(&hops, Destination::Node(hop_node_id), None).unwrap_err(); assert_eq!(err, SendError::TooBigPacket); } @@ -129,13 +129,38 @@ fn invalid_blinded_route_error() { let secp_ctx = Secp256k1::new(); let mut blinded_route = BlindedRoute::new::(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap(); blinded_route.blinded_hops.clear(); - let err = nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap_err(); + let err = nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route), None).unwrap_err(); assert_eq!(err, SendError::TooFewBlindedHops); // 1 hop let mut blinded_route = BlindedRoute::new::(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap(); blinded_route.blinded_hops.remove(0); assert_eq!(blinded_route.blinded_hops.len(), 1); - let err = nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route)).unwrap_err(); + let err = nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route), None).unwrap_err(); assert_eq!(err, SendError::TooFewBlindedHops); } + +#[test] +fn reply_path() { + let mut nodes = create_nodes(4); + let secp_ctx = Secp256k1::new(); + + // Destination::Node + let reply_path = BlindedRoute::new::(&[nodes[2].get_node_pk(), nodes[1].get_node_pk(), nodes[0].get_node_pk()], &*nodes[0].keys_manager, &secp_ctx).unwrap(); + nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], Destination::Node(nodes[3].get_node_pk()), Some(reply_path)).unwrap(); + pass_along_path(&nodes, None); + // Make sure the last node successfully decoded the reply path. + nodes[3].logger.assert_log_contains( + "lightning::onion_message::messenger".to_string(), + format!("Received an onion message with path_id: None and reply_path").to_string(), 1); + + // Destination::BlindedRoute + let blinded_route = BlindedRoute::new::(&[nodes[1].get_node_pk(), nodes[2].get_node_pk(), nodes[3].get_node_pk()], &*nodes[3].keys_manager, &secp_ctx).unwrap(); + let reply_path = BlindedRoute::new::(&[nodes[2].get_node_pk(), nodes[1].get_node_pk(), nodes[0].get_node_pk()], &*nodes[0].keys_manager, &secp_ctx).unwrap(); + + nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route), Some(reply_path)).unwrap(); + pass_along_path(&nodes, None); + nodes[3].logger.assert_log_contains( + "lightning::onion_message::messenger".to_string(), + format!("Received an onion message with path_id: None and reply_path").to_string(), 2); +} diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index c264cbc387f..a5438afbb8e 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -142,7 +142,7 @@ impl OnionMessenger /// Send an empty onion message to `destination`, routing it through `intermediate_nodes`. /// See [`OnionMessenger`] for example usage. - pub fn send_onion_message(&self, intermediate_nodes: &[PublicKey], destination: Destination) -> Result<(), SendError> { + pub fn send_onion_message(&self, intermediate_nodes: &[PublicKey], destination: Destination, reply_path: Option) -> Result<(), SendError> { if let Destination::BlindedRoute(BlindedRoute { ref blinded_hops, .. }) = destination { if blinded_hops.len() < 2 { return Err(SendError::TooFewBlindedHops); @@ -160,7 +160,7 @@ impl OnionMessenger } }; let (packet_payloads, packet_keys) = packet_payloads_and_keys( - &self.secp_ctx, intermediate_nodes, destination, &blinding_secret) + &self.secp_ctx, intermediate_nodes, destination, reply_path, &blinding_secret) .map_err(|e| SendError::Secp256k1(e))?; let prng_seed = self.keys_manager.get_secure_random_bytes(); @@ -209,9 +209,11 @@ impl OnionMessenger msg.onion_routing_packet.hmac, control_tlvs_ss) { Ok((Payload::Receive { - control_tlvs: ReceiveControlTlvs::Unblinded(ReceiveTlvs { path_id }) + control_tlvs: ReceiveControlTlvs::Unblinded(ReceiveTlvs { path_id }), reply_path, }, None)) => { - log_info!(self.logger, "Received an onion message with path_id: {:02x?}", path_id); + log_info!(self.logger, + "Received an onion message with path_id: {:02x?} and {}reply_path", + path_id, if reply_path.is_some() { "" } else { "no " }); }, Ok((Payload::Forward(ForwardControlTlvs::Unblinded(ForwardTlvs { next_node_id, next_blinding_override @@ -299,7 +301,8 @@ pub type SimpleRefOnionMessenger<'a, 'b, L> = OnionMessenger( - secp_ctx: &Secp256k1, unblinded_path: &[PublicKey], destination: Destination, session_priv: &SecretKey + secp_ctx: &Secp256k1, unblinded_path: &[PublicKey], destination: Destination, mut reply_path: + Option, session_priv: &SecretKey ) -> Result<(Vec<(Payload, [u8; 32])>, Vec), secp256k1::Error> { let num_hops = unblinded_path.len() + destination.num_hops(); let mut payloads = Vec::with_capacity(num_hops); @@ -344,6 +347,7 @@ fn packet_payloads_and_keys( } else if let Some(encrypted_payload) = enc_payload_opt { payloads.push((Payload::Receive { control_tlvs: ReceiveControlTlvs::Blinded(encrypted_payload), + reply_path: reply_path.take(), }, control_tlvs_ss)); } @@ -361,7 +365,8 @@ fn packet_payloads_and_keys( if let Some(control_tlvs_ss) = prev_control_tlvs_ss { payloads.push((Payload::Receive { - control_tlvs: ReceiveControlTlvs::Unblinded(ReceiveTlvs { path_id: None, }) + control_tlvs: ReceiveControlTlvs::Unblinded(ReceiveTlvs { path_id: None, }), + reply_path: reply_path.take(), }, control_tlvs_ss)); } diff --git a/lightning/src/onion_message/packet.rs b/lightning/src/onion_message/packet.rs index 5afe578121b..4ab53735ed6 100644 --- a/lightning/src/onion_message/packet.rs +++ b/lightning/src/onion_message/packet.rs @@ -14,7 +14,7 @@ use bitcoin::secp256k1::ecdh::SharedSecret; use ln::msgs::DecodeError; use ln::onion_utils; -use super::blinded_route::{ForwardTlvs, ReceiveTlvs}; +use super::blinded_route::{BlindedRoute, ForwardTlvs, ReceiveTlvs}; use util::chacha20poly1305rfc::{ChaChaPolyReadAdapter, ChaChaPolyWriteAdapter}; use util::ser::{BigSize, FixedLengthReader, LengthRead, LengthReadable, LengthReadableArgs, Readable, ReadableArgs, Writeable, Writer}; @@ -98,8 +98,8 @@ pub(super) enum Payload { /// This payload is for the final hop. Receive { control_tlvs: ReceiveControlTlvs, + reply_path: Option, // Coming soon: - // reply_path: Option, // message: Message, } } @@ -135,21 +135,31 @@ pub(super) enum ReceiveControlTlvs { impl Writeable for (Payload, [u8; 32]) { fn write(&self, w: &mut W) -> Result<(), io::Error> { match &self.0 { - Payload::Forward(ForwardControlTlvs::Blinded(encrypted_bytes)) | - Payload::Receive { control_tlvs: ReceiveControlTlvs::Blinded(encrypted_bytes)} => { + Payload::Forward(ForwardControlTlvs::Blinded(encrypted_bytes)) => { encode_varint_length_prefixed_tlv!(w, { (4, encrypted_bytes, vec_type) }) }, + Payload::Receive { + control_tlvs: ReceiveControlTlvs::Blinded(encrypted_bytes), reply_path + } => { + encode_varint_length_prefixed_tlv!(w, { + (2, reply_path, option), + (4, encrypted_bytes, vec_type) + }) + }, Payload::Forward(ForwardControlTlvs::Unblinded(control_tlvs)) => { let write_adapter = ChaChaPolyWriteAdapter::new(self.1, &control_tlvs); encode_varint_length_prefixed_tlv!(w, { (4, write_adapter, required) }) }, - Payload::Receive { control_tlvs: ReceiveControlTlvs::Unblinded(control_tlvs)} => { + Payload::Receive { + control_tlvs: ReceiveControlTlvs::Unblinded(control_tlvs), reply_path, + } => { let write_adapter = ChaChaPolyWriteAdapter::new(self.1, &control_tlvs); encode_varint_length_prefixed_tlv!(w, { + (2, reply_path, option), (4, write_adapter, required) }) }, @@ -163,12 +173,11 @@ impl ReadableArgs for Payload { fn read(mut r: &mut R, encrypted_tlvs_ss: SharedSecret) -> Result { let v: BigSize = Readable::read(r)?; let mut rd = FixedLengthReader::new(r, v.0); - // TODO: support reply paths - let mut _reply_path_bytes: Option> = Some(Vec::new()); + let mut reply_path: Option = None; let mut read_adapter: Option> = None; let rho = onion_utils::gen_rho_from_shared_secret(&encrypted_tlvs_ss.secret_bytes()); decode_tlv_stream!(&mut rd, { - (2, _reply_path_bytes, vec_type), + (2, reply_path, option), (4, read_adapter, (option: LengthReadableArgs, rho)) }); rd.eat_remaining().map_err(|_| DecodeError::ShortRead)?; @@ -179,7 +188,7 @@ impl ReadableArgs for Payload { Ok(Payload::Forward(ForwardControlTlvs::Unblinded(tlvs))) }, Some(ChaChaPolyReadAdapter { readable: ControlTlvs::Receive(tlvs)}) => { - Ok(Payload::Receive { control_tlvs: ReceiveControlTlvs::Unblinded(tlvs)}) + Ok(Payload::Receive { control_tlvs: ReceiveControlTlvs::Unblinded(tlvs), reply_path }) }, } } From c7c88a6572400ccdc60def75d24f0b169d6e3353 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 24 Aug 2022 13:59:58 +0200 Subject: [PATCH 73/91] Export and document all `log` macros. Previously, only `log_error` and `log_trace` macros have been exported. This change exports the macros of all log levels, which enables them to be used downstream. --- lightning/src/util/macro_logger.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lightning/src/util/macro_logger.rs b/lightning/src/util/macro_logger.rs index 63496b28362..cd79a3f7bba 100644 --- a/lightning/src/util/macro_logger.rs +++ b/lightning/src/util/macro_logger.rs @@ -160,6 +160,7 @@ macro_rules! log_internal { } /// Logs an entry at the given level. +#[doc(hidden)] #[macro_export] macro_rules! log_given_level { ($logger: expr, $lvl:expr, $($arg:tt)+) => ( @@ -185,7 +186,7 @@ macro_rules! log_given_level { ); } -/// Log an error. +/// Log at the `ERROR` level. #[macro_export] macro_rules! log_error { ($logger: expr, $($arg:tt)*) => ( @@ -193,25 +194,31 @@ macro_rules! log_error { ) } +/// Log at the `WARN` level. +#[macro_export] macro_rules! log_warn { ($logger: expr, $($arg:tt)*) => ( log_given_level!($logger, $crate::util::logger::Level::Warn, $($arg)*); ) } +/// Log at the `INFO` level. +#[macro_export] macro_rules! log_info { ($logger: expr, $($arg:tt)*) => ( log_given_level!($logger, $crate::util::logger::Level::Info, $($arg)*); ) } +/// Log at the `DEBUG` level. +#[macro_export] macro_rules! log_debug { ($logger: expr, $($arg:tt)*) => ( log_given_level!($logger, $crate::util::logger::Level::Debug, $($arg)*); ) } -/// Log a trace log. +/// Log at the `TRACE` level. #[macro_export] macro_rules! log_trace { ($logger: expr, $($arg:tt)*) => ( @@ -219,7 +226,8 @@ macro_rules! log_trace { ) } -/// Log a gossip log. +/// Log at the `GOSSIP` level. +#[macro_export] macro_rules! log_gossip { ($logger: expr, $($arg:tt)*) => ( log_given_level!($logger, $crate::util::logger::Level::Gossip, $($arg)*); From 4c9231371b8496699479e65debfcf342a1a11edd Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Wed, 17 Aug 2022 20:15:23 +0000 Subject: [PATCH 74/91] Expose a `Balance` for inbound HTLCs even without a preimage If we don't currently have the preimage for an inbound HTLC, that does not guarantee we can never claim it, but instead only that we cannot claim it unless we receive the preimage from the channel we forwarded the channel out on. Thus, we cannot consider a channel to have no claimable balances if the only remaining output on the commitment ransaction is an inbound HTLC for which we do not have the preimage, as we may be able to claim it in the future. This commit addresses this issue by adding a new `Balance` variant - `MaybePreimageClaimableHTLCAwaitingTimeout`, which is generated until the HTLC output is spent. Fixes #1620 --- lightning/src/chain/channelmonitor.rs | 27 ++- lightning/src/ln/monitor_tests.rs | 244 +++++++++++++++++++++++++- 2 files changed, 266 insertions(+), 5 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 500f2b521dd..a27c3af8890 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -579,13 +579,24 @@ pub enum Balance { /// fees) if the counterparty does not know the preimage for the HTLCs. These are somewhat /// likely to be claimed by our counterparty before we do. MaybeClaimableHTLCAwaitingTimeout { - /// The amount available to claim, in satoshis, excluding the on-chain fees which will be - /// required to do so. + /// The amount potentially available to claim, in satoshis, excluding the on-chain fees + /// which will be required to do so. claimable_amount_satoshis: u64, /// The height at which we will be able to claim the balance if our counterparty has not /// done so. claimable_height: u32, }, + /// HTLCs which we received from our counterparty which are claimable with a preimage which we + /// do not currently have. This will only be claimable if we receive the preimage from the node + /// to which we forwarded this HTLC before the timeout. + MaybePreimageClaimableHTLC { + /// The amount potentially available to claim, in satoshis, excluding the on-chain fees + /// which will be required to do so. + claimable_amount_satoshis: u64, + /// The height at which our counterparty will be able to claim the balance if we have not + /// yet received the preimage and claimed it ourselves. + expiry_height: u32, + }, /// The channel has been closed, and our counterparty broadcasted a revoked commitment /// transaction. /// @@ -1565,6 +1576,11 @@ impl ChannelMonitorImpl { timeout_height: htlc.cltv_expiry, }); } + } else if htlc_resolved.is_none() { + return Some(Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: htlc.amount_msat / 1000, + expiry_height: htlc.cltv_expiry, + }); } None } @@ -1728,6 +1744,13 @@ impl ChannelMonitor { }); } else if us.payment_preimages.get(&htlc.payment_hash).is_some() { claimable_inbound_htlc_value_sat += htlc.amount_msat / 1000; + } else { + // As long as the HTLC is still in our latest commitment state, treat + // it as potentially claimable, even if it has long-since expired. + res.push(Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: htlc.amount_msat / 1000, + expiry_height: htlc.cltv_expiry, + }); } } res.push(Balance::ClaimableOnChannelClose { diff --git a/lightning/src/ln/monitor_tests.rs b/lightning/src/ln/monitor_tests.rs index 67ea07f2abd..1f49aed77de 100644 --- a/lightning/src/ln/monitor_tests.rs +++ b/lightning/src/ln/monitor_tests.rs @@ -288,10 +288,16 @@ fn do_test_claim_value_force_close(prev_commitment_tx: bool) { claimable_height: htlc_cltv_timeout, }]), sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); - assert_eq!(vec![Balance::ClaimableOnChannelClose { + assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose { claimable_amount_satoshis: 1_000, - }], - nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances()); + }, Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 3_000, + expiry_height: htlc_cltv_timeout, + }, Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 4_000, + expiry_height: htlc_cltv_timeout, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); nodes[1].node.claim_funds(payment_preimage); check_added_monitors!(nodes[1], 1); @@ -747,6 +753,238 @@ fn test_balances_on_local_commitment_htlcs() { test_spendable_output(&nodes[0], &as_txn[1]); } +#[test] +fn test_no_preimage_inbound_htlc_balances() { + // Tests that MaybePreimageClaimableHTLC are generated for inbound HTLCs for which we do not + // have a preimage. + let chanmon_cfgs = create_chanmon_cfgs(2); + let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs); + + let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 500_000_000, InitFeatures::known(), InitFeatures::known()); + let funding_outpoint = OutPoint { txid: funding_tx.txid(), index: 0 }; + + // Send two HTLCs, one from A to B, and one from B to A. + let to_b_failed_payment_hash = route_payment(&nodes[0], &[&nodes[1]], 10_000_000).1; + let to_a_failed_payment_hash = route_payment(&nodes[1], &[&nodes[0]], 20_000_000).1; + let htlc_cltv_timeout = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1; // Note ChannelManager adds one to CLTV timeouts for safety + + let chan_feerate = get_feerate!(nodes[0], chan_id) as u64; + let opt_anchors = get_opt_anchors!(nodes[0], chan_id); + + // Both A and B will have an HTLC that's claimable on timeout and one that's claimable if they + // receive the preimage. These will remain the same through the channel closure and until the + // HTLC output is spent. + + assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose { + claimable_amount_satoshis: 1_000_000 - 500_000 - 10_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + }, Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 20_000, + expiry_height: htlc_cltv_timeout, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 10_000, + claimable_height: htlc_cltv_timeout, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose { + claimable_amount_satoshis: 500_000 - 20_000, + }, Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 10_000, + expiry_height: htlc_cltv_timeout, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 20_000, + claimable_height: htlc_cltv_timeout, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + // Get nodes[0]'s commitment transaction and HTLC-Timeout transaction + let as_txn = get_local_commitment_txn!(nodes[0], chan_id); + assert_eq!(as_txn.len(), 2); + check_spends!(as_txn[1], as_txn[0]); + check_spends!(as_txn[0], funding_tx); + + // Now close the channel by confirming A's commitment transaction on both nodes, checking the + // claimable balances remain the same except for the non-HTLC balance changing variant. + let node_a_commitment_claimable = nodes[0].best_block_info().1 + BREAKDOWN_TIMEOUT as u32; + let as_pre_spend_claims = sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 1_000_000 - 500_000 - 10_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + confirmation_height: node_a_commitment_claimable, + }, Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 20_000, + expiry_height: htlc_cltv_timeout, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 10_000, + claimable_height: htlc_cltv_timeout, + }]); + + mine_transaction(&nodes[0], &as_txn[0]); + nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear(); + check_added_monitors!(nodes[0], 1); + check_closed_broadcast!(nodes[0], true); + check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed); + + assert_eq!(as_pre_spend_claims, + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[1], &as_txn[0]); + check_added_monitors!(nodes[1], 1); + check_closed_broadcast!(nodes[1], true); + check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed); + + let node_b_commitment_claimable = nodes[1].best_block_info().1 + ANTI_REORG_DELAY - 1; + let mut bs_pre_spend_claims = sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 500_000 - 20_000, + confirmation_height: node_b_commitment_claimable, + }, Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 10_000, + expiry_height: htlc_cltv_timeout, + }, Balance::MaybeClaimableHTLCAwaitingTimeout { + claimable_amount_satoshis: 20_000, + claimable_height: htlc_cltv_timeout, + }]); + assert_eq!(bs_pre_spend_claims, + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + // We'll broadcast the HTLC-Timeout transaction one block prior to the htlc's expiration (as it + // is confirmable in the next block), but will still include the same claimable balances as no + // HTLC has been spent, even after the HTLC expires. We'll also fail the inbound HTLC, but it + // won't do anything as the channel is already closed. + + connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); + let as_htlc_timeout_claim = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + assert_eq!(as_htlc_timeout_claim.len(), 1); + check_spends!(as_htlc_timeout_claim[0], as_txn[0]); + expect_pending_htlcs_forwardable_conditions!(nodes[0], + [HTLCDestination::FailedPayment { payment_hash: to_a_failed_payment_hash }]); + + assert_eq!(as_pre_spend_claims, + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[0], 1); + assert_eq!(as_pre_spend_claims, + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + // For node B, we'll get the non-HTLC funds claimable after ANTI_REORG_DELAY confirmations + connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1); + test_spendable_output(&nodes[1], &as_txn[0]); + bs_pre_spend_claims.retain(|e| if let Balance::ClaimableAwaitingConfirmations { .. } = e { false } else { true }); + + // The next few blocks for B look the same as for A, though for the opposite HTLC + nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clear(); + connect_blocks(&nodes[1], TEST_FINAL_CLTV - (ANTI_REORG_DELAY - 1) - 1); + expect_pending_htlcs_forwardable_conditions!(nodes[1], + [HTLCDestination::FailedPayment { payment_hash: to_b_failed_payment_hash }]); + let bs_htlc_timeout_claim = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0); + assert_eq!(bs_htlc_timeout_claim.len(), 1); + check_spends!(bs_htlc_timeout_claim[0], as_txn[0]); + + assert_eq!(bs_pre_spend_claims, + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[1], 1); + assert_eq!(bs_pre_spend_claims, + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + // Now confirm the two HTLC timeout transactions for A, checking that the inbound HTLC resolves + // after ANTI_REORG_DELAY confirmations and the other takes BREAKDOWN_TIMEOUT confirmations. + mine_transaction(&nodes[0], &as_htlc_timeout_claim[0]); + let as_timeout_claimable_height = nodes[0].best_block_info().1 + (BREAKDOWN_TIMEOUT as u32) - 1; + assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 1_000_000 - 500_000 - 10_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + confirmation_height: node_a_commitment_claimable, + }, Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 20_000, + expiry_height: htlc_cltv_timeout, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 10_000, + confirmation_height: as_timeout_claimable_height, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[0], &bs_htlc_timeout_claim[0]); + assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 1_000_000 - 500_000 - 10_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + confirmation_height: node_a_commitment_claimable, + }, Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 20_000, + expiry_height: htlc_cltv_timeout, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 10_000, + confirmation_height: as_timeout_claimable_height, + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + // Once as_htlc_timeout_claim[0] reaches ANTI_REORG_DELAY confirmations, we should get a + // payment failure event. + connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2); + expect_payment_failed!(nodes[0], to_b_failed_payment_hash, true); + + connect_blocks(&nodes[0], 1); + assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 1_000_000 - 500_000 - 10_000 - chan_feerate * + (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, + confirmation_height: node_a_commitment_claimable, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 10_000, + confirmation_height: core::cmp::max(as_timeout_claimable_height, htlc_cltv_timeout), + }]), + sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[0], node_a_commitment_claimable - nodes[0].best_block_info().1); + assert_eq!(vec![Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 10_000, + confirmation_height: core::cmp::max(as_timeout_claimable_height, htlc_cltv_timeout), + }], + nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances()); + test_spendable_output(&nodes[0], &as_txn[0]); + + connect_blocks(&nodes[0], as_timeout_claimable_height - nodes[0].best_block_info().1); + assert!(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty()); + test_spendable_output(&nodes[0], &as_htlc_timeout_claim[0]); + + // The process for B should be completely identical as well, noting that the non-HTLC-balance + // was already claimed. + mine_transaction(&nodes[1], &bs_htlc_timeout_claim[0]); + let bs_timeout_claimable_height = nodes[1].best_block_info().1 + ANTI_REORG_DELAY - 1; + assert_eq!(sorted_vec(vec![Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 10_000, + expiry_height: htlc_cltv_timeout, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 20_000, + confirmation_height: bs_timeout_claimable_height, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + mine_transaction(&nodes[1], &as_htlc_timeout_claim[0]); + assert_eq!(sorted_vec(vec![Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 10_000, + expiry_height: htlc_cltv_timeout, + }, Balance::ClaimableAwaitingConfirmations { + claimable_amount_satoshis: 20_000, + confirmation_height: bs_timeout_claimable_height, + }]), + sorted_vec(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); + + connect_blocks(&nodes[1], ANTI_REORG_DELAY - 2); + expect_payment_failed!(nodes[1], to_a_failed_payment_hash, true); + + assert_eq!(vec![Balance::MaybePreimageClaimableHTLC { + claimable_amount_satoshis: 10_000, + expiry_height: htlc_cltv_timeout, + }], + nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances()); + test_spendable_output(&nodes[1], &bs_htlc_timeout_claim[0]); + + connect_blocks(&nodes[1], 1); + assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty()); +} + fn sorted_vec_with_additions(v_orig: &Vec, extra_ts: &[&T]) -> Vec { let mut v = v_orig.clone(); for t in extra_ts { From 39cede60d0bc8890bca0418371ec4e3b801d2541 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Wed, 24 Aug 2022 22:16:04 +0000 Subject: [PATCH 75/91] Rename MaybeClaimableHTLCAwaitingTimeout for consistency As we now have a MaybePreimageClaimableHTLC, its more consistent to rename `MaybeClaimableHTLCAwaitingTimeout` to `MaybeTimeoutClaimableHTLC`. --- lightning/src/chain/channelmonitor.rs | 6 ++-- lightning/src/ln/monitor_tests.rs | 52 +++++++++++++-------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index a27c3af8890..46779073fe1 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -578,7 +578,7 @@ pub enum Balance { /// HTLCs which we sent to our counterparty which are claimable after a timeout (less on-chain /// fees) if the counterparty does not know the preimage for the HTLCs. These are somewhat /// likely to be claimed by our counterparty before we do. - MaybeClaimableHTLCAwaitingTimeout { + MaybeTimeoutClaimableHTLC { /// The amount potentially available to claim, in satoshis, excluding the on-chain fees /// which will be required to do so. claimable_amount_satoshis: u64, @@ -1553,7 +1553,7 @@ impl ChannelMonitorImpl { confirmation_height: conf_thresh, }); } else { - return Some(Balance::MaybeClaimableHTLCAwaitingTimeout { + return Some(Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: htlc.amount_msat / 1000, claimable_height: htlc.cltv_expiry, }); @@ -1738,7 +1738,7 @@ impl ChannelMonitor { for (htlc, _, _) in us.current_holder_commitment_tx.htlc_outputs.iter() { if htlc.transaction_output_index.is_none() { continue; } if htlc.offered { - res.push(Balance::MaybeClaimableHTLCAwaitingTimeout { + res.push(Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: htlc.amount_msat / 1000, claimable_height: htlc.cltv_expiry, }); diff --git a/lightning/src/ln/monitor_tests.rs b/lightning/src/ln/monitor_tests.rs index 1f49aed77de..a2fccbbc388 100644 --- a/lightning/src/ln/monitor_tests.rs +++ b/lightning/src/ln/monitor_tests.rs @@ -280,10 +280,10 @@ fn do_test_claim_value_force_close(prev_commitment_tx: bool) { assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose { claimable_amount_satoshis: 1_000_000 - 3_000 - 4_000 - 1_000 - 3 - chan_feerate * (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 3_000, claimable_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 4_000, claimable_height: htlc_cltv_timeout, }]), @@ -341,12 +341,12 @@ fn do_test_claim_value_force_close(prev_commitment_tx: bool) { chan_feerate * (channel::commitment_tx_base_weight(opt_anchors) + if prev_commitment_tx { 1 } else { 2 } * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 4_000, claimable_height: htlc_cltv_timeout, }]; if !prev_commitment_tx { - a_expected_balances.push(Balance::MaybeClaimableHTLCAwaitingTimeout { + a_expected_balances.push(Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 3_000, claimable_height: htlc_cltv_timeout, }); @@ -403,10 +403,10 @@ fn do_test_claim_value_force_close(prev_commitment_tx: bool) { claimable_amount_satoshis: 1_000_000 - 3_000 - 4_000 - 1_000 - 3 - chan_feerate * (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, confirmation_height: nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 1, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 3_000, claimable_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 4_000, claimable_height: htlc_cltv_timeout, }]), @@ -434,10 +434,10 @@ fn do_test_claim_value_force_close(prev_commitment_tx: bool) { // After ANTI_REORG_DELAY, A will consider its balance fully spendable and generate a // `SpendableOutputs` event. However, B still has to wait for the CSV delay. - assert_eq!(sorted_vec(vec![Balance::MaybeClaimableHTLCAwaitingTimeout { + assert_eq!(sorted_vec(vec![Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 3_000, claimable_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 4_000, claimable_height: htlc_cltv_timeout, }]), @@ -465,16 +465,16 @@ fn do_test_claim_value_force_close(prev_commitment_tx: bool) { } else { expect_payment_sent!(nodes[0], payment_preimage); } - assert_eq!(sorted_vec(vec![Balance::MaybeClaimableHTLCAwaitingTimeout { + assert_eq!(sorted_vec(vec![Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 3_000, claimable_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 4_000, claimable_height: htlc_cltv_timeout, }]), sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances())); connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1); - assert_eq!(vec![Balance::MaybeClaimableHTLCAwaitingTimeout { + assert_eq!(vec![Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 4_000, claimable_height: htlc_cltv_timeout, }], @@ -646,10 +646,10 @@ fn test_balances_on_local_commitment_htlcs() { claimable_amount_satoshis: 1_000_000 - 10_000 - 20_000 - chan_feerate * (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, confirmation_height: node_a_commitment_claimable, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 10_000, claimable_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 20_000, claimable_height: htlc_cltv_timeout, }]), @@ -673,10 +673,10 @@ fn test_balances_on_local_commitment_htlcs() { claimable_amount_satoshis: 1_000_000 - 10_000 - 20_000 - chan_feerate * (channel::commitment_tx_base_weight(opt_anchors) + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000, confirmation_height: node_a_commitment_claimable, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 10_000, claimable_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 20_000, claimable_height: htlc_cltv_timeout, }]), @@ -697,7 +697,7 @@ fn test_balances_on_local_commitment_htlcs() { }, Balance::ClaimableAwaitingConfirmations { claimable_amount_satoshis: 10_000, confirmation_height: node_a_htlc_claimable, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 20_000, claimable_height: htlc_cltv_timeout, }]), @@ -714,7 +714,7 @@ fn test_balances_on_local_commitment_htlcs() { }, Balance::ClaimableAwaitingConfirmations { claimable_amount_satoshis: 10_000, confirmation_height: node_a_htlc_claimable, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 20_000, claimable_height: htlc_cltv_timeout, }]), @@ -783,7 +783,7 @@ fn test_no_preimage_inbound_htlc_balances() { }, Balance::MaybePreimageClaimableHTLC { claimable_amount_satoshis: 20_000, expiry_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 10_000, claimable_height: htlc_cltv_timeout, }]), @@ -794,7 +794,7 @@ fn test_no_preimage_inbound_htlc_balances() { }, Balance::MaybePreimageClaimableHTLC { claimable_amount_satoshis: 10_000, expiry_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 20_000, claimable_height: htlc_cltv_timeout, }]), @@ -816,7 +816,7 @@ fn test_no_preimage_inbound_htlc_balances() { }, Balance::MaybePreimageClaimableHTLC { claimable_amount_satoshis: 20_000, expiry_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 10_000, claimable_height: htlc_cltv_timeout, }]); @@ -842,7 +842,7 @@ fn test_no_preimage_inbound_htlc_balances() { }, Balance::MaybePreimageClaimableHTLC { claimable_amount_satoshis: 10_000, expiry_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 20_000, claimable_height: htlc_cltv_timeout, }]); @@ -1089,13 +1089,13 @@ fn do_test_revoked_counterparty_commitment_balances(confirm_htlc_spend_first: bo // lists the two on-chain timeout-able HTLCs as claimable balances. assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose { claimable_amount_satoshis: 100_000 - 5_000 - 4_000 - 3 - 2_000 + 3_000, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 2_000, claimable_height: missing_htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 4_000, claimable_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 5_000, claimable_height: live_htlc_cltv_timeout, }]), @@ -1501,10 +1501,10 @@ fn test_revoked_counterparty_aggregated_claims() { assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose { claimable_amount_satoshis: 100_000 - 4_000 - 3_000, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 4_000, claimable_height: htlc_cltv_timeout, - }, Balance::MaybeClaimableHTLCAwaitingTimeout { + }, Balance::MaybeTimeoutClaimableHTLC { claimable_amount_satoshis: 3_000, claimable_height: htlc_cltv_timeout, }]), From 1de698fdd9274dc47e18fb680ea15166742785a1 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 25 Aug 2022 13:56:19 -0400 Subject: [PATCH 76/91] PeerMan: fix bug in drop_gossip util Fixes a flipped bool that was introduced in 4a1ee5f9a984c9b0c0892025d624ade734337b1a --- lightning/src/ln/peer_handler.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 623aa969dfd..925d7e5946d 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -399,11 +399,8 @@ impl Peer { /// Returns whether this peer's buffer is full and we should drop gossip messages. fn buffer_full_drop_gossip(&self) -> bool { - if self.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP - || self.msgs_sent_since_pong > BUFFER_DRAIN_MSGS_PER_TICK * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO { - return false - } - true + self.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP || + self.msgs_sent_since_pong > BUFFER_DRAIN_MSGS_PER_TICK * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO } } From ab149dc9d51b4d560b7ef7d86c9718044b998264 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 25 Aug 2022 13:58:49 -0400 Subject: [PATCH 77/91] PeerMan: rename drop_gossip util to be more accurate It's more accurate to name it as dropping gossip broadcasts, as it won't drop all gossip. --- lightning/src/ln/peer_handler.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 925d7e5946d..08efffe9641 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -398,7 +398,7 @@ impl Peer { } /// Returns whether this peer's buffer is full and we should drop gossip messages. - fn buffer_full_drop_gossip(&self) -> bool { + fn buffer_full_drop_gossip_broadcast(&self) -> bool { self.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP || self.msgs_sent_since_pong > BUFFER_DRAIN_MSGS_PER_TICK * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO } @@ -1322,7 +1322,7 @@ impl P !peer.should_forward_channel_announcement(msg.contents.short_channel_id) { continue } - if peer.buffer_full_drop_gossip() { + if peer.buffer_full_drop_gossip_broadcast() { log_gossip!(self.logger, "Skipping broadcast message to {:?} as its outbound buffer is full", peer.their_node_id); continue; } @@ -1346,7 +1346,7 @@ impl P !peer.should_forward_node_announcement(msg.contents.node_id) { continue } - if peer.buffer_full_drop_gossip() { + if peer.buffer_full_drop_gossip_broadcast() { log_gossip!(self.logger, "Skipping broadcast message to {:?} as its outbound buffer is full", peer.their_node_id); continue; } @@ -1369,7 +1369,7 @@ impl P !peer.should_forward_channel_announcement(msg.contents.short_channel_id) { continue } - if peer.buffer_full_drop_gossip() { + if peer.buffer_full_drop_gossip_broadcast() { log_gossip!(self.logger, "Skipping broadcast message to {:?} as its outbound buffer is full", peer.their_node_id); continue; } From 47e818f198abafba01b9ad278582886f9007dac2 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Wed, 24 Aug 2022 18:04:58 -0400 Subject: [PATCH 78/91] Separate gossip broadcasts into their own queue in PeerManager This allows us to better prioritize channel messages over gossip broadcasts and lays groundwork for rate limiting onion messages more simply, since they won't be competing with gossip broadcasts for space in the main message queue. --- lightning/src/ln/peer_handler.rs | 54 ++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 08efffe9641..573e910ab08 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -337,6 +337,9 @@ struct Peer { pending_outbound_buffer: LinkedList>, pending_outbound_buffer_first_msg_offset: usize, + // Queue gossip broadcasts separately from `pending_outbound_buffer` so we can easily prioritize + // channel messages over them. + gossip_broadcast_buffer: LinkedList>, awaiting_write_event: bool, pending_read_buffer: Vec, @@ -389,17 +392,26 @@ impl Peer { self.pending_outbound_buffer.len() < OUTBOUND_BUFFER_LIMIT_READ_PAUSE } - /// Determines if we should push additional gossip messages onto a peer's outbound buffer for - /// backfilling gossip data to the peer. This is checked every time the peer's buffer may have - /// been drained. + /// Determines if we should push additional gossip background sync (aka "backfill") onto a peer's + /// outbound buffer. This is checked every time the peer's buffer may have been drained. fn should_buffer_gossip_backfill(&self) -> bool { - self.pending_outbound_buffer.is_empty() && - self.msgs_sent_since_pong < BUFFER_DRAIN_MSGS_PER_TICK + self.pending_outbound_buffer.is_empty() && self.gossip_broadcast_buffer.is_empty() + && self.msgs_sent_since_pong < BUFFER_DRAIN_MSGS_PER_TICK } - /// Returns whether this peer's buffer is full and we should drop gossip messages. + /// Determines if we should push additional gossip broadcast messages onto a peer's outbound + /// buffer. This is checked every time the peer's buffer may have been drained. + fn should_buffer_gossip_broadcast(&self) -> bool { + self.pending_outbound_buffer.is_empty() + && self.msgs_sent_since_pong < BUFFER_DRAIN_MSGS_PER_TICK + } + + /// Returns whether this peer's outbound buffers are full and we should drop gossip broadcasts. fn buffer_full_drop_gossip_broadcast(&self) -> bool { - self.pending_outbound_buffer.len() > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP || + let total_outbound_buffered = + self.gossip_broadcast_buffer.len() + self.pending_outbound_buffer.len(); + + total_outbound_buffered > OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP || self.msgs_sent_since_pong > BUFFER_DRAIN_MSGS_PER_TICK * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO } } @@ -668,6 +680,7 @@ impl P pending_outbound_buffer: LinkedList::new(), pending_outbound_buffer_first_msg_offset: 0, + gossip_broadcast_buffer: LinkedList::new(), awaiting_write_event: false, pending_read_buffer, @@ -714,6 +727,7 @@ impl P pending_outbound_buffer: LinkedList::new(), pending_outbound_buffer_first_msg_offset: 0, + gossip_broadcast_buffer: LinkedList::new(), awaiting_write_event: false, pending_read_buffer, @@ -734,6 +748,11 @@ impl P fn do_attempt_write_data(&self, descriptor: &mut Descriptor, peer: &mut Peer) { while !peer.awaiting_write_event { + if peer.should_buffer_gossip_broadcast() { + if let Some(msg) = peer.gossip_broadcast_buffer.pop_front() { + peer.pending_outbound_buffer.push_back(msg); + } + } if peer.should_buffer_gossip_backfill() { match peer.sync_status { InitSyncTracker::NoSyncRequested => {}, @@ -848,12 +867,6 @@ impl P } } - /// Append a message to a peer's pending outbound/write buffer - fn enqueue_encoded_message(&self, peer: &mut Peer, encoded_message: &Vec) { - peer.msgs_sent_since_pong += 1; - peer.pending_outbound_buffer.push_back(peer.channel_encryptor.encrypt_message(&encoded_message[..])); - } - /// Append a message to a peer's pending outbound/write buffer fn enqueue_message(&self, peer: &mut Peer, message: &M) { let mut buffer = VecWriter(Vec::with_capacity(2048)); @@ -864,7 +877,14 @@ impl P } else { log_trace!(self.logger, "Enqueueing message {:?} to {}", message, log_pubkey!(peer.their_node_id.unwrap())) } - self.enqueue_encoded_message(peer, &buffer.0); + peer.msgs_sent_since_pong += 1; + peer.pending_outbound_buffer.push_back(peer.channel_encryptor.encrypt_message(&buffer.0[..])); + } + + /// Append a message to a peer's pending outbound/write gossip broadcast buffer + fn enqueue_encoded_gossip_broadcast(&self, peer: &mut Peer, encoded_message: &Vec) { + peer.msgs_sent_since_pong += 1; + peer.gossip_broadcast_buffer.push_back(peer.channel_encryptor.encrypt_message(&encoded_message[..])); } fn do_read_event(&self, peer_descriptor: &mut Descriptor, data: &[u8]) -> Result { @@ -1333,7 +1353,7 @@ impl P if except_node.is_some() && peer.their_node_id.as_ref() == except_node { continue; } - self.enqueue_encoded_message(&mut *peer, &encoded_msg); + self.enqueue_encoded_gossip_broadcast(&mut *peer, &encoded_msg); } }, wire::Message::NodeAnnouncement(ref msg) => { @@ -1356,7 +1376,7 @@ impl P if except_node.is_some() && peer.their_node_id.as_ref() == except_node { continue; } - self.enqueue_encoded_message(&mut *peer, &encoded_msg); + self.enqueue_encoded_gossip_broadcast(&mut *peer, &encoded_msg); } }, wire::Message::ChannelUpdate(ref msg) => { @@ -1376,7 +1396,7 @@ impl P if except_node.is_some() && peer.their_node_id.as_ref() == except_node { continue; } - self.enqueue_encoded_message(&mut *peer, &encoded_msg); + self.enqueue_encoded_gossip_broadcast(&mut *peer, &encoded_msg); } }, _ => debug_assert!(false, "We shouldn't attempt to forward anything but gossip messages"), From 4adff1039fd9ad0d7fca787af8fd6eb638c8e37a Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Sat, 6 Aug 2022 00:33:48 -0400 Subject: [PATCH 79/91] Add boilerplate for sending and receiving onion messages in PeerManager Adds the boilerplate needed for PeerManager and OnionMessenger to work together, with some corresponding docs and misc updates mostly due to the PeerManager public API changing. --- fuzz/src/full_stack.rs | 3 +- fuzz/src/onion_message.rs | 2 +- lightning-background-processor/src/lib.rs | 10 +-- lightning-net-tokio/src/lib.rs | 25 +++++-- lightning/src/ln/msgs.rs | 8 ++- lightning/src/ln/peer_handler.rs | 67 +++++++++++++------ lightning/src/ln/wire.rs | 11 ++- .../src/onion_message/functional_tests.rs | 1 + lightning/src/onion_message/messenger.rs | 32 ++++++--- lightning/src/util/events.rs | 6 ++ 10 files changed, 121 insertions(+), 44 deletions(-) diff --git a/fuzz/src/full_stack.rs b/fuzz/src/full_stack.rs index c1d797ea579..f506acc9fe8 100644 --- a/fuzz/src/full_stack.rs +++ b/fuzz/src/full_stack.rs @@ -166,7 +166,7 @@ type ChannelMan = ChannelManager< EnforcingSigner, Arc, Arc, Arc, Arc, Arc>>, Arc, Arc, Arc, Arc>; -type PeerMan<'a> = PeerManager, Arc, Arc>>, Arc, Arc>>, Arc, IgnoringMessageHandler>; +type PeerMan<'a> = PeerManager, Arc, Arc>>, Arc, Arc>>, IgnoringMessageHandler, Arc, IgnoringMessageHandler>; struct MoneyLossDetector<'a> { manager: Arc, @@ -414,6 +414,7 @@ pub fn do_test(data: &[u8], logger: &Arc) { let mut loss_detector = MoneyLossDetector::new(&peers, channelmanager.clone(), monitor.clone(), PeerManager::new(MessageHandler { chan_handler: channelmanager.clone(), route_handler: gossip_sync.clone(), + onion_message_handler: IgnoringMessageHandler {}, }, our_network_key, &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0], Arc::clone(&logger), IgnoringMessageHandler{})); let mut should_forward = false; diff --git a/fuzz/src/onion_message.rs b/fuzz/src/onion_message.rs index 57603dde110..a2fe88afc83 100644 --- a/fuzz/src/onion_message.rs +++ b/fuzz/src/onion_message.rs @@ -6,7 +6,7 @@ use bitcoin::secp256k1::ecdh::SharedSecret; use bitcoin::secp256k1::ecdsa::RecoverableSignature; use lightning::chain::keysinterface::{Recipient, KeyMaterial, KeysInterface}; -use lightning::ln::msgs::{self, DecodeError}; +use lightning::ln::msgs::{self, DecodeError, OnionMessageHandler}; use lightning::ln::script::ShutdownScript; use lightning::util::enforcing_trait_impls::EnforcingSigner; use lightning::util::logger::Logger; diff --git a/lightning-background-processor/src/lib.rs b/lightning-background-processor/src/lib.rs index 8f6f0c49c1d..e95c9c3709e 100644 --- a/lightning-background-processor/src/lib.rs +++ b/lightning-background-processor/src/lib.rs @@ -19,7 +19,7 @@ use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; use lightning::chain::chainmonitor::{ChainMonitor, Persist}; use lightning::chain::keysinterface::{Sign, KeysInterface}; use lightning::ln::channelmanager::ChannelManager; -use lightning::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler}; +use lightning::ln::msgs::{ChannelMessageHandler, OnionMessageHandler, RoutingMessageHandler}; use lightning::ln::peer_handler::{CustomMessageHandler, PeerManager, SocketDescriptor}; use lightning::routing::gossip::{NetworkGraph, P2PGossipSync}; use lightning::routing::scoring::WriteableScore; @@ -281,6 +281,7 @@ impl BackgroundProcessor { P: 'static + Deref + Send + Sync, Descriptor: 'static + SocketDescriptor + Send + Sync, CMH: 'static + Deref + Send + Sync, + OMH: 'static + Deref + Send + Sync, RMH: 'static + Deref + Send + Sync, EH: 'static + EventHandler + Send, PS: 'static + Deref + Send, @@ -289,7 +290,7 @@ impl BackgroundProcessor { PGS: 'static + Deref> + Send + Sync, RGS: 'static + Deref> + Send, UMH: 'static + Deref + Send + Sync, - PM: 'static + Deref> + Send + Sync, + PM: 'static + Deref> + Send + Sync, S: 'static + Deref + Send + Sync, SC: WriteableScore<'a>, >( @@ -306,6 +307,7 @@ impl BackgroundProcessor { L::Target: 'static + Logger, P::Target: 'static + Persist, CMH::Target: 'static + ChannelMessageHandler, + OMH::Target: 'static + OnionMessageHandler, RMH::Target: 'static + RoutingMessageHandler, UMH::Target: 'static + CustomMessageHandler, PS::Target: 'static + Persister<'a, Signer, CW, T, K, F, L, SC>, @@ -544,7 +546,7 @@ mod tests { node: Arc>, p2p_gossip_sync: PGS, rapid_gossip_sync: RGS, - peer_manager: Arc, Arc, Arc, IgnoringMessageHandler>>, + peer_manager: Arc, Arc, IgnoringMessageHandler, Arc, IgnoringMessageHandler>>, chain_monitor: Arc, persister: Arc, tx_broadcaster: Arc, @@ -663,7 +665,7 @@ mod tests { let network_graph = Arc::new(NetworkGraph::new(genesis_block.header.block_hash(), logger.clone())); let p2p_gossip_sync = Arc::new(P2PGossipSync::new(network_graph.clone(), Some(chain_source.clone()), logger.clone())); let rapid_gossip_sync = Arc::new(RapidGossipSync::new(network_graph.clone())); - let msg_handler = MessageHandler { chan_handler: Arc::new(test_utils::TestChannelMessageHandler::new()), route_handler: Arc::new(test_utils::TestRoutingMessageHandler::new() )}; + let msg_handler = MessageHandler { chan_handler: Arc::new(test_utils::TestChannelMessageHandler::new()), route_handler: Arc::new(test_utils::TestRoutingMessageHandler::new()), onion_message_handler: IgnoringMessageHandler{}}; let peer_manager = Arc::new(PeerManager::new(msg_handler, keys_manager.get_node_secret(Recipient::Node).unwrap(), &seed, logger.clone(), IgnoringMessageHandler{})); let scorer = Arc::new(Mutex::new(test_utils::TestScorer::with_penalty(0))); let node = Node { node: manager, p2p_gossip_sync, rapid_gossip_sync, peer_manager, chain_monitor, persister, tx_broadcaster, network_graph, logger, best_block, scorer }; diff --git a/lightning-net-tokio/src/lib.rs b/lightning-net-tokio/src/lib.rs index ac9d4bb3bd5..7dfa38e95e8 100644 --- a/lightning-net-tokio/src/lib.rs +++ b/lightning-net-tokio/src/lib.rs @@ -83,7 +83,7 @@ use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}; use lightning::ln::peer_handler; use lightning::ln::peer_handler::SocketDescriptor as LnSocketTrait; use lightning::ln::peer_handler::CustomMessageHandler; -use lightning::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, NetAddress}; +use lightning::ln::msgs::{ChannelMessageHandler, NetAddress, OnionMessageHandler, RoutingMessageHandler}; use lightning::util::logger::Logger; use std::ops::Deref; @@ -123,13 +123,15 @@ struct Connection { id: u64, } impl Connection { - async fn poll_event_process(peer_manager: Arc>, mut event_receiver: mpsc::Receiver<()>) where + async fn poll_event_process(peer_manager: Arc>, mut event_receiver: mpsc::Receiver<()>) where CMH: Deref + 'static + Send + Sync, RMH: Deref + 'static + Send + Sync, + OMH: Deref + 'static + Send + Sync, L: Deref + 'static + Send + Sync, UMH: Deref + 'static + Send + Sync, CMH::Target: ChannelMessageHandler + Send + Sync, RMH::Target: RoutingMessageHandler + Send + Sync, + OMH::Target: OnionMessageHandler + Send + Sync, L::Target: Logger + Send + Sync, UMH::Target: CustomMessageHandler + Send + Sync, { @@ -141,13 +143,15 @@ impl Connection { } } - async fn schedule_read(peer_manager: Arc>, us: Arc>, mut reader: io::ReadHalf, mut read_wake_receiver: mpsc::Receiver<()>, mut write_avail_receiver: mpsc::Receiver<()>) where + async fn schedule_read(peer_manager: Arc>, us: Arc>, mut reader: io::ReadHalf, mut read_wake_receiver: mpsc::Receiver<()>, mut write_avail_receiver: mpsc::Receiver<()>) where CMH: Deref + 'static + Send + Sync, RMH: Deref + 'static + Send + Sync, + OMH: Deref + 'static + Send + Sync, L: Deref + 'static + Send + Sync, UMH: Deref + 'static + Send + Sync, CMH::Target: ChannelMessageHandler + 'static + Send + Sync, RMH::Target: RoutingMessageHandler + 'static + Send + Sync, + OMH::Target: OnionMessageHandler + 'static + Send + Sync, L::Target: Logger + 'static + Send + Sync, UMH::Target: CustomMessageHandler + 'static + Send + Sync, { @@ -268,13 +272,15 @@ fn get_addr_from_stream(stream: &StdTcpStream) -> Option { /// The returned future will complete when the peer is disconnected and associated handling /// futures are freed, though, because all processing futures are spawned with tokio::spawn, you do /// not need to poll the provided future in order to make progress. -pub fn setup_inbound(peer_manager: Arc>, stream: StdTcpStream) -> impl std::future::Future where +pub fn setup_inbound(peer_manager: Arc>, stream: StdTcpStream) -> impl std::future::Future where CMH: Deref + 'static + Send + Sync, RMH: Deref + 'static + Send + Sync, + OMH: Deref + 'static + Send + Sync, L: Deref + 'static + Send + Sync, UMH: Deref + 'static + Send + Sync, CMH::Target: ChannelMessageHandler + Send + Sync, RMH::Target: RoutingMessageHandler + Send + Sync, + OMH::Target: OnionMessageHandler + Send + Sync, L::Target: Logger + Send + Sync, UMH::Target: CustomMessageHandler + Send + Sync, { @@ -315,13 +321,15 @@ pub fn setup_inbound(peer_manager: Arc(peer_manager: Arc>, their_node_id: PublicKey, stream: StdTcpStream) -> impl std::future::Future where +pub fn setup_outbound(peer_manager: Arc>, their_node_id: PublicKey, stream: StdTcpStream) -> impl std::future::Future where CMH: Deref + 'static + Send + Sync, RMH: Deref + 'static + Send + Sync, + OMH: Deref + 'static + Send + Sync, L: Deref + 'static + Send + Sync, UMH: Deref + 'static + Send + Sync, CMH::Target: ChannelMessageHandler + Send + Sync, RMH::Target: RoutingMessageHandler + Send + Sync, + OMH::Target: OnionMessageHandler + Send + Sync, L::Target: Logger + Send + Sync, UMH::Target: CustomMessageHandler + Send + Sync, { @@ -391,13 +399,15 @@ pub fn setup_outbound(peer_manager: Arc(peer_manager: Arc>, their_node_id: PublicKey, addr: SocketAddr) -> Option> where +pub async fn connect_outbound(peer_manager: Arc>, their_node_id: PublicKey, addr: SocketAddr) -> Option> where CMH: Deref + 'static + Send + Sync, RMH: Deref + 'static + Send + Sync, + OMH: Deref + 'static + Send + Sync, L: Deref + 'static + Send + Sync, UMH: Deref + 'static + Send + Sync, CMH::Target: ChannelMessageHandler + Send + Sync, RMH::Target: RoutingMessageHandler + Send + Sync, + OMH::Target: OnionMessageHandler + Send + Sync, L::Target: Logger + Send + Sync, UMH::Target: CustomMessageHandler + Send + Sync, { @@ -646,6 +656,7 @@ mod tests { let a_manager = Arc::new(PeerManager::new(MessageHandler { chan_handler: Arc::clone(&a_handler), route_handler: Arc::clone(&a_handler), + onion_message_handler: Arc::new(lightning::ln::peer_handler::IgnoringMessageHandler{}), }, a_key.clone(), &[1; 32], Arc::new(TestLogger()), Arc::new(lightning::ln::peer_handler::IgnoringMessageHandler{}))); let (b_connected_sender, mut b_connected) = mpsc::channel(1); @@ -660,6 +671,7 @@ mod tests { let b_manager = Arc::new(PeerManager::new(MessageHandler { chan_handler: Arc::clone(&b_handler), route_handler: Arc::clone(&b_handler), + onion_message_handler: Arc::new(lightning::ln::peer_handler::IgnoringMessageHandler{}), }, b_key.clone(), &[2; 32], Arc::new(TestLogger()), Arc::new(lightning::ln::peer_handler::IgnoringMessageHandler{}))); // We bind on localhost, hoping the environment is properly configured with a local @@ -711,6 +723,7 @@ mod tests { let a_manager = Arc::new(PeerManager::new(MessageHandler { chan_handler: Arc::new(lightning::ln::peer_handler::ErroringMessageHandler::new()), + onion_message_handler: Arc::new(lightning::ln::peer_handler::IgnoringMessageHandler{}), route_handler: Arc::new(lightning::ln::peer_handler::IgnoringMessageHandler{}), }, a_key, &[1; 32], Arc::new(TestLogger()), Arc::new(lightning::ln::peer_handler::IgnoringMessageHandler{}))); diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index ffc595cceda..190907ce26e 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -40,7 +40,7 @@ use core::fmt::Debug; use io::{self, Read}; use io_extras::read_to_end; -use util::events::MessageSendEventsProvider; +use util::events::{MessageSendEventsProvider, OnionMessageProvider}; use util::logger; use util::ser::{BigSize, LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname}; @@ -945,6 +945,12 @@ pub trait RoutingMessageHandler : MessageSendEventsProvider { fn handle_query_short_channel_ids(&self, their_node_id: &PublicKey, msg: QueryShortChannelIds) -> Result<(), LightningError>; } +/// A trait to describe an object that can receive onion messages. +pub trait OnionMessageHandler : OnionMessageProvider { + /// Handle an incoming onion_message message from the given peer. + fn handle_onion_message(&self, peer_node_id: &PublicKey, msg: &OnionMessage); +} + mod fuzzy_internal_msgs { use prelude::*; use ln::{PaymentPreimage, PaymentSecret}; diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 573e910ab08..58c4f11f02c 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -19,7 +19,7 @@ use bitcoin::secp256k1::{self, Secp256k1, SecretKey, PublicKey}; use ln::features::InitFeatures; use ln::msgs; -use ln::msgs::{ChannelMessageHandler, LightningError, NetAddress, RoutingMessageHandler}; +use ln::msgs::{ChannelMessageHandler, LightningError, NetAddress, OnionMessageHandler, RoutingMessageHandler}; use ln::channelmanager::{SimpleArcChannelManager, SimpleRefChannelManager}; use util::ser::{VecWriter, Writeable, Writer}; use ln::peer_channel_encryptor::{PeerChannelEncryptor,NextNoiseStep}; @@ -27,7 +27,7 @@ use ln::wire; use ln::wire::Encode; use routing::gossip::{NetworkGraph, P2PGossipSync}; use util::atomic_counter::AtomicCounter; -use util::events::{MessageSendEvent, MessageSendEventsProvider}; +use util::events::{MessageSendEvent, MessageSendEventsProvider, OnionMessageProvider}; use util::logger::Logger; use prelude::*; @@ -76,6 +76,12 @@ impl RoutingMessageHandler for IgnoringMessageHandler { fn handle_query_channel_range(&self, _their_node_id: &PublicKey, _msg: msgs::QueryChannelRange) -> Result<(), LightningError> { Ok(()) } fn handle_query_short_channel_ids(&self, _their_node_id: &PublicKey, _msg: msgs::QueryShortChannelIds) -> Result<(), LightningError> { Ok(()) } } +impl OnionMessageProvider for IgnoringMessageHandler { + fn next_onion_message_for_peer(&self, _peer_node_id: PublicKey) -> Option { None } +} +impl OnionMessageHandler for IgnoringMessageHandler { + fn handle_onion_message(&self, _their_node_id: &PublicKey, _msg: &msgs::OnionMessage) {} +} impl Deref for IgnoringMessageHandler { type Target = IgnoringMessageHandler; fn deref(&self) -> &Self { self } @@ -199,9 +205,11 @@ impl Deref for ErroringMessageHandler { } /// Provides references to trait impls which handle different types of messages. -pub struct MessageHandler where +pub struct MessageHandler where CM::Target: ChannelMessageHandler, - RM::Target: RoutingMessageHandler { + RM::Target: RoutingMessageHandler, + OM::Target: OnionMessageHandler, +{ /// A message handler which handles messages specific to channels. Usually this is just a /// [`ChannelManager`] object or an [`ErroringMessageHandler`]. /// @@ -212,6 +220,10 @@ pub struct MessageHandler where /// /// [`P2PGossipSync`]: crate::routing::gossip::P2PGossipSync pub route_handler: RM, + + /// A message handler which handles onion messages. For now, this can only be an + /// [`IgnoringMessageHandler`]. + pub onion_message_handler: OM, } /// Provides an object which can be used to send data to and which uniquely identifies a connection @@ -423,7 +435,7 @@ impl Peer { /// issues such as overly long function definitions. /// /// (C-not exported) as Arcs don't make sense in bindings -pub type SimpleArcPeerManager = PeerManager>, Arc>>, Arc, Arc>>, Arc, Arc>; +pub type SimpleArcPeerManager = PeerManager>, Arc>>, Arc, Arc>>, IgnoringMessageHandler, Arc, Arc>; /// SimpleRefPeerManager is a type alias for a PeerManager reference, and is the reference /// counterpart to the SimpleArcPeerManager type alias. Use this type by default when you don't @@ -433,7 +445,7 @@ pub type SimpleArcPeerManager = PeerManager = PeerManager, &'e P2PGossipSync<&'g NetworkGraph<&'f L>, &'h C, &'f L>, &'f L, IgnoringMessageHandler>; +pub type SimpleRefPeerManager<'a, 'b, 'c, 'd, 'e, 'f, 'g, 'h, SD, M, T, F, C, L> = PeerManager, &'e P2PGossipSync<&'g NetworkGraph<&'f L>, &'h C, &'f L>, IgnoringMessageHandler, &'f L, IgnoringMessageHandler>; /// A PeerManager manages a set of peers, described by their [`SocketDescriptor`] and marshalls /// socket events into messages which it passes on to its [`MessageHandler`]. @@ -454,12 +466,13 @@ pub type SimpleRefPeerManager<'a, 'b, 'c, 'd, 'e, 'f, 'g, 'h, SD, M, T, F, C, L> /// you're using lightning-net-tokio. /// /// [`read_event`]: PeerManager::read_event -pub struct PeerManager where +pub struct PeerManager where CM::Target: ChannelMessageHandler, RM::Target: RoutingMessageHandler, + OM::Target: OnionMessageHandler, L::Target: Logger, CMH::Target: CustomMessageHandler { - message_handler: MessageHandler, + message_handler: MessageHandler, /// Connection state for each connected peer - we have an outer read-write lock which is taken /// as read while we're doing processing for a peer and taken write when a peer is being added /// or removed. @@ -518,31 +531,34 @@ macro_rules! encode_msg { }} } -impl PeerManager where +impl PeerManager where CM::Target: ChannelMessageHandler, + OM::Target: OnionMessageHandler, L::Target: Logger { - /// Constructs a new PeerManager with the given ChannelMessageHandler. No routing message - /// handler is used and network graph messages are ignored. + /// Constructs a new `PeerManager` with the given `ChannelMessageHandler` and + /// `OnionMessageHandler`. No routing message handler is used and network graph messages are + /// ignored. /// /// ephemeral_random_data is used to derive per-connection ephemeral keys and must be /// cryptographically secure random bytes. /// /// (C-not exported) as we can't export a PeerManager with a dummy route handler - pub fn new_channel_only(channel_message_handler: CM, our_node_secret: SecretKey, ephemeral_random_data: &[u8; 32], logger: L) -> Self { + pub fn new_channel_only(channel_message_handler: CM, onion_message_handler: OM, our_node_secret: SecretKey, ephemeral_random_data: &[u8; 32], logger: L) -> Self { Self::new(MessageHandler { chan_handler: channel_message_handler, route_handler: IgnoringMessageHandler{}, + onion_message_handler, }, our_node_secret, ephemeral_random_data, logger, IgnoringMessageHandler{}) } } -impl PeerManager where +impl PeerManager where RM::Target: RoutingMessageHandler, L::Target: Logger { - /// Constructs a new PeerManager with the given RoutingMessageHandler. No channel message - /// handler is used and messages related to channels will be ignored (or generate error - /// messages). Note that some other lightning implementations time-out connections after some - /// time if no channel is built with the peer. + /// Constructs a new `PeerManager` with the given `RoutingMessageHandler`. No channel message + /// handler or onion message handler is used and onion and channel messages will be ignored (or + /// generate error messages). Note that some other lightning implementations time-out connections + /// after some time if no channel is built with the peer. /// /// ephemeral_random_data is used to derive per-connection ephemeral keys and must be /// cryptographically secure random bytes. @@ -552,6 +568,7 @@ impl PeerManager) -> Option { } } -impl PeerManager where +impl PeerManager where CM::Target: ChannelMessageHandler, RM::Target: RoutingMessageHandler, + OM::Target: OnionMessageHandler, L::Target: Logger, CMH::Target: CustomMessageHandler { /// Constructs a new PeerManager with the given message handlers and node_id secret key /// ephemeral_random_data is used to derive per-connection ephemeral keys and must be /// cryptographically secure random bytes. - pub fn new(message_handler: MessageHandler, our_node_secret: SecretKey, ephemeral_random_data: &[u8; 32], logger: L, custom_message_handler: CMH) -> Self { + pub fn new(message_handler: MessageHandler, our_node_secret: SecretKey, ephemeral_random_data: &[u8; 32], logger: L, custom_message_handler: CMH) -> Self { let mut ephemeral_key_midstate = Sha256::engine(); ephemeral_key_midstate.input(ephemeral_random_data); @@ -1314,6 +1332,11 @@ impl P self.message_handler.route_handler.handle_reply_channel_range(&their_node_id, msg)?; }, + // Onion message: + wire::Message::OnionMessage(msg) => { + self.message_handler.onion_message_handler.handle_onion_message(&their_node_id, &msg); + }, + // Unknown messages: wire::Message::Unknown(type_id) if message.is_even() => { log_debug!(self.logger, "Received unknown even message of type {}, disconnecting peer!", type_id); @@ -1930,12 +1953,12 @@ mod tests { cfgs } - fn create_network<'a>(peer_count: usize, cfgs: &'a Vec) -> Vec> { + fn create_network<'a>(peer_count: usize, cfgs: &'a Vec) -> Vec> { let mut peers = Vec::new(); for i in 0..peer_count { let node_secret = SecretKey::from_slice(&[42 + i as u8; 32]).unwrap(); let ephemeral_bytes = [i as u8; 32]; - let msg_handler = MessageHandler { chan_handler: &cfgs[i].chan_handler, route_handler: &cfgs[i].routing_handler }; + let msg_handler = MessageHandler { chan_handler: &cfgs[i].chan_handler, route_handler: &cfgs[i].routing_handler, onion_message_handler: IgnoringMessageHandler {} }; let peer = PeerManager::new(msg_handler, node_secret, &ephemeral_bytes, &cfgs[i].logger, IgnoringMessageHandler {}); peers.push(peer); } @@ -1943,7 +1966,7 @@ mod tests { peers } - fn establish_connection<'a>(peer_a: &PeerManager, peer_b: &PeerManager) -> (FileDescriptor, FileDescriptor) { + fn establish_connection<'a>(peer_a: &PeerManager, peer_b: &PeerManager) -> (FileDescriptor, FileDescriptor) { let secp_ctx = Secp256k1::new(); let a_id = PublicKey::from_secret_key(&secp_ctx, &peer_a.our_node_secret); let mut fd_a = FileDescriptor { fd: 1, outbound_data: Arc::new(Mutex::new(Vec::new())) }; diff --git a/lightning/src/ln/wire.rs b/lightning/src/ln/wire.rs index cbf5c77d600..1191a8d3d53 100644 --- a/lightning/src/ln/wire.rs +++ b/lightning/src/ln/wire.rs @@ -9,7 +9,7 @@ //! Wire encoding/decoding for Lightning messages according to [BOLT #1], and for //! custom message through the [`CustomMessageReader`] trait. -//! +//! //! [BOLT #1]: https://github.com/lightning/bolts/blob/master/01-messaging.md use io; @@ -60,6 +60,7 @@ pub(crate) enum Message where T: core::fmt::Debug + Type + TestEq { ChannelReady(msgs::ChannelReady), Shutdown(msgs::Shutdown), ClosingSigned(msgs::ClosingSigned), + OnionMessage(msgs::OnionMessage), UpdateAddHTLC(msgs::UpdateAddHTLC), UpdateFulfillHTLC(msgs::UpdateFulfillHTLC), UpdateFailHTLC(msgs::UpdateFailHTLC), @@ -100,6 +101,7 @@ impl Message where T: core::fmt::Debug + Type + TestEq { &Message::ChannelReady(ref msg) => msg.type_id(), &Message::Shutdown(ref msg) => msg.type_id(), &Message::ClosingSigned(ref msg) => msg.type_id(), + &Message::OnionMessage(ref msg) => msg.type_id(), &Message::UpdateAddHTLC(ref msg) => msg.type_id(), &Message::UpdateFulfillHTLC(ref msg) => msg.type_id(), &Message::UpdateFailHTLC(ref msg) => msg.type_id(), @@ -185,6 +187,9 @@ fn do_read(buffer: &mut R, message_type: u1 msgs::ClosingSigned::TYPE => { Ok(Message::ClosingSigned(Readable::read(buffer)?)) }, + msgs::OnionMessage::TYPE => { + Ok(Message::OnionMessage(Readable::read(buffer)?)) + }, msgs::UpdateAddHTLC::TYPE => { Ok(Message::UpdateAddHTLC(Readable::read(buffer)?)) }, @@ -344,6 +349,10 @@ impl Encode for msgs::ClosingSigned { const TYPE: u16 = 39; } +impl Encode for msgs::OnionMessage { + const TYPE: u16 = 513; +} + impl Encode for msgs::UpdateAddHTLC { const TYPE: u16 = 128; } diff --git a/lightning/src/onion_message/functional_tests.rs b/lightning/src/onion_message/functional_tests.rs index ccc834434a1..5a83a0b2b21 100644 --- a/lightning/src/onion_message/functional_tests.rs +++ b/lightning/src/onion_message/functional_tests.rs @@ -10,6 +10,7 @@ //! Onion message testing and test utilities live here. use chain::keysinterface::{KeysInterface, Recipient}; +use ln::msgs::OnionMessageHandler; use super::{BlindedRoute, Destination, OnionMessenger, SendError}; use util::enforcing_trait_impls::EnforcingSigner; use util::test_utils; diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index a5438afbb8e..2684ab8b3e8 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -16,13 +16,15 @@ use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::secp256k1::{self, PublicKey, Scalar, Secp256k1, SecretKey}; use chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Recipient, Sign}; -use ln::msgs; +use ln::msgs::{self, OnionMessageHandler}; use ln::onion_utils; use super::blinded_route::{BlindedRoute, ForwardTlvs, ReceiveTlvs}; use super::packet::{BIG_PACKET_HOP_DATA_LEN, ForwardControlTlvs, Packet, Payload, ReceiveControlTlvs, SMALL_PACKET_HOP_DATA_LEN}; use super::utils; +use util::events::OnionMessageProvider; use util::logger::Logger; +use core::mem; use core::ops::Deref; use sync::{Arc, Mutex}; use prelude::*; @@ -178,10 +180,23 @@ impl OnionMessenger Ok(()) } + #[cfg(test)] + pub(super) fn release_pending_msgs(&self) -> HashMap> { + let mut pending_msgs = self.pending_messages.lock().unwrap(); + let mut msgs = HashMap::new(); + core::mem::swap(&mut *pending_msgs, &mut msgs); + msgs + } +} + +impl OnionMessageHandler for OnionMessenger + where K::Target: KeysInterface, + L::Target: Logger, +{ /// Handle an incoming onion message. Currently, if a message was destined for us we will log, but /// soon we'll delegate the onion message to a handler that can generate invoices or send /// payments. - pub fn handle_onion_message(&self, _peer_node_id: &PublicKey, msg: &msgs::OnionMessage) { + fn handle_onion_message(&self, _peer_node_id: &PublicKey, msg: &msgs::OnionMessage) { let control_tlvs_ss = match self.keys_manager.ecdh(Recipient::Node, &msg.blinding_point, None) { Ok(ss) => ss, Err(e) => { @@ -273,13 +288,14 @@ impl OnionMessenger }, }; } +} - #[cfg(test)] - pub(super) fn release_pending_msgs(&self) -> HashMap> { - let mut pending_msgs = self.pending_messages.lock().unwrap(); - let mut msgs = HashMap::new(); - core::mem::swap(&mut *pending_msgs, &mut msgs); - msgs +impl OnionMessageProvider for OnionMessenger + where K::Target: KeysInterface, + L::Target: Logger, +{ + fn next_onion_message_for_peer(&self, peer_node_id: PublicKey) -> Option { + None } } diff --git a/lightning/src/util/events.rs b/lightning/src/util/events.rs index d5674759856..e86eae3c813 100644 --- a/lightning/src/util/events.rs +++ b/lightning/src/util/events.rs @@ -1195,6 +1195,12 @@ pub trait MessageSendEventsProvider { fn get_and_clear_pending_msg_events(&self) -> Vec; } +/// A trait indicating an object may generate onion messages to send +pub trait OnionMessageProvider { + /// Gets the next pending onion message for the peer with the given node id. + fn next_onion_message_for_peer(&self, peer_node_id: PublicKey) -> Option; +} + /// A trait indicating an object may generate events. /// /// Events are processed by passing an [`EventHandler`] to [`process_pending_events`]. From 544f72b37c90f824ea07fdf8ca0262b0fe0a9488 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Tue, 16 Aug 2022 14:33:10 -0400 Subject: [PATCH 80/91] PeerManager: bump the read pause limit ...to make sure we can still get channel messages out after enqueuing some big gossip messages. --- lightning/src/ln/peer_handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 58c4f11f02c..6da0bae27d0 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -310,7 +310,7 @@ const FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO: usize = 2; /// we have fewer than this many messages in the outbound buffer again. /// We also use this as the target number of outbound gossip messages to keep in the write buffer, /// refilled as we send bytes. -const OUTBOUND_BUFFER_LIMIT_READ_PAUSE: usize = 10; +const OUTBOUND_BUFFER_LIMIT_READ_PAUSE: usize = 12; /// When the outbound buffer has this many messages, we'll simply skip relaying gossip messages to /// the peer. const OUTBOUND_BUFFER_LIMIT_DROP_GOSSIP: usize = OUTBOUND_BUFFER_LIMIT_READ_PAUSE * FORWARD_INIT_SYNC_BUFFER_LIMIT_RATIO; From b28783a0b5b537acb4891438adcf5f83c239256a Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Mon, 22 Aug 2022 12:22:22 -0400 Subject: [PATCH 81/91] Implement OnionMessageProvider for OnionMessenger --- lightning/src/onion_message/messenger.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 2684ab8b3e8..75eb4619b8f 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -86,7 +86,7 @@ pub struct OnionMessenger { keys_manager: K, logger: L, - pending_messages: Mutex>>, + pending_messages: Mutex>>, secp_ctx: Secp256k1, // Coming soon: // invoice_handler: InvoiceHandler, @@ -170,8 +170,8 @@ impl OnionMessenger packet_payloads, packet_keys, prng_seed).map_err(|()| SendError::TooBigPacket)?; let mut pending_per_peer_msgs = self.pending_messages.lock().unwrap(); - let pending_msgs = pending_per_peer_msgs.entry(introduction_node_id).or_insert(Vec::new()); - pending_msgs.push( + let pending_msgs = pending_per_peer_msgs.entry(introduction_node_id).or_insert_with(VecDeque::new); + pending_msgs.push_back( msgs::OnionMessage { blinding_point, onion_routing_packet: onion_packet, @@ -181,7 +181,7 @@ impl OnionMessenger } #[cfg(test)] - pub(super) fn release_pending_msgs(&self) -> HashMap> { + pub(super) fn release_pending_msgs(&self) -> HashMap> { let mut pending_msgs = self.pending_messages.lock().unwrap(); let mut msgs = HashMap::new(); core::mem::swap(&mut *pending_msgs, &mut msgs); @@ -253,8 +253,8 @@ impl OnionMessageHandler for OnionMessenger blinding_point, @@ -295,6 +295,10 @@ impl OnionMessageProvider for OnionMessenger Option { + let mut pending_msgs = self.pending_messages.lock().unwrap(); + if let Some(msgs) = pending_msgs.get_mut(&peer_node_id) { + return msgs.pop_front() + } None } } From f4834def9d304002642cc1907432d2053519857d Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 25 Aug 2022 14:30:29 -0400 Subject: [PATCH 82/91] Implement buffering onion messages for peers. In this commit, we check if a peer's outbound buffer has room for onion messages, and if so pulls them from an implementer of a new trait, OnionMessageProvider. Makes sure channel messages are prioritized over OMs, and OMs are prioritized over gossip. The onion_message module remains private until further rate limiting is added. --- lightning/src/ln/peer_handler.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 6da0bae27d0..1258026e17e 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -411,6 +411,13 @@ impl Peer { && self.msgs_sent_since_pong < BUFFER_DRAIN_MSGS_PER_TICK } + /// Determines if we should push an onion message onto a peer's outbound buffer. This is checked + /// every time the peer's buffer may have been drained. + fn should_buffer_onion_message(&self) -> bool { + self.pending_outbound_buffer.is_empty() + && self.msgs_sent_since_pong < BUFFER_DRAIN_MSGS_PER_TICK + } + /// Determines if we should push additional gossip broadcast messages onto a peer's outbound /// buffer. This is checked every time the peer's buffer may have been drained. fn should_buffer_gossip_broadcast(&self) -> bool { @@ -766,6 +773,14 @@ impl Date: Mon, 29 Aug 2022 22:49:24 -0700 Subject: [PATCH 83/91] Change `payment_cache` to accept `PaymentInfo` Introduces a new `PaymentInfo` struct that contains both the previous `attempts` count that was tracked as well as the paths that are also currently inflight. --- lightning-invoice/src/payment.rs | 47 ++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/lightning-invoice/src/payment.rs b/lightning-invoice/src/payment.rs index 476d12a3475..a2bdb8876a4 100644 --- a/lightning-invoice/src/payment.rs +++ b/lightning-invoice/src/payment.rs @@ -145,7 +145,7 @@ use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning::ln::channelmanager::{ChannelDetails, PaymentId, PaymentSendFailure}; use lightning::ln::msgs::LightningError; use lightning::routing::scoring::{LockableScore, Score}; -use lightning::routing::router::{PaymentParameters, Route, RouteParameters}; +use lightning::routing::router::{PaymentParameters, Route, RouteHop, RouteParameters}; use lightning::util::events::{Event, EventHandler}; use lightning::util::logger::Logger; use time_utils::Time; @@ -188,10 +188,25 @@ where logger: L, event_handler: E, /// Caches the overall attempts at making a payment, which is updated prior to retrying. - payment_cache: Mutex>>, + payment_cache: Mutex>>, retry: Retry, } +/// Used by [`InvoicePayerUsingTime::payment_cache`] to track the payments that are either +/// currently being made, or have outstanding paths that need retrying. +struct PaymentInfo { + attempts: PaymentAttempts, + paths: Vec>, +} + +impl PaymentInfo { + fn new() -> Self { + PaymentInfo { + attempts: PaymentAttempts::new(), + paths: vec![], + } + } +} /// Storing minimal payment attempts information required for determining if a outbound payment can /// be retried. #[derive(Clone, Copy)] @@ -361,7 +376,7 @@ where let payment_hash = PaymentHash(invoice.payment_hash().clone().into_inner()); match self.payment_cache.lock().unwrap().entry(payment_hash) { hash_map::Entry::Occupied(_) => return Err(PaymentError::Invoice("payment pending")), - hash_map::Entry::Vacant(entry) => entry.insert(PaymentAttempts::new()), + hash_map::Entry::Vacant(entry) => entry.insert(PaymentInfo::new()), }; let payment_secret = Some(invoice.payment_secret().clone()); @@ -397,7 +412,7 @@ where let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner()); match self.payment_cache.lock().unwrap().entry(payment_hash) { hash_map::Entry::Occupied(_) => return Err(PaymentError::Invoice("payment pending")), - hash_map::Entry::Vacant(entry) => entry.insert(PaymentAttempts::new()), + hash_map::Entry::Vacant(entry) => entry.insert(PaymentInfo::new()), }; let route_params = RouteParameters { @@ -437,9 +452,9 @@ where PaymentSendFailure::PathParameterError(_) => Err(e), PaymentSendFailure::AllFailedRetrySafe(_) => { let mut payment_cache = self.payment_cache.lock().unwrap(); - let payment_attempts = payment_cache.get_mut(&payment_hash).unwrap(); - payment_attempts.count += 1; - if self.retry.is_retryable_now(payment_attempts) { + let payment_info = payment_cache.get_mut(&payment_hash).unwrap(); + payment_info.attempts.count += 1; + if self.retry.is_retryable_now(&payment_info.attempts) { core::mem::drop(payment_cache); Ok(self.pay_internal(params, payment_hash, send_payment)?) } else { @@ -469,13 +484,15 @@ where fn retry_payment( &self, payment_id: PaymentId, payment_hash: PaymentHash, params: &RouteParameters ) -> Result<(), ()> { - let attempts = - *self.payment_cache.lock().unwrap().entry(payment_hash) - .and_modify(|attempts| attempts.count += 1) - .or_insert(PaymentAttempts { - count: 1, - first_attempted_at: T::now() - }); + let attempts = self.payment_cache.lock().unwrap().entry(payment_hash) + .and_modify(|info| info.attempts.count += 1 ) + .or_insert_with(|| PaymentInfo { + attempts: PaymentAttempts { + count: 1, + first_attempted_at: T::now(), + }, + paths: vec![], + }).attempts; if !self.retry.is_retryable_now(&attempts) { log_trace!(self.logger, "Payment {} exceeded maximum attempts; not retrying ({})", log_bytes!(payment_hash.0), attempts); @@ -583,7 +600,7 @@ where let mut payment_cache = self.payment_cache.lock().unwrap(); let attempts = payment_cache .remove(payment_hash) - .map_or(1, |attempts| attempts.count + 1); + .map_or(1, |payment_info| payment_info.attempts.count + 1); log_trace!(self.logger, "Payment {} succeeded (attempts: {})", log_bytes!(payment_hash.0), attempts); }, Event::ProbeSuccessful { payment_hash, path, .. } => { From 7223f7061630bb55cf9f769e97b672a8afaf8e2b Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 30 Aug 2022 04:03:47 +0000 Subject: [PATCH 84/91] Drop honggfuzz `arbitrary` dependency to meet MSRV --- fuzz/Cargo.toml | 2 +- fuzz/ci-fuzz.sh | 2 +- lightning-invoice/fuzz/Cargo.toml | 2 +- lightning-invoice/fuzz/ci-fuzz.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index be37fb83cdd..95308e1bf5c 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -22,7 +22,7 @@ lightning = { path = "../lightning", features = ["regex"] } lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync" } bitcoin = { version = "0.29.0", features = ["secp-lowmemory"] } hex = "0.3" -honggfuzz = { version = "0.5", optional = true } +honggfuzz = { version = "0.5", optional = true, default-features = false } libfuzzer-sys = { version = "0.4", optional = true } [build-dependencies] diff --git a/fuzz/ci-fuzz.sh b/fuzz/ci-fuzz.sh index 6f0074b6a1a..969505ca88d 100755 --- a/fuzz/ci-fuzz.sh +++ b/fuzz/ci-fuzz.sh @@ -13,7 +13,7 @@ rm *_target.rs [ "$(git diff)" != "" ] && exit 1 popd -cargo install --color always --force honggfuzz +cargo install --color always --force honggfuzz --no-default-features sed -i 's/lto = true//' Cargo.toml HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" cargo --color always hfuzz build for TARGET in src/bin/*.rs; do diff --git a/lightning-invoice/fuzz/Cargo.toml b/lightning-invoice/fuzz/Cargo.toml index 833606c1ad8..6f79757c22b 100644 --- a/lightning-invoice/fuzz/Cargo.toml +++ b/lightning-invoice/fuzz/Cargo.toml @@ -12,7 +12,7 @@ afl_fuzz = ["afl"] honggfuzz_fuzz = ["honggfuzz"] [dependencies] -honggfuzz = { version = "0.5", optional = true } +honggfuzz = { version = "0.5", optional = true, default-features = false } afl = { version = "0.4", optional = true } lightning-invoice = { path = ".." } lightning = { path = "../../lightning", features = ["regex"] } diff --git a/lightning-invoice/fuzz/ci-fuzz.sh b/lightning-invoice/fuzz/ci-fuzz.sh index ae85ea91301..db1b9eb388c 100755 --- a/lightning-invoice/fuzz/ci-fuzz.sh +++ b/lightning-invoice/fuzz/ci-fuzz.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -cargo install --force honggfuzz +cargo install --force honggfuzz --no-default-features for TARGET in fuzz_targets/*; do FILENAME=$(basename $TARGET) FILE="${FILENAME%.*}" From c70bd1fd558513e72fa03dc16f9556882883046e Mon Sep 17 00:00:00 2001 From: jurvis Date: Mon, 29 Aug 2022 22:50:44 -0700 Subject: [PATCH 85/91] Keep track of inflight HTLCs across payments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added two methods, `process_path_inflight_htlcs` and `remove_path_inflight_htlcs`, that updates that `payment_cache` map with path information that may have failed, succeeded, or have been given up on. Introduced `AccountForInflightHtlcs`, which will wrap our user-provided scorer. We move the `S:Score` type parameterization from the `Router` to `find_route`, so we can use our newly introduced `AccountForInflightHtlcs`. `AccountForInflightHtlcs` keeps track of a map of inflight HTLCs by their short channel id, direction, and give us the value that is being used up. This map will in turn be populated prior to calling `find_route`, where we’ll use `create_inflight_map`, to generate a current map of all inflight HTLCs based on what was stored in `payment_cache`. --- lightning-invoice/src/payment.rs | 533 ++++++++++++++++++++++++++++--- lightning-invoice/src/utils.rs | 4 +- lightning/src/routing/gossip.rs | 2 +- lightning/src/routing/scoring.rs | 2 +- 4 files changed, 487 insertions(+), 54 deletions(-) diff --git a/lightning-invoice/src/payment.rs b/lightning-invoice/src/payment.rs index a2bdb8876a4..8854fcbc9af 100644 --- a/lightning-invoice/src/payment.rs +++ b/lightning-invoice/src/payment.rs @@ -77,8 +77,8 @@ //! # } //! # //! # struct FakeRouter {} -//! # impl Router for FakeRouter { -//! # fn find_route( +//! # impl Router for FakeRouter { +//! # fn find_route( //! # &self, payer: &PublicKey, params: &RouteParameters, payment_hash: &PaymentHash, //! # first_hops: Option<&[&ChannelDetails]>, scorer: &S //! # ) -> Result { unimplemented!() } @@ -144,8 +144,10 @@ use crate::prelude::*; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning::ln::channelmanager::{ChannelDetails, PaymentId, PaymentSendFailure}; use lightning::ln::msgs::LightningError; -use lightning::routing::scoring::{LockableScore, Score}; +use lightning::routing::gossip::NodeId; +use lightning::routing::scoring::{ChannelUsage, LockableScore, Score}; use lightning::routing::router::{PaymentParameters, Route, RouteHop, RouteParameters}; +use lightning::util::errors::APIError; use lightning::util::events::{Event, EventHandler}; use lightning::util::logger::Logger; use time_utils::Time; @@ -175,10 +177,9 @@ use time_utils; type ConfiguredTime = time_utils::Eternity; /// (C-not exported) generally all users should use the [`InvoicePayer`] type alias. -pub struct InvoicePayerUsingTime +pub struct InvoicePayerUsingTime where P::Target: Payer, - R: for <'a> Router<<::Target as LockableScore<'a>>::Locked>, S::Target: for <'a> LockableScore<'a>, L::Target: Logger, { @@ -207,6 +208,50 @@ impl PaymentInfo { } } } + +/// Used to store information about all the HTLCs that are inflight across all payment attempts +struct AccountForInFlightHtlcs<'a, S: Score> { + scorer: &'a mut S, + /// Maps a channel's short channel id and its direction to the liquidity used up. + inflight_htlcs: HashMap<(u64, bool), u64>, +} + +#[cfg(c_bindings)] +impl<'a, S:Score> lightning::util::ser::Writeable for AccountForInFlightHtlcs<'a, S> { + fn write(&self, writer: &mut W) -> Result<(), std::io::Error> { self.scorer.write(writer) } +} + +impl<'a, S: Score> Score for AccountForInFlightHtlcs<'a, S> { + fn channel_penalty_msat(&self, short_channel_id: u64, source: &NodeId, target: &NodeId, usage: ChannelUsage) -> u64 { + if let Some(used_liqudity) = self.inflight_htlcs.get(&(short_channel_id, source < target)) { + let usage = ChannelUsage { + inflight_htlc_msat: usage.inflight_htlc_msat + used_liqudity, + ..usage + }; + + self.scorer.channel_penalty_msat(short_channel_id, source, target, usage) + } else { + self.scorer.channel_penalty_msat(short_channel_id, source, target, usage) + } + } + + fn payment_path_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) { + self.scorer.payment_path_failed(path, short_channel_id) + } + + fn payment_path_successful(&mut self, path: &[&RouteHop]) { + self.scorer.payment_path_successful(path) + } + + fn probe_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) { + self.scorer.probe_failed(path, short_channel_id) + } + + fn probe_successful(&mut self, path: &[&RouteHop]) { + self.scorer.probe_successful(path) + } +} + /// Storing minimal payment attempts information required for determining if a outbound payment can /// be retried. #[derive(Clone, Copy)] @@ -267,9 +312,9 @@ pub trait Payer { } /// A trait defining behavior for routing an [`Invoice`] payment. -pub trait Router { +pub trait Router { /// Finds a [`Route`] between `payer` and `payee` for a payment with the given values. - fn find_route( + fn find_route( &self, payer: &PublicKey, route_params: &RouteParameters, payment_hash: &PaymentHash, first_hops: Option<&[&ChannelDetails]>, scorer: &S ) -> Result; @@ -314,10 +359,9 @@ pub enum PaymentError { Sending(PaymentSendFailure), } -impl InvoicePayerUsingTime +impl InvoicePayerUsingTime where P::Target: Payer, - R: for <'a> Router<<::Target as LockableScore<'a>>::Locked>, S::Target: for <'a> LockableScore<'a>, L::Target: Logger, { @@ -440,13 +484,19 @@ where let payer = self.payer.node_id(); let first_hops = self.payer.first_hops(); + let inflight_htlcs = self.create_inflight_map(); let route = self.router.find_route( - &payer, params, &payment_hash, Some(&first_hops.iter().collect::>()), - &self.scorer.lock() + &payer, ¶ms, &payment_hash, Some(&first_hops.iter().collect::>()), + &AccountForInFlightHtlcs { scorer: &mut self.scorer.lock(), inflight_htlcs } ).map_err(|e| PaymentError::Routing(e))?; match send_payment(&route) { - Ok(payment_id) => Ok(payment_id), + Ok(payment_id) => { + for path in route.paths { + self.process_path_inflight_htlcs(payment_hash, path); + } + Ok(payment_id) + }, Err(e) => match e { PaymentSendFailure::ParameterError(_) => Err(e), PaymentSendFailure::PathParameterError(_) => Err(e), @@ -461,7 +511,19 @@ where Err(e) } }, - PaymentSendFailure::PartialFailure { failed_paths_retry, payment_id, .. } => { + PaymentSendFailure::PartialFailure { failed_paths_retry, payment_id, results } => { + // If a `PartialFailure` event returns a result that is an `Ok()`, it means that + // part of our payment is retried. When we receive `MonitorUpdateFailed`, it + // means that we are still waiting for our channel monitor update to be completed. + for (result, path) in results.iter().zip(route.paths.into_iter()) { + match result { + Ok(_) | Err(APIError::MonitorUpdateFailed) => { + self.process_path_inflight_htlcs(payment_hash, path); + }, + _ => {}, + } + } + if let Some(retry_data) = failed_paths_retry { // Some paths were sent, even if we failed to send the full MPP value our // recipient may misbehave and claim the funds, at which point we have to @@ -481,6 +543,24 @@ where }.map_err(|e| PaymentError::Sending(e)) } + // Takes in a path to have its information stored in `payment_cache`. This is done for paths + // that are pending retry. + fn process_path_inflight_htlcs(&self, payment_hash: PaymentHash, path: Vec) { + self.payment_cache.lock().unwrap().entry(payment_hash) + .or_insert_with(|| PaymentInfo::new()) + .paths.push(path); + } + + // Find the path we want to remove in `payment_cache`. If it doesn't exist, do nothing. + fn remove_path_inflight_htlcs(&self, payment_hash: PaymentHash, path: &Vec) { + self.payment_cache.lock().unwrap().entry(payment_hash) + .and_modify(|payment_info| { + if let Some(idx) = payment_info.paths.iter().position(|p| p == path) { + payment_info.paths.swap_remove(idx); + } + }); + } + fn retry_payment( &self, payment_id: PaymentId, payment_hash: PaymentHash, params: &RouteParameters ) -> Result<(), ()> { @@ -508,17 +588,25 @@ where let payer = self.payer.node_id(); let first_hops = self.payer.first_hops(); + let inflight_htlcs = self.create_inflight_map(); + let route = self.router.find_route( &payer, ¶ms, &payment_hash, Some(&first_hops.iter().collect::>()), - &self.scorer.lock() + &AccountForInFlightHtlcs { scorer: &mut self.scorer.lock(), inflight_htlcs } ); + if route.is_err() { log_trace!(self.logger, "Failed to find a route for payment {}; not retrying ({:})", log_bytes!(payment_hash.0), attempts); return Err(()); } - match self.payer.retry_payment(&route.unwrap(), payment_id) { - Ok(()) => Ok(()), + match self.payer.retry_payment(&route.as_ref().unwrap(), payment_id) { + Ok(()) => { + for path in route.unwrap().paths.into_iter() { + self.process_path_inflight_htlcs(payment_hash, path); + } + Ok(()) + }, Err(PaymentSendFailure::ParameterError(_)) | Err(PaymentSendFailure::PathParameterError(_)) => { log_trace!(self.logger, "Failed to retry for payment {} due to bogus route/payment data, not retrying.", log_bytes!(payment_hash.0)); @@ -527,7 +615,19 @@ where Err(PaymentSendFailure::AllFailedRetrySafe(_)) => { self.retry_payment(payment_id, payment_hash, params) }, - Err(PaymentSendFailure::PartialFailure { failed_paths_retry, .. }) => { + Err(PaymentSendFailure::PartialFailure { failed_paths_retry, results, .. }) => { + // If a `PartialFailure` error contains a result that is an `Ok()`, it means that + // part of our payment is retried. When we receive `MonitorUpdateFailed`, it + // means that we are still waiting for our channel monitor update to complete. + for (result, path) in results.iter().zip(route.unwrap().paths.into_iter()) { + match result { + Ok(_) | Err(APIError::MonitorUpdateFailed) => { + self.process_path_inflight_htlcs(payment_hash, path); + }, + _ => {}, + } + } + if let Some(retry) = failed_paths_retry { // Always return Ok for the same reason as noted in pay_internal. let _ = self.retry_payment(payment_id, payment_hash, &retry); @@ -544,6 +644,47 @@ where pub fn remove_cached_payment(&self, payment_hash: &PaymentHash) { self.payment_cache.lock().unwrap().remove(payment_hash); } + + /// Given a [`PaymentHash`], this function looks up inflight path attempts in the payment_cache. + /// Then, it uses the path information inside the cache to construct a HashMap mapping a channel's + /// short channel id and direction to the amount being sent through it. + /// + /// This function should be called whenever we need information about currently used up liquidity + /// across payments. + fn create_inflight_map(&self) -> HashMap<(u64, bool), u64> { + let mut total_inflight_map: HashMap<(u64, bool), u64> = HashMap::new(); + // Make an attempt at finding existing payment information from `payment_cache`. If it + // does not exist, it probably is a fresh payment and we can just return an empty + // HashMap. + for payment_info in self.payment_cache.lock().unwrap().values() { + for path in &payment_info.paths { + if path.is_empty() { break }; + // total_inflight_map needs to be direction-sensitive when keeping track of the HTLC value + // that is held up. However, the `hops` array, which is a path returned by `find_route` in + // the router excludes the payer node. In the following lines, the payer's information is + // hardcoded with an inflight value of 0 so that we can correctly represent the first hop + // in our sliding window of two. + let our_node_id: PublicKey = self.payer.node_id(); + let reversed_hops_with_payer = path.iter().rev().skip(1) + .map(|hop| hop.pubkey) + .chain(core::iter::once(our_node_id)); + let mut cumulative_msat = 0; + + // Taking the reversed vector from above, we zip it with just the reversed hops list to + // work "backwards" of the given path, since the last hop's `fee_msat` actually represents + // the total amount sent. + for (next_hop, prev_hop) in path.iter().rev().zip(reversed_hops_with_payer) { + cumulative_msat += next_hop.fee_msat; + total_inflight_map + .entry((next_hop.short_channel_id, NodeId::from_pubkey(&prev_hop) < NodeId::from_pubkey(&next_hop.pubkey))) + .and_modify(|used_liquidity_msat| *used_liquidity_msat += cumulative_msat) + .or_insert(cumulative_msat); + } + } + } + + total_inflight_map + } } fn expiry_time_from_unix_epoch(invoice: &Invoice) -> Duration { @@ -557,14 +698,23 @@ fn has_expired(route_params: &RouteParameters) -> bool { } else { false } } -impl EventHandler for InvoicePayerUsingTime +impl EventHandler for InvoicePayerUsingTime where P::Target: Payer, - R: for <'a> Router<<::Target as LockableScore<'a>>::Locked>, S::Target: for <'a> LockableScore<'a>, L::Target: Logger, { fn handle_event(&self, event: &Event) { + match event { + Event::PaymentPathFailed { payment_hash, path, .. } + | Event::PaymentPathSuccessful { path, payment_hash: Some(payment_hash), .. } + | Event::ProbeSuccessful { payment_hash, path, .. } + | Event::ProbeFailed { payment_hash, path, .. } => { + self.remove_path_inflight_htlcs(*payment_hash, path); + }, + _ => {}, + } + match event { Event::PaymentPathFailed { payment_id, payment_hash, rejected_by_dest, path, short_channel_id, retry, .. @@ -633,7 +783,7 @@ mod tests { use lightning::ln::features::{ChannelFeatures, NodeFeatures, InitFeatures}; use lightning::ln::functional_test_utils::*; use lightning::ln::msgs::{ChannelMessageHandler, ErrorAction, LightningError}; - use lightning::routing::gossip::NodeId; + use lightning::routing::gossip::{EffectiveCapacity, NodeId}; use lightning::routing::router::{PaymentParameters, Route, RouteHop}; use lightning::routing::scoring::ChannelUsage; use lightning::util::test_utils::TestLogger; @@ -645,6 +795,7 @@ mod tests { use std::time::{SystemTime, Duration}; use time_utils::tests::SinceEpoch; use DEFAULT_EXPIRY_TIME; + use lightning::util::errors::APIError::{ChannelUnavailable, MonitorUpdateFailed}; fn invoice(payment_preimage: PaymentPreimage) -> Invoice { let payment_hash = Sha256::hash(&payment_preimage.0); @@ -792,8 +943,8 @@ mod tests { let final_value_msat = invoice.amount_milli_satoshis().unwrap(); let payer = TestPayer::new() - .fails_with_partial_failure(retry.clone(), OnAttempt(1)) - .fails_with_partial_failure(retry, OnAttempt(2)) + .fails_with_partial_failure(retry.clone(), OnAttempt(1), None) + .fails_with_partial_failure(retry, OnAttempt(2), None) .expect_send(Amount::ForInvoice(final_value_msat)) .expect_send(Amount::OnRetry(final_value_msat / 2)) .expect_send(Amount::OnRetry(final_value_msat / 2)); @@ -1381,18 +1532,263 @@ mod tests { invoice_payer.handle_event(&event); } + #[test] + fn generates_correct_inflight_map_data() { + let event_handled = core::cell::RefCell::new(false); + let event_handler = |_: &_| { *event_handled.borrow_mut() = true; }; + + let payment_preimage = PaymentPreimage([1; 32]); + let invoice = invoice(payment_preimage); + let payment_hash = Some(PaymentHash(invoice.payment_hash().clone().into_inner())); + let final_value_msat = invoice.amount_milli_satoshis().unwrap(); + + let payer = TestPayer::new().expect_send(Amount::ForInvoice(final_value_msat)); + let final_value_msat = invoice.amount_milli_satoshis().unwrap(); + let route = TestRouter::route_for_value(final_value_msat); + let router = TestRouter {}; + let scorer = RefCell::new(TestScorer::new()); + let logger = TestLogger::new(); + let invoice_payer = + InvoicePayer::new(&payer, router, &scorer, &logger, event_handler, Retry::Attempts(0)); + + let payment_id = invoice_payer.pay_invoice(&invoice).unwrap(); + + let inflight_map = invoice_payer.create_inflight_map(); + // First path check + assert_eq!(inflight_map.get(&(0, false)).unwrap().clone(), 94); + assert_eq!(inflight_map.get(&(1, true)).unwrap().clone(), 84); + assert_eq!(inflight_map.get(&(2, false)).unwrap().clone(), 64); + + // Second path check + assert_eq!(inflight_map.get(&(1, false)).unwrap().clone(), 64); + + invoice_payer.handle_event(&Event::PaymentPathSuccessful { + payment_id, payment_hash, path: route.paths[0].clone() + }); + + let inflight_map = invoice_payer.create_inflight_map(); + + assert_eq!(inflight_map.get(&(0, false)), None); + assert_eq!(inflight_map.get(&(1, true)), None); + assert_eq!(inflight_map.get(&(2, false)), None); + + // Second path should still be inflight + assert_eq!(inflight_map.get(&(1, false)).unwrap().clone(), 64) + } + + #[test] + fn considers_inflight_htlcs_between_invoice_payments_when_path_succeeds() { + // First, let's just send a payment through, but only make sure one of the path completes + let event_handled = core::cell::RefCell::new(false); + let event_handler = |_: &_| { *event_handled.borrow_mut() = true; }; + + let payment_preimage = PaymentPreimage([1; 32]); + let payment_invoice = invoice(payment_preimage); + let payment_hash = Some(PaymentHash(payment_invoice.payment_hash().clone().into_inner())); + let final_value_msat = payment_invoice.amount_milli_satoshis().unwrap(); + + let payer = TestPayer::new() + .expect_send(Amount::ForInvoice(final_value_msat)) + .expect_send(Amount::ForInvoice(final_value_msat)); + let final_value_msat = payment_invoice.amount_milli_satoshis().unwrap(); + let route = TestRouter::route_for_value(final_value_msat); + let router = TestRouter {}; + let scorer = RefCell::new(TestScorer::new() + // 1st invoice, 1st path + .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + // 1st invoice, 2nd path + .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + // 2nd invoice, 1st path + .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + // 2nd invoice, 2nd path + .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } ) + ); + let logger = TestLogger::new(); + let invoice_payer = + InvoicePayer::new(&payer, router, &scorer, &logger, event_handler, Retry::Attempts(0)); + + // Succeed 1st path, leave 2nd path inflight + let payment_id = invoice_payer.pay_invoice(&payment_invoice).unwrap(); + invoice_payer.handle_event(&Event::PaymentPathSuccessful { + payment_id, payment_hash, path: route.paths[0].clone() + }); + + // Let's pay a second invoice that will be using the same path. This should trigger the + // assertions that expect the last 4 ChannelUsage values above where TestScorer is initialized. + // Particularly, the 2nd path of the 1st payment, since it is not yet complete, should still + // have 64 msats inflight for paths considering the channel with scid of 1. + let payment_preimage_2 = PaymentPreimage([2; 32]); + let payment_invoice_2 = invoice(payment_preimage_2); + invoice_payer.pay_invoice(&payment_invoice_2).unwrap(); + } + + #[test] + fn considers_inflight_htlcs_between_retries() { + // First, let's just send a payment through, but only make sure one of the path completes + let event_handled = core::cell::RefCell::new(false); + let event_handler = |_: &_| { *event_handled.borrow_mut() = true; }; + + let payment_preimage = PaymentPreimage([1; 32]); + let payment_invoice = invoice(payment_preimage); + let payment_hash = PaymentHash(payment_invoice.payment_hash().clone().into_inner()); + let final_value_msat = payment_invoice.amount_milli_satoshis().unwrap(); + + let payer = TestPayer::new() + .expect_send(Amount::ForInvoice(final_value_msat)) + .expect_send(Amount::OnRetry(final_value_msat / 2)) + .expect_send(Amount::OnRetry(final_value_msat / 2)); + let final_value_msat = payment_invoice.amount_milli_satoshis().unwrap(); + let router = TestRouter {}; + let scorer = RefCell::new(TestScorer::new() + // 1st invoice, 1st path + .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + // 1st invoice, 2nd path + .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + // Retry 1 + .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 32, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 32, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } ) + // Retry 2 + .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 96, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 32, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 32, inflight_htlc_msat: 96, effective_capacity: EffectiveCapacity::Unknown } ) + ); + let logger = TestLogger::new(); + let invoice_payer = + InvoicePayer::new(&payer, router, &scorer, &logger, event_handler, Retry::Attempts(2)); + + // Fail 1st path, leave 2nd path inflight + let payment_id = Some(invoice_payer.pay_invoice(&payment_invoice).unwrap()); + invoice_payer.handle_event(&Event::PaymentPathFailed { + payment_id, + payment_hash, + network_update: None, + rejected_by_dest: false, + all_paths_failed: false, + path: TestRouter::path_for_value(final_value_msat), + short_channel_id: None, + retry: Some(TestRouter::retry_for_invoice(&payment_invoice)), + }); + + // Fails again the 1st path of our retry + invoice_payer.handle_event(&Event::PaymentPathFailed { + payment_id, + payment_hash, + network_update: None, + rejected_by_dest: false, + all_paths_failed: false, + path: TestRouter::path_for_value(final_value_msat / 2), + short_channel_id: None, + retry: Some(RouteParameters { + final_value_msat: final_value_msat / 2, + ..TestRouter::retry_for_invoice(&payment_invoice) + }), + }); + } + + #[test] + fn accounts_for_some_inflight_htlcs_sent_during_partial_failure() { + let event_handled = core::cell::RefCell::new(false); + let event_handler = |_: &_| { *event_handled.borrow_mut() = true; }; + + let payment_preimage = PaymentPreimage([1; 32]); + let invoice_to_pay = invoice(payment_preimage); + let final_value_msat = invoice_to_pay.amount_milli_satoshis().unwrap(); + + let retry = TestRouter::retry_for_invoice(&invoice_to_pay); + let payer = TestPayer::new() + .fails_with_partial_failure( + retry.clone(), OnAttempt(1), + Some(vec![ + Err(ChannelUnavailable { err: "abc".to_string() }), Err(MonitorUpdateFailed) + ])) + .expect_send(Amount::ForInvoice(final_value_msat)); + + let router = TestRouter {}; + let scorer = RefCell::new(TestScorer::new()); + let logger = TestLogger::new(); + let invoice_payer = + InvoicePayer::new(&payer, router, &scorer, &logger, event_handler, Retry::Attempts(0)); + + invoice_payer.pay_invoice(&invoice_to_pay).unwrap(); + let inflight_map = invoice_payer.create_inflight_map(); + + // Only the second path, which failed with `MonitorUpdateFailed` should be added to our + // inflight map because retries are disabled. + assert_eq!(inflight_map.len(), 1); + } + + #[test] + fn accounts_for_all_inflight_htlcs_sent_during_partial_failure() { + let event_handled = core::cell::RefCell::new(false); + let event_handler = |_: &_| { *event_handled.borrow_mut() = true; }; + + let payment_preimage = PaymentPreimage([1; 32]); + let invoice_to_pay = invoice(payment_preimage); + let final_value_msat = invoice_to_pay.amount_milli_satoshis().unwrap(); + + let retry = TestRouter::retry_for_invoice(&invoice_to_pay); + let payer = TestPayer::new() + .fails_with_partial_failure( + retry.clone(), OnAttempt(1), + Some(vec![ + Ok(()), Err(MonitorUpdateFailed) + ])) + .expect_send(Amount::ForInvoice(final_value_msat)); + + let router = TestRouter {}; + let scorer = RefCell::new(TestScorer::new()); + let logger = TestLogger::new(); + let invoice_payer = + InvoicePayer::new(&payer, router, &scorer, &logger, event_handler, Retry::Attempts(0)); + + invoice_payer.pay_invoice(&invoice_to_pay).unwrap(); + let inflight_map = invoice_payer.create_inflight_map(); + + // All paths successful, hence we check of the existence of all 4 hops. + assert_eq!(inflight_map.len(), 4); + } + struct TestRouter; impl TestRouter { fn route_for_value(final_value_msat: u64) -> Route { Route { paths: vec![ - vec![RouteHop { - pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(), - channel_features: ChannelFeatures::empty(), - node_features: NodeFeatures::empty(), - short_channel_id: 0, fee_msat: final_value_msat / 2, cltv_expiry_delta: 144 - }], + vec![ + RouteHop { + pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(), + channel_features: ChannelFeatures::empty(), + node_features: NodeFeatures::empty(), + short_channel_id: 0, + fee_msat: 10, + cltv_expiry_delta: 0 + }, + RouteHop { + pubkey: PublicKey::from_slice(&hex::decode("0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c").unwrap()[..]).unwrap(), + channel_features: ChannelFeatures::empty(), + node_features: NodeFeatures::empty(), + short_channel_id: 1, + fee_msat: 20, + cltv_expiry_delta: 0 + }, + RouteHop { + pubkey: PublicKey::from_slice(&hex::decode("027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007").unwrap()[..]).unwrap(), + channel_features: ChannelFeatures::empty(), + node_features: NodeFeatures::empty(), + short_channel_id: 2, + fee_msat: final_value_msat / 2, + cltv_expiry_delta: 0 + }, + ], vec![RouteHop { pubkey: PublicKey::from_slice(&hex::decode("0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c").unwrap()[..]).unwrap(), channel_features: ChannelFeatures::empty(), @@ -1424,11 +1820,24 @@ mod tests { } } - impl Router for TestRouter { - fn find_route( - &self, _payer: &PublicKey, route_params: &RouteParameters, _payment_hash: &PaymentHash, - _first_hops: Option<&[&ChannelDetails]>, _scorer: &S + impl Router for TestRouter { + fn find_route( + &self, payer: &PublicKey, route_params: &RouteParameters, _payment_hash: &PaymentHash, + _first_hops: Option<&[&ChannelDetails]>, scorer: &S ) -> Result { + // Simulate calling the Scorer just as you would in find_route + let route = Self::route_for_value(route_params.final_value_msat); + for path in route.paths { + for hop in path { + let usage = ChannelUsage { + amount_msat: hop.fee_msat, + inflight_htlc_msat: 0, + effective_capacity: EffectiveCapacity::Unknown, + }; + scorer.channel_penalty_msat(hop.short_channel_id, &NodeId::from_pubkey(payer), &NodeId::from_pubkey(&hop.pubkey), usage); + } + } + Ok(Route { payment_params: Some(route_params.payment_params.clone()), ..Self::route_for_value(route_params.final_value_msat) }) @@ -1437,8 +1846,8 @@ mod tests { struct FailingRouter; - impl Router for FailingRouter { - fn find_route( + impl Router for FailingRouter { + fn find_route( &self, _payer: &PublicKey, _params: &RouteParameters, _payment_hash: &PaymentHash, _first_hops: Option<&[&ChannelDetails]>, _scorer: &S ) -> Result { @@ -1447,7 +1856,8 @@ mod tests { } struct TestScorer { - expectations: Option>, + event_expectations: Option>, + scorer_expectations: RefCell>>, } #[derive(Debug)] @@ -1459,12 +1869,18 @@ mod tests { impl TestScorer { fn new() -> Self { Self { - expectations: None, + event_expectations: None, + scorer_expectations: RefCell::new(None), } } fn expect(mut self, expectation: TestResult) -> Self { - self.expectations.get_or_insert_with(|| VecDeque::new()).push_back(expectation); + self.event_expectations.get_or_insert_with(|| VecDeque::new()).push_back(expectation); + self + } + + fn expect_usage(self, expectation: ChannelUsage) -> Self { + self.scorer_expectations.borrow_mut().get_or_insert_with(|| VecDeque::new()).push_back(expectation); self } } @@ -1476,11 +1892,22 @@ mod tests { impl Score for TestScorer { fn channel_penalty_msat( - &self, _short_channel_id: u64, _source: &NodeId, _target: &NodeId, _usage: ChannelUsage - ) -> u64 { 0 } + &self, _short_channel_id: u64, _source: &NodeId, _target: &NodeId, usage: ChannelUsage + ) -> u64 { + if let Some(scorer_expectations) = self.scorer_expectations.borrow_mut().as_mut() { + match scorer_expectations.pop_front() { + Some(expectation) => { + assert_eq!(expectation.amount_msat, usage.amount_msat); + assert_eq!(expectation.inflight_htlc_msat, usage.inflight_htlc_msat); + }, + None => {}, + } + } + 0 + } fn payment_path_failed(&mut self, actual_path: &[&RouteHop], actual_short_channel_id: u64) { - if let Some(expectations) = &mut self.expectations { + if let Some(expectations) = &mut self.event_expectations { match expectations.pop_front() { Some(TestResult::PaymentFailure { path, short_channel_id }) => { assert_eq!(actual_path, &path.iter().collect::>()[..]); @@ -1495,7 +1922,7 @@ mod tests { } fn payment_path_successful(&mut self, actual_path: &[&RouteHop]) { - if let Some(expectations) = &mut self.expectations { + if let Some(expectations) = &mut self.event_expectations { match expectations.pop_front() { Some(TestResult::PaymentFailure { path, .. }) => { panic!("Unexpected payment path failure: {:?}", path) @@ -1509,7 +1936,7 @@ mod tests { } fn probe_failed(&mut self, actual_path: &[&RouteHop], _: u64) { - if let Some(expectations) = &mut self.expectations { + if let Some(expectations) = &mut self.event_expectations { match expectations.pop_front() { Some(TestResult::PaymentFailure { path, .. }) => { panic!("Unexpected failed payment path: {:?}", path) @@ -1522,7 +1949,7 @@ mod tests { } } fn probe_successful(&mut self, actual_path: &[&RouteHop]) { - if let Some(expectations) = &mut self.expectations { + if let Some(expectations) = &mut self.event_expectations { match expectations.pop_front() { Some(TestResult::PaymentFailure { path, .. }) => { panic!("Unexpected payment path failure: {:?}", path) @@ -1542,9 +1969,15 @@ mod tests { return; } - if let Some(expectations) = &self.expectations { - if !expectations.is_empty() { - panic!("Unsatisfied scorer expectations: {:?}", expectations); + if let Some(event_expectations) = &self.event_expectations { + if !event_expectations.is_empty() { + panic!("Unsatisfied event expectations: {:?}", event_expectations); + } + } + + if let Some(scorer_expectations) = self.scorer_expectations.borrow().as_ref() { + if !scorer_expectations.is_empty() { + panic!("Unsatisfied scorer expectations: {:?}", scorer_expectations) } } } @@ -1584,9 +2017,9 @@ mod tests { self.fails_with(failure, OnAttempt(attempt)) } - fn fails_with_partial_failure(self, retry: RouteParameters, attempt: OnAttempt) -> Self { + fn fails_with_partial_failure(self, retry: RouteParameters, attempt: OnAttempt, results: Option>>) -> Self { self.fails_with(PaymentSendFailure::PartialFailure { - results: vec![], + results: results.unwrap_or(vec![]), failed_paths_retry: Some(retry), payment_id: PaymentId([1; 32]), }, attempt) @@ -1667,8 +2100,8 @@ mod tests { // *** Full Featured Functional Tests with a Real ChannelManager *** struct ManualRouter(RefCell>>); - impl Router for ManualRouter { - fn find_route( + impl Router for ManualRouter { + fn find_route( &self, _payer: &PublicKey, _params: &RouteParameters, _payment_hash: &PaymentHash, _first_hops: Option<&[&ChannelDetails]>, _scorer: &S ) -> Result { diff --git a/lightning-invoice/src/utils.rs b/lightning-invoice/src/utils.rs index 9ef2d2c8d39..72d022eeab1 100644 --- a/lightning-invoice/src/utils.rs +++ b/lightning-invoice/src/utils.rs @@ -455,9 +455,9 @@ impl>, L: Deref> DefaultRouter where L:: } } -impl>, L: Deref, S: Score> Router for DefaultRouter +impl>, L: Deref> Router for DefaultRouter where L::Target: Logger { - fn find_route( + fn find_route( &self, payer: &PublicKey, params: &RouteParameters, _payment_hash: &PaymentHash, first_hops: Option<&[&ChannelDetails]>, scorer: &S ) -> Result { diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 160609216e8..637a8c046ed 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -915,7 +915,7 @@ impl<'a> fmt::Debug for DirectedChannelInfoWithUpdate<'a> { /// /// While this may be smaller than the actual channel capacity, amounts greater than /// [`Self::as_msat`] should not be routed through the channel. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub enum EffectiveCapacity { /// The available liquidity in the channel known from being a channel counterparty, and thus a /// direct hop. diff --git a/lightning/src/routing/scoring.rs b/lightning/src/routing/scoring.rs index 6ae339eae2b..a2492accf59 100644 --- a/lightning/src/routing/scoring.rs +++ b/lightning/src/routing/scoring.rs @@ -221,7 +221,7 @@ impl<'a, S: Writeable> Writeable for MutexGuard<'a, S> { } /// Proposed use of a channel passed as a parameter to [`Score::channel_penalty_msat`]. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct ChannelUsage { /// The amount to send through the channel, denominated in millisatoshis. pub amount_msat: u64, From 80daf942b2be483d6470223be31df49e0056a6f9 Mon Sep 17 00:00:00 2001 From: jurvis Date: Sat, 27 Aug 2022 23:07:50 -0700 Subject: [PATCH 86/91] Make payment tests more realistic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made sure that every hop has a unique receipient. When we simulate calling `channel_penalty_msat` in `TestRouter`’s find route, use actual previous node ids instead of just using the payer’s. --- lightning-invoice/src/payment.rs | 90 +++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/lightning-invoice/src/payment.rs b/lightning-invoice/src/payment.rs index 8854fcbc9af..e7cc43508d9 100644 --- a/lightning-invoice/src/payment.rs +++ b/lightning-invoice/src/payment.rs @@ -1560,7 +1560,8 @@ mod tests { assert_eq!(inflight_map.get(&(2, false)).unwrap().clone(), 64); // Second path check - assert_eq!(inflight_map.get(&(1, false)).unwrap().clone(), 64); + assert_eq!(inflight_map.get(&(3, false)).unwrap().clone(), 74); + assert_eq!(inflight_map.get(&(4, false)).unwrap().clone(), 64); invoice_payer.handle_event(&Event::PaymentPathSuccessful { payment_id, payment_hash, path: route.paths[0].clone() @@ -1573,7 +1574,8 @@ mod tests { assert_eq!(inflight_map.get(&(2, false)), None); // Second path should still be inflight - assert_eq!(inflight_map.get(&(1, false)).unwrap().clone(), 64) + assert_eq!(inflight_map.get(&(3, false)).unwrap().clone(), 74); + assert_eq!(inflight_map.get(&(4, false)).unwrap().clone(), 64) } #[test] @@ -1595,17 +1597,19 @@ mod tests { let router = TestRouter {}; let scorer = RefCell::new(TestScorer::new() // 1st invoice, 1st path - .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) - .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 84, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 94, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) // 1st invoice, 2nd path .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 74, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) // 2nd invoice, 1st path - .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) - .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } ) .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 84, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 94, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) // 2nd invoice, 2nd path .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 74, inflight_htlc_msat: 74, effective_capacity: EffectiveCapacity::Unknown } ) ); let logger = TestLogger::new(); let invoice_payer = @@ -1640,26 +1644,31 @@ mod tests { let payer = TestPayer::new() .expect_send(Amount::ForInvoice(final_value_msat)) .expect_send(Amount::OnRetry(final_value_msat / 2)) - .expect_send(Amount::OnRetry(final_value_msat / 2)); + .expect_send(Amount::OnRetry(final_value_msat / 4)); let final_value_msat = payment_invoice.amount_milli_satoshis().unwrap(); let router = TestRouter {}; let scorer = RefCell::new(TestScorer::new() // 1st invoice, 1st path - .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) - .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 84, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 94, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) // 1st invoice, 2nd path .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) - // Retry 1 - .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) - .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 74, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + // Retry 1, 1st path .expect_usage(ChannelUsage { amount_msat: 32, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 52, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 62, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + // Retry 1, 2nd path .expect_usage(ChannelUsage { amount_msat: 32, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } ) - // Retry 2 - .expect_usage(ChannelUsage { amount_msat: 10, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) - .expect_usage(ChannelUsage { amount_msat: 20, inflight_htlc_msat: 96, effective_capacity: EffectiveCapacity::Unknown } ) - .expect_usage(ChannelUsage { amount_msat: 32, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) - .expect_usage(ChannelUsage { amount_msat: 32, inflight_htlc_msat: 96, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 42, inflight_htlc_msat: 64 + 10, effective_capacity: EffectiveCapacity::Unknown } ) + // Retry 2, 1st path + .expect_usage(ChannelUsage { amount_msat: 16, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 36, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 46, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } ) + // Retry 2, 2nd path + .expect_usage(ChannelUsage { amount_msat: 16, inflight_htlc_msat: 64 + 32, effective_capacity: EffectiveCapacity::Unknown } ) + .expect_usage(ChannelUsage { amount_msat: 26, inflight_htlc_msat: 74 + 32 + 10, effective_capacity: EffectiveCapacity::Unknown } ) ); let logger = TestLogger::new(); let invoice_payer = @@ -1688,7 +1697,7 @@ mod tests { path: TestRouter::path_for_value(final_value_msat / 2), short_channel_id: None, retry: Some(RouteParameters { - final_value_msat: final_value_msat / 2, + final_value_msat: final_value_msat / 4, ..TestRouter::retry_for_invoice(&payment_invoice) }), }); @@ -1723,7 +1732,7 @@ mod tests { // Only the second path, which failed with `MonitorUpdateFailed` should be added to our // inflight map because retries are disabled. - assert_eq!(inflight_map.len(), 1); + assert_eq!(inflight_map.len(), 2); } #[test] @@ -1753,8 +1762,8 @@ mod tests { invoice_payer.pay_invoice(&invoice_to_pay).unwrap(); let inflight_map = invoice_payer.create_inflight_map(); - // All paths successful, hence we check of the existence of all 4 hops. - assert_eq!(inflight_map.len(), 4); + // All paths successful, hence we check of the existence of all 5 hops. + assert_eq!(inflight_map.len(), 5); } struct TestRouter; @@ -1789,12 +1798,24 @@ mod tests { cltv_expiry_delta: 0 }, ], - vec![RouteHop { - pubkey: PublicKey::from_slice(&hex::decode("0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c").unwrap()[..]).unwrap(), - channel_features: ChannelFeatures::empty(), - node_features: NodeFeatures::empty(), - short_channel_id: 1, fee_msat: final_value_msat / 2, cltv_expiry_delta: 144 - }], + vec![ + RouteHop { + pubkey: PublicKey::from_slice(&hex::decode("029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255").unwrap()[..]).unwrap(), + channel_features: ChannelFeatures::empty(), + node_features: NodeFeatures::empty(), + short_channel_id: 3, + fee_msat: 10, + cltv_expiry_delta: 144 + }, + RouteHop { + pubkey: PublicKey::from_slice(&hex::decode("027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007").unwrap()[..]).unwrap(), + channel_features: ChannelFeatures::empty(), + node_features: NodeFeatures::empty(), + short_channel_id: 4, + fee_msat: final_value_msat / 2, + cltv_expiry_delta: 144 + } + ], ], payment_params: None, } @@ -1828,13 +1849,22 @@ mod tests { // Simulate calling the Scorer just as you would in find_route let route = Self::route_for_value(route_params.final_value_msat); for path in route.paths { - for hop in path { + let mut aggregate_msat = 0u64; + for (idx, hop) in path.iter().rev().enumerate() { + aggregate_msat += hop.fee_msat; let usage = ChannelUsage { - amount_msat: hop.fee_msat, + amount_msat: aggregate_msat, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown, }; - scorer.channel_penalty_msat(hop.short_channel_id, &NodeId::from_pubkey(payer), &NodeId::from_pubkey(&hop.pubkey), usage); + + // Since the path is reversed, the last element in our iteration is the first + // hop. + if idx == path.len() - 1 { + scorer.channel_penalty_msat(hop.short_channel_id, &NodeId::from_pubkey(payer), &NodeId::from_pubkey(&hop.pubkey), usage); + } else { + scorer.channel_penalty_msat(hop.short_channel_id, &NodeId::from_pubkey(&path[idx + 1].pubkey), &NodeId::from_pubkey(&hop.pubkey), usage); + } } } From 0b1f476b83756bb9667cb9c54186669ac1f16f74 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Wed, 3 Aug 2022 11:42:54 -0400 Subject: [PATCH 87/91] Expose onion message module as public And fix warnings --- lightning/src/lib.rs | 4 ---- lightning/src/onion_message/functional_tests.rs | 4 ++-- lightning/src/onion_message/messenger.rs | 11 +++++------ lightning/src/onion_message/packet.rs | 4 ++-- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index 199a3cbee4b..1000966d5a5 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -79,11 +79,7 @@ pub mod util; pub mod chain; pub mod ln; pub mod routing; -#[cfg(fuzzing)] pub mod onion_message; -#[cfg(not(fuzzing))] -#[allow(unused)] -mod onion_message; // To be exposed after sending/receiving OMs is supported in PeerManager. #[cfg(feature = "std")] /// Re-export of either `core2::io` or `std::io`, depending on the `std` feature flag. diff --git a/lightning/src/onion_message/functional_tests.rs b/lightning/src/onion_message/functional_tests.rs index 5a83a0b2b21..001d96e8342 100644 --- a/lightning/src/onion_message/functional_tests.rs +++ b/lightning/src/onion_message/functional_tests.rs @@ -124,7 +124,7 @@ fn too_big_packet_error() { #[test] fn invalid_blinded_route_error() { // Make sure we error as expected if a provided blinded route has 0 or 1 hops. - let mut nodes = create_nodes(3); + let nodes = create_nodes(3); // 0 hops let secp_ctx = Secp256k1::new(); @@ -143,7 +143,7 @@ fn invalid_blinded_route_error() { #[test] fn reply_path() { - let mut nodes = create_nodes(4); + let nodes = create_nodes(4); let secp_ctx = Secp256k1::new(); // Destination::Node diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 75eb4619b8f..3a14c78d0fb 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -24,7 +24,6 @@ use super::utils; use util::events::OnionMessageProvider; use util::logger::Logger; -use core::mem; use core::ops::Deref; use sync::{Arc, Mutex}; use prelude::*; @@ -35,9 +34,7 @@ use prelude::*; /// /// # Example /// -// Needs to be `ignore` until the `onion_message` module is made public, otherwise this is a test -// failure. -/// ```ignore +/// ``` /// # extern crate bitcoin; /// # use bitcoin::hashes::_export::_core::time::Duration; /// # use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey}; @@ -66,7 +63,8 @@ use prelude::*; /// /// // Send an empty onion message to a node id. /// let intermediate_hops = [hop_node_id1, hop_node_id2]; -/// onion_messenger.send_onion_message(&intermediate_hops, Destination::Node(destination_node_id)); +/// let reply_path = None; +/// onion_messenger.send_onion_message(&intermediate_hops, Destination::Node(destination_node_id), reply_path); /// /// // Create a blinded route to yourself, for someone to send an onion message to. /// # let your_node_id = hop_node_id1; @@ -75,7 +73,8 @@ use prelude::*; /// /// // Send an empty onion message to a blinded route. /// # let intermediate_hops = [hop_node_id1, hop_node_id2]; -/// onion_messenger.send_onion_message(&intermediate_hops, Destination::BlindedRoute(blinded_route)); +/// let reply_path = None; +/// onion_messenger.send_onion_message(&intermediate_hops, Destination::BlindedRoute(blinded_route), reply_path); /// ``` /// /// [offers]: diff --git a/lightning/src/onion_message/packet.rs b/lightning/src/onion_message/packet.rs index 4ab53735ed6..1337bdb14d5 100644 --- a/lightning/src/onion_message/packet.rs +++ b/lightning/src/onion_message/packet.rs @@ -74,7 +74,7 @@ impl LengthReadable for Packet { while read_idx < hop_data_len { let mut read_buffer = [0; READ_BUFFER_SIZE]; let read_amt = cmp::min(hop_data_len - read_idx, READ_BUFFER_SIZE); - r.read_exact(&mut read_buffer[..read_amt]); + r.read_exact(&mut read_buffer[..read_amt])?; hop_data.extend_from_slice(&read_buffer[..read_amt]); read_idx += read_amt; } @@ -170,7 +170,7 @@ impl Writeable for (Payload, [u8; 32]) { // Uses the provided secret to simultaneously decode and decrypt the control TLVs. impl ReadableArgs for Payload { - fn read(mut r: &mut R, encrypted_tlvs_ss: SharedSecret) -> Result { + fn read(r: &mut R, encrypted_tlvs_ss: SharedSecret) -> Result { let v: BigSize = Readable::read(r)?; let mut rd = FixedLengthReader::new(r, v.0); let mut reply_path: Option = None; From b9b02b48fb4dd66181f61739fa0047c29dead697 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 4 Aug 2022 11:05:07 -0400 Subject: [PATCH 88/91] Refuse to send and forward OMs to disconnected peers We also refuse to connect to peers that don't advertise onion message forwarding support. --- lightning/src/ln/features.rs | 7 +- lightning/src/ln/msgs.rs | 6 ++ lightning/src/ln/peer_handler.rs | 10 +- .../src/onion_message/functional_tests.rs | 27 +++--- lightning/src/onion_message/messenger.rs | 96 ++++++++++++------- 5 files changed, 99 insertions(+), 47 deletions(-) diff --git a/lightning/src/ln/features.rs b/lightning/src/ln/features.rs index dbc17a34138..c978c61a3a5 100644 --- a/lightning/src/ln/features.rs +++ b/lightning/src/ln/features.rs @@ -433,6 +433,11 @@ mod sealed { define_feature!(27, ShutdownAnySegwit, [InitContext, NodeContext], "Feature flags for `opt_shutdown_anysegwit`.", set_shutdown_any_segwit_optional, set_shutdown_any_segwit_required, supports_shutdown_anysegwit, requires_shutdown_anysegwit); + // We do not yet advertise the onion messages feature bit, but we need to detect when peers + // support it. + define_feature!(39, OnionMessages, [InitContext, NodeContext], + "Feature flags for `option_onion_messages`.", set_onion_messages_optional, + set_onion_messages_required, supports_onion_messages, requires_onion_messages); define_feature!(45, ChannelType, [InitContext, NodeContext], "Feature flags for `option_channel_type`.", set_channel_type_optional, set_channel_type_required, supports_channel_type, requires_channel_type); @@ -767,7 +772,7 @@ impl Features { impl Features { // We are no longer setting initial_routing_sync now that gossip_queries - // is enabled. This feature is ignored by a peer when gossip_queries has + // is enabled. This feature is ignored by a peer when gossip_queries has // been negotiated. #[cfg(test)] pub(crate) fn clear_initial_routing_sync(&mut self) { diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 190907ce26e..a32b17b9f0d 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -949,6 +949,12 @@ pub trait RoutingMessageHandler : MessageSendEventsProvider { pub trait OnionMessageHandler : OnionMessageProvider { /// Handle an incoming onion_message message from the given peer. fn handle_onion_message(&self, peer_node_id: &PublicKey, msg: &OnionMessage); + /// Called when a connection is established with a peer. Can be used to track which peers + /// advertise onion message support and are online. + fn peer_connected(&self, their_node_id: &PublicKey, init: &Init); + /// Indicates a connection to the peer failed/an existing connection was lost. Allows handlers to + /// drop and refuse to forward onion messages to this peer. + fn peer_disconnected(&self, their_node_id: &PublicKey, no_connection_possible: bool); } mod fuzzy_internal_msgs { diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 1258026e17e..f80c8984c1c 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -81,6 +81,8 @@ impl OnionMessageProvider for IgnoringMessageHandler { } impl OnionMessageHandler for IgnoringMessageHandler { fn handle_onion_message(&self, _their_node_id: &PublicKey, _msg: &msgs::OnionMessage) {} + fn peer_connected(&self, _their_node_id: &PublicKey, _init: &msgs::Init) {} + fn peer_disconnected(&self, _their_node_id: &PublicKey, _no_connection_possible: bool) {} } impl Deref for IgnoringMessageHandler { type Target = IgnoringMessageHandler; @@ -1173,8 +1175,9 @@ impl Vec { - let mut res = Vec::new(); + let mut nodes = Vec::new(); for i in 0..num_messengers { let logger = Arc::new(test_utils::TestLogger::with_id(format!("node {}", i))); let seed = [i as u8; 32]; let keys_manager = Arc::new(test_utils::TestKeysInterface::new(&seed, Network::Testnet)); - res.push(MessengerNode { + nodes.push(MessengerNode { keys_manager: keys_manager.clone(), messenger: OnionMessenger::new(keys_manager, logger.clone()), logger, }); } - res + for idx in 0..num_messengers - 1 { + let i = idx as usize; + let mut features = InitFeatures::known(); + features.set_onion_messages_optional(); + let init_msg = msgs::Init { features, remote_network_address: None }; + nodes[i].messenger.peer_connected(&nodes[i + 1].get_node_pk(), &init_msg.clone()); + nodes[i + 1].messenger.peer_connected(&nodes[i].get_node_pk(), &init_msg.clone()); + } + nodes } fn pass_along_path(path: &Vec, expected_path_id: Option<[u8; 32]>) { @@ -53,7 +62,6 @@ fn pass_along_path(path: &Vec, expected_path_id: Option<[u8; 32]> let num_nodes = path.len(); for (idx, node) in path.into_iter().skip(1).enumerate() { let events = prev_node.messenger.release_pending_msgs(); - assert_eq!(events.len(), 1); let onion_msg = { let msgs = events.get(&node.get_node_pk()).unwrap(); assert_eq!(msgs.len(), 1); @@ -110,12 +118,9 @@ fn three_blinded_hops() { #[test] fn too_big_packet_error() { // Make sure we error as expected if a packet is too big to send. - let nodes = create_nodes(1); - - let hop_secret = SecretKey::from_slice(&hex::decode("0101010101010101010101010101010101010101010101010101010101010101").unwrap()[..]).unwrap(); - let secp_ctx = Secp256k1::new(); - let hop_node_id = PublicKey::from_secret_key(&secp_ctx, &hop_secret); + let nodes = create_nodes(2); + let hop_node_id = nodes[1].get_node_pk(); let hops = [hop_node_id; 400]; let err = nodes[0].messenger.send_onion_message(&hops, Destination::Node(hop_node_id), None).unwrap_err(); assert_eq!(err, SendError::TooBigPacket); diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index 3a14c78d0fb..b9da9246cb1 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -122,6 +122,8 @@ pub enum SendError { /// The provided [`Destination`] was an invalid [`BlindedRoute`], due to having fewer than two /// blinded hops. TooFewBlindedHops, + /// Our next-hop peer was offline or does not support onion message forwarding. + InvalidFirstHop, } impl OnionMessenger @@ -165,25 +167,28 @@ impl OnionMessenger .map_err(|e| SendError::Secp256k1(e))?; let prng_seed = self.keys_manager.get_secure_random_bytes(); - let onion_packet = construct_onion_message_packet( + let onion_routing_packet = construct_onion_message_packet( packet_payloads, packet_keys, prng_seed).map_err(|()| SendError::TooBigPacket)?; let mut pending_per_peer_msgs = self.pending_messages.lock().unwrap(); - let pending_msgs = pending_per_peer_msgs.entry(introduction_node_id).or_insert_with(VecDeque::new); - pending_msgs.push_back( - msgs::OnionMessage { - blinding_point, - onion_routing_packet: onion_packet, + match pending_per_peer_msgs.entry(introduction_node_id) { + hash_map::Entry::Vacant(_) => Err(SendError::InvalidFirstHop), + hash_map::Entry::Occupied(mut e) => { + e.get_mut().push_back(msgs::OnionMessage { blinding_point, onion_routing_packet }); + Ok(()) } - ); - Ok(()) + } } #[cfg(test)] pub(super) fn release_pending_msgs(&self) -> HashMap> { let mut pending_msgs = self.pending_messages.lock().unwrap(); let mut msgs = HashMap::new(); - core::mem::swap(&mut *pending_msgs, &mut msgs); + // We don't want to disconnect the peers by removing them entirely from the original map, so we + // swap the pending message buffers individually. + for (peer_node_id, pending_messages) in &mut *pending_msgs { + msgs.insert(*peer_node_id, core::mem::take(pending_messages)); + } msgs } } @@ -252,32 +257,43 @@ impl OnionMessageHandler for OnionMessenger blinding_point, - None => { - let blinding_factor = { - let mut sha = Sha256::engine(); - sha.input(&msg.blinding_point.serialize()[..]); - sha.input(control_tlvs_ss.as_ref()); - Sha256::from_engine(sha).into_inner() - }; - let next_blinding_point = msg.blinding_point; - match next_blinding_point.mul_tweak(&self.secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap()) { - Ok(bp) => bp, - Err(e) => { - log_trace!(self.logger, "Failed to compute next blinding point: {}", e); - return - } - } - }, - }, - onion_routing_packet: outgoing_packet, + + #[cfg(fuzzing)] + pending_per_peer_msgs.entry(next_node_id).or_insert_with(VecDeque::new); + + match pending_per_peer_msgs.entry(next_node_id) { + hash_map::Entry::Vacant(_) => { + log_trace!(self.logger, "Dropping forwarded onion message to disconnected peer {:?}", next_node_id); + return }, - ); - log_trace!(self.logger, "Forwarding an onion message to peer {}", next_node_id); + hash_map::Entry::Occupied(mut e) => { + e.get_mut().push_back( + msgs::OnionMessage { + blinding_point: match next_blinding_override { + Some(blinding_point) => blinding_point, + None => { + let blinding_factor = { + let mut sha = Sha256::engine(); + sha.input(&msg.blinding_point.serialize()[..]); + sha.input(control_tlvs_ss.as_ref()); + Sha256::from_engine(sha).into_inner() + }; + let next_blinding_point = msg.blinding_point; + match next_blinding_point.mul_tweak(&self.secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap()) { + Ok(bp) => bp, + Err(e) => { + log_trace!(self.logger, "Failed to compute next blinding point: {}", e); + return + } + } + }, + }, + onion_routing_packet: outgoing_packet, + }, + ); + log_trace!(self.logger, "Forwarding an onion message to peer {}", next_node_id); + } + }; }, Err(e) => { log_trace!(self.logger, "Errored decoding onion message packet: {:?}", e); @@ -287,6 +303,18 @@ impl OnionMessageHandler for OnionMessenger OnionMessageProvider for OnionMessenger From 04f467342851691d368df1eb318bcfaf6c58a607 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 2 Sep 2022 13:41:41 -0400 Subject: [PATCH 89/91] Don't construct OnionMessage while holding peer lock --- lightning/src/onion_message/messenger.rs | 47 ++++++++++++------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index b9da9246cb1..a3320f10d21 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -255,6 +255,28 @@ impl OnionMessageHandler for OnionMessenger blinding_point, + None => { + let blinding_factor = { + let mut sha = Sha256::engine(); + sha.input(&msg.blinding_point.serialize()[..]); + sha.input(control_tlvs_ss.as_ref()); + Sha256::from_engine(sha).into_inner() + }; + let next_blinding_point = msg.blinding_point; + match next_blinding_point.mul_tweak(&self.secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap()) { + Ok(bp) => bp, + Err(e) => { + log_trace!(self.logger, "Failed to compute next blinding point: {}", e); + return + } + } + }, + }, + onion_routing_packet: outgoing_packet, + }; let mut pending_per_peer_msgs = self.pending_messages.lock().unwrap(); @@ -267,30 +289,7 @@ impl OnionMessageHandler for OnionMessenger { - e.get_mut().push_back( - msgs::OnionMessage { - blinding_point: match next_blinding_override { - Some(blinding_point) => blinding_point, - None => { - let blinding_factor = { - let mut sha = Sha256::engine(); - sha.input(&msg.blinding_point.serialize()[..]); - sha.input(control_tlvs_ss.as_ref()); - Sha256::from_engine(sha).into_inner() - }; - let next_blinding_point = msg.blinding_point; - match next_blinding_point.mul_tweak(&self.secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap()) { - Ok(bp) => bp, - Err(e) => { - log_trace!(self.logger, "Failed to compute next blinding point: {}", e); - return - } - } - }, - }, - onion_routing_packet: outgoing_packet, - }, - ); + e.get_mut().push_back(onion_message); log_trace!(self.logger, "Forwarding an onion message to peer {}", next_node_id); } }; From a8ea0bde59b2f74925742c639915e86ceb18045b Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 1 Sep 2022 15:26:17 -0400 Subject: [PATCH 90/91] Limit OnionMessenger outbound buffer size Drop OMs if they push us over the max OnionMessenger outbound buffer size --- .../src/onion_message/functional_tests.rs | 10 ++++++ lightning/src/onion_message/messenger.rs | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/lightning/src/onion_message/functional_tests.rs b/lightning/src/onion_message/functional_tests.rs index b18431a66dd..5fec5be51c4 100644 --- a/lightning/src/onion_message/functional_tests.rs +++ b/lightning/src/onion_message/functional_tests.rs @@ -170,3 +170,13 @@ fn reply_path() { "lightning::onion_message::messenger".to_string(), format!("Received an onion message with path_id: None and reply_path").to_string(), 2); } + +#[test] +fn peer_buffer_full() { + let nodes = create_nodes(2); + for _ in 0..188 { // Based on MAX_PER_PEER_BUFFER_SIZE in OnionMessenger + nodes[0].messenger.send_onion_message(&[], Destination::Node(nodes[1].get_node_pk()), None).unwrap(); + } + let err = nodes[0].messenger.send_onion_message(&[], Destination::Node(nodes[1].get_node_pk()), None).unwrap_err(); + assert_eq!(err, SendError::BufferFull); +} diff --git a/lightning/src/onion_message/messenger.rs b/lightning/src/onion_message/messenger.rs index a3320f10d21..e38a0d10f6a 100644 --- a/lightning/src/onion_message/messenger.rs +++ b/lightning/src/onion_message/messenger.rs @@ -23,6 +23,7 @@ use super::packet::{BIG_PACKET_HOP_DATA_LEN, ForwardControlTlvs, Packet, Payload use super::utils; use util::events::OnionMessageProvider; use util::logger::Logger; +use util::ser::Writeable; use core::ops::Deref; use sync::{Arc, Mutex}; @@ -124,6 +125,8 @@ pub enum SendError { TooFewBlindedHops, /// Our next-hop peer was offline or does not support onion message forwarding. InvalidFirstHop, + /// Our next-hop peer's buffer was full or our total outbound buffer was full. + BufferFull, } impl OnionMessenger @@ -171,6 +174,7 @@ impl OnionMessenger packet_payloads, packet_keys, prng_seed).map_err(|()| SendError::TooBigPacket)?; let mut pending_per_peer_msgs = self.pending_messages.lock().unwrap(); + if outbound_buffer_full(&introduction_node_id, &pending_per_peer_msgs) { return Err(SendError::BufferFull) } match pending_per_peer_msgs.entry(introduction_node_id) { hash_map::Entry::Vacant(_) => Err(SendError::InvalidFirstHop), hash_map::Entry::Occupied(mut e) => { @@ -193,6 +197,29 @@ impl OnionMessenger } } +fn outbound_buffer_full(peer_node_id: &PublicKey, buffer: &HashMap>) -> bool { + const MAX_TOTAL_BUFFER_SIZE: usize = (1 << 20) * 128; + const MAX_PER_PEER_BUFFER_SIZE: usize = (1 << 10) * 256; + let mut total_buffered_bytes = 0; + let mut peer_buffered_bytes = 0; + for (pk, peer_buf) in buffer { + for om in peer_buf { + let om_len = om.serialized_length(); + if pk == peer_node_id { + peer_buffered_bytes += om_len; + } + total_buffered_bytes += om_len; + + if total_buffered_bytes >= MAX_TOTAL_BUFFER_SIZE || + peer_buffered_bytes >= MAX_PER_PEER_BUFFER_SIZE + { + return true + } + } + } + false +} + impl OnionMessageHandler for OnionMessenger where K::Target: KeysInterface, L::Target: Logger, @@ -279,6 +306,10 @@ impl OnionMessageHandler for OnionMessenger Date: Fri, 2 Sep 2022 21:57:32 +0000 Subject: [PATCH 91/91] Handle monotonic clock going backwards during runtime We've had some users complain that `duration_since` is panic'ing for them. This is possible if the machine being run on is buggy and the "monotonic clock" goes backwards, which sadly some ancient systems can do. Rust addressed this issue in 1.60 by forcing `Instant::duration_since` to not panic if the machine is buggy (and time goes backwards), but for users on older rust versions we do the same by hand here. --- lightning/src/util/time.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lightning/src/util/time.rs b/lightning/src/util/time.rs index d3768aa7ca6..f450dc2c301 100644 --- a/lightning/src/util/time.rs +++ b/lightning/src/util/time.rs @@ -65,7 +65,12 @@ impl Time for std::time::Instant { } fn duration_since(&self, earlier: Self) -> Duration { - self.duration_since(earlier) + // On rust prior to 1.60 `Instant::duration_since` will panic if time goes backwards. + // However, we support rust versions prior to 1.60 and some users appear to have "monotonic + // clocks" that go backwards in practice (likely relatively ancient kernels/etc). Thus, we + // manually check for time going backwards here and return a duration of zero in that case. + let now = Self::now(); + if now > earlier { now - earlier } else { Duration::from_secs(0) } } fn duration_since_epoch() -> Duration {