From 160f49eb5d8c93b3ae07e171d5132b27e39d960a Mon Sep 17 00:00:00 2001 From: Carsten Otto Date: Fri, 7 May 2021 17:31:46 +0100 Subject: [PATCH] add "Economy Fee Mode" fixes #161 --- README.md | 46 +++++++++++++++++++++++++++++-- lnd.py | 17 +++++++++++- logic.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++----- rebalance.py | 9 +++++-- routes.py | 6 ++--- 5 files changed, 140 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 455455f..4790027 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ If this fails, make sure you're running Python 3. usage: rebalance.py [-h] [--lnddir LNDDIR] [--grpc GRPC] [-r RATIO] [-l] [-o | -i] [-f CHANNEL] [-t CHANNEL] [-a AMOUNT | -p PERCENTAGE] [-e EXCLUDE] - [--max-fee-factor MAX_FEE_FACTOR] + [--max-fee-factor MAX_FEE_FACTOR | --econ-fee | --no-econ-fee] optional arguments: -h, --help show this help message and exit @@ -81,6 +81,8 @@ rebalance: (default: 10) Reject routes that cost more than x times the lnd default (base: 1 sat, rate: 1 millionth sat) per hop on average + --econ-fee, --no-econ-fee + (default: disabled) Economy Fee Mode, see README.md" ``` ### List of channels @@ -135,12 +137,52 @@ It is also possible to indicate the `--to/-t` channel by the number shown next t If you do not specify the amount, the rebalance amount is determined automatically. As an alternative to specify the amount you may also set a percentage of the rebalance amount using `-p`. -For example, the following command tries to sends 20% of the amount required to rebalance the channel: +For example, the following command tries to send 20% of the amount required to rebalance the channel: `rebalance.py -t 23 -p 20` The maximum amount you can send in one transaction currently is limited (by the protocol) to 4,294,967 satoshis. +### Fees + +In order for the network to route your rebalance transaction, you have to pay fees. +To protect yourself from "greedy" peers, certain routes are rejected by default by using `--max-fee-factor` with a +default value of 10: +> Reject routes that cost more than 10 times the lnd default (base: 1 sat, rate: 1 millionth sat) per hop on average. + +Note that it may be necessary to pay higher fees, i.e. you'd need to set a higher `--max-fee-factor`. + +#### Economy Fee Mode +As an alternative to `--max-fee-factor`, you can also enable the Economy Fee Mode using `--econ-fee`. +If this mode is enabled, three different fees are taken into account: + +1. The actual fee you'd have to pay for the rebalance transaction +2. The fee you'd earn if, instead of sending the funds due to your own rebalance transaction, your node is paid to forward the amount through the outbound (`--from`) channel +3. The fee you'd earn if, after the rebalance transaction is done, your node forwards the amount back again through the inbound (`--to`) channel + +The rebalance transaction is not performed if the direct costs (1) plus the implicit costs (2) are higher than the +possible future income (3). + +#### Example +You have lots of funds in channel A and nothing in channel B. You would like to send half of the funds through +channel A (outbound channel) through the lightning network, and finally back into channel B. This incurs a transaction +fee you'd have to pay for the transaction. This is the direct cost (1). + +Furthermore, if in the future there is demand for your node to route funds +through channel A, you cannot do that as much (because you reduced outbound liquidity in the channel). +This is the implicit cost (2). + +Finally, you rebalance channel B in the hope that lateron someone reqeusts your node to forward funds from your own node +through channel B towards the peer at the other end, so that you can earn fees for this. +These fees are the possible future income (3). + +#### Warning +To determine the future income, the fee set as part of the channel policy is used in the computation. +As such, if you set a too high fee rate, with `--econ-mode` you'd allow more expensive rebalance +transactions. Please make sure to set realistic fee rates, which at best are already known to attract forwardings. +To protect you from making mistakes, `--econ-mode` only works if the inbound channel has a fee rate of less than +10,000 sat per 1M sat (which corresponds to a 1% fee). + ## Contributing Contributions are highly welcome! diff --git a/lnd.py b/lnd.py index f9251d3..9bf60eb 100644 --- a/lnd.py +++ b/lnd.py @@ -77,7 +77,11 @@ def get_channels(self): self.channels = self.stub.ListChannels(request).channels return self.channels - def get_route(self, pub_key, amount, ignored_pairs, ignored_nodes, first_hop_channel_id): + def get_route(self, pub_key, amount, ignored_pairs, ignored_nodes, first_hop_channel_id, fee_limit_sat): + if fee_limit_sat: + fee_limit = {"fixed": int(fee_limit_sat)} + else: + fee_limit = None if pub_key: last_hop_pubkey = base64.b16decode(pub_key, True) else: @@ -87,6 +91,7 @@ def get_route(self, pub_key, amount, ignored_pairs, ignored_nodes, first_hop_cha last_hop_pubkey=last_hop_pubkey, amt=amount, ignored_pairs=ignored_pairs, + fee_limit=fee_limit, ignored_nodes=ignored_nodes, use_mission_control=True, outgoing_chan_id=first_hop_channel_id, @@ -97,6 +102,16 @@ def get_route(self, pub_key, amount, ignored_pairs, ignored_nodes, first_hop_cha except: return None + def get_policy_to(self, channel_id): + # node1_policy contains the fee base and rate for payments from node1 to node2 + for edge in self.get_edges(): + if edge.channel_id == channel_id: + if edge.node1_pub == self.get_own_pubkey(): + result = edge.node1_policy + else: + result = edge.node2_policy + return result + def send_payment(self, payment_request, route): last_hop = route.hops[-1] last_hop.mpp_record.payment_addr = payment_request.payment_addr diff --git a/logic.py b/logic.py index 78cbdd9..ccbd722 100644 --- a/logic.py +++ b/logic.py @@ -4,6 +4,7 @@ DEFAULT_BASE_FEE_SAT_MSAT = 1000 DEFAULT_FEE_RATE_MSAT = 0.001 +MAX_FEE_RATE = 10000 def debug(message): @@ -15,7 +16,9 @@ def debugnobreak(message): class Logic: - def __init__(self, lnd, first_hop_channel, last_hop_channel, amount, channel_ratio, excluded, max_fee_factor): + def __init__(self, + lnd, first_hop_channel, last_hop_channel, amount, channel_ratio, excluded, max_fee_factor, econ_fee + ): self.lnd = lnd self.first_hop_channel = first_hop_channel self.last_hop_channel = last_hop_channel @@ -25,8 +28,10 @@ def __init__(self, lnd, first_hop_channel, last_hop_channel, amount, channel_rat if excluded: self.excluded = excluded self.max_fee_factor = max_fee_factor + self.econ_fee = econ_fee def rebalance(self): + fee_limit = self.get_fee_limit() if self.last_hop_channel: debug(("Sending {:,} satoshis to rebalance to channel with ID %d" % self.last_hop_channel.chan_id).format(self.amount)) @@ -38,7 +43,7 @@ def rebalance(self): debug("Forced first channel has ID %d" % self.first_hop_channel.chan_id) payment_request = self.generate_invoice() - routes = Routes(self.lnd, payment_request, self.first_hop_channel, self.last_hop_channel) + routes = Routes(self.lnd, payment_request, self.first_hop_channel, self.last_hop_channel, fee_limit) self.initialize_ignored_channels(routes) @@ -52,6 +57,19 @@ def rebalance(self): debug("Could not find any suitable route") return False + def get_fee_limit(self): + fee_limit = None + if self.last_hop_channel and self.econ_fee: + policy = self.lnd.get_policy_to(self.last_hop_channel.chan_id) + if policy.fee_rate_milli_msat > MAX_FEE_RATE: + debug("Unable to use --econ-fee, policy for inbound channel looks odd (fee rate %s)" + % policy.fee_rate_milli_msat) + sys.exit(1) + fee_limit = self.fee_for_policy(self.amount, policy) + debug("Setting fee limit to %s (due to --econ-fee)" % int(fee_limit)) + + return fee_limit + def try_route(self, payment_request, route, routes, tried_routes): if self.route_is_invalid(route, routes): return False @@ -110,16 +128,16 @@ def route_is_invalid(self, route, routes): first_hop = route.hops[0] last_hop = route.hops[-1] if self.low_local_ratio_after_sending(first_hop, route.total_amt): - debugnobreak("First hop would have low local ratio after sending, ") + debugnobreak("Outbound channel would have low local ratio after sending, ") routes.ignore_first_hop(self.get_channel_for_channel_id(first_hop.chan_id)) return True if self.first_hop_and_last_hop_use_same_channel(first_hop, last_hop): - debugnobreak("First hop and last hop use same channel, ") + debugnobreak("Outbound and inbound channel are identical, ") hop_before_last_hop = route.hops[-2] routes.ignore_edge_from_to(last_hop.chan_id, hop_before_last_hop.pub_key, last_hop.pub_key) return True if self.high_local_ratio_after_receiving(last_hop): - debugnobreak("Last hop would have high local ratio after receiving, ") + debugnobreak("Inbound channel would have high local ratio after receiving, ") hop_before_last_hop = route.hops[-2] routes.ignore_edge_from_to(last_hop.chan_id, hop_before_last_hop.pub_key, last_hop.pub_key) return True @@ -163,10 +181,43 @@ def first_hop_and_last_hop_use_same_channel(first_hop, last_hop): return first_hop.chan_id == last_hop.chan_id def fees_too_high(self, route): + if self.econ_fee: + return self.fees_too_high_econ_fee(route) hops_with_fees = len(route.hops) - 1 lnd_fees = hops_with_fees * (DEFAULT_BASE_FEE_SAT_MSAT + (self.amount * DEFAULT_FEE_RATE_MSAT)) limit = self.max_fee_factor * lnd_fees - return route.total_fees_msat > limit + high_fees = route.total_fees_msat > limit + if high_fees: + debugnobreak("High fees (%s sat over limit of %s), " + % (int((route.total_fees_msat - limit) / 1000), int(limit/1000))) + return high_fees + + def fees_too_high_econ_fee(self, route): + policy_first_hop = self.lnd.get_policy_to(route.hops[0].chan_id) + amount = route.total_amt + missed_fee = self.fee_for_policy(amount, policy_first_hop) + policy_last_hop = self.lnd.get_policy_to(route.hops[-1].chan_id) + if policy_last_hop.fee_rate_milli_msat > MAX_FEE_RATE: + debug("Ignoring route, policy for inbound channel looks odd (fee rate %s)" + % policy_last_hop.fee_rate_milli_msat) + return True + expected_fee = self.fee_for_policy(amount, policy_last_hop) + rebalance_fee = route.total_fees + high_fees = rebalance_fee + missed_fee > expected_fee + if high_fees: + difference = rebalance_fee + missed_fee - expected_fee + debugnobreak("High fees (" + "%s expected future fee income for inbound channel, " + "have to pay %s now, " + "missing out on %s future fees for outbound channel, " + "difference %s), " + % (int(expected_fee), int(rebalance_fee), int(missed_fee), int(difference))) + return high_fees + + @staticmethod + def fee_for_policy(amount, policy): + expected_fee_msat = amount / 1000000 * policy.fee_rate_milli_msat + policy.fee_base_msat / 1000 + return expected_fee_msat def generate_invoice(self): if self.last_hop_channel: @@ -196,9 +247,22 @@ def initialize_ignored_channels(self, routes): from_pub_key = self.lnd.get_own_pubkey() to_pub_key = self.last_hop_channel.remote_pubkey routes.ignore_edge_from_to(chan_id, from_pub_key, to_pub_key, show_message=False) + if self.econ_fee: + self.ignore_first_hops_with_fee_rate_higher_than_last_hop(routes) for channel in self.lnd.get_channels(): if self.low_local_ratio_after_sending(channel, self.amount): routes.ignore_first_hop(channel, show_message=False) if channel.chan_id in self.excluded: debugnobreak("Channel is excluded, ") routes.ignore_first_hop(channel) + + def ignore_first_hops_with_fee_rate_higher_than_last_hop(self, routes): + policy_last_hop = self.lnd.get_policy_to(self.last_hop_channel.chan_id) + last_hop_fee_rate = policy_last_hop.fee_rate_milli_msat + from_pub_key = self.lnd.get_own_pubkey() + for channel in self.lnd.get_channels(): + chan_id = channel.chan_id + policy = self.lnd.get_policy_to(chan_id) + if policy.fee_rate_milli_msat > last_hop_fee_rate: + to_pub_key = channel.remote_pubkey + routes.ignore_edge_from_to(chan_id, from_pub_key, to_pub_key, show_message=False) diff --git a/rebalance.py b/rebalance.py index 7405026..879f711 100755 --- a/rebalance.py +++ b/rebalance.py @@ -81,9 +81,10 @@ def main(): sys.exit(0) max_fee_factor = arguments.max_fee_factor + econ_fee = arguments.econ_fee excluded = arguments.exclude return Logic(lnd, first_hop_channel, last_hop_channel, amount, channel_ratio, excluded, - max_fee_factor).rebalance() + max_fee_factor, econ_fee).rebalance() def get_amount(arguments, first_hop_channel, last_hop_channel): @@ -180,11 +181,15 @@ def get_argument_parser(): action="append", help="Exclude the given channel ID as the outgoing channel (no funds will be taken " "out of excluded channels)") - rebalance_group.add_argument("--max-fee-factor", + fee_group = rebalance_group.add_mutually_exclusive_group() + fee_group.add_argument("--max-fee-factor", type=float, default=10, help="(default: 10) Reject routes that cost more than x times the lnd default " "(base: 1 sat, rate: 1 millionth sat) per hop on average") + fee_group.add_argument("--econ-fee", + action=argparse.BooleanOptionalAction, + help="(default: disabled) Economy Fee Mode, see README.md") return parser diff --git a/routes.py b/routes.py index e149249..0174bd2 100644 --- a/routes.py +++ b/routes.py @@ -13,7 +13,7 @@ def debugnobreak(message): class Routes: - def __init__(self, lnd, payment_request, first_hop_channel, last_hop_channel): + def __init__(self, lnd, payment_request, first_hop_channel, last_hop_channel, fee_limit): self.lnd = lnd self.payment_request = payment_request self.first_hop_channel = first_hop_channel @@ -23,6 +23,7 @@ def __init__(self, lnd, payment_request, first_hop_channel, last_hop_channel): self.ignored_pairs = [] self.ignored_nodes = [] self.num_requested_routes = 0 + self.fee_limit = fee_limit def has_next(self): self.update_routes() @@ -55,7 +56,7 @@ def request_route(self): else: first_hop_channel_id = None routes = self.lnd.get_route(last_hop_pubkey, amount, self.ignored_pairs, - self.ignored_nodes, first_hop_channel_id) + self.ignored_nodes, first_hop_channel_id, self.fee_limit) if routes is None: self.num_requested_routes = MAX_ROUTES_TO_REQUEST else: @@ -100,7 +101,6 @@ def ignore_node_with_highest_fee(self, route): max_fee_hop = hop pub_key = max_fee_hop.pub_key - debugnobreak("High fees (%s msat), " % max_fee_msat) self.ignore_node(pub_key) def ignore_edge_from_to(self, chan_id, from_pubkey, to_pubkey, show_message=True):