Skip to content

Commit

Permalink
Merge branch 'master' into patch-1
Browse files Browse the repository at this point in the history
  • Loading branch information
chrisguida authored Feb 7, 2024
2 parents f87f550 + b000c70 commit f763076
Show file tree
Hide file tree
Showing 110 changed files with 371 additions and 246 deletions.
28 changes: 14 additions & 14 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,41 +12,41 @@ on:

jobs:
build-and-test:
name: Test PY=${{ matrix.python-version }}, DEV=${{ matrix.developer }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
name: Test PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, CLN=${{ matrix.cln-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
python-version: [3.7, 3.8, 3.9]
developer: [0,1]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
bitcoind-version: ["26.0"]
cln-version: ["master", "v23.11"]
experimental: [1]
deprecated: [0]

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Checkout c-lightning@master
uses: actions/checkout@v3
- name: Checkout c-lightning@${{ matrix.cln-version }}
uses: actions/checkout@v4
with:
repository: 'ElementsProject/lightning'
path: 'lightning'
ref: 'master'
ref: ${{ matrix.cln-version }}
submodules: 'recursive'
fetch-depth: 0 # Required for pyln versions to be recognized

- name: Download runtime dependencies
run: |
export BITCOIND_VERSION="0.20.1"
wget https://storage.googleapis.com/c-lightning-tests/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.bz2
tar -xjf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.bz2
export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
tar -xzf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
rm -rf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz bitcoin-${BITCOIND_VERSION}
- name: Compile & install c-lightning@master
run: |
export EXPERIMENTAL_FEATURES=${{ matrix.experimental }}
export DEVELOPER=${{ matrix.developer }}
export COMPAT=${{ matrix.deprecated }}
export VALGRIND=0
sudo apt-get install -y \
Expand Down Expand Up @@ -82,19 +82,19 @@ jobs:
sudo make install
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Test with pytest
run: |
export EXPERIMENTAL_FEATURES=${{ matrix.experimental }}
export DEVELOPER=${{ matrix.developer }}
export COMPAT=${{ matrix.deprecated }}
export SLOW_MACHINE=1
export TEST_DEBUG=1
export TRAVIS=1
export CLN_PATH=${{ github.workspace }}/lightning
pip3 install --upgrade pip
pip3 install --user -U virtualenv pip > /dev/null
python3 .ci/test.py
Expand All @@ -103,7 +103,7 @@ jobs:
# signals successful completion. Used for the PR status to pass
# before merging.
name: CI completion
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
needs:
- build-and-test
steps:
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
ARG CLN_VERSION="0.11.1"
ARG CLN_VERSION="23.11.2"

FROM elementsproject/lightningd:v${CLN_VERSION}

Expand Down
145 changes: 80 additions & 65 deletions README.md

Large diffs are not rendered by default.

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
13 changes: 13 additions & 0 deletions archive/commando/README.md → archived/commando/README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,18 @@
# Commando plugin

Commando has been **included in Core Lightning as first class C plugin**.

It has been actively developed since and has more cool new features added
than listed below.

Checkout latest updates on commando at:
https://docs.corelightning.org/docs/commando &
https://docs.corelightning.org/reference/lightning-commando

------------------------------------------------------------------------------------------------------

# Archived Commando python plugin

This plugin allows other nodes to send your node commands, and allows you
to send them to other nodes. The nodes must be authorized, and must be
directly connected.
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
73 changes: 42 additions & 31 deletions prometheus/prometheus.py → archived/prometheus/prometheus.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@ def collect(self):
value=blockheight,
)

fees_msat = info["msatoshi_fees_collected"]
fees_msat = int(info.get(
"fees_collected_msat",
info.get("msatoshi_fees_collected", None)
))
yield GaugeMetricFamily(
'lightning_fees_collected_msat',
'How much have we been paid to route payments?',
Expand Down Expand Up @@ -84,10 +87,19 @@ def collect(self):
labels=['id'],
)

for p in peers:
labels = [p['id']]
count.add_metric(labels, len(p['channels']))
connected.add_metric(labels, int(p['connected']))
channels = self.rpc.listpeerchannels()['channels']
# Associate each channel with a peer
peers = {}
conn = {}
for c in channels:
peer_id = c['peer_id']
peers[peer_id] = peers.get(peer_id, 0) + 1
conn[peer_id] = conn.get(peer_id, 0) + c['peer_connected']

for p in peers.keys():
labels = [p]
count.add_metric(labels, peers[p])
connected.add_metric(labels, conn.get(p, 0))

return [count, connected]

Expand Down Expand Up @@ -159,32 +171,31 @@ def collect(self):
labels=['id', 'scid', 'alias'],
)

peers = self.rpc.listpeers()['peers']
for p in peers:
for c in p['channels']:
# append alias for human readable labels, if no label is found fill with shortid.
node = self.rpc.listnodes(p['id'])['nodes']
if len(node) != 0 and 'alias' in node[0]:
alias = node[0]['alias']
else:
alias = 'unknown'

labels = [p['id'], c.get('short_channel_id', c.get('channel_id')), alias]
balance_gauge.add_metric(labels, c['to_us_msat'].to_satoshi())
spendable_gauge.add_metric(labels,
c['spendable_msat'].to_satoshi())
total_gauge.add_metric(labels, c['total_msat'].to_satoshi())
htlc_gauge.add_metric(labels, len(c['htlcs']))

in_payments_offered_gauge.add_metric(labels, c['in_payments_offered'])
in_payments_fulfilled_gauge.add_metric(labels, c['in_payments_fulfilled'])
in_msatoshi_offered_gauge.add_metric(labels, c['in_msatoshi_offered'])
in_msatoshi_fulfilled_gauge.add_metric(labels, c['in_msatoshi_fulfilled'])

out_payments_offered_gauge.add_metric(labels, c['out_payments_offered'])
out_payments_fulfilled_gauge.add_metric(labels, c['out_payments_fulfilled'])
out_msatoshi_offered_gauge.add_metric(labels, c['out_msatoshi_offered'])
out_msatoshi_fulfilled_gauge.add_metric(labels, c['out_msatoshi_fulfilled'])
channels = self.rpc.listpeerchannels()['channels']
for c in channels:
# append alias for human readable labels, if no label is found fill with shortid.
node = self.rpc.listnodes(c['peer_id'])['nodes']
if len(node) != 0 and 'alias' in node[0]:
alias = node[0]['alias']
else:
alias = 'unknown'

labels = [c['peer_id'], c.get('short_channel_id', c.get('channel_id')), alias]
balance_gauge.add_metric(labels, c['to_us_msat'].to_satoshi())
spendable_gauge.add_metric(labels,
c['spendable_msat'].to_satoshi())
total_gauge.add_metric(labels, c['total_msat'].to_satoshi())
htlc_gauge.add_metric(labels, len(c['htlcs']))

in_payments_offered_gauge.add_metric(labels, c['in_payments_offered'])
in_payments_fulfilled_gauge.add_metric(labels, c['in_payments_fulfilled'])
in_msatoshi_offered_gauge.add_metric(labels, int(c['in_offered_msat']))
in_msatoshi_fulfilled_gauge.add_metric(labels, int(c['in_fulfilled_msat']))

out_payments_offered_gauge.add_metric(labels, c['out_payments_offered'])
out_payments_fulfilled_gauge.add_metric(labels, c['out_payments_fulfilled'])
out_msatoshi_offered_gauge.add_metric(labels, int(c['out_offered_msat']))
out_msatoshi_fulfilled_gauge.add_metric(labels, int(c['out_fulfilled_msat']))

return [
htlc_gauge,
Expand Down
File renamed without changes.
40 changes: 40 additions & 0 deletions archived/prometheus/test_prometheus.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import os
from pyln.testing.fixtures import * # noqa: F401,F403
import urllib
from ephemeral_port_reserve import reserve

plugin_path = os.path.join(os.path.dirname(__file__), "prometheus.py")


def test_prometheus_starts(node_factory):
l1 = node_factory.get_node()
# Test dynamically
l1.rpc.plugin_start(plugin_path)
l1.rpc.plugin_stop(plugin_path)
l1.rpc.plugin_start(plugin_path)
l1.stop()
# Then statically
l1.daemon.opts["plugin"] = plugin_path
l1.start()


def test_prometheus_scrape(node_factory):
"""Test that we can scrape correctly.
"""
l1 = node_factory.get_node(options={'plugin': plugin_path})
scrape = urllib.request.urlopen("http://localhost:9750")



def test_prometheus_channels(node_factory):
port = reserve()
l1, l2, l3 = node_factory.line_graph(
3,
opts=[
{},
{'plugin': plugin_path, 'prometheus-listen': f'127.0.0.1:{port}'},
{}
]
)
scrape = urllib.request.urlopen(f'http://localhost:{port}')
print(scrape)
File renamed without changes.
File renamed without changes.
11 changes: 6 additions & 5 deletions rebalance/rebalance.py → archived/rebalance/rebalance.py
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@ def get_avg_forward_fees(intervals):


@plugin.method("rebalancereport")
def rebalancereport(plugin: Plugin):
def rebalancereport(plugin: Plugin, include_avg_fees: bool = True):
"""Show information about rebalance
"""
res = {}
Expand Down Expand Up @@ -961,10 +961,11 @@ def rebalancereport(plugin: Plugin):
else:
res["average_rebalance_fee_ppm"] = 0

avg_forward_fees = get_avg_forward_fees([1, 7, 30])
res['average_forward_fee_ppm_1d'] = avg_forward_fees[0]
res['average_forward_fee_ppm_7d'] = avg_forward_fees[1]
res['average_forward_fee_ppm_30d'] = avg_forward_fees[2]
if include_avg_fees:
avg_forward_fees = get_avg_forward_fees([1, 7, 30])
res['average_forward_fee_ppm_1d'] = avg_forward_fees[0]
res['average_forward_fee_ppm_7d'] = avg_forward_fees[1]
res['average_forward_fee_ppm_30d'] = avg_forward_fees[2]

return res

Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
10 changes: 10 additions & 0 deletions clearnet/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# clearnet enforcer plugin
This plugin aims to prefer usage over clearnet connections.
It does so by disconnecing TOR connections when there are known and usable
clearnet addresses.

# Options

# Methods
## clearnet-enforce [peer_id]
Tries to enforce clearnet on all peer or on a given peer_id
114 changes: 114 additions & 0 deletions clearnet/clearnet.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
import socket
from contextlib import closing
from pyln.client import Plugin, RpcError

plugin = Plugin()


def get_address_type(addrstr: str):
""" I know this can be more sophisticated, but works """
if ".onion:" in addrstr:
return 'tor'
if addrstr[0].isdigit():
return 'ipv4'
if addrstr.startswith("["):
return 'ipv6'
return 'dns'


# taken from:
# https://stackoverflow.com/questions/19196105/how-to-check-if-a-network-port-is-open
def check_socket(host: str, port: int, timeout: float = None):
""" Checks if a socket can be opened to a host """
if host.count('.') == 3:
proto = socket.AF_INET
if host.count(':') > 1:
proto = socket.AF_INET6
with closing(socket.socket(proto, socket.SOCK_STREAM)) as sock:
if timeout is not None:
sock.settimeout(timeout) # seconds (float)
if sock.connect_ex((host, port)) == 0:
return True
else:
return False


def clearnet_pid(peer: dict, messages: list):
peer_id = peer['id']
if not peer['connected']:
messages += [f"Peer is not conencted: {peer_id}"]
return False
if get_address_type(peer['netaddr'][0]) != 'tor':
messages += [f"Already connected via clearnet: {peer_id}"]
return True

# lets check what gossip knows about this peer
nodes = plugin.rpc.listnodes(peer_id)['nodes']
if len(nodes) == 0:
messages += [f"Error: No gossip for: {peer_id}"]
return
addrs = [a for a in nodes[0]['addresses'] if not a['type'].startswith("tor")]
if len(addrs) == 0:
messages += [f"Error: No clearnet addresses known for: {peer_id}"]
return

# now check addrs for open ports
for addr in addrs:
if addr['type'] == 'dns':
messages += [f"TODO: DNS lookups for: {addr['address']}"]
continue
if check_socket(addr['address'], addr['port'], 2.0):
# disconnect
result = plugin.rpc.disconnect(peer_id, True)
if len(result) != 0:
messages += [f"Error: Can't disconnect: {peer_id} {result}"]
continue

# try clearnet connection
try:
result = plugin.rpc.connect(peer_id, addr['address'], addr['port'])
newtype = result['address']['type']
if not newtype.startswith('tor'):
messages += [f"Established clearnet connection for: {peer_id} with {newtype}"]
return True
except RpcError: # we got an connection error, try reconnect
messages += [f"Error: Connection failed for: {peer_id} with {addr['type']}"]
try:
result = plugin.rpc.connect(peer_id) # without address
newtype = result['address']['type']
if not newtype.startswith('tor'):
messages += [f"Established clearnet connection for: {peer_id} with {newtype}"]
return True
except RpcError: # we got a reconnection error
messages += [f"Error: Reconnection failed for: {peer_id}"]
continue
messages += [f"Reconnected: {peer_id} with {newtype}"]
continue
return False


@plugin.method("clearnet")
def clearnet(plugin: Plugin, peer_id: str = None):
""" Enforce a clearnet connection on all peers or a given `peer_id`."""
if peer_id is None:
peers = plugin.rpc.listpeers(peer_id)['peers']
else:
if not isinstance(peer_id, str) or len(peer_id) != 66:
return f"Error: Invalid peer_id: {peer_id}"
peers = plugin.rpc.listpeers(peer_id)['peers']
if len(peers) == 0:
return f"Error: peer not found: {peer_id}"

messages = []
for peer in peers:
clearnet_pid(peer, messages)
return messages


@plugin.init()
def init(options: dict, configuration: dict, plugin: Plugin, **kwargs):
plugin.log(f"clearnet enforcer plugin initialized")


plugin.run()
1 change: 1 addition & 0 deletions clearnet/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pyln-client>=0.12
Loading

0 comments on commit f763076

Please sign in to comment.