diff --git a/docs/logging_monitoring.md b/docs/logging_monitoring.md index f5bc705fa..6ed43b9ea 100644 --- a/docs/logging_monitoring.md +++ b/docs/logging_monitoring.md @@ -116,6 +116,16 @@ outbounds 0.0 mempool_size 0.0 ``` +### Defining lnd metrics to capture + +Lightning nodes can also be configured to export metrics to prometheus using `lnd-exporter`. +Example configuration is provided in `test/data/ln/`. Review `node-defauts.yaml` for a typical logging configuration. All default metrics reported to prometheus are prefixed with `lnd_` + +[lnd-exporter configuration reference](https://github.com/macgyver13/lnd-exporter/tree/main?tab=readme-ov-file#configuration) +lnd-exporter assumes same macaroon referenced in ln_framework (can be overridden by env variable) + +**Note: `test/data/ln` and `test/data/logging` take advantage of **extraContainers** configuration option to add containers to default `lnd/templates/pod`* + ### Grafana Data from Prometheus exporters is collected and fed into Grafana for a diff --git a/resources/charts/bitcoincore/charts/lnd/templates/configmap.yaml b/resources/charts/bitcoincore/charts/lnd/templates/configmap.yaml index 65cd54cd6..096c764ff 100644 --- a/resources/charts/bitcoincore/charts/lnd/templates/configmap.yaml +++ b/resources/charts/bitcoincore/charts/lnd/templates/configmap.yaml @@ -4,6 +4,7 @@ metadata: name: {{ include "lnd.fullname" . }} labels: {{- include "lnd.labels" . | nindent 4 }} + role: macaroon-ref data: lnd.conf: | {{- .Values.baseConfig | nindent 4 }} @@ -37,7 +38,7 @@ data: AwEHoUQDQgAEBVltIvaTlAQI/3FFatTqVflZuZdRJ0SmRMSJrFLPtp0fxE7hmteS t6gjQriy90fP8j9OJXBNAjt915kLY4zVvg== -----END EC PRIVATE KEY----- - + MACAROON_HEX: 0201036c6e6402f801030a1062beabbf2a614b112128afa0c0b4fdd61201301a160a0761646472657373120472656164120577726974651a130a04696e666f120472656164120577726974651a170a08696e766f69636573120472656164120577726974651a210a086d616361726f6f6e120867656e6572617465120472656164120577726974651a160a076d657373616765120472656164120577726974651a170a086f6666636861696e120472656164120577726974651a160a076f6e636861696e120472656164120577726974651a140a057065657273120472656164120577726974651a180a067369676e6572120867656e657261746512047265616400000620b17be53e367290871681055d0de15587f6d1cd47d1248fe2662ae27f62cfbdc6 --- apiVersion: v1 kind: ConfigMap diff --git a/resources/charts/bitcoincore/charts/lnd/templates/pod.yaml b/resources/charts/bitcoincore/charts/lnd/templates/pod.yaml index e3b9782d7..c5d66851a 100644 --- a/resources/charts/bitcoincore/charts/lnd/templates/pod.yaml +++ b/resources/charts/bitcoincore/charts/lnd/templates/pod.yaml @@ -12,6 +12,8 @@ metadata: collect_logs: "true" {{- end }} chain: {{ .Values.global.chain }} + annotations: + kubectl.kubernetes.io/default-container: "lnd" spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -56,11 +58,9 @@ spec: - mountPath: /root/.lnd/tls.cert name: config subPath: tls.cert - {{- if .Values.circuitBreaker }} - - name: circuitbreaker - image: pinheadmz/circuitbreaker:278737d - imagePullPolicy: IfNotPresent - {{- end}} + {{- with .Values.extraContainers }} + {{- toYaml . | nindent 4 }} + {{- end }} volumes: {{- with .Values.volumes }} {{- toYaml . | nindent 4 }} diff --git a/resources/charts/bitcoincore/charts/lnd/templates/service.yaml b/resources/charts/bitcoincore/charts/lnd/templates/service.yaml index 51826ee9b..aecf301fe 100644 --- a/resources/charts/bitcoincore/charts/lnd/templates/service.yaml +++ b/resources/charts/bitcoincore/charts/lnd/templates/service.yaml @@ -20,5 +20,11 @@ spec: targetPort: rest protocol: TCP name: rest +{{- if .Values.metricsExport }} + - port: {{ .Values.prometheusMetricsPort }} + targetPort: prom-metrics + protocol: TCP + name: prometheus-metrics +{{- end }} selector: {{- include "lnd.selectorLabels" . | nindent 4 }} diff --git a/resources/charts/bitcoincore/charts/lnd/templates/servicemonitor.yaml b/resources/charts/bitcoincore/charts/lnd/templates/servicemonitor.yaml new file mode 100644 index 000000000..d032d4e15 --- /dev/null +++ b/resources/charts/bitcoincore/charts/lnd/templates/servicemonitor.yaml @@ -0,0 +1,15 @@ +{{- if .Values.metricsExport }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "lnd.fullname" . }} + labels: + app.kubernetes.io/name: lnd-metrics + release: prometheus +spec: + endpoints: + - port: prometheus-metrics + selector: + matchLabels: + app: {{ include "lnd.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/resources/plugins/simln/charts/simln/templates/configmap.yaml b/resources/plugins/simln/charts/simln/templates/configmap.yaml index 9688722b6..9df3f0543 100644 --- a/resources/plugins/simln/charts/simln/templates/configmap.yaml +++ b/resources/plugins/simln/charts/simln/templates/configmap.yaml @@ -17,5 +17,9 @@ data: IQDPofN0fEl5gTwCYhk3nZbjMqJhZ8BsSJ6K8XRhxr7zbwIhAPsgQCFOqUWg632O NEO53OQ6CIqnpxSskjsFNH4ZBQOE -----END CERTIFICATE----- - admin.macaroon.hex: | - 0201036c6e6402f801030a1062beabbf2a614b112128afa0c0b4fdd61201301a160a0761646472657373120472656164120577726974651a130a04696e666f120472656164120577726974651a170a08696e766f69636573120472656164120577726974651a210a086d616361726f6f6e120867656e6572617465120472656164120577726974651a160a076d657373616765120472656164120577726974651a170a086f6666636861696e120472656164120577726974651a160a076f6e636861696e120472656164120577726974651a140a057065657273120472656164120577726974651a180a067369676e6572120867656e657261746512047265616400000620b17be53e367290871681055d0de15587f6d1cd47d1248fe2662ae27f62cfbdc6 +{{- $configMaps := lookup "v1" "ConfigMap" .Release.Namespace "" }} +{{- range $configMaps.items }} + {{- if and .metadata.labels (hasKey .metadata.labels "role") (eq (index .metadata.labels "role") "macaroon-ref") }} + admin.macaroon.hex: {{ index .data "MACAROON_HEX" | quote }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/test/data/ln/network.yaml b/test/data/ln/network.yaml index 792861da2..5d30686d4 100644 --- a/test/data/ln/network.yaml +++ b/test/data/ln/network.yaml @@ -2,26 +2,15 @@ nodes: - name: tank-0000 addnode: - tank-0001 - ln: - lnd: true - - name: tank-0001 addnode: - tank-0002 - ln: - lnd: true - - name: tank-0002 addnode: - tank-0000 - ln: - lnd: true - - name: tank-0003 addnode: - tank-0000 - ln: - lnd: true lnd: config: | bitcoin.timelockdelta=33 @@ -32,12 +21,9 @@ nodes: target: tank-0004-ln capacity: 100000 push_amt: 50000 - - name: tank-0004 addnode: - tank-0000 - ln: - lnd: true lnd: channels: - id: @@ -46,9 +32,6 @@ nodes: target: tank-0005-ln capacity: 50000 push_amt: 25000 - - name: tank-0005 addnode: - - tank-0000 - ln: - lnd: true + - tank-0000 \ No newline at end of file diff --git a/test/data/ln/node-defaults.yaml b/test/data/ln/node-defaults.yaml index 884ad1343..62e05199f 100644 --- a/test/data/ln/node-defaults.yaml +++ b/test/data/ln/node-defaults.yaml @@ -1,8 +1,40 @@ +# enable collectLogs and metricsExport to activate publish lnd-exporter metrics + +#Core configs image: repository: bitcoindevproject/bitcoin pullPolicy: IfNotPresent tag: "27.0" +collectLogs: false +metricsExport: false +#LN configs +ln: + lnd: true lnd: defaultConfig: | - color=#000000 \ No newline at end of file + color=#000000 + config: | + bitcoin.timelockdelta=33 + metricsExport: false + prometheusMetricsPort: 9332 + extraContainers: + - name: lnd-exporter + image: bitdonkey/lnd-exporter:0.1.3 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /macaroon.hex + subPath: MACAROON_HEX + env: + - name: METRICS + value: > + lnd_balance_channels=parse("/v1/balance/channels","balance") + lnd_local_balance_channels=parse("/v1/balance/channels","local_balance.sat") + lnd_remote_balance_channels=parse("/v1/balance/channels","remote_balance.sat") + lnd_block_height=parse("/v1/getinfo","block_height") + lnd_peers=parse("/v1/getinfo","num_peers") + ports: + - name: prom-metrics + containerPort: 9332 + protocol: TCP \ No newline at end of file diff --git a/test/data/logging/network.yaml b/test/data/logging/network.yaml index fb79c030e..067161cae 100644 --- a/test/data/logging/network.yaml +++ b/test/data/logging/network.yaml @@ -11,5 +11,27 @@ nodes: - name: tank-0002 addnode: - tank-0000 + ln: + lnd: true + lnd: + metricsExport: true + prometheusMetricsPort: 9332 + extraContainers: + - name: lnd-exporter + image: bitdonkey/lnd-exporter:0.1.3 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /macaroon.hex + subPath: MACAROON_HEX + env: + - name: METRICS + value: > + lnd_block_height=parse("/v1/getinfo","block_height") + lnd_peers=parse("/v1/getinfo","num_peers") + ports: + - name: prom-metrics + containerPort: 9332 + protocol: TCP caddy: enabled: true \ No newline at end of file diff --git a/test/logging_test.py b/test/logging_test.py index bfec6c25c..230a55d41 100755 --- a/test/logging_test.py +++ b/test/logging_test.py @@ -94,6 +94,7 @@ def get_five_values_for_metric(metric): self.wait_for_predicate(lambda: get_five_values_for_metric("blocks")) self.wait_for_predicate(lambda: get_five_values_for_metric("txrate")) + self.wait_for_predicate(lambda: get_five_values_for_metric("lnd_block_height")) # Verify default dashboard exists dbs = requests.get(f"{self.grafana_url}/api/search").json()