From 48b097b03804a6a7be95d1d59e0b03dc88f139fc Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 28 Feb 2024 11:29:05 +0100 Subject: [PATCH] fixes --- .../actions/configure-module/20configure | 75 ++++++++++++++----- .../configure-module/validate-input.json | 8 +- imageroot/actions/get-configuration/20read | 2 +- .../get-configuration/validate-output.json | 3 +- imageroot/etc/loki.yaml | 5 +- imageroot/systemd/user/prometheus.service | 2 +- 6 files changed, 72 insertions(+), 23 deletions(-) diff --git a/imageroot/actions/configure-module/20configure b/imageroot/actions/configure-module/20configure index 0ab84ea..9c423dd 100755 --- a/imageroot/actions/configure-module/20configure +++ b/imageroot/actions/configure-module/20configure @@ -15,7 +15,10 @@ import hashlib request = json.load(sys.stdin) (start,end) = os.environ["TCP_PORTS_RANGE"].split('-') -ports = [*range(int(start), int(end))] +ports = [*range(int(start), int(end)+1)] + +loki_path = '/loki' +prometheus_path = '/prometheus' # Configure Traefik to route requests to the nethsec-controller service response = agent.tasks.run( @@ -23,12 +26,53 @@ response = agent.tasks.run( action='set-route', data={ 'instance': os.environ['MODULE_ID'], - 'url': 'http://127.0.0.1:' + ports[3], + 'url': f'http://127.0.0.1:{ports[3]}', 'http2https': True, 'lets_encrypt': request["lets_encrypt"], 'host': request["host"], }, ) +agent.assert_exp(response['exit_code'] == 0) +response = agent.tasks.run( + agent_id=agent.resolve_agent_id('traefik@node'), + action='set-route', + data={ + 'instance': os.environ['MODULE_ID'], + 'url': f'http://127.0.0.1:{ports[8]}', + 'http2https': True, + 'lets_encrypt': request["lets_encrypt"], + 'host': request["host"], + 'path': '/grafana' + }, +) +agent.assert_exp(response['exit_code'] == 0) +response = agent.tasks.run( + agent_id=agent.resolve_agent_id('traefik@node'), + action='set-route', + data={ + 'instance': os.environ['MODULE_ID'], + 'url': f'http://127.0.0.1:{ports[5]}', + 'http2https': True, + 'lets_encrypt': request["lets_encrypt"], + 'host': request["host"], + 'path': loki_path + }, +) +agent.assert_exp(response['exit_code'] == 0) +response = agent.tasks.run( + agent_id=agent.resolve_agent_id('traefik@node'), + action='set-route', + data={ + 'instance': os.environ['MODULE_ID'], + 'url': f'http://127.0.0.1:{ports[7]}', + 'http2https': True, + 'lets_encrypt': request["lets_encrypt"], + 'host': request["host"], + 'path': prometheus_path + }, +) +agent.assert_exp(response['exit_code'] == 0) + # Replace password if passed as parameter, otherwise read the old one if 'api_password' in request and request['api_password'] != '': @@ -38,8 +82,6 @@ else: tmp = json.load(tmp) request['api_password'] = tmp['api_password'] -# Check if traefik configuration has been successfull -agent.assert_exp(response['exit_code'] == 0) # Save configuration to JSON for later user and backup with open('config.json', 'w') as config: @@ -55,32 +97,29 @@ with open('config.env', 'w') as env: server_address = request["ovpn_network"].removesuffix('.0') + '.1' with open('promtail.env', 'w') as promtail: - redis_client = agent.redis_connect() - loki_instance = redis_client.get('cluster/default_instance/loki') - loki_addr = redis_client.hget(f'module/{loki_instance}/environment', 'LOKI_ADDR') - loki_http_port = redis_client.hget(f'module/{loki_instance}/environment', 'LOKI_HTTP_PORT') - loki_logs_ingress_token = redis_client.hget(f'module/{loki_instance}/environment', 'LOKI_LOGS_INGRESS_TOKEN') - promtail_address = server_address - promtail_port = ports[4] - - promtail.write(f'LOKI_ADDR={loki_addr}\n') - promtail.write(f'LOKI_HTTP_PORT={loki_http_port}\n') - promtail.write(f'LOKI_LOGS_INGRESS_TOKEN={loki_logs_ingress_token}\n') - promtail.write(f'PROMTAIL_ADDRESS={promtail_address}\n') - promtail.write(f'PROMTAIL_PORT={promtail_port}\n') + promtail.write('LOKI_ADDR=127.0.0.1\n') + promtail.write(f'LOKI_HTTP_PORT={ports[5]}\n') + #promtail.write(f'LOKI_LOGS_INGRESS_TOKEN={loki_logs_ingress_token}\n') + promtail.write(f'PROMTAIL_ADDRESS={server_address}\n') + promtail.write(f'PROMTAIL_PORT={ports[4]}\n') with open('loki.env', 'w') as lfp: lfp.write(f"LOKI_HTTP_PORT={ports[5]}\n") lfp.write(f"LOKI_GRPC_PORT={ports[6]}\n") + lfp.write(f"LOKI_PATH={loki_path}\n") lfp.write(f"LOKI_RETENTION={request.get('loki_rentention', '180')}d\n") # retention in days with open('grafana.env', 'w') as gfp: gfp.write(f"GF_DEFAULT_INSTANCE_NAME={request['host']}\n") - gfp.write(f"GF_SERVER_HTTP_PORT={ports['8']}\n") + gfp.write(f"GF_SERVER_ROOT_URL=https://{request['host']}/grafana\n") + gfp.write("GF_SERVER_SERVE_FROM_SUB_PATH=true\n") + gfp.write(f"GF_SERVER_HTTP_PORT={ports[8]}\n") gfp.write("GF_SERVER_HTTP_ADDR=127.0.0.1\n") with open('prometheus.env', 'w') as pfp: pfp.write(f"PROMETHEUS_PORT={ports[7]}\n") + pfp.write(f"PROMETHEUS_PATH={prometheus_path}\n") + pfp.write(f"PROMETHEUS_RETENTION={request.get('prometheus_retention', '15')}d\n") with open('prometheus.yml', 'w', encoding='utf-8') as fp: fp.write("global:\n") diff --git a/imageroot/actions/configure-module/validate-input.json b/imageroot/actions/configure-module/validate-input.json index 96f2d34..63b16b6 100644 --- a/imageroot/actions/configure-module/validate-input.json +++ b/imageroot/actions/configure-module/validate-input.json @@ -12,7 +12,8 @@ "ovpn_network": "127.2.10.0", "ovpn_netmask": "255.255.0.0", "ovpn_cn": "nethsec", - "loki_retention": 180 + "loki_retention": 180, + "prometheus_retention": 15, } ], "type": "object", @@ -57,6 +58,11 @@ "type": "integer", "description": "Retention policy for Loki logs, default is 180 days", "minimum": 1 + }, + "prometheus_retention": { + "type": "integer", + "description": "Retention policy for Prometehus metrics, default is 15 days", + "minimum": 1 } } } diff --git a/imageroot/actions/get-configuration/20read b/imageroot/actions/get-configuration/20read index c311e34..0135d24 100755 --- a/imageroot/actions/get-configuration/20read +++ b/imageroot/actions/get-configuration/20read @@ -26,7 +26,7 @@ else: foctet=random.randrange(16,31) soctet=random.randrange(1,254) - config = {'host': '', 'lets_encrypt': False, 'ovpn_network': f'172.{foctet}.{soctet}.0', 'ovpn_netmask': '255.255.255.0', 'ovpn_cn': 'nethsec', 'api_user': 'admin', 'api_password': ''} + config = {'host': '', 'lets_encrypt': False, 'ovpn_network': f'172.{foctet}.{soctet}.0', 'ovpn_netmask': '255.255.255.0', 'ovpn_cn': 'nethsec', 'api_user': 'admin', 'api_password': '', 'loki_retention': 180} characters = list(string.ascii_letters + string.digits + "!@#$%^&*()") random.shuffle(characters) diff --git a/imageroot/actions/get-configuration/validate-output.json b/imageroot/actions/get-configuration/validate-output.json index e68c506..cd50662 100644 --- a/imageroot/actions/get-configuration/validate-output.json +++ b/imageroot/actions/get-configuration/validate-output.json @@ -11,7 +11,8 @@ "lets_encrypt": true, "ovpn_network": "127.2.10.0", "ovpn_netmask": "255.255.0.0", - "ovpn_cn": "nethsec" + "ovpn_cn": "nethsec", + "loki_retention": 180 } ], "type": "object" diff --git a/imageroot/etc/loki.yaml b/imageroot/etc/loki.yaml index c3617e9..f371309 100644 --- a/imageroot/etc/loki.yaml +++ b/imageroot/etc/loki.yaml @@ -3,6 +3,9 @@ auth_enabled: false server: http_listen_port: ${LOKI_HTTP_PORT} grpc_listen_port: ${LOKI_GRPC_PORT} + http_listen_address: 127.0.0.1 + grpc_listen_address: 127.0.0.1 + http_path_prefix: ${LOKI_PATH} common: instance_addr: 127.0.0.1 @@ -27,7 +30,7 @@ limits_config: retention_period: ${LOKI_RETENTION} compactor: - working_directory: /data/retention + working_directory: /loki/retention compaction_interval: 10m retention_enabled: true retention_delete_delay: 5m diff --git a/imageroot/systemd/user/prometheus.service b/imageroot/systemd/user/prometheus.service index 7d25b6a..6a5cfe4 100644 --- a/imageroot/systemd/user/prometheus.service +++ b/imageroot/systemd/user/prometheus.service @@ -22,7 +22,7 @@ ExecStart=/usr/bin/podman run \ --volume=./prometheus.d/:/prometheus/prometheus.d/:z \ --volume=prometheus-data:/prometheus:z \ --network=host \ - ${PROMETHEUS_IMAGE} --web.listen-address=127.0.0.1:${PROMETHEUS_PORT} + ${PROMETHEUS_IMAGE} --web.listen-address=127.0.0.1:${PROMETHEUS_PORT} --web.external-url=${PROMETHEUS_PATH} --storage.tsdb.retention.time=${PROMETHEUS_RETENTION} ExecStop=/usr/bin/podman stop --ignore --cidfile %t/prometheus.ctr-id -t 10 ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/prometheus.ctr-id PIDFile=%t/prometheus.pid