Merge pull request #36 from NethServer/report

build and action: add timescaledb

NethServer/nethsecurity#754

README.md

@@ -14,6 +14,7 @@ The module is composed by the following containers:
 - [loki](#loki): log storage, it stores logs from promtail
 - [grafana](#grafana): metrics visualization, it visualizes metrics from prometheus and logs from loki
 - [webssh](#webssh): web-based ssh client
+- [timescale](#timescale): time-series database for storing metrics
 
 
 ## Install
@@ -42,11 +43,12 @@ Launch `configure-module`, by setting the following parameters:
 - `api_user`: controller admin user
 - `api_password`: controller admin password, change it after first login
 - `loki_retention`: Loki retention period in days (default: ``180`` days)
-- `promtail_retention`: Promtail retention period in days (default: ``15`` days)
+- `prometheus_retention`: Promtail and Timescale retention period in days (default: 15 days)
+- `maxmind_license`: [MaxMind](https://www.maxmind.com/) license key to download the GEO IP database, the database is loaded every time the API server is started
 
 Example:
 
-    api-cli run  module/nethsecurity-controller1/configure-module --data '{"host": "mycontroller.nethsecurity.org", "lets_encrypt": false, "ovpn_network": "172.19.64.0", "ovpn_netmask": "255.255.255.0", "ovpn_cn": "nethsec", "api_user": "admin", "api_password": "password", "loki_retention": 180, "prometheus_retention": 15}'
+    api-cli run  module/nethsecurity-controller1/configure-module --data '{"host": "mycontroller.nethsecurity.org", "lets_encrypt": false, "ovpn_network": "172.19.64.0", "ovpn_netmask": "255.255.255.0", "ovpn_cn": "nethsec", "api_user": "admin", "api_password": "password", "loki_retention": 180, "prometheus_retention": 15, ""maxmind_license": "xxx"}'
 
 The above command will:
 - start and configure the nethsecurity-controller instance
@@ -160,6 +162,11 @@ It has also some pre-configured dashboards:
 - nethsecurity.json: a dashboard with the most important metrics from the connected machines, like CPU, memory, disk, network, and system load
 - logs.json: a dashboard where you can visualize the logs from all the connected machines and filter them by hostname, application, and priority
 - loki.json: a dashboard with the most important metrics from Loki, like the number of logs ingested, the number of logs dropped, and the status of queriers
+- network_traffic.json: this dashboard uses data from Timescale database and shows the global network traffic by unit
+- network_traffic_by_client.json: this dashboard uses data from Timescale database and shows the network traffic by unit and client (a client is a machine connected to the unit local network)
+- network_traffic_by_host.json: this dashboard uses data from Timescale database and shows the network traffic by unit and host (a host is a machien on the internet)
+- malware.json: this dashboard uses data from Timescale database and shows the malware blocked by the unit
+- vpn.json: this dashboard uses data from Timescale database and shows the VPN connections
 
 Grafana is accessible at `https://<controller-host>/grafana/`, default credentials are the same set for the controller. You should change them on the first login.
 
@@ -169,6 +176,16 @@ Grafana is accessible at `https://<controller-host>/grafana/`, default credentia
 
 Access to WebSSH is protected using a random generated URL, you can find it inside the module configuration file at `/home/nethsecurity-controller1/.config/state/config.json`.
 
+### Timescale
+
+[Timescale](https://docs.timescale.com/latest/main) is a time-series database for storing metrics. It's configured via environment variables and the configuration is available at `/home/nethsecurity-controller1/.config/state/db.env`.
+
+You can connect to the database with the following command:
+```
+runagent -m nethsecurity-controller1
+source db.env; podman exec -it timescale psql -U "${POSTGRES_USER}" -p "${POSTGRES_PORT}"
+```
+
 ## Uninstall
 
 To uninstall the instance:

build-images.sh

@@ -9,12 +9,13 @@ images=()
 repobase="${REPOBASE:-ghcr.io/nethserver}"
 # Configure the image name
 reponame="nethsecurity-controller"
-controller_version="1.0.1"
+controller_version="1.1.0"
 promtail_version=2.7.1
 loki_version=2.9.4
 prometheus_version=2.50.1
-grafana_version=10.3.3
+grafana_version=11.2.0
 webssh_version=1.6.2
+timescale_version="2.16.1-pg16"
 
 # Create a new empty container for webssh
 echo "Build webssh container" # from https://github.com/huashengdun/webssh
@@ -74,8 +75,8 @@ buildah add "${container}" ui/dist /ui
 # Setup the entrypoint, ask to reserve one TCP port with the label and set a rootless container
 buildah config --entrypoint=/ \
     --label="org.nethserver.authorizations=traefik@any:routeadm node:tunadm" \
-    --label="org.nethserver.tcp-ports-demand=10" \
-    --label="org.nethserver.images=ghcr.io/nethserver/nethsecurity-vpn:$controller_version ghcr.io/nethserver/nethsecurity-api:$controller_version ghcr.io/nethserver/nethsecurity-ui:$controller_version ghcr.io/nethserver/nethsecurity-proxy:$controller_version docker.io/grafana/promtail:$promtail_version docker.io/grafana/loki:$loki_version docker.io/prom/prometheus:v$prometheus_version docker.io/grafana/grafana:$grafana_version ghcr.io/nethserver/webssh:${IMAGETAG:-latest}" \
+    --label="org.nethserver.tcp-ports-demand=11" \
+    --label="org.nethserver.images=ghcr.io/nethserver/nethsecurity-vpn:$controller_version ghcr.io/nethserver/nethsecurity-api:$controller_version ghcr.io/nethserver/nethsecurity-ui:$controller_version ghcr.io/nethserver/nethsecurity-proxy:$controller_version docker.io/grafana/promtail:$promtail_version docker.io/grafana/loki:$loki_version docker.io/prom/prometheus:v$prometheus_version docker.io/grafana/grafana:$grafana_version ghcr.io/nethserver/webssh:${IMAGETAG:-latest} docker.io/timescale/timescaledb:$timescale_version" \
     "${container}"
 # Commit the image
 buildah commit "${container}" "${repobase}/${reponame}"

imageroot/actions/configure-module/20configure

@@ -103,6 +103,7 @@ with open('config.json', 'w') as cfp:
 # Load subscription info
 rdb = agent.redis_connect(privileged=False)
 subscription = rdb.hgetall('cluster/subscription')
+metrics_retention_days = request.get('prometheus_retention', '15')
 
 with open('config.env', 'w') as env:
     env.write(f'ADMIN_USER={request["api_user"]}\n')
@@ -119,6 +120,9 @@ with open('config.env', 'w') as env:
         env.write(f'VALID_SUBSCRIPTION=true\n')
     else:
         env.write(f'VALID_SUBSCRIPTION=false\n')
+    if 'maxmind_license' in request:
+        env.write(f'MAXMIND_LICENSE={request["maxmind_license"]}\n')
+    env.write(f'RETENTION_DAYS={metrics_retention_days}\n')
 
 server_address = request["ovpn_network"].removesuffix('.0') + '.1'
 with open('promtail.env', 'w') as promtail:
@@ -140,11 +144,12 @@ with open('grafana.env', 'w') as gfp:
     gfp.write("GF_SERVER_HTTP_ADDR=127.0.0.1\n")
     gfp.write(f'GF_SECURITY_ADMIN_USER={request["api_user"]}\n')
     gfp.write(f'GF_SECURITY_ADMIN_PASSWORD={request.get("api_password", config["api_password"])}\n')
+    gfp.write('GF_DATE_FORMATS_USE_BROWSER_LOCALE=true\n')
 
 with open('prometheus.env', 'w') as pfp:
     pfp.write(f"PROMETHEUS_PORT={ports[7]}\n")
     pfp.write(f"PROMETHEUS_PATH={config['prometheus_path']}\n")
-    pfp.write(f"PROMETHEUS_RETENTION={request.get('prometheus_retention', '15')}d\n")
+    pfp.write(f"PROMETHEUS_RETENTION={metrics_retention_days}d\n")
 
 with open('prometheus.yml', 'w', encoding='utf-8') as fp:
     fp.write("global:\n")
@@ -159,6 +164,7 @@ with open('prometheus.yml', 'w', encoding='utf-8') as fp:
     fp.write(f'        - 127.0.0.1:{ports[5]}\n')
 
 # Grafana configuration
+db = agent.read_envfile('db.env')
 with open('grafana.yml', 'w') as fp:
     fp.write("apiVersion: 1\n")
     fp.write("datasources:\n")
@@ -174,6 +180,23 @@ with open('grafana.yml', 'w') as fp:
     fp.write('    access: proxy\n')
     fp.write(f'    url: http://127.0.0.1:{ports[5]}\n')
 
+    fp.write('  - name: Local Timescale\n')
+    fp.write('    type: postgres\n')
+    fp.write('    uid: timescale\n')
+    fp.write(f'    url: 127.0.0.1:{db.get("POSTGRES_PORT")}\n')
+    fp.write(f'    user: grafana\n')
+    fp.write('    secureJsonData:\n')
+    fp.write(f'      password: {db.get("GRAFANA_POSTGRES_PASSWORD")}\n')
+    fp.write('    jsonData:\n')
+    fp.write('      database: report\n')
+    fp.write('      sslmode: disable\n')
+    fp.write('      maxOpenConns: 100\n')
+    fp.write('      maxIdleConns: 100\n')
+    fp.write('      maxIdleConnsAuto: true\n')
+    fp.write('      connMaxLifetime: 14400\n')
+    fp.write('      postgresVersion: 1500\n')
+    fp.write('      timescaledb: true\n')
+
 network = agent.read_envfile('network.env')
 tun = network.get('OVPN_TUN')
 bits = sum(bin(int(x)).count('1') for x in request["ovpn_netmask"].split('.'))

imageroot/actions/configure-module/validate-input.json

@@ -13,7 +13,8 @@
             "ovpn_netmask": "255.255.0.0",
             "ovpn_cn": "nethsec",
             "loki_retention": 180,
-            "prometheus_retention": 15
+            "prometheus_retention": 15,
+            "maxmind_license": "1234567890"
         }
     ],
     "type": "object",
@@ -63,6 +64,11 @@
             "type": "integer",
             "description": "Retention policy for Prometehus metrics, default is 15 days",
             "minimum": 1
+        },
+        "maxmind_license": {
+            "type": "string",
+            "description": "MaxMind API key, required for GeoIP database updates",
+            "minLength": 1
         }
     }
 }

imageroot/actions/create-module/20initialize

@@ -19,11 +19,14 @@ promtail_port=$(($start+4))
 # port 8 is reserved for prometheus
 # port 9 is reserved for grafana
 webssh_port=$(($start+9))
+db_port=$(($start+10))
 
 num=$(echo $MODULE_ID | sed 's/nethsecurity\-controller//')
 
 jwt_secret=$(uuidgen | sha256sum | awk '{print $1}')
 reg_secret=$(uuidgen | sha256sum | awk '{print $1}')
+db_secret=$(uuidgen | sha256sum | awk '{print $1}')
+grafana_postgres_password=$(uuidgen | sha256sum | awk '{print $1}')
 
 cat << EOF > network.env
 OVPN_UDP_PORT=$ovpn_udp_port
@@ -43,4 +46,13 @@ SECRET_JWT=$jwt_secret
 REGISTRATION_TOKEN=$reg_secret
 EOF
 
+cat << EOF > db.env
+POSTGRES_USER=report
+POSTGRES_PORT=$db_port
+POSTGRES_PASSWORD=$db_secret
+GRAFANA_POSTGRES_PASSWORD=$grafana_postgres_password
+REPORT_DB_URI=postgres://report:$db_secret@127.0.0.1:$db_port/report
+TS_TUNE_MAX_BG_WORKERS=100
+EOF
+
 mkdir -p clients

imageroot/actions/restore-module/30restore_database

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Load database credentials
+. ./db.env
+
+# Wait for database to be ready pooling every 5 seconds
+until podman exec timescale pg_isready -U "${POSTGRES_USER}" -p "${POSTGRES_PORT}" -d "${POSTGRES_DB}"; do
+  sleep 5
+done
+
+# Dump the DB from timescale
+podman exec -i timescale psql -U "${POSTGRES_USER}" -p "${POSTGRES_PORT}" < backup.sql

imageroot/bin/module-cleanup-state

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+set -e
+
+rm -f backup.sql

imageroot/bin/module-dump-state

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+set -e
+
+# Load database credentials
+. ./db.env
+
+# Dump the DB from timescale
+podman exec -i timescale pg_dump -U "${POSTGRES_USER}" -p "${POSTGRES_PORT}" > timescale.sql