From f58e454a436fa334dcce2b1875e1728ad57e1901 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 21 Aug 2024 15:13:15 +0200 Subject: [PATCH 01/42] build and config: add timescaledb Timescale database is used to store report data --- build-images.sh | 5 ++-- imageroot/actions/create-module/20initialize | 9 ++++++ imageroot/systemd/user/api.service | 1 + imageroot/systemd/user/controller.service | 4 +-- imageroot/systemd/user/timescale.service | 31 ++++++++++++++++++++ 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 imageroot/systemd/user/timescale.service diff --git a/build-images.sh b/build-images.sh index 48507e8..3564976 100755 --- a/build-images.sh +++ b/build-images.sh @@ -15,6 +15,7 @@ loki_version=2.9.4 prometheus_version=2.50.1 grafana_version=10.3.3 webssh_version=1.6.2 +timescale_version="latest-pg16" # Create a new empty container for webssh echo "Build webssh container" # from https://github.com/huashengdun/webssh @@ -74,8 +75,8 @@ buildah add "${container}" ui/dist /ui # Setup the entrypoint, ask to reserve one TCP port with the label and set a rootless container buildah config --entrypoint=/ \ --label="org.nethserver.authorizations=traefik@any:routeadm node:tunadm" \ - --label="org.nethserver.tcp-ports-demand=10" \ - --label="org.nethserver.images=ghcr.io/nethserver/nethsecurity-vpn:$controller_version ghcr.io/nethserver/nethsecurity-api:$controller_version ghcr.io/nethserver/nethsecurity-ui:$controller_version ghcr.io/nethserver/nethsecurity-proxy:$controller_version docker.io/grafana/promtail:$promtail_version docker.io/grafana/loki:$loki_version docker.io/prom/prometheus:v$prometheus_version docker.io/grafana/grafana:$grafana_version ghcr.io/nethserver/webssh:${IMAGETAG:-latest}" \ + --label="org.nethserver.tcp-ports-demand=11" \ + --label="org.nethserver.images=ghcr.io/nethserver/nethsecurity-vpn:$controller_version ghcr.io/nethserver/nethsecurity-api:$controller_version ghcr.io/nethserver/nethsecurity-ui:$controller_version ghcr.io/nethserver/nethsecurity-proxy:$controller_version docker.io/grafana/promtail:$promtail_version docker.io/grafana/loki:$loki_version docker.io/prom/prometheus:v$prometheus_version docker.io/grafana/grafana:$grafana_version ghcr.io/nethserver/webssh:${IMAGETAG:-latest} docker.io/timescale/timescaledb:$timescale_version" \ "${container}" # Commit the image buildah commit "${container}" "${repobase}/${reponame}" diff --git a/imageroot/actions/create-module/20initialize b/imageroot/actions/create-module/20initialize index 6e40e1c..b34d325 100755 --- a/imageroot/actions/create-module/20initialize +++ b/imageroot/actions/create-module/20initialize @@ -19,11 +19,13 @@ promtail_port=$(($start+4)) # port 8 is reserved for prometheus # port 9 is reserved for grafana webssh_port=$(($start+9)) +db_port=$(($start+10)) num=$(echo $MODULE_ID | sed 's/nethsecurity\-controller//') jwt_secret=$(uuidgen | sha256sum | awk '{print $1}') reg_secret=$(uuidgen | sha256sum | awk '{print $1}') +db_secret=$(uuidgen | sha256sum | awk '{print $1}') cat << EOF > network.env OVPN_UDP_PORT=$ovpn_udp_port @@ -43,4 +45,11 @@ SECRET_JWT=$jwt_secret REGISTRATION_TOKEN=$reg_secret EOF +cat << EOF > db.env +POSTGRES_USER=report +POSTGRES_PORT=$db_port +POSTGRES_PASSWORD=$db_secret +REPORT_DB_URI=postgres://report:$db_secret@127.0.0.1:$db_port/report +EOF + mkdir -p clients diff --git a/imageroot/systemd/user/api.service b/imageroot/systemd/user/api.service index bb50911..ac0b20d 100644 --- a/imageroot/systemd/user/api.service +++ b/imageroot/systemd/user/api.service @@ -25,6 +25,7 @@ ExecStart=/usr/bin/podman run \ --env-file=%S/state/config.env \ --env-file=%S/state/promtail.env \ --env-file=%S/state/subscription.env \ + --env-file=%S/state/db.env \ ${NETHSECURITY_API_IMAGE} ExecStop=/usr/bin/podman stop --ignore --cidfile %t/api.ctr-id -t 10 ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/api.ctr-id diff --git a/imageroot/systemd/user/controller.service b/imageroot/systemd/user/controller.service index 8839724..7c1a94c 100644 --- a/imageroot/systemd/user/controller.service +++ b/imageroot/systemd/user/controller.service @@ -1,7 +1,7 @@ [Unit] Description=Podman controller.service -Requires=vpn.service api.service ui.service proxy.service promtail.service metrics-exporter.path loki.service prometheus.service grafana.service webssh.service -Before=vpn.service api.service ui.service proxy.service promtail.service metrics-exporter.path loki.service prometheus.service grafana.service webssh.service +Requires=vpn.service api.service ui.service proxy.service promtail.service metrics-exporter.path loki.service prometheus.service grafana.service webssh.service timescale.service +Before=vpn.service api.service ui.service proxy.service promtail.service metrics-exporter.path loki.service prometheus.service grafana.service webssh.service timescale.service ConditionPathExists=%S/state/environment ConditionPathExists=%S/state/network.env diff --git a/imageroot/systemd/user/timescale.service b/imageroot/systemd/user/timescale.service new file mode 100644 index 0000000..5e6157e --- /dev/null +++ b/imageroot/systemd/user/timescale.service @@ -0,0 +1,31 @@ +[Unit] +Description=Podman timescale.service +BindsTo=controller.service +Before=api.service + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +EnvironmentFile=%S/state/environment +EnvironmentFile=%S/state/secret.env +EnvironmentFile=%S/state/db.env +WorkingDirectory=%S/state +Restart=always +TimeoutStopSec=70 +ExecStartPre=/bin/rm -f %t/timescale.pid %t/timescale.ctr-id +ExecStart=/usr/bin/podman run \ + --conmon-pidfile %t/timescale.pid \ + --cidfile %t/timescale.ctr-id \ + --cgroups=no-conmon \ + --pod-id-file %t/controller.pod-id \ + --replace -d --name timescale \ + --volume=timescale-data:/var/lib/postgresql/data:z \ + --env-file=%S/state/db.env \ + --network=host \ + ${TIMESCALEDB_IMAGE} -p ${POSTGRES_PORT} -h 127.0.0.1 +ExecStop=/usr/bin/podman stop --ignore --cidfile %t/timescale.ctr-id -t 10 +ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/timescale.ctr-id +PIDFile=%t/timescale.pid +Type=forking + +[Install] +WantedBy=default.target From c56eb8c6f5a8747c8322f3736f683e601c091aa2 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 21 Aug 2024 16:31:27 +0200 Subject: [PATCH 02/42] systemd: fix webssh dependency --- imageroot/systemd/user/webssh.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imageroot/systemd/user/webssh.service b/imageroot/systemd/user/webssh.service index 733216f..bdb0c8f 100644 --- a/imageroot/systemd/user/webssh.service +++ b/imageroot/systemd/user/webssh.service @@ -2,7 +2,7 @@ [Unit] Description=Podman webssh.service BindsTo=controller.service -After=vpm.service +After=vpn.service [Service] Environment=PODMAN_SYSTEMD_UNIT=%n From 9ccd05a5491de724e82fc0280f9d57bfbdcb349d Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 4 Sep 2024 15:28:45 +0200 Subject: [PATCH 03/42] grafana: add new monitoring dashboards --- imageroot/etc/dashboards/malware.json | 1326 +++++++++++++++++ imageroot/etc/dashboards/network_traffic.json | 899 +++++++++++ .../dashboards/network_traffic_by_client.json | 736 +++++++++ .../dashboards/network_traffic_by_host.json | 543 +++++++ 4 files changed, 3504 insertions(+) create mode 100644 imageroot/etc/dashboards/malware.json create mode 100644 imageroot/etc/dashboards/network_traffic.json create mode 100644 imageroot/etc/dashboards/network_traffic_by_client.json create mode 100644 imageroot/etc/dashboards/network_traffic_by_host.json diff --git a/imageroot/etc/dashboards/malware.json b/imageroot/etc/dashboards/malware.json new file mode 100644 index 0000000..f611dee --- /dev/null +++ b/imageroot/etc/dashboards/malware.json @@ -0,0 +1,1326 @@ +{ + "__inputs": [ + { + "name": "DS_LOCAL_TIMESCALE", + "label": "Local Timescale", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "panel", + "id": "geomap", + "name": "Geomap", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND unit_id = $unitid ", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked threats", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 5, + "y": 0 + }, + "id": 6, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT src, count(*) AS count FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by src\nORDER by count DESC\nLIMIT 10", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Top 10 blocked threats by IPs", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^sum$/", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT chain, SUM(count) FROM ca_ts_malware_hourly_chain\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by chain", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked threats by chain", + "type": "piechart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^sum$/", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT category, SUM(count) FROM ca_ts_malware_hourly_category\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by category", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked threats by category", + "type": "piechart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 10, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": true, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "field": "count", + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 5, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "symbolAlign": { + "horizontal": "center", + "vertical": "center" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "location": { + "lookup": "lookup", + "mode": "auto" + }, + "name": "Layer 1", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "fit", + "lat": 0, + "lon": 0, + "zoom": 15 + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) as count, country AS lookup FROM ca_ts_malware_hourly_country\nWHERE $__timeFilter(bucket) AND unit_id = $unitid \nGROUP BY country", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked threats", + "type": "geomap" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 14, + "x": 10, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, count(*) FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by bucket", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked threats by time", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 21 + }, + "id": 7, + "options": { + "barRadius": 0, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT country, count FROM ca_ts_malware_hourly_country\nWHERE $__timeFilter(bucket) AND unit_id = $unitid \nGROUP BY country, count\nORDER BY count DESC\nLIMIT 20", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Top 20 blocked threats by countries", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 0, + "y": 29 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND unit_id = $unitid ", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked attacks", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 5, + "y": 29 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, count(*) FROM ca_ts_attacks_hourly\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by bucket", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked attacks by hour", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 10, + "x": 0, + "y": 37 + }, + "id": 11, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": true, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "field": "count", + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 5, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "symbolAlign": { + "horizontal": "center", + "vertical": "center" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "location": { + "lookup": "lookup", + "mode": "auto" + }, + "name": "Layer 1", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "fit", + "lat": 0, + "lon": 0, + "zoom": 15 + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) as count, country AS lookup FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND unit_id = $unitid \nGROUP BY country", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked attacks", + "type": "geomap" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 10, + "y": 37 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ip, count(*) AS count FROM ts_attacks \nWHERE $__timeFilter(time) AND $unitid = $unitid\nGROUP by ip\nORDER by count DESC\nLIMIT 10", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Top 10 blocked attacks by IPs", + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Unit", + "multi": false, + "name": "unitid", + "options": [], + "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "DS_TIMESCALE", + "options": [], + "query": "grafana-postgresql-datasource", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Malware", + "uid": "dd395331-5dc0-4172-b243-8646c0ca3ccd", + "version": 11, + "weekStart": "" +} \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic.json b/imageroot/etc/dashboards/network_traffic.json new file mode 100644 index 0000000..4301890 --- /dev/null +++ b/imageroot/etc/dashboards/network_traffic.json @@ -0,0 +1,899 @@ +{ + "__inputs": [ + { + "name": "DS_LOCAL_TIMESCALE", + "label": "Local Timescale", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT sum(bytes) FROM ca_dpi_stats_hourly_bytes WHERE unit_id=$unitid ", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total traffic", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 0 + }, + "id": 3, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, bytes FROM ca_dpi_stats_hourly_bytes \nWHERE $__timeFilter(bucket) and unit_id =$unitid\nGROUP BY bucket, bytes", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by hour", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 8 + }, + "id": 11, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 10, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT replace(application, 'netify.', '') AS application, sum(bytes) FROM ca_dpi_stats_hourly_application\nWHERE unit_id=$unitid\nGROUP BY application", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "By application", + "type": "piechart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 8 + }, + "id": 12, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 10, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT protocol, sum(bytes) as bytes FROM ca_dpi_stats_hourly_protocol\nWHERE unit_id=$unitid\nGROUP BY protocol", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "By protocol", + "type": "piechart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 8 + }, + "id": 13, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 10, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT client_name, sum(bytes) FROM ca_dpi_stats_hourly_client\nWHERE unit_id=$unitid\nGROUP BY client_name", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "By client", + "type": "piechart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "id": 14, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 10, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT host, sum(bytes) FROM ca_dpi_stats_hourly_host\nWHERE unit_id=$unitid\nGROUP BY host", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "By host", + "type": "piechart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "client_address" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT client_address, client_name, bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND unit_id=$unitid\nORDER BY bytes DESC\nLIMIT 20", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Top 20 active clients", + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "client_address" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 15, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT host, bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND unit_id=$unitid\nORDER BY bytes DESC\nLIMIT 20", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Top 20 contacted hosts", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "unit", + "timescale", + "monitoring", + "traffic" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Unit", + "multi": false, + "name": "unitid", + "options": [], + "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "DS_TIMESCALE", + "options": [], + "query": "grafana-postgresql-datasource", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Network traffic", + "uid": "fe0af3cb-9be0-4b2a-8ccb-86704956cf2e", + "version": 11, + "weekStart": "" +} \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_client.json b/imageroot/etc/dashboards/network_traffic_by_client.json new file mode 100644 index 0000000..e616d27 --- /dev/null +++ b/imageroot/etc/dashboards/network_traffic_by_client.json @@ -0,0 +1,736 @@ +{ + "__inputs": [ + { + "name": "DS_LOCAL_TIMESCALE", + "label": "Local Timescale", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^client_name$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT client_name \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND client_address = '$client_address'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Hostname", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, bytes \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND client_address = '$client_address'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by hour", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 0, + "y": 3 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bytes \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND client_address = '$client_address'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total traffic", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 3, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "never", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT application, SUM(bytes) FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY application", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by application", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 7, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "never", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT protocol, SUM(bytes) FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY protocol", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by protocol", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT host, SUM(bytes) as bytes FROM dpi_stats \nWHERE host!='' AND $__timeFilter(time) AND unit_id = $unitid AND client_address like '$client_address'\nGROUP BY host\nORDER BY bytes DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Visited host", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "monitoring", + "traffic", + "unit", + "timescale", + "client", + "details" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Unit", + "multi": false, + "name": "unitid", + "options": [], + "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "name": "client_address", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": {}, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "DS_TIMESCALE", + "options": [], + "query": "grafana-postgresql-datasource", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Network traffic by client", + "uid": "b14a1181-a2ee-4df4-a732-888e0190037f", + "version": 10, + "weekStart": "" +} \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_host.json b/imageroot/etc/dashboards/network_traffic_by_host.json new file mode 100644 index 0000000..a3d6e7f --- /dev/null +++ b/imageroot/etc/dashboards/network_traffic_by_host.json @@ -0,0 +1,543 @@ +{ + "__inputs": [ + { + "name": "DS_LOCAL_TIMESCALE", + "label": "Local Timescale", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND host like '$host'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total traffic", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 7, + "y": 0 + }, + "id": 3, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, sum(bytes) as bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND host like '$host'\nGROUP BY bucket, bytes\nORDER BY bucket", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by hour", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 19, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n time_bucket('30m', time) AS time,\n client_address as client,\n SUM(bytes) AS value\nFROM dpi_stats\nWHERE $__timeFilter(time) AND host LIKE '$host' AND unit_id='$unitid'\nGROUP BY time, client\nORDER BY time", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by hour per client", + "transformations": [ + { + "id": "partitionByValues", + "options": { + "fields": [ + "client" + ], + "naming": { + "asLabels": true + } + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT client_address, client_name, SUM(bytes) as bytes FROM dpi_stats \nWHERE $__timeFilter(time) AND unit_id = '$unitid' AND host LIKE '$host'\nGROUP BY client_address, client_name\nORDER BY bytes DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Visited by client", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "monitoring", + "timescale", + "traffic", + "host" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "hide": 0, + "includeAll": false, + "label": "Unit", + "multi": false, + "name": "unitid", + "options": [], + "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "pod-031-2010-09.backblaze.com", + "value": "pod-031-2010-09.backblaze.com" + }, + "hide": 0, + "name": "host", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": {}, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "DS_TIMESCALE", + "options": [], + "query": "grafana-postgresql-datasource", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Network traffic by host", + "uid": "c8d43e5f-068e-4cac-9283-2318d9e1911b", + "version": 9, + "weekStart": "" +} \ No newline at end of file From 982c5a06d3e748b939c3e413d3e0ab3fdb623817 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 11 Sep 2024 10:04:47 +0200 Subject: [PATCH 04/42] build: bump grafana version Required to make it work the image render plugin on a subpath --- build-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-images.sh b/build-images.sh index 3564976..f6a3b17 100755 --- a/build-images.sh +++ b/build-images.sh @@ -13,7 +13,7 @@ controller_version="1.0.1" promtail_version=2.7.1 loki_version=2.9.4 prometheus_version=2.50.1 -grafana_version=10.3.3 +grafana_version=11.2.0 webssh_version=1.6.2 timescale_version="latest-pg16" From e9f94796e6ef80d1e3b59fd5b7a288819e84e6f3 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 11 Sep 2024 10:58:42 +0200 Subject: [PATCH 05/42] grafana: provision timescale datasource --- imageroot/actions/configure-module/20configure | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/imageroot/actions/configure-module/20configure b/imageroot/actions/configure-module/20configure index 8774077..93f34cf 100755 --- a/imageroot/actions/configure-module/20configure +++ b/imageroot/actions/configure-module/20configure @@ -159,6 +159,7 @@ with open('prometheus.yml', 'w', encoding='utf-8') as fp: fp.write(f' - 127.0.0.1:{ports[5]}\n') # Grafana configuration +db = agent.read_envfile('db.env') with open('grafana.yml', 'w') as fp: fp.write("apiVersion: 1\n") fp.write("datasources:\n") @@ -174,6 +175,23 @@ with open('grafana.yml', 'w') as fp: fp.write(' access: proxy\n') fp.write(f' url: http://127.0.0.1:{ports[5]}\n') + fp.write(' - name: Local Timescale\n') + fp.write(' type: postgres\n') + fp.write(' uid: timescale\n') + fp.write(f' url: 127.0.0.1:{db.get("POSTGRES_PORT")}\n') + fp.write(f' user: {db.get("POSTGRES_USER")}\n') + fp.write(' secureJsonData:\n') + fp.write(f' password: {db.get("POSTGRES_PASSWORD")}\n') + fp.write(' jsonData:\n') + fp.write(' database: report\n') + fp.write(' sslmode: disable\n') + fp.write(' maxOpenConns: 100\n') + fp.write(' maxIdleConns: 100\n') + fp.write(' maxIdleConnsAuto: true\n') + fp.write(' connMaxLifetime: 14400\n') + fp.write(' postgresVersion: 1500\n') + fp.write(' timescaledb: true\n') + network = agent.read_envfile('network.env') tun = network.get('OVPN_TUN') bits = sum(bin(int(x)).count('1') for x in request["ovpn_netmask"].split('.')) From 75733c0d1e459434b87c537f79903829ab291312 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 11 Sep 2024 16:19:21 +0200 Subject: [PATCH 06/42] configure: add maxmind_license --- imageroot/actions/configure-module/20configure | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imageroot/actions/configure-module/20configure b/imageroot/actions/configure-module/20configure index 93f34cf..2b61605 100755 --- a/imageroot/actions/configure-module/20configure +++ b/imageroot/actions/configure-module/20configure @@ -119,6 +119,8 @@ with open('config.env', 'w') as env: env.write(f'VALID_SUBSCRIPTION=true\n') else: env.write(f'VALID_SUBSCRIPTION=false\n') + if 'maxmind_license' in request: + env.write(f'MAXMIND_LICENSE={request["maxmind_license"]}\n') server_address = request["ovpn_network"].removesuffix('.0') + '.1' with open('promtail.env', 'w') as promtail: From bd11415aa0b6229a4c10d08af903c0114eb85d71 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 11 Sep 2024 16:19:30 +0200 Subject: [PATCH 07/42] README: add report info --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 19feb8a..a631a16 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ The module is composed by the following containers: - [loki](#loki): log storage, it stores logs from promtail - [grafana](#grafana): metrics visualization, it visualizes metrics from prometheus and logs from loki - [webssh](#webssh): web-based ssh client +- [timescale](#timescale): time-series database for storing metrics ## Install @@ -43,10 +44,11 @@ Launch `configure-module`, by setting the following parameters: - `api_password`: controller admin password, change it after first login - `loki_retention`: Loki retention period in days (default: ``180`` days) - `promtail_retention`: Promtail retention period in days (default: ``15`` days) +- `maxmind_license`: [MaxMind](https://www.maxmind.com/) license key to download the GEO IP database, the database is loaded every time the API server is started Example: - api-cli run module/nethsecurity-controller1/configure-module --data '{"host": "mycontroller.nethsecurity.org", "lets_encrypt": false, "ovpn_network": "172.19.64.0", "ovpn_netmask": "255.255.255.0", "ovpn_cn": "nethsec", "api_user": "admin", "api_password": "password", "loki_retention": 180, "prometheus_retention": 15}' + api-cli run module/nethsecurity-controller1/configure-module --data '{"host": "mycontroller.nethsecurity.org", "lets_encrypt": false, "ovpn_network": "172.19.64.0", "ovpn_netmask": "255.255.255.0", "ovpn_cn": "nethsec", "api_user": "admin", "api_password": "password", "loki_retention": 180, "prometheus_retention": 15, ""maxmind_license": "xxx"}' The above command will: - start and configure the nethsecurity-controller instance @@ -160,6 +162,11 @@ It has also some pre-configured dashboards: - nethsecurity.json: a dashboard with the most important metrics from the connected machines, like CPU, memory, disk, network, and system load - logs.json: a dashboard where you can visualize the logs from all the connected machines and filter them by hostname, application, and priority - loki.json: a dashboard with the most important metrics from Loki, like the number of logs ingested, the number of logs dropped, and the status of queriers +- network_traffic.json: this dashboard uses data from Timescale database and shows the global network traffic by unit +- network_traffic_by_client.json: this dashboard uses data from Timescale database and shows the network traffic by unit and client (a client is a machine connected to the unit local network) +- network_traffic_by_host.json: this dashboard uses data from Timescale database and shows the network traffic by unit and host (a host is a machien on the internet) +- malware.json: this dashboard uses data from Timescale database and shows the malware blocked by the unit +- vpn.json: this dashboard uses data from Timescale database and shows the VPN connections Grafana is accessible at `https:///grafana/`, default credentials are the same set for the controller. You should change them on the first login. @@ -169,6 +176,16 @@ Grafana is accessible at `https:///grafana/`, default credentia Access to WebSSH is protected using a random generated URL, you can find it inside the module configuration file at `/home/nethsecurity-controller1/.config/state/config.json`. +### Timescale + +[Timescale](https://docs.timescale.com/latest/main) is a time-series database for storing metrics. It's configured via environment variables and the configuration is available at `/home/nethsecurity-controller1/.config/state/db.env`. + +If Postgres client is installed, you can connect to the database with the following command: +``` +runagent -m nethsecurity-controller1 +source db.env; psql -d $REPORT_DB_URI +``` + ## Uninstall To uninstall the instance: From c1161e1221c7303d53857f3af05eeaca67c4e32d Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 11 Sep 2024 18:16:28 +0200 Subject: [PATCH 08/42] add vpn dashboard --- imageroot/etc/dashboards/vpn.json | 492 ++++++++++++++++++++++++++++++ 1 file changed, 492 insertions(+) create mode 100644 imageroot/etc/dashboards/vpn.json diff --git a/imageroot/etc/dashboards/vpn.json b/imageroot/etc/dashboards/vpn.json new file mode 100644 index 0000000..ce0de86 --- /dev/null +++ b/imageroot/etc/dashboards/vpn.json @@ -0,0 +1,492 @@ +{ + "__inputs": [ + { + "name": "DS_LOCAL_TIMESCALE", + "label": "Local Timescale", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "geomap", + "name": "Geomap", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "repeat": "instance", + "repeatDirection": "h", + "title": "Instance: $instance", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND unit_id = $unitid AND instance = '$instance'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Number of connections", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 7, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT time, count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND unit_id = $unitid AND instance = '$instance'\nGROUP BY time", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Panel Title", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "controls": { + "mouseWheelZoom": true, + "showAttribution": true, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 5, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "symbolAlign": { + "horizontal": "center", + "vertical": "center" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "location": { + "mode": "auto" + }, + "name": "Layer 1", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "zero", + "lat": 0, + "lon": 0, + "zoom": 1 + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) AS count, country AS lookup FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND unit_id = $unitid AND instance = '$instance'\nGROUP BY lookup", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Panel Title", + "type": "geomap" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "DS_TIMESCALE", + "options": [], + "query": "grafana-postgresql-datasource", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "hide": 0, + "includeAll": false, + "label": "Unit", + "multi": false, + "name": "unitid", + "options": [], + "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "definition": "SELECT instance FROM ovpnrw_connections WHERE unit_id = $unitid", + "description": "", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": "SELECT instance FROM ovpnrw_connections WHERE unit_id = $unitid", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "VPN", + "uid": "fdxlb58zxvpj4f", + "version": 4, + "weekStart": "" +} \ No newline at end of file From 3c68ab7785d0424d696d4c1c30f24c96c41b722f Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 12 Sep 2024 14:44:53 +0200 Subject: [PATCH 09/42] fix: expanding tailscale workers to 100 --- imageroot/actions/create-module/20initialize | 1 + 1 file changed, 1 insertion(+) diff --git a/imageroot/actions/create-module/20initialize b/imageroot/actions/create-module/20initialize index b34d325..be803c9 100755 --- a/imageroot/actions/create-module/20initialize +++ b/imageroot/actions/create-module/20initialize @@ -50,6 +50,7 @@ POSTGRES_USER=report POSTGRES_PORT=$db_port POSTGRES_PASSWORD=$db_secret REPORT_DB_URI=postgres://report:$db_secret@127.0.0.1:$db_port/report +TS_TUNE_MAX_BG_WORKERS=100 EOF mkdir -p clients From 0977b90cda64d4118cc09fb9744007e4bc628557 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 12 Sep 2024 16:05:37 +0200 Subject: [PATCH 10/42] feat: adding backup of timescale --- imageroot/actions/restore-module/30restore_database | 7 +++++++ imageroot/bin/module-cleanup-state | 3 +++ imageroot/bin/module-dump-state | 7 +++++++ imageroot/etc/state-include.conf | 3 ++- 4 files changed, 19 insertions(+), 1 deletion(-) create mode 100755 imageroot/actions/restore-module/30restore_database create mode 100755 imageroot/bin/module-cleanup-state create mode 100755 imageroot/bin/module-dump-state diff --git a/imageroot/actions/restore-module/30restore_database b/imageroot/actions/restore-module/30restore_database new file mode 100755 index 0000000..f1a1d4d --- /dev/null +++ b/imageroot/actions/restore-module/30restore_database @@ -0,0 +1,7 @@ +#!/bin/sh + +# Load database credentials +. db.env + +# Dump the DB from timescale +podman exec -i timescale pd_restore -d "${REPORT_DB_URI}" < backup.sql diff --git a/imageroot/bin/module-cleanup-state b/imageroot/bin/module-cleanup-state new file mode 100755 index 0000000..ceff87c --- /dev/null +++ b/imageroot/bin/module-cleanup-state @@ -0,0 +1,3 @@ +#!/bin/sh + +rm -f backup.sql diff --git a/imageroot/bin/module-dump-state b/imageroot/bin/module-dump-state new file mode 100755 index 0000000..27b6e16 --- /dev/null +++ b/imageroot/bin/module-dump-state @@ -0,0 +1,7 @@ +#!/bin/sh + +# Load database credentials +. db.env + +# Dump the DB from timescale +podman exec -i timescale pg_dump -d "${REPORT_DB_URI}" > backup.sql diff --git a/imageroot/etc/state-include.conf b/imageroot/etc/state-include.conf index c2d26fe..3e12002 100644 --- a/imageroot/etc/state-include.conf +++ b/imageroot/etc/state-include.conf @@ -1,8 +1,9 @@ volumes/vpn-data volumes/grafana-data volumes/loki-data -volumns/prometheus-data +volumes/prometheus-data volumes/api-credentials volumes/api-data state/secret.env state/config.json +state/backup.sql From a94794235e58c409e50eac184a26dad93c4f5947 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 12 Sep 2024 16:24:56 +0200 Subject: [PATCH 11/42] fix: solving backup issues --- imageroot/bin/module-cleanup-state | 4 +++- imageroot/bin/module-dump-state | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/imageroot/bin/module-cleanup-state b/imageroot/bin/module-cleanup-state index ceff87c..c0ffb39 100755 --- a/imageroot/bin/module-cleanup-state +++ b/imageroot/bin/module-cleanup-state @@ -1,3 +1,5 @@ -#!/bin/sh +#!/usr/bin/env sh + +set -e rm -f backup.sql diff --git a/imageroot/bin/module-dump-state b/imageroot/bin/module-dump-state index 27b6e16..001ea7e 100755 --- a/imageroot/bin/module-dump-state +++ b/imageroot/bin/module-dump-state @@ -1,7 +1,9 @@ -#!/bin/sh +#!/usr/bin/env sh + +set -e # Load database credentials -. db.env +. ./db.env # Dump the DB from timescale podman exec -i timescale pg_dump -d "${REPORT_DB_URI}" > backup.sql From ce55c2d1f143c1d170992a34d2cf2e66676e988a Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 12 Sep 2024 16:37:00 +0200 Subject: [PATCH 12/42] fix: hammering down restore process --- imageroot/actions/restore-module/30restore_database | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imageroot/actions/restore-module/30restore_database b/imageroot/actions/restore-module/30restore_database index f1a1d4d..beddd2e 100755 --- a/imageroot/actions/restore-module/30restore_database +++ b/imageroot/actions/restore-module/30restore_database @@ -1,7 +1,7 @@ #!/bin/sh # Load database credentials -. db.env +. ./db.env # Dump the DB from timescale podman exec -i timescale pd_restore -d "${REPORT_DB_URI}" < backup.sql From 6b4f8f1825f2a8ab88fd8877363abfb2eefb0f4d Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 12 Sep 2024 16:55:36 +0200 Subject: [PATCH 13/42] fix: dumping only data and restoring using psql --- imageroot/actions/restore-module/30restore_database | 2 +- imageroot/bin/module-dump-state | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/imageroot/actions/restore-module/30restore_database b/imageroot/actions/restore-module/30restore_database index beddd2e..1ab9c9e 100755 --- a/imageroot/actions/restore-module/30restore_database +++ b/imageroot/actions/restore-module/30restore_database @@ -4,4 +4,4 @@ . ./db.env # Dump the DB from timescale -podman exec -i timescale pd_restore -d "${REPORT_DB_URI}" < backup.sql +podman exec -i timescale psql --dbname "${REPORT_DB_URI}" < backup.sql diff --git a/imageroot/bin/module-dump-state b/imageroot/bin/module-dump-state index 001ea7e..4e66f4a 100755 --- a/imageroot/bin/module-dump-state +++ b/imageroot/bin/module-dump-state @@ -6,4 +6,4 @@ set -e . ./db.env # Dump the DB from timescale -podman exec -i timescale pg_dump -d "${REPORT_DB_URI}" > backup.sql +podman exec -i timescale pg_dump --dbname "${REPORT_DB_URI}" --data-only > backup.sql From 1cf67d16fc3b7db38f0cace9b634cc8dee0dcdb3 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Mon, 16 Sep 2024 15:36:40 +0200 Subject: [PATCH 14/42] grafana: add VPN network traffic chart The char has been added to the Metrics dashboard: it is not possible to add it inside the reporting dashboard because Prometheus uses the uuid to select the unit, while Timescale uses an integer id --- imageroot/etc/dashboards/nethsecurity.json | 1021 +++++++++++--------- 1 file changed, 573 insertions(+), 448 deletions(-) diff --git a/imageroot/etc/dashboards/nethsecurity.json b/imageroot/etc/dashboards/nethsecurity.json index cfcbaf1..911dc29 100644 --- a/imageroot/etc/dashboards/nethsecurity.json +++ b/imageroot/etc/dashboards/nethsecurity.json @@ -26,7 +26,7 @@ "fiscalYearStartMonth": 0, "gnetId": 7107, "graphTooltip": 0, - "id": 1, + "id": 4, "links": [], "liveNow": false, "panels": [ @@ -58,6 +58,7 @@ }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, @@ -74,6 +75,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -114,8 +116,7 @@ } ] }, - "unit": "none", - "unitScale": true + "unit": "none" }, "overrides": [] }, @@ -126,7 +127,6 @@ "y": 1 }, "id": 11, - "links": [], "options": { "legend": { "calcs": [ @@ -220,6 +220,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 50, "gradientMode": "none", @@ -261,8 +262,7 @@ } ] }, - "unit": "percent", - "unitScale": true + "unit": "percent" }, "overrides": [ { @@ -294,7 +294,6 @@ "y": 8 }, "id": 8, - "links": [], "options": { "legend": { "calcs": [ @@ -333,7 +332,7 @@ "type": "timeseries" }, { - "collapsed": false, + "collapsed": true, "datasource": { "type": "datasource", "uid": "grafana" @@ -345,239 +344,236 @@ "y": 14 }, "id": 45, - "panels": [], - "targets": [ + "panels": [ { "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "System overview: network", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total IPv4 Traffic.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "Total IPv4 Traffic.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "KBs" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "KBs", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 24, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min", - "sum" + "editorMode": "code", + "exemplar": true, + "expr": "netdata_system_ip_kilobits_persec_average{unit=~\"$unit\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{dimension}}", + "range": true, + "refId": "A" + } ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "title": "IPv4 bandwidth", + "transparent": true, + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.3.3", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "netdata_system_ip_kilobits_persec_average{unit=~\"$unit\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{dimension}}", - "range": true, - "refId": "A" - } - ], - "title": "IPv4 bandwidth", - "transparent": true, - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total IPv6 Traffic.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "Total IPv6 Traffic.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "KBs" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "KBs", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 25, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min", - "sum" + "editorMode": "code", + "expr": "netdata_system_ipv6_kilobits_persec_average{unit=~\"$unit\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{dimension}}", + "range": true, + "refId": "A" + } ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "title": "IPv6 bandwidth", + "transparent": true, + "type": "timeseries" } - }, - "pluginVersion": "10.3.3", + ], "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "datasource", + "uid": "grafana" }, - "editorMode": "code", - "expr": "netdata_system_ipv6_kilobits_persec_average{unit=~\"$unit\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{dimension}}", - "range": true, "refId": "A" } ], - "title": "IPv6 bandwidth", - "transparent": true, - "type": "timeseries" + "title": "System overview: network", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "datasource": { "type": "datasource", "uid": "grafana" @@ -586,141 +582,140 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 15 }, "id": 41, - "panels": [], - "targets": [ + "panels": [ { "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "System overview: RAM", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "System Random Access Memory (i.e. physical memory) usage.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "System Random Access Memory (i.e. physical memory) usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decmbytes", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 16, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" + "editorMode": "code", + "expr": "netdata_system_ram_MiB_average{dimension!=\"free\", unit=~\"$unit\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{dimension}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "netdata_system_ram_MiB_average{dimension=\"free\", unit=~\"$unit\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{dimension}}", + "range": true, + "refId": "B" + } ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "title": "System RAM", + "transparent": true, + "type": "timeseries" } - }, - "pluginVersion": "10.3.3", + ], "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "datasource", + "uid": "grafana" }, - "editorMode": "code", - "expr": "netdata_system_ram_MiB_average{dimension!=\"free\", unit=~\"$unit\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{dimension}}", - "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "netdata_system_ram_MiB_average{dimension=\"free\", unit=~\"$unit\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{dimension}}", - "range": true, - "refId": "B" } ], - "title": "System RAM", - "transparent": true, - "type": "timeseries" + "title": "System overview: RAM", + "type": "row" }, { "collapsed": true, @@ -732,7 +727,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 16 }, "id": 586, "panels": [ @@ -783,8 +778,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -801,7 +795,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 39 }, "id": 1461, "options": { @@ -881,8 +875,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -899,7 +892,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 39 }, "id": 1462, "options": { @@ -979,8 +972,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -997,7 +989,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 47 }, "id": 1463, "options": { @@ -1077,8 +1069,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1095,7 +1086,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 47 }, "id": 1464, "options": { @@ -1153,7 +1144,7 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 17 }, "id": 39, "panels": [ @@ -1205,8 +1196,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1223,7 +1213,7 @@ "h": 7, "w": 12, "x": 0, - "y": 32 + "y": 40 }, "id": 13, "links": [], @@ -1313,8 +1303,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1331,7 +1320,7 @@ "h": 7, "w": 12, "x": 12, - "y": 32 + "y": 40 }, "id": 14, "links": [], @@ -1398,7 +1387,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 18 }, "id": 47, "panels": [ @@ -1468,7 +1457,7 @@ "h": 7, "w": 12, "x": 0, - "y": 22 + "y": 30 }, "id": 29, "links": [], @@ -1571,7 +1560,7 @@ "h": 7, "w": 12, "x": 12, - "y": 22 + "y": 30 }, "id": 31, "links": [], @@ -1672,7 +1661,7 @@ "h": 7, "w": 12, "x": 0, - "y": 29 + "y": 37 }, "id": 33, "links": [], @@ -1773,7 +1762,7 @@ "h": 7, "w": 12, "x": 12, - "y": 29 + "y": 37 }, "id": 35, "links": [], @@ -1832,7 +1821,7 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 19 }, "id": 49, "panels": [ @@ -1901,7 +1890,7 @@ "h": 6, "w": 24, "x": 0, - "y": 23 + "y": 31 }, "id": 51, "links": [], @@ -1965,7 +1954,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 20 }, "id": 59, "panels": [ @@ -2035,7 +2024,7 @@ "h": 7, "w": 11, "x": 0, - "y": 24 + "y": 32 }, "id": 55, "links": [], @@ -2158,7 +2147,7 @@ "h": 7, "w": 13, "x": 11, - "y": 24 + "y": 32 }, "id": 57, "links": [], @@ -2219,7 +2208,7 @@ "h": 1, "w": 24, "x": 0, - "y": 35 + "y": 21 }, "id": 63, "panels": [ @@ -2309,7 +2298,7 @@ "h": 7, "w": 24, "x": 0, - "y": 25 + "y": 33 }, "id": 113, "links": [], @@ -2370,7 +2359,7 @@ "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 22 }, "id": 81, "panels": [ @@ -2438,7 +2427,7 @@ "h": 6, "w": 24, "x": 0, - "y": 26 + "y": 34 }, "id": 79, "links": [], @@ -2502,7 +2491,7 @@ "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 23 }, "id": 43, "panels": [ @@ -2571,7 +2560,7 @@ "h": 7, "w": 24, "x": 0, - "y": 39 + "y": 47 }, "id": 18, "links": [], @@ -2635,7 +2624,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 24 }, "id": 146, "panels": [ @@ -2706,7 +2695,7 @@ "h": 7, "w": 12, "x": 0, - "y": 40 + "y": 48 }, "id": 140, "links": [], @@ -2814,7 +2803,7 @@ "h": 7, "w": 12, "x": 12, - "y": 40 + "y": 48 }, "id": 142, "links": [], @@ -2919,7 +2908,7 @@ "h": 7, "w": 24, "x": 0, - "y": 47 + "y": 55 }, "id": 144, "links": [], @@ -2983,7 +2972,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 25 }, "id": 152, "panels": [ @@ -3074,7 +3063,7 @@ "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 49 }, "id": 148, "links": [], @@ -3137,7 +3126,7 @@ "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 49 }, "hiddenSeries": false, "id": 150, @@ -3238,7 +3227,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 26 }, "id": 178, "panels": [ @@ -3260,6 +3249,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -3298,8 +3288,7 @@ } ] }, - "unit": "decmbytes", - "unitScale": true + "unit": "decmbytes" }, "overrides": [] }, @@ -3307,10 +3296,9 @@ "h": 7, "w": 12, "x": 0, - "y": 42 + "y": 23 }, "id": 158, - "links": [], "options": { "legend": { "calcs": [ @@ -3366,6 +3354,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -3404,8 +3393,7 @@ } ] }, - "unit": "ms", - "unitScale": true + "unit": "ms" }, "overrides": [] }, @@ -3413,10 +3401,9 @@ "h": 7, "w": 12, "x": 12, - "y": 42 + "y": 23 }, "id": 164, - "links": [], "options": { "legend": { "calcs": [ @@ -3472,6 +3459,7 @@ "axisLabel": "operations", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -3510,8 +3498,7 @@ } ] }, - "unit": "none", - "unitScale": true + "unit": "none" }, "overrides": [] }, @@ -3519,10 +3506,9 @@ "h": 7, "w": 12, "x": 0, - "y": 49 + "y": 30 }, "id": 162, - "links": [], "options": { "legend": { "calcs": [ @@ -3578,6 +3564,7 @@ "axisLabel": "ms per operation", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -3616,8 +3603,7 @@ } ] }, - "unit": "ms", - "unitScale": true + "unit": "ms" }, "overrides": [] }, @@ -3625,10 +3611,9 @@ "h": 7, "w": 12, "x": 12, - "y": 49 + "y": 30 }, "id": 172, - "links": [], "options": { "legend": { "calcs": [ @@ -3731,10 +3716,9 @@ "h": 7, "w": 12, "x": 0, - "y": 56 + "y": 37 }, "id": 166, - "links": [], "options": { "legend": { "calcs": [ @@ -3836,10 +3820,9 @@ "h": 7, "w": 12, "x": 12, - "y": 56 + "y": 37 }, "id": 176, - "links": [], "options": { "legend": { "calcs": [ @@ -3902,7 +3885,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 27 }, "id": 185, "panels": [ @@ -3924,6 +3907,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 50, "gradientMode": "none", @@ -3963,8 +3947,7 @@ } ] }, - "unit": "kbytes", - "unitScale": true + "unit": "kbytes" }, "overrides": [] }, @@ -3972,10 +3955,9 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 24 }, "id": 183, - "links": [], "options": { "legend": { "calcs": [ @@ -4028,6 +4010,120 @@ "title": "Disks: filesystem", "type": "row" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 1465, + "panels": [], + "title": "VPN networking", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 29 + }, + "id": 1466, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "repeat": "vpninterface", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "netdata_net_net_kilobits_persec_average{unit=~\"$unit\",chart=~\"$vpninterface\"}", + "instant": true, + "legendFormat": " {{dimension}}", + "range": false, + "refId": "A" + } + ], + "title": "VPN interface: $vpninterface", + "type": "timeseries" + }, { "collapsed": true, "datasource": { @@ -4038,7 +4134,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 40 }, "id": 264, "panels": [ @@ -4107,7 +4203,7 @@ "h": 7, "w": 24, "x": 0, - "y": 51 + "y": 59 }, "id": 458, "links": [], @@ -4170,7 +4266,7 @@ "h": 1, "w": 24, "x": 0, - "y": 43 + "y": 41 }, "id": 285, "panels": [ @@ -4259,7 +4355,7 @@ "h": 7, "w": 24, "x": 0, - "y": 52 + "y": 60 }, "id": 283, "links": [], @@ -4323,7 +4419,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 42 }, "id": 308, "panels": [ @@ -4392,7 +4488,7 @@ "h": 7, "w": 24, "x": 0, - "y": 53 + "y": 61 }, "id": 304, "links": [], @@ -4456,7 +4552,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 43 }, "id": 364, "panels": [ @@ -4525,7 +4621,7 @@ "h": 7, "w": 24, "x": 0, - "y": 54 + "y": 62 }, "id": 327, "links": [], @@ -4631,7 +4727,7 @@ "h": 8, "w": 12, "x": 0, - "y": 61 + "y": 69 }, "id": 329, "links": [], @@ -4737,7 +4833,7 @@ "h": 8, "w": 12, "x": 12, - "y": 61 + "y": 69 }, "id": 331, "links": [], @@ -4841,7 +4937,7 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 77 }, "id": 350, "links": [], @@ -4948,7 +5044,7 @@ "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 77 }, "id": 352, "links": [], @@ -5055,7 +5151,7 @@ "h": 7, "w": 24, "x": 0, - "y": 77 + "y": 85 }, "id": 356, "links": [], @@ -5162,7 +5258,7 @@ "h": 9, "w": 12, "x": 0, - "y": 84 + "y": 92 }, "id": 354, "links": [], @@ -5268,7 +5364,7 @@ "h": 9, "w": 12, "x": 12, - "y": 84 + "y": 92 }, "id": 362, "links": [], @@ -5331,7 +5427,7 @@ "h": 1, "w": 24, "x": 0, - "y": 46 + "y": 44 }, "id": 391, "panels": [ @@ -5400,7 +5496,7 @@ "h": 7, "w": 12, "x": 0, - "y": 55 + "y": 63 }, "id": 383, "links": [], @@ -5504,7 +5600,7 @@ "h": 7, "w": 12, "x": 12, - "y": 55 + "y": 63 }, "id": 385, "links": [], @@ -5610,7 +5706,7 @@ "h": 7, "w": 12, "x": 0, - "y": 62 + "y": 70 }, "id": 387, "links": [], @@ -5736,7 +5832,7 @@ "h": 7, "w": 12, "x": 12, - "y": 62 + "y": 70 }, "id": 389, "links": [], @@ -5799,7 +5895,7 @@ "h": 1, "w": 24, "x": 0, - "y": 47 + "y": 45 }, "id": 416, "panels": [ @@ -5867,7 +5963,7 @@ "h": 8, "w": 12, "x": 0, - "y": 56 + "y": 64 }, "id": 410, "links": [], @@ -5973,7 +6069,7 @@ "h": 8, "w": 12, "x": 12, - "y": 56 + "y": 64 }, "id": 412, "links": [], @@ -6099,7 +6195,7 @@ "h": 7, "w": 24, "x": 0, - "y": 64 + "y": 72 }, "id": 414, "links": [], @@ -6163,7 +6259,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 46 }, "id": 523, "panels": [ @@ -6231,7 +6327,7 @@ "h": 7, "w": 12, "x": 0, - "y": 57 + "y": 65 }, "id": 502, "links": [], @@ -6335,7 +6431,7 @@ "h": 7, "w": 12, "x": 12, - "y": 57 + "y": 65 }, "id": 504, "links": [], @@ -6399,7 +6495,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 47 }, "id": 567, "panels": [ @@ -6468,7 +6564,7 @@ "h": 7, "w": 24, "x": 0, - "y": 58 + "y": 66 }, "id": 565, "links": [], @@ -6532,7 +6628,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 48 }, "id": 1106, "panels": [ @@ -6621,7 +6717,7 @@ "h": 7, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 1104, "links": [], @@ -6747,7 +6843,7 @@ "h": 7, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 1107, "links": [], @@ -6811,7 +6907,7 @@ "h": 1, "w": 24, "x": 0, - "y": 51 + "y": 49 }, "id": 1174, "panels": [ @@ -6880,7 +6976,7 @@ "h": 7, "w": 12, "x": 0, - "y": 52 + "y": 60 }, "id": 1172, "links": [], @@ -6943,7 +7039,7 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 50 }, "id": 1245, "panels": [ @@ -7012,7 +7108,7 @@ "h": 7, "w": 12, "x": 0, - "y": 53 + "y": 61 }, "id": 1239, "links": [], @@ -7116,7 +7212,7 @@ "h": 7, "w": 12, "x": 12, - "y": 53 + "y": 61 }, "id": 1241, "links": [], @@ -7222,7 +7318,7 @@ "h": 7, "w": 24, "x": 0, - "y": 60 + "y": 68 }, "id": 1243, "links": [], @@ -7286,7 +7382,7 @@ "h": 1, "w": 24, "x": 0, - "y": 53 + "y": 51 }, "id": 1381, "panels": [ @@ -7395,7 +7491,7 @@ "h": 7, "w": 12, "x": 0, - "y": 54 + "y": 62 }, "id": 1377, "links": [], @@ -7540,7 +7636,7 @@ "h": 7, "w": 12, "x": 12, - "y": 54 + "y": 62 }, "id": 1379, "links": [], @@ -7604,7 +7700,7 @@ "h": 1, "w": 24, "x": 0, - "y": 54 + "y": 52 }, "id": 1460, "panels": [ @@ -7672,7 +7768,7 @@ "h": 7, "w": 24, "x": 0, - "y": 55 + "y": 63 }, "id": 1446, "links": [], @@ -7778,7 +7874,7 @@ "h": 7, "w": 24, "x": 0, - "y": 62 + "y": 70 }, "id": 1448, "links": [], @@ -7883,7 +7979,7 @@ "h": 9, "w": 12, "x": 0, - "y": 69 + "y": 77 }, "id": 1450, "links": [], @@ -7988,7 +8084,7 @@ "h": 9, "w": 12, "x": 12, - "y": 69 + "y": 77 }, "id": 1452, "links": [], @@ -8093,7 +8189,7 @@ "h": 9, "w": 12, "x": 0, - "y": 78 + "y": 86 }, "id": 1454, "links": [], @@ -8198,7 +8294,7 @@ "h": 9, "w": 12, "x": 12, - "y": 78 + "y": 86 }, "id": 1456, "links": [], @@ -8303,7 +8399,7 @@ "h": 7, "w": 24, "x": 0, - "y": 87 + "y": 95 }, "id": 1458, "links": [], @@ -8370,9 +8466,9 @@ "list": [ { "current": { - "selected": true, - "text": "7BD33F72-F9D7-4A78-80A0-7ACD15991B92", - "value": "7BD33F72-F9D7-4A78-80A0-7ACD15991B92" + "selected": false, + "text": "181ff9b0-e2be-4c71-897a-6575e3991a1c", + "value": "181ff9b0-e2be-4c71-897a-6575e3991a1c" }, "datasource": { "type": "prometheus", @@ -8414,6 +8510,35 @@ "regex": "", "skipUrlSync": false, "type": "datasource" + }, + { + "allValue": "", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(netdata_net_net_kilobits_persec_average{chart=~\"^net.tun.*|^net.tap.*\"},chart)", + "hide": 0, + "includeAll": true, + "label": "VPN interface", + "multi": false, + "name": "vpninterface", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(netdata_net_net_kilobits_persec_average{chart=~\"^net.tun.*|^net.tap.*\"},chart)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, @@ -8449,6 +8574,6 @@ "timezone": "", "title": "Unit metrics", "uid": "W3S__804z", - "version": 25, + "version": 11, "weekStart": "" } From 8108257517c4386e14eb75b8daf46b57effdfb0f Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Fri, 13 Sep 2024 11:37:49 +0200 Subject: [PATCH 15/42] fix: final touchup of database restore --- imageroot/actions/restore-module/30restore_database | 7 ++++++- imageroot/bin/module-dump-state | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/imageroot/actions/restore-module/30restore_database b/imageroot/actions/restore-module/30restore_database index 1ab9c9e..a6aa9aa 100755 --- a/imageroot/actions/restore-module/30restore_database +++ b/imageroot/actions/restore-module/30restore_database @@ -3,5 +3,10 @@ # Load database credentials . ./db.env +# Wait for database to be ready pooling every 5 seconds +until podman exec timescale pg_isready -U "${POSTGRES_USER}" -p "${POSTGRES_PORT}" -d "${POSTGRES_DB}"; do + sleep 5 +done + # Dump the DB from timescale -podman exec -i timescale psql --dbname "${REPORT_DB_URI}" < backup.sql +podman exec -i timescale psql -U "${POSTGRES_USER}" -p "${POSTGRES_PORT}" < backup.sql diff --git a/imageroot/bin/module-dump-state b/imageroot/bin/module-dump-state index 4e66f4a..682f68a 100755 --- a/imageroot/bin/module-dump-state +++ b/imageroot/bin/module-dump-state @@ -6,4 +6,4 @@ set -e . ./db.env # Dump the DB from timescale -podman exec -i timescale pg_dump --dbname "${REPORT_DB_URI}" --data-only > backup.sql +podman exec -i timescale pg_dump -U "${POSTGRES_USER}" -p "${POSTGRES_PORT}" > timescale.sql From b3a7f831ab65b5941fe0ae7a266c2a71b67d27ff Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Mon, 16 Sep 2024 15:44:21 +0200 Subject: [PATCH 16/42] feat: added grafana user to postgresql --- imageroot/actions/configure-module/20configure | 4 ++-- imageroot/actions/create-module/20initialize | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/imageroot/actions/configure-module/20configure b/imageroot/actions/configure-module/20configure index 2b61605..dd1a44d 100755 --- a/imageroot/actions/configure-module/20configure +++ b/imageroot/actions/configure-module/20configure @@ -181,9 +181,9 @@ with open('grafana.yml', 'w') as fp: fp.write(' type: postgres\n') fp.write(' uid: timescale\n') fp.write(f' url: 127.0.0.1:{db.get("POSTGRES_PORT")}\n') - fp.write(f' user: {db.get("POSTGRES_USER")}\n') + fp.write(f' user: grafana\n') fp.write(' secureJsonData:\n') - fp.write(f' password: {db.get("POSTGRES_PASSWORD")}\n') + fp.write(f' password: {db.get("GRAFANA_POSTGRES_PASSWORD")}\n') fp.write(' jsonData:\n') fp.write(' database: report\n') fp.write(' sslmode: disable\n') diff --git a/imageroot/actions/create-module/20initialize b/imageroot/actions/create-module/20initialize index be803c9..976be46 100755 --- a/imageroot/actions/create-module/20initialize +++ b/imageroot/actions/create-module/20initialize @@ -26,6 +26,7 @@ num=$(echo $MODULE_ID | sed 's/nethsecurity\-controller//') jwt_secret=$(uuidgen | sha256sum | awk '{print $1}') reg_secret=$(uuidgen | sha256sum | awk '{print $1}') db_secret=$(uuidgen | sha256sum | awk '{print $1}') +grafana_postgres_password=$(uuidgen | sha256sum | awk '{print $1}') cat << EOF > network.env OVPN_UDP_PORT=$ovpn_udp_port @@ -49,6 +50,7 @@ cat << EOF > db.env POSTGRES_USER=report POSTGRES_PORT=$db_port POSTGRES_PASSWORD=$db_secret +GRAFANA_POSTGRES_PASSWORD=$grafana_postgres_password REPORT_DB_URI=postgres://report:$db_secret@127.0.0.1:$db_port/report TS_TUNE_MAX_BG_WORKERS=100 EOF From 2c3d44cda31fd368328d09798194ab65c3491828 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Mon, 16 Sep 2024 16:37:08 +0200 Subject: [PATCH 17/42] feat: added configurable retention period --- README.md | 1 + imageroot/actions/configure-module/20configure | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index a631a16..bed98b9 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Launch `configure-module`, by setting the following parameters: - `loki_retention`: Loki retention period in days (default: ``180`` days) - `promtail_retention`: Promtail retention period in days (default: ``15`` days) - `maxmind_license`: [MaxMind](https://www.maxmind.com/) license key to download the GEO IP database, the database is loaded every time the API server is started +- `retention_days`: retention period for the metrics in the Timescale database (default: `60` days) Example: diff --git a/imageroot/actions/configure-module/20configure b/imageroot/actions/configure-module/20configure index dd1a44d..500202e 100755 --- a/imageroot/actions/configure-module/20configure +++ b/imageroot/actions/configure-module/20configure @@ -121,6 +121,7 @@ with open('config.env', 'w') as env: env.write(f'VALID_SUBSCRIPTION=false\n') if 'maxmind_license' in request: env.write(f'MAXMIND_LICENSE={request["maxmind_license"]}\n') + env.write(f'RETENTION_DAYS={request.get("retention_days", "60")}\n') server_address = request["ovpn_network"].removesuffix('.0') + '.1' with open('promtail.env', 'w') as promtail: From 12fa51801391a8f05aace9a7137dd0f41de62772 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Mon, 16 Sep 2024 17:32:56 +0200 Subject: [PATCH 18/42] fix: using prometheus retention to configure api server --- README.md | 3 +-- imageroot/actions/configure-module/20configure | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bed98b9..1966271 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,8 @@ Launch `configure-module`, by setting the following parameters: - `api_user`: controller admin user - `api_password`: controller admin password, change it after first login - `loki_retention`: Loki retention period in days (default: ``180`` days) -- `promtail_retention`: Promtail retention period in days (default: ``15`` days) +- `prometheus_retention`: Promtail retention period in days (default: ``15`` days) - `maxmind_license`: [MaxMind](https://www.maxmind.com/) license key to download the GEO IP database, the database is loaded every time the API server is started -- `retention_days`: retention period for the metrics in the Timescale database (default: `60` days) Example: diff --git a/imageroot/actions/configure-module/20configure b/imageroot/actions/configure-module/20configure index 500202e..a23d271 100755 --- a/imageroot/actions/configure-module/20configure +++ b/imageroot/actions/configure-module/20configure @@ -103,6 +103,7 @@ with open('config.json', 'w') as cfp: # Load subscription info rdb = agent.redis_connect(privileged=False) subscription = rdb.hgetall('cluster/subscription') +metrics_retention_days = request.get('prometheus_retention', '15') with open('config.env', 'w') as env: env.write(f'ADMIN_USER={request["api_user"]}\n') @@ -121,7 +122,7 @@ with open('config.env', 'w') as env: env.write(f'VALID_SUBSCRIPTION=false\n') if 'maxmind_license' in request: env.write(f'MAXMIND_LICENSE={request["maxmind_license"]}\n') - env.write(f'RETENTION_DAYS={request.get("retention_days", "60")}\n') + env.write(f'RETENTION_DAYS={metrics_retention_days}\n') server_address = request["ovpn_network"].removesuffix('.0') + '.1' with open('promtail.env', 'w') as promtail: @@ -147,7 +148,7 @@ with open('grafana.env', 'w') as gfp: with open('prometheus.env', 'w') as pfp: pfp.write(f"PROMETHEUS_PORT={ports[7]}\n") pfp.write(f"PROMETHEUS_PATH={config['prometheus_path']}\n") - pfp.write(f"PROMETHEUS_RETENTION={request.get('prometheus_retention', '15')}d\n") + pfp.write(f"PROMETHEUS_RETENTION={metrics_retention_days}d\n") with open('prometheus.yml', 'w', encoding='utf-8') as fp: fp.write("global:\n") From bcb44d5660bc0bd2960055ed4e6d5496de4a6b6d Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Mon, 16 Sep 2024 17:43:42 +0200 Subject: [PATCH 19/42] docs: added Timescale reference to `prometheus_retention` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1966271..1c2b139 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Launch `configure-module`, by setting the following parameters: - `api_user`: controller admin user - `api_password`: controller admin password, change it after first login - `loki_retention`: Loki retention period in days (default: ``180`` days) -- `prometheus_retention`: Promtail retention period in days (default: ``15`` days) +- `prometheus_retention`: Promtail and Timescale retention period in days (default: 15 days) - `maxmind_license`: [MaxMind](https://www.maxmind.com/) license key to download the GEO IP database, the database is loaded every time the API server is started Example: From d0631345d6644d9e39269441b4e9de80fd812e40 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 17 Sep 2024 10:10:12 +0200 Subject: [PATCH 20/42] grafana: vpn, add missing charts --- imageroot/etc/dashboards/vpn.json | 484 +++++++++++++++++++++++++++++- 1 file changed, 474 insertions(+), 10 deletions(-) diff --git a/imageroot/etc/dashboards/vpn.json b/imageroot/etc/dashboards/vpn.json index ce0de86..1bafd8d 100644 --- a/imageroot/etc/dashboards/vpn.json +++ b/imageroot/etc/dashboards/vpn.json @@ -11,6 +11,12 @@ ], "__elements": {}, "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, { "type": "panel", "id": "geomap", @@ -76,7 +82,7 @@ "panels": [], "repeat": "instance", "repeatDirection": "h", - "title": "Instance: $instance", + "title": "OpenVPN Road Warrior instance: $instance", "type": "row" }, { @@ -108,7 +114,7 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 8, "w": 7, "x": 0, "y": 1 @@ -275,9 +281,200 @@ } } ], - "title": "Panel Title", + "title": "Connections by time", "type": "timeseries" }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 0, + "y": 9 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND instance = '$instance'\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total traffic", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 7, + "y": 9 + }, + "id": 6, + "options": { + "barRadius": 0, + "barWidth": 0.88, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes, common_name as user FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND instance = '$instance'\nGROUP BY common_name\nORDER by bytes DESC\nLIMIT 20", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Top 20 user by traffic", + "type": "barchart" + }, { "datasource": { "default": false, @@ -314,13 +511,18 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 9, + "w": 19, "x": 0, - "y": 9 + "y": 17 }, "id": 4, "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "type": "default" + }, "controls": { "mouseWheelZoom": true, "showAttribution": true, @@ -369,7 +571,7 @@ "location": { "mode": "auto" }, - "name": "Layer 1", + "name": "Country", "tooltip": true, "type": "markers" } @@ -416,10 +618,237 @@ } } ], - "title": "Panel Title", + "title": "Public IP connections", "type": "geomap" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 0, + "y": 26 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND instance = '$instance'\nGROUP BY bucket\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by hour", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 34 + }, + "id": 7, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "vertical", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.2.0", + "repeat": "user", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND instance = '$instance' AND common_name like $user \n AND (bytes_sent + bytes_received) > 0\nGROUP BY bucket\nORDER BY bucket", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by user: $user", + "type": "barchart" } ], + "refresh": "", "schemaVersion": 39, "tags": [], "templating": { @@ -476,17 +905,52 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Local Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "%", + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "definition": "SELECT common_name as __value FROM ovpnrw_connections WHERE unit_id = $unitid", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": false, + "name": "user", + "options": [], + "query": "SELECT common_name as __value FROM ovpnrw_connections WHERE unit_id = $unitid", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, "time": { - "from": "now-24h", + "from": "now-30d", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "VPN", "uid": "fdxlb58zxvpj4f", - "version": 4, + "version": 30, "weekStart": "" } \ No newline at end of file From 35379fcf1d0ca10a343b743f87cabcdb32869bb2 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 17 Sep 2024 10:52:03 +0200 Subject: [PATCH 21/42] fix: api, set GIN_MODE --- imageroot/systemd/user/api.service | 1 + 1 file changed, 1 insertion(+) diff --git a/imageroot/systemd/user/api.service b/imageroot/systemd/user/api.service index ac0b20d..f2b9125 100644 --- a/imageroot/systemd/user/api.service +++ b/imageroot/systemd/user/api.service @@ -26,6 +26,7 @@ ExecStart=/usr/bin/podman run \ --env-file=%S/state/promtail.env \ --env-file=%S/state/subscription.env \ --env-file=%S/state/db.env \ + --env 'GIN_MODE=release' \ ${NETHSECURITY_API_IMAGE} ExecStop=/usr/bin/podman stop --ignore --cidfile %t/api.ctr-id -t 10 ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/api.ctr-id From 6d24dd518f7cc00ac02a3abb9cfe685f6787c500 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 17 Sep 2024 11:51:03 +0200 Subject: [PATCH 22/42] grafana: rename malware dashboard to security --- imageroot/etc/dashboards/{malware.json => security.json} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename imageroot/etc/dashboards/{malware.json => security.json} (99%) diff --git a/imageroot/etc/dashboards/malware.json b/imageroot/etc/dashboards/security.json similarity index 99% rename from imageroot/etc/dashboards/malware.json rename to imageroot/etc/dashboards/security.json index f611dee..ed96705 100644 --- a/imageroot/etc/dashboards/malware.json +++ b/imageroot/etc/dashboards/security.json @@ -1314,13 +1314,13 @@ ] }, "time": { - "from": "now-6h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "utc", - "title": "Malware", + "title": "Security", "uid": "dd395331-5dc0-4172-b243-8646c0ca3ccd", - "version": 11, + "version": 12, "weekStart": "" } \ No newline at end of file From cee4ad28f97d1b1f80a959c4523be934bf9f8c29 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 17 Sep 2024 12:09:54 +0200 Subject: [PATCH 23/42] grafana: improve traffic_by_client --- .../dashboards/network_traffic_by_client.json | 318 +++++++++++++++++- 1 file changed, 304 insertions(+), 14 deletions(-) diff --git a/imageroot/etc/dashboards/network_traffic_by_client.json b/imageroot/etc/dashboards/network_traffic_by_client.json index e616d27..f879f26 100644 --- a/imageroot/etc/dashboards/network_traffic_by_client.json +++ b/imageroot/etc/dashboards/network_traffic_by_client.json @@ -404,7 +404,7 @@ "id": 3, "options": { "barRadius": 0, - "barWidth": 0.97, + "barWidth": 0.89, "fullHighlight": false, "groupWidth": 0.7, "legend": { @@ -433,7 +433,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT application, SUM(bytes) FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY application", + "rawSql": "SELECT REPLACE(application, 'netify.', '') as application, SUM(bytes) AS BYTES FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY application\nORDER BY bytes desc\nLIMIT 10", "refId": "A", "sql": { "columns": [ @@ -454,9 +454,98 @@ } } ], - "title": "Traffic by application", + "title": "Traffic by application (top 10)", "type": "barchart" }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT REPLACE(application, 'netify.', '') as application, SUM(bytes) as bytes FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY application\nORDER BY bytes DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by application", + "type": "table" + }, { "datasource": { "default": false, @@ -510,8 +599,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 9 + "x": 0, + "y": 17 }, "id": 7, "options": { @@ -566,7 +655,7 @@ } } ], - "title": "Traffic by protocol", + "title": "Traffic by protocol (top 10)", "type": "barchart" }, { @@ -577,6 +666,9 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "custom": { "align": "auto", "cellOptions": { @@ -598,6 +690,109 @@ } ] }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 9, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT protocol, SUM(bytes) as bytes FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY protocol\nORDER BY bytes DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by protocol", + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "bytes" }, "overrides": [] @@ -606,7 +801,102 @@ "h": 12, "w": 12, "x": 0, - "y": 17 + "y": 25 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.88, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT host, SUM(bytes) as bytes FROM dpi_stats \nWHERE host!='' AND $__timeFilter(time) AND unit_id = $unitid AND client_address like '$client_address'\nGROUP BY host\nORDER BY bytes DESC\nLIMIT 10", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic by host (top 10)", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 25 }, "id": 1, "options": { @@ -652,7 +942,7 @@ } } ], - "title": "Visited host", + "title": "Traffic by host", "type": "table" } ], @@ -692,19 +982,19 @@ { "current": { "selected": false, - "text": "", - "value": "" + "text": "192.168.5.28", + "value": "192.168.5.28" }, "hide": 0, "name": "client_address", "options": [ { - "selected": true, + "selected": false, "text": "", "value": "" } ], - "query": "", + "query": "192.168.5.28", "skipUrlSync": false, "type": "textbox" }, @@ -724,13 +1014,13 @@ ] }, "time": { - "from": "now-6h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Network traffic by client", "uid": "b14a1181-a2ee-4df4-a732-888e0190037f", - "version": 10, + "version": 11, "weekStart": "" } \ No newline at end of file From e3147aa18f33ae52c13189ff21de955a8950e8ff Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 17 Sep 2024 12:20:28 +0200 Subject: [PATCH 24/42] grafana: traffic dashboard, add host link --- imageroot/etc/dashboards/network_traffic.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/imageroot/etc/dashboards/network_traffic.json b/imageroot/etc/dashboards/network_traffic.json index 4301890..b39feb6 100644 --- a/imageroot/etc/dashboards/network_traffic.json +++ b/imageroot/etc/dashboards/network_traffic.json @@ -769,7 +769,7 @@ { "matcher": { "id": "byName", - "options": "client_address" + "options": "host" }, "properties": [ { @@ -777,7 +777,7 @@ "value": [ { "title": "", - "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}" + "url": "/grafana/d/c8d43e5f-068e-4cac-9283-2318d9e1911b/network-traffic-by-host?${__all_variables}&var-host=${__value.raw}" } ] } @@ -887,13 +887,13 @@ ] }, "time": { - "from": "now-12h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Network traffic", "uid": "fe0af3cb-9be0-4b2a-8ccb-86704956cf2e", - "version": 11, + "version": 13, "weekStart": "" } \ No newline at end of file From aaa632b749dcafb0d09d022e6054742a8daa71a4 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 17 Sep 2024 15:58:24 +0200 Subject: [PATCH 25/42] feat(ui): added maxmind token field and tooltip for metrics --- .../configure-module/validate-input.json | 8 +++++++- ui/public/i18n/en/translation.json | 7 +++++-- ui/src/views/Settings.vue | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/imageroot/actions/configure-module/validate-input.json b/imageroot/actions/configure-module/validate-input.json index ca55a2c..2aabc86 100644 --- a/imageroot/actions/configure-module/validate-input.json +++ b/imageroot/actions/configure-module/validate-input.json @@ -13,7 +13,8 @@ "ovpn_netmask": "255.255.0.0", "ovpn_cn": "nethsec", "loki_retention": 180, - "prometheus_retention": 15 + "prometheus_retention": 15, + "maxmind_license": "1234567890" } ], "type": "object", @@ -63,6 +64,11 @@ "type": "integer", "description": "Retention policy for Prometehus metrics, default is 15 days", "minimum": 1 + }, + "maxmind_license": { + "type": "string", + "description": "MaxMind API key, required for GeoIP database updates", + "minLength": 1 } } } diff --git a/ui/public/i18n/en/translation.json b/ui/public/i18n/en/translation.json index c7ca787..26c1818 100644 --- a/ui/public/i18n/en/translation.json +++ b/ui/public/i18n/en/translation.json @@ -38,7 +38,8 @@ "loki_retention": "Logs retention", "loki_retention_helper": "Days to keep logs in Loki", "prometheus_retention": "Metrics retention", - "prometheus_retention_helper": "Days to keep metrics in Prometheus", + "prometheus_retention_helper": "Days to keep metrics in Prometheus and Timescale", + "prometheus_retention_tooltip": "Timescale is a time-series database used to store metrics. It is used by the controller to store and aggregate additional metrics on top of the metrics provided by Prometheus.", "cn_helper": "The name of the controller, used to create the VPN certification authority", "network_helper": "The VPN network used to connect the units to the controller.", "user_helper": "This is the only user that can create and manage other users inside the controller", @@ -53,7 +54,9 @@ "password_placeholder": "Password can be modified from the controller webapp", "password_information_title": "Default password", "password_information_description": "The default administrator password is displayed only once, please store it in a safe place", - "vpn_port_description": "Units will connect to controller using a VPN connection. Make sure the units can reach the controller on UDP port {port}" + "vpn_port_description": "Units will connect to controller using a VPN connection. Make sure the units can reach the controller on UDP port {port}", + "maxmind_license": "MaxMind license key", + "maxmind_license_helper": "The license key is required to use the MaxMind GeoIP2 database" }, "about": { "title": "About" diff --git a/ui/src/views/Settings.vue b/ui/src/views/Settings.vue index 674514a..e2f2189 100644 --- a/ui/src/views/Settings.vue +++ b/ui/src/views/Settings.vue @@ -96,6 +96,19 @@ :invalid-message="$t(error.prometheus_retention)" type="number" :label="$t('settings.prometheus_retention')" :helper-text="$t('settings.prometheus_retention_helper')" :disabled="loading.configureModule"> + + + @@ -151,6 +164,7 @@ export default { firstConfig: true, loki_retention: "180", prometheus_retention: "15", + maxmind_license: "", passwordPlaceholder: "", loading: { getConfiguration: false, @@ -168,6 +182,7 @@ export default { cn: "", loki_retention: "", prometheus_retention: "", + maxmind_license: "", }, }; }, @@ -249,6 +264,7 @@ export default { this.password = config.api_password; this.loki_retention = config.loki_retention.toString(); this.prometheus_retention = config.prometheus_retention.toString(); + this.maxmind_license = config.maxmind_license; this.vpn_port = config.vpn_port; this.focusElement("host"); @@ -386,6 +402,7 @@ export default { api_user: this.user, loki_retention: parseInt(this.loki_retention), prometheus_retention: parseInt(this.prometheus_retention), + maxmind_license: this.maxmind_license, }; if (this.password) { params.api_password = this.password; From 4e0260d03eaf3f04b97056137b78826102d52ab5 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 17 Sep 2024 12:45:32 +0200 Subject: [PATCH 26/42] grafana: add connectivity dashboard --- imageroot/etc/dashboards/connectivity.json | 498 +++++++++++++++++++++ 1 file changed, 498 insertions(+) create mode 100644 imageroot/etc/dashboards/connectivity.json diff --git a/imageroot/etc/dashboards/connectivity.json b/imageroot/etc/dashboards/connectivity.json new file mode 100644 index 0000000..1b079ab --- /dev/null +++ b/imageroot/etc/dashboards/connectivity.json @@ -0,0 +1,498 @@ +{ + "__inputs": [ + { + "name": "DS_LOCAL_TIMESCALE", + "label": "Local Timescale", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT distinct(wan) as wan, interface FROM ca_mwan_events_hourly\nWHERE $__timeFilter(bucket) AND unit_id = $unitid\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "MultiWAN list", + "type": "table" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 5, + "y": 0 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT \n time,\n wan,\n interface\nFROM mwan_events\nWHERE event = 'offline' AND $__timeFilter(time) AND unit_id = $unitid \nORDER BY time DESC\nLIMIT 3;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Last 3 downtime events", + "type": "table" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "offline": { + "color": "dark-red", + "index": 0, + "text": "Offline" + }, + "online": { + "color": "dark-green", + "index": 1, + "text": "Online" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 11, + "x": 13, + "y": 0 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT DISTINCT ON (unit_id, wan, interface)\n wan,\n interface,\n event AS last_known_status,\n time AS last_update_time\nFROM mwan_events\nWHERE $__timeFilter(time) AND unit_id = $unitid \nORDER BY unit_id, wan, interface, time DESC;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "MultiWAN status", + "type": "table" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, count(*) as count FROM ca_mwan_events_hourly\nWHERE $__timeFilter(bucket) AND unit_id = $unitid\nGROUP by bucket", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "MultiWAN events by hour", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "DS_TIMESCALE", + "options": [], + "query": "grafana-postgresql-datasource", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "hide": 0, + "includeAll": false, + "label": "Unit", + "multi": false, + "name": "unitid", + "options": [], + "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Connectivity", + "uid": "ady5vjxjqywowd", + "version": 7, + "weekStart": "" +} \ No newline at end of file From 15ba2ac60ca1551fd40948f20238c4fbe0e4e6a7 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 18 Sep 2024 09:33:34 +0200 Subject: [PATCH 27/42] grafana: update dashboard queries Use uuid instead unit_id to identify the units. --- imageroot/etc/dashboards/connectivity.json | 21 +- imageroot/etc/dashboards/network_traffic.json | 426 ++++++++++++- .../dashboards/network_traffic_by_client.json | 60 +- .../dashboards/network_traffic_by_host.json | 42 +- imageroot/etc/dashboards/security.json | 573 +++++++++++++----- imageroot/etc/dashboards/vpn.json | 49 +- 6 files changed, 894 insertions(+), 277 deletions(-) diff --git a/imageroot/etc/dashboards/connectivity.json b/imageroot/etc/dashboards/connectivity.json index 1b079ab..da3eb7f 100644 --- a/imageroot/etc/dashboards/connectivity.json +++ b/imageroot/etc/dashboards/connectivity.json @@ -121,7 +121,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT distinct(wan) as wan, interface FROM ca_mwan_events_hourly\nWHERE $__timeFilter(bucket) AND unit_id = $unitid\n", + "rawSql": "SELECT distinct(wan) as wan, interface FROM ca_mwan_events_hourly\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\n", "refId": "A", "sql": { "columns": [ @@ -208,7 +208,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \n time,\n wan,\n interface\nFROM mwan_events\nWHERE event = 'offline' AND $__timeFilter(time) AND unit_id = $unitid \nORDER BY time DESC\nLIMIT 3;", + "rawSql": "SELECT \n time,\n wan,\n interface\nFROM mwan_events\nWHERE event = 'offline' AND $__timeFilter(time) AND uuid = '$uuid'\nORDER BY time DESC\nLIMIT 3;", "refId": "A", "sql": { "columns": [ @@ -308,7 +308,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT DISTINCT ON (unit_id, wan, interface)\n wan,\n interface,\n event AS last_known_status,\n time AS last_update_time\nFROM mwan_events\nWHERE $__timeFilter(time) AND unit_id = $unitid \nORDER BY unit_id, wan, interface, time DESC;\n", + "rawSql": "SELECT DISTINCT ON (uuid, wan, interface)\n wan,\n interface,\n event AS last_known_status,\n time AS last_update_time\nFROM mwan_events\nWHERE $__timeFilter(time) AND uuid = '$uuid'\nORDER BY uuid, wan, interface, time DESC;\n", "refId": "A", "sql": { "columns": [ @@ -421,7 +421,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, count(*) as count FROM ca_mwan_events_hourly\nWHERE $__timeFilter(bucket) AND unit_id = $unitid\nGROUP by bucket", + "rawSql": "SELECT bucket, count(*) as count FROM ca_mwan_events_hourly\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP by bucket", "refId": "A", "sql": { "columns": [ @@ -452,8 +452,9 @@ "list": [ { "current": {}, - "hide": 0, + "hide": 2, "includeAll": false, + "label": "Local Timescale", "multi": false, "name": "DS_TIMESCALE", "options": [], @@ -469,14 +470,14 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "definition": "SELECT uuid AS __value, name AS __text FROM units", "hide": 0, "includeAll": false, "label": "Unit", "multi": false, - "name": "unitid", + "name": "uuid", "options": [], - "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "query": "SELECT uuid AS __value, name AS __text FROM units", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -486,13 +487,13 @@ ] }, "time": { - "from": "now-7d", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Connectivity", "uid": "ady5vjxjqywowd", - "version": 7, + "version": 10, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic.json b/imageroot/etc/dashboards/network_traffic.json index b39feb6..60a088a 100644 --- a/imageroot/etc/dashboards/network_traffic.json +++ b/imageroot/etc/dashboards/network_traffic.json @@ -134,7 +134,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT sum(bytes) FROM ca_dpi_stats_hourly_bytes WHERE unit_id=$unitid ", + "rawSql": "SELECT sum(bytes) FROM ca_dpi_stats_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'", "refId": "A", "sql": { "columns": [ @@ -246,7 +246,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, bytes FROM ca_dpi_stats_hourly_bytes \nWHERE $__timeFilter(bucket) and unit_id =$unitid\nGROUP BY bucket, bytes", + "rawSql": "SELECT bucket, bytes FROM ca_dpi_stats_hourly_bytes \nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY bucket, bytes", "refId": "A", "sql": { "columns": [ @@ -334,7 +334,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT replace(application, 'netify.', '') AS application, sum(bytes) FROM ca_dpi_stats_hourly_application\nWHERE unit_id=$unitid\nGROUP BY application", + "rawSql": "SELECT\n CASE\n WHEN application_rank <= 10 THEN application\n ELSE 'Other'\n END AS application,\n SUM(bytes) AS total_bytes\nFROM (\n SELECT\n replace(application, 'netify.', '') AS application,\n SUM(bytes) AS bytes,\n ROW_NUMBER() OVER (ORDER BY SUM(bytes) DESC) AS application_rank\n FROM\n ca_dpi_stats_hourly_application\n WHERE\n $__timeFilter(bucket) AND uuid = '$uuid'\n GROUP BY\n application\n) AS ranked_applications\nGROUP BY\n CASE\n WHEN application_rank <= 10 THEN application\n ELSE 'Other'\n END\nORDER BY\n total_bytes DESC;\n", "refId": "A", "sql": { "columns": [ @@ -355,9 +355,101 @@ } } ], - "title": "By application", + "title": "Top 10 applications", "type": "piechart" }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 8 + }, + "id": 16, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT replace(application, 'netify.', '') AS application, sum(bytes) as bytes FROM ca_dpi_stats_hourly_application\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY application\nORDER BY bytes DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "By application", + "type": "table" + }, { "datasource": { "default": false, @@ -385,10 +477,10 @@ "gridPos": { "h": 8, "w": 6, - "x": 6, + "x": 12, "y": 8 }, - "id": 12, + "id": 13, "options": { "displayLabels": [], "legend": { @@ -422,7 +514,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT protocol, sum(bytes) as bytes FROM ca_dpi_stats_hourly_protocol\nWHERE unit_id=$unitid\nGROUP BY protocol", + "rawSql": "SELECT client_name, sum(bytes) FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY client_name", "refId": "A", "sql": { "columns": [ @@ -443,9 +535,101 @@ } } ], - "title": "By protocol", + "title": "By client", "type": "piechart" }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT client_name, sum(bytes) as bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY client_name\nORDER BY bytes DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "By client", + "type": "table" + }, { "datasource": { "default": false, @@ -473,10 +657,10 @@ "gridPos": { "h": 8, "w": 6, - "x": 12, - "y": 8 + "x": 0, + "y": 16 }, - "id": 13, + "id": 12, "options": { "displayLabels": [], "legend": { @@ -510,7 +694,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT client_name, sum(bytes) FROM ca_dpi_stats_hourly_client\nWHERE unit_id=$unitid\nGROUP BY client_name", + "rawSql": "SELECT\n CASE\n WHEN protocol_rank <= 10 THEN protocol\n ELSE 'Other'\n END AS protocol,\n SUM(bytes) AS total_bytes\nFROM (\n SELECT\n protocol,\n SUM(bytes) AS bytes,\n ROW_NUMBER() OVER (ORDER BY SUM(bytes) DESC) AS protocol_rank\n FROM\n ca_dpi_stats_hourly_protocol\n WHERE\n $__timeFilter(bucket) AND uuid = '$uuid'\n GROUP BY\n protocol\n) AS ranked_protocols\nGROUP BY\n CASE\n WHEN protocol_rank <= 10 THEN protocol\n ELSE 'Other'\n END\nORDER BY\n total_bytes DESC;\n", "refId": "A", "sql": { "columns": [ @@ -531,9 +715,101 @@ } } ], - "title": "By client", + "title": "Top 10 protocols", "type": "piechart" }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 18, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT protocol, sum(bytes) as bytes FROM ca_dpi_stats_hourly_protocol\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY protocol\nORDER BY bytes DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "By application", + "type": "table" + }, { "datasource": { "default": false, @@ -561,8 +837,8 @@ "gridPos": { "h": 8, "w": 6, - "x": 18, - "y": 8 + "x": 12, + "y": 16 }, "id": 14, "options": { @@ -598,7 +874,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT host, sum(bytes) FROM ca_dpi_stats_hourly_host\nWHERE unit_id=$unitid\nGROUP BY host", + "rawSql": "SELECT\n CASE\n WHEN host_rank <= 10 THEN host\n ELSE 'Other'\n END AS host,\n SUM(bytes) AS total_bytes\nFROM (\n SELECT\n host,\n SUM(bytes) AS bytes,\n ROW_NUMBER() OVER (ORDER BY SUM(bytes) DESC) AS host_rank\n FROM\n ca_dpi_stats_hourly_host\n WHERE\n $__timeFilter(bucket) AND uuid = '$uuid'\n GROUP BY\n host\n) AS ranked_hosts\nGROUP BY\n CASE\n WHEN host_rank <= 10 THEN host\n ELSE 'Other'\n END\nORDER BY\n total_bytes DESC;\n", "refId": "A", "sql": { "columns": [ @@ -619,7 +895,7 @@ } } ], - "title": "By host", + "title": "Top 10 hosts", "type": "piechart" }, { @@ -628,6 +904,7 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -638,9 +915,9 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, - "links": [], "mappings": [], "thresholds": { "mode": "absolute", @@ -655,6 +932,96 @@ } ] }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT host, sum(bytes) as bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY host\nORDER BY bytes DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "By host", + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "bytes" }, "overrides": [ @@ -681,7 +1048,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 24 }, "id": 5, "options": { @@ -706,7 +1073,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT client_address, client_name, bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND unit_id=$unitid\nORDER BY bytes DESC\nLIMIT 20", + "rawSql": "SELECT client_address, client_name, bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nORDER BY bytes DESC\nLIMIT 20", "refId": "A", "sql": { "columns": [ @@ -754,8 +1121,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -789,7 +1155,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 24 }, "id": 15, "options": { @@ -814,7 +1180,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT host, bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND unit_id=$unitid\nORDER BY bytes DESC\nLIMIT 20", + "rawSql": "SELECT host, bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nORDER BY bytes DESC\nLIMIT 20", "refId": "A", "sql": { "columns": [ @@ -844,8 +1210,7 @@ "tags": [ "unit", "timescale", - "monitoring", - "traffic" + "monitoring" ], "templating": { "list": [ @@ -855,15 +1220,15 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "definition": "SELECT uuid AS __value, name AS __text FROM units", "description": "", "hide": 0, "includeAll": false, "label": "Unit", "multi": false, - "name": "unitid", + "name": "uuid", "options": [], - "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "query": "SELECT uuid AS __value, name AS __text FROM units", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -874,6 +1239,7 @@ "current": {}, "hide": 2, "includeAll": false, + "label": "Local Timescale", "multi": false, "name": "DS_TIMESCALE", "options": [], @@ -894,6 +1260,6 @@ "timezone": "", "title": "Network traffic", "uid": "fe0af3cb-9be0-4b2a-8ccb-86704956cf2e", - "version": 13, + "version": 19, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_client.json b/imageroot/etc/dashboards/network_traffic_by_client.json index f879f26..c11a06b 100644 --- a/imageroot/etc/dashboards/network_traffic_by_client.json +++ b/imageroot/etc/dashboards/network_traffic_by_client.json @@ -127,7 +127,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT client_name \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND client_address = '$client_address'", + "rawSql": "SELECT client_name \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND client_address = '$client_address'", "refId": "A", "sql": { "columns": [ @@ -238,7 +238,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, bytes \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND client_address = '$client_address'", + "rawSql": "SELECT bucket, bytes \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND client_address = '$client_address'", "refId": "A", "sql": { "columns": [ @@ -321,7 +321,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bytes \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND client_address = '$client_address'", + "rawSql": "SELECT bytes \nFROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND client_address = '$client_address'", "refId": "A", "sql": { "columns": [ @@ -433,7 +433,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT REPLACE(application, 'netify.', '') as application, SUM(bytes) AS BYTES FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY application\nORDER BY bytes desc\nLIMIT 10", + "rawSql": "SELECT REPLACE(application, 'netify.', '') as application, SUM(bytes) AS BYTES FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY application\nORDER BY bytes desc\nLIMIT 10", "refId": "A", "sql": { "columns": [ @@ -473,6 +473,7 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, "mappings": [], @@ -504,11 +505,12 @@ "cellHeight": "sm", "footer": { "countRows": false, + "enablePagination": true, "fields": "", "reducer": [ "sum" ], - "show": false + "show": true }, "showHeader": true }, @@ -522,7 +524,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT REPLACE(application, 'netify.', '') as application, SUM(bytes) as bytes FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY application\nORDER BY bytes DESC", + "rawSql": "SELECT REPLACE(application, 'netify.', '') as application, SUM(bytes) as bytes FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY application\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -634,7 +636,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT protocol, SUM(bytes) FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY protocol", + "rawSql": "SELECT protocol, SUM(bytes) FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY protocol", "refId": "A", "sql": { "columns": [ @@ -674,6 +676,7 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, "mappings": [], @@ -705,11 +708,12 @@ "cellHeight": "sm", "footer": { "countRows": false, + "enablePagination": true, "fields": "", "reducer": [ "sum" ], - "show": false + "show": true }, "showHeader": true }, @@ -723,7 +727,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT protocol, SUM(bytes) as bytes FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND unit_id='$unitid'\nGROUP BY protocol\nORDER BY bytes DESC", + "rawSql": "SELECT protocol, SUM(bytes) as bytes FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY protocol\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -784,8 +788,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -835,7 +838,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT host, SUM(bytes) as bytes FROM dpi_stats \nWHERE host!='' AND $__timeFilter(time) AND unit_id = $unitid AND client_address like '$client_address'\nGROUP BY host\nORDER BY bytes DESC\nLIMIT 10", + "rawSql": "SELECT host, SUM(bytes) as bytes FROM dpi_stats \nWHERE host!='' AND $__timeFilter(time) AND uuid = '$uuid' AND client_address like '$client_address'\nGROUP BY host\nORDER BY bytes DESC\nLIMIT 10", "refId": "A", "sql": { "columns": [ @@ -872,6 +875,7 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, "mappings": [], @@ -879,8 +883,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -903,11 +906,12 @@ "cellHeight": "sm", "footer": { "countRows": false, + "enablePagination": true, "fields": "", "reducer": [ "sum" ], - "show": false + "show": true }, "showHeader": true }, @@ -921,7 +925,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT host, SUM(bytes) as bytes FROM dpi_stats \nWHERE host!='' AND $__timeFilter(time) AND unit_id = $unitid AND client_address like '$client_address'\nGROUP BY host\nORDER BY bytes DESC", + "rawSql": "SELECT host, SUM(bytes) as bytes FROM dpi_stats \nWHERE host!='' AND $__timeFilter(time) AND uuid = '$uuid' AND client_address like '$client_address'\nGROUP BY host\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -950,11 +954,8 @@ "schemaVersion": 39, "tags": [ "monitoring", - "traffic", "unit", - "timescale", - "client", - "details" + "timescale" ], "templating": { "list": [ @@ -964,15 +965,15 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "definition": "SELECT uuid AS __value, name AS __text FROM units", "description": "", "hide": 0, "includeAll": false, "label": "Unit", "multi": false, - "name": "unitid", + "name": "uuid", "options": [], - "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "query": "SELECT uuid AS __value, name AS __text FROM units", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -982,19 +983,19 @@ { "current": { "selected": false, - "text": "192.168.5.28", - "value": "192.168.5.28" + "text": "", + "value": "" }, "hide": 0, "name": "client_address", "options": [ { - "selected": false, + "selected": true, "text": "", "value": "" } ], - "query": "192.168.5.28", + "query": "", "skipUrlSync": false, "type": "textbox" }, @@ -1002,6 +1003,7 @@ "current": {}, "hide": 2, "includeAll": false, + "label": "Local Timescale", "multi": false, "name": "DS_TIMESCALE", "options": [], @@ -1014,13 +1016,13 @@ ] }, "time": { - "from": "now-24h", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Network traffic by client", "uid": "b14a1181-a2ee-4df4-a732-888e0190037f", - "version": 11, + "version": 17, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_host.json b/imageroot/etc/dashboards/network_traffic_by_host.json index a3d6e7f..53d630e 100644 --- a/imageroot/etc/dashboards/network_traffic_by_host.json +++ b/imageroot/etc/dashboards/network_traffic_by_host.json @@ -124,7 +124,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND host like '$host'", + "rawSql": "SELECT bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND host like '$host'", "refId": "A", "sql": { "columns": [ @@ -157,7 +157,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -236,7 +236,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, sum(bytes) as bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND host like '$host'\nGROUP BY bucket, bytes\nORDER BY bucket", + "rawSql": "SELECT bucket, sum(bytes) as bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND host like '$host'\nGROUP BY bucket\nORDER BY bucket", "refId": "A", "sql": { "columns": [ @@ -348,7 +348,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n time_bucket('30m', time) AS time,\n client_address as client,\n SUM(bytes) AS value\nFROM dpi_stats\nWHERE $__timeFilter(time) AND host LIKE '$host' AND unit_id='$unitid'\nGROUP BY time, client\nORDER BY time", + "rawSql": "SELECT\n time_bucket('30m', time) AS time,\n client_address as client,\n SUM(bytes) AS value\nFROM dpi_stats\nWHERE $__timeFilter(time) AND host LIKE '$host' AND uuid = '$uuid'\nGROUP BY time, client\nORDER BY time", "refId": "A", "sql": { "columns": [ @@ -398,6 +398,7 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, "mappings": [], @@ -429,11 +430,12 @@ "cellHeight": "sm", "footer": { "countRows": false, + "enablePagination": true, "fields": "", "reducer": [ "sum" ], - "show": false + "show": true }, "showHeader": true }, @@ -447,7 +449,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT client_address, client_name, SUM(bytes) as bytes FROM dpi_stats \nWHERE $__timeFilter(time) AND unit_id = '$unitid' AND host LIKE '$host'\nGROUP BY client_address, client_name\nORDER BY bytes DESC", + "rawSql": "SELECT client_address, client_name, SUM(bytes) as bytes FROM dpi_stats \nWHERE $__timeFilter(time) AND uuid = '$uuid' AND host LIKE '$host'\nGROUP BY client_address, client_name\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -477,8 +479,7 @@ "tags": [ "monitoring", "timescale", - "traffic", - "host" + "unit" ], "templating": { "list": [ @@ -488,14 +489,14 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "definition": "SELECT uuid AS __value, name AS __text FROM units", "hide": 0, "includeAll": false, "label": "Unit", "multi": false, - "name": "unitid", + "name": "uuid", "options": [], - "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "query": "SELECT uuid AS __value, name AS __text FROM units", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -505,13 +506,19 @@ { "current": { "selected": false, - "text": "pod-031-2010-09.backblaze.com", - "value": "pod-031-2010-09.backblaze.com" + "text": "%", + "value": "%" }, "hide": 0, "name": "host", - "options": [], - "query": "", + "options": [ + { + "selected": true, + "text": "%", + "value": "%" + } + ], + "query": "%", "skipUrlSync": false, "type": "textbox" }, @@ -519,6 +526,7 @@ "current": {}, "hide": 2, "includeAll": false, + "label": "Local Timescale", "multi": false, "name": "DS_TIMESCALE", "options": [], @@ -531,13 +539,13 @@ ] }, "time": { - "from": "now-6h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Network traffic by host", "uid": "c8d43e5f-068e-4cac-9283-2318d9e1911b", - "version": 9, + "version": 15, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/security.json b/imageroot/etc/dashboards/security.json index ed96705..7051aa4 100644 --- a/imageroot/etc/dashboards/security.json +++ b/imageroot/etc/dashboards/security.json @@ -47,6 +47,18 @@ "name": "Stat", "version": "" }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, { "type": "panel", "id": "timeseries", @@ -79,10 +91,34 @@ "panels": [ { "datasource": { - "default": false, "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, + "description": "", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Threats\n\nBlocked IPs based on loaded blocklist categories.", + "mode": "markdown" + }, + "pluginVersion": "11.2.0", + "type": "text" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, "fieldConfig": { "defaults": { "color": { @@ -109,7 +145,7 @@ "h": 8, "w": 5, "x": 0, - "y": 0 + "y": 3 }, "id": 3, "options": { @@ -134,12 +170,12 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" + "uid": "${DS_TIMESCALE}" }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND unit_id = $unitid ", + "rawSql": "SELECT count(*) FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'", "refId": "A", "sql": { "columns": [ @@ -165,9 +201,8 @@ }, { "datasource": { - "default": false, "type": "grafana-postgresql-datasource", - "uid": "${DS_TIMESCALE}" + "uid": "${DS_LOCAL_TIMESCALE}" }, "fieldConfig": { "defaults": { @@ -216,7 +251,7 @@ "h": 8, "w": 7, "x": 5, - "y": 0 + "y": 3 }, "id": 6, "options": { @@ -245,12 +280,12 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" + "uid": "${DS_TIMESCALE}" }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT src, count(*) AS count FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by src\nORDER by count DESC\nLIMIT 10", + "rawSql": "SELECT src, count(*) AS count FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP by src\nORDER by count DESC\nLIMIT 10", "refId": "A", "sql": { "columns": [ @@ -276,9 +311,8 @@ }, { "datasource": { - "default": false, "type": "grafana-postgresql-datasource", - "uid": "${DS_TIMESCALE}" + "uid": "${DS_LOCAL_TIMESCALE}" }, "fieldConfig": { "defaults": { @@ -300,7 +334,7 @@ "h": 8, "w": 6, "x": 12, - "y": 0 + "y": 3 }, "id": 4, "options": { @@ -327,12 +361,12 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" + "uid": "${DS_TIMESCALE}" }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT chain, SUM(count) FROM ca_ts_malware_hourly_chain\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by chain", + "rawSql": "SELECT chain, SUM(count) FROM ca_ts_malware_hourly_chain\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP by chain", "refId": "A", "sql": { "columns": [ @@ -358,9 +392,8 @@ }, { "datasource": { - "default": false, "type": "grafana-postgresql-datasource", - "uid": "${DS_TIMESCALE}" + "uid": "${DS_LOCAL_TIMESCALE}" }, "fieldConfig": { "defaults": { @@ -382,7 +415,7 @@ "h": 8, "w": 6, "x": 18, - "y": 0 + "y": 3 }, "id": 8, "options": { @@ -409,12 +442,12 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" + "uid": "${DS_TIMESCALE}" }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT category, SUM(count) FROM ca_ts_malware_hourly_category\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by category", + "rawSql": "SELECT category, SUM(count) FROM ca_ts_malware_hourly_category\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP by category", "refId": "A", "sql": { "columns": [ @@ -440,9 +473,8 @@ }, { "datasource": { - "default": false, "type": "grafana-postgresql-datasource", - "uid": "${DS_TIMESCALE}" + "uid": "${DS_LOCAL_TIMESCALE}" }, "fieldConfig": { "defaults": { @@ -476,9 +508,9 @@ }, "gridPos": { "h": 13, - "w": 10, + "w": 12, "x": 0, - "y": 8 + "y": 11 }, "id": 1, "options": { @@ -537,7 +569,7 @@ "lookup": "lookup", "mode": "auto" }, - "name": "Layer 1", + "name": "Blocked", "tooltip": true, "type": "markers" } @@ -558,12 +590,12 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" + "uid": "${DS_TIMESCALE}" }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) as count, country AS lookup FROM ca_ts_malware_hourly_country\nWHERE $__timeFilter(bucket) AND unit_id = $unitid \nGROUP BY country", + "rawSql": "SELECT count(*) as count, country AS lookup FROM ca_ts_malware_hourly_country\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY country", "refId": "A", "sql": { "columns": [ @@ -589,9 +621,8 @@ }, { "datasource": { - "default": false, "type": "grafana-postgresql-datasource", - "uid": "${DS_TIMESCALE}" + "uid": "${DS_LOCAL_TIMESCALE}" }, "fieldConfig": { "defaults": { @@ -650,9 +681,9 @@ }, "gridPos": { "h": 13, - "w": 14, - "x": 10, - "y": 8 + "w": 12, + "x": 12, + "y": 11 }, "id": 2, "options": { @@ -671,12 +702,12 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" + "uid": "${DS_TIMESCALE}" }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, count(*) FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by bucket", + "rawSql": "SELECT bucket, count(*) FROM ca_ts_malware_hourly_direction\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP by bucket", "refId": "A", "sql": { "columns": [ @@ -700,6 +731,31 @@ "title": "Blocked threats by time", "type": "timeseries" }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "description": "", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 15, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Attacks\n\nBlocked IPs based on brute force attacks.", + "mode": "markdown" + }, + "pluginVersion": "11.2.0", + "type": "text" + }, { "datasource": { "default": false, @@ -709,76 +765,49 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { "h": 8, - "w": 10, + "w": 6, "x": 0, - "y": 21 + "y": 27 }, - "id": 7, + "id": 9, "options": { - "barRadius": 0, - "barWidth": 0.8, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.3.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -788,7 +817,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT country, count FROM ca_ts_malware_hourly_country\nWHERE $__timeFilter(bucket) AND unit_id = $unitid \nGROUP BY country, count\nORDER BY count DESC\nLIMIT 20", + "rawSql": "SELECT count(*) FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND uuid = '$uuid'", "refId": "A", "sql": { "columns": [ @@ -809,8 +838,8 @@ } } ], - "title": "Top 20 blocked threats by countries", - "type": "barchart" + "title": "Blocked attacks", + "type": "stat" }, { "datasource": { @@ -821,50 +850,75 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 5, - "x": 0, - "y": 29 + "w": 9, + "x": 6, + "y": 27 }, - "id": 9, + "id": 7, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "barRadius": 0, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, - "pluginVersion": "11.2.0", + "pluginVersion": "10.3.3", "targets": [ { "datasource": { @@ -874,7 +928,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND unit_id = $unitid ", + "rawSql": "SELECT country, SUM(count) as count FROM ca_ts_attacks_hourly\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND country != ''\nGROUP BY country\nORDER BY count DESC\nLIMIT 20", "refId": "A", "sql": { "columns": [ @@ -895,8 +949,8 @@ } } ], - "title": "Blocked attacks", - "type": "stat" + "title": "Top 20 blocked attacks by countries", + "type": "barchart" }, { "datasource": { @@ -915,29 +969,17 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, "thresholdsStyle": { "mode": "off" } @@ -947,37 +989,47 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 18, - "x": 5, - "y": 29 + "w": 9, + "x": 15, + "y": 27 }, - "id": 12, + "id": 16, "options": { + "barRadius": 0, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom", + "placement": "right", "showLegend": true }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - } + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, + "pluginVersion": "10.3.3", "targets": [ { "datasource": { @@ -987,7 +1039,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, count(*) FROM ca_ts_attacks_hourly\nWHERE $__timeFilter(bucket) AND $unitid = $unitid\nGROUP by bucket", + "rawSql": "SELECT ip, count(*) as count FROM ts_attacks\nWHERE $__timeFilter(time) AND uuid = '$uuid'\nGROUP BY ip\nORDER BY count DESC\nLIMIT 20", "refId": "A", "sql": { "columns": [ @@ -1008,8 +1060,8 @@ } } ], - "title": "Blocked attacks by hour", - "type": "timeseries" + "title": "Top 20 blocked attacks by IP", + "type": "barchart" }, { "datasource": { @@ -1035,8 +1087,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1049,9 +1100,9 @@ }, "gridPos": { "h": 13, - "w": 10, + "w": 12, "x": 0, - "y": 37 + "y": 35 }, "id": 11, "options": { @@ -1110,7 +1161,7 @@ "lookup": "lookup", "mode": "auto" }, - "name": "Layer 1", + "name": "Blocked", "tooltip": true, "type": "markers" } @@ -1136,7 +1187,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) as count, country AS lookup FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND unit_id = $unitid \nGROUP BY country", + "rawSql": "SELECT count(*) as count, country AS lookup FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY country", "refId": "A", "sql": { "columns": [ @@ -1177,17 +1228,29 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "fillOpacity": 80, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, "thresholdsStyle": { "mode": "off" } @@ -1197,8 +1260,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1210,34 +1272,114 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 7, - "x": 10, - "y": 37 + "h": 13, + "w": 12, + "x": 12, + "y": 35 }, - "id": 10, + "id": 12, "options": { - "barRadius": 0, - "barWidth": 0.7, - "fullHighlight": false, - "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", "tooltip": { "mode": "single", "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT bucket, count(*) FROM ca_ts_attacks_hourly\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP by bucket", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked attacks by hour", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "overrides": [] }, - "pluginVersion": "10.3.3", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 13, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -1247,7 +1389,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ip, count(*) AS count FROM ts_attacks \nWHERE $__timeFilter(time) AND $unitid = $unitid\nGROUP by ip\nORDER by count DESC\nLIMIT 10", + "rawSql": "SELECT country, count(*) FROM ca_ts_malware_hourly_country\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY country\nORDER BY count DESC", "refId": "A", "sql": { "columns": [ @@ -1268,13 +1410,107 @@ } } ], - "title": "Top 10 blocked attacks by IPs", - "type": "barchart" + "title": "Blocked by countries", + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ip, count(*) as count FROM ts_attacks\nWHERE $__timeFilter(time) AND uuid = '$uuid'\nGROUP BY ip\nORDER BY count DESC\nLIMIT 20", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Blocked by IP", + "type": "table" } ], "refresh": "", "schemaVersion": 39, - "tags": [], + "tags": [ + "monitoring", + "unit", + "timescale" + ], "templating": { "list": [ { @@ -1283,15 +1519,15 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "definition": "SELECT uuid AS __value, name AS __text FROM units", "description": "", "hide": 0, "includeAll": false, "label": "Unit", "multi": false, - "name": "unitid", + "name": "uuid", "options": [], - "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "query": "SELECT uuid AS __value, name AS __text FROM units", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1302,6 +1538,7 @@ "current": {}, "hide": 2, "includeAll": false, + "label": "Local Timescale", "multi": false, "name": "DS_TIMESCALE", "options": [], @@ -1318,9 +1555,9 @@ "to": "now" }, "timepicker": {}, - "timezone": "utc", + "timezone": "", "title": "Security", "uid": "dd395331-5dc0-4172-b243-8646c0ca3ccd", - "version": 12, + "version": 21, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/vpn.json b/imageroot/etc/dashboards/vpn.json index 1bafd8d..577cfe0 100644 --- a/imageroot/etc/dashboards/vpn.json +++ b/imageroot/etc/dashboards/vpn.json @@ -147,7 +147,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND unit_id = $unitid AND instance = '$instance'", + "rawSql": "SELECT count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = '$instance'", "refId": "A", "sql": { "columns": [ @@ -260,7 +260,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT time, count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND unit_id = $unitid AND instance = '$instance'\nGROUP BY time", + "rawSql": "SELECT time, count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = '$instance'\nGROUP BY time", "refId": "A", "sql": { "columns": [ @@ -343,7 +343,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND instance = '$instance'\n", + "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = '$instance'\n", "refId": "A", "sql": { "columns": [ @@ -451,7 +451,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes, common_name as user FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND instance = '$instance'\nGROUP BY common_name\nORDER by bytes DESC\nLIMIT 20", + "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes, common_name as user FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = '$instance'\nGROUP BY common_name\nORDER by bytes DESC\nLIMIT 20", "refId": "A", "sql": { "columns": [ @@ -597,7 +597,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) AS count, country AS lookup FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND unit_id = $unitid AND instance = '$instance'\nGROUP BY lookup", + "rawSql": "SELECT count(*) AS count, country AS lookup FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = '$instance'\nGROUP BY lookup", "refId": "A", "sql": { "columns": [ @@ -670,8 +670,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -712,7 +711,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND instance = '$instance'\nGROUP BY bucket\n", + "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = '$instance'\nGROUP BY bucket\n", "refId": "A", "sql": { "columns": [ @@ -774,8 +773,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -785,7 +783,7 @@ }, "gridPos": { "h": 8, - "w": 6, + "w": 24, "x": 0, "y": 34 }, @@ -823,7 +821,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND unit_id = $unitid AND instance = '$instance' AND common_name like $user \n AND (bytes_sent + bytes_received) > 0\nGROUP BY bucket\nORDER BY bucket", + "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = '$instance' AND common_name like $user\n AND (bytes_sent + bytes_received) > 0\nGROUP BY bucket\nORDER BY bucket", "refId": "A", "sql": { "columns": [ @@ -850,13 +848,18 @@ ], "refresh": "", "schemaVersion": 39, - "tags": [], + "tags": [ + "timescale", + "unit", + "monitoring" + ], "templating": { "list": [ { "current": {}, "hide": 2, "includeAll": false, + "label": "Local Timescale", "multi": false, "name": "DS_TIMESCALE", "options": [], @@ -872,14 +875,14 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "definition": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "definition": "SELECT uuid AS __value, name AS __text FROM units", "hide": 0, "includeAll": false, "label": "Unit", "multi": false, - "name": "unitid", + "name": "uuid", "options": [], - "query": "SELECT id AS __value, CONCAT(name,' (',uuid,')') AS __text FROM units", + "query": "SELECT uuid AS __value, name AS __text FROM units", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -892,14 +895,14 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_LOCAL_TIMESCALE}" }, - "definition": "SELECT instance FROM ovpnrw_connections WHERE unit_id = $unitid", + "definition": "SELECT instance FROM ovpnrw_connections WHERE uuid = '$uuid'", "description": "", "hide": 0, "includeAll": false, "multi": false, "name": "instance", "options": [], - "query": "SELECT instance FROM ovpnrw_connections WHERE unit_id = $unitid", + "query": "SELECT instance FROM ovpnrw_connections WHERE uuid = '$uuid'", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -908,7 +911,7 @@ }, { "current": {}, - "hide": 0, + "hide": 2, "includeAll": false, "label": "Local Prometheus", "multi": false, @@ -927,14 +930,14 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "definition": "SELECT common_name as __value FROM ovpnrw_connections WHERE unit_id = $unitid", + "definition": "SELECT common_name as __value FROM ovpnrw_connections WHERE uuid = '$uuid'", "hide": 0, "includeAll": true, "label": "User", "multi": false, "name": "user", "options": [], - "query": "SELECT common_name as __value FROM ovpnrw_connections WHERE unit_id = $unitid", + "query": "SELECT common_name as __value FROM ovpnrw_connections WHERE uuid = '$uuid'", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -944,13 +947,13 @@ ] }, "time": { - "from": "now-30d", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "VPN", "uid": "fdxlb58zxvpj4f", - "version": 30, + "version": 37, "weekStart": "" } \ No newline at end of file From 49523a5d2488c7b6709961cc5427cd6a58a695ce Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 18 Sep 2024 12:00:28 +0200 Subject: [PATCH 28/42] grafana: fix VPN in unit dashboard --- imageroot/etc/dashboards/nethsecurity.json | 288 +++++++++++---------- 1 file changed, 157 insertions(+), 131 deletions(-) diff --git a/imageroot/etc/dashboards/nethsecurity.json b/imageroot/etc/dashboards/nethsecurity.json index 911dc29..9b16490 100644 --- a/imageroot/etc/dashboards/nethsecurity.json +++ b/imageroot/etc/dashboards/nethsecurity.json @@ -1,4 +1,41 @@ { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Local Promethus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph (old)", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -26,7 +63,7 @@ "fiscalYearStartMonth": 0, "gnetId": 7107, "graphTooltip": 0, - "id": 4, + "id": null, "links": [], "liveNow": false, "panels": [ @@ -814,7 +851,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "netdata_net_net_kilobits_persec_average{unit=~\"$unit\",dimension=\"received\"}", @@ -911,7 +948,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "abs(netdata_net_net_kilobits_persec_average{unit=~\"$unit\",dimension=\"sent\"})", @@ -1008,7 +1045,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "netdata_net_packets_packets_persec_average{unit=~\"$unit\", dimension=~\"received\"}", @@ -1105,7 +1142,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "abs(netdata_net_packets_packets_persec_average{unit=~\"$unit\", dimension=~\"sent\"})", @@ -4011,7 +4048,7 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -4019,110 +4056,111 @@ "y": 28 }, "id": 1465, - "panels": [], - "title": "VPN networking", - "type": "row" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "panels": [ + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 29 + }, + "id": 1466, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "repeat": "vpninterface", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 8, - "x": 0, - "y": 29 - }, - "id": 1466, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "repeat": "vpninterface", - "repeatDirection": "h", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "netdata_net_net_kilobits_persec_average{unit=~\"$unit\",chart=~\"$vpninterface\"}", - "instant": true, - "legendFormat": " {{dimension}}", - "range": false, - "refId": "A" + "editorMode": "code", + "exemplar": false, + "expr": "netdata_net_net_kilobits_persec_average{unit=~\"$unit\",chart=~\"$vpninterface\"}", + "instant": true, + "legendFormat": " {{dimension}}", + "range": false, + "refId": "A" + } + ], + "title": "VPN interface: $vpninterface", + "type": "timeseries" } ], - "title": "VPN interface: $vpninterface", - "type": "timeseries" + "title": "VPN networking", + "type": "row" }, { "collapsed": true, @@ -4134,7 +4172,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 29 }, "id": 264, "panels": [ @@ -4266,7 +4304,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 30 }, "id": 285, "panels": [ @@ -4419,7 +4457,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 31 }, "id": 308, "panels": [ @@ -4552,7 +4590,7 @@ "h": 1, "w": 24, "x": 0, - "y": 43 + "y": 32 }, "id": 364, "panels": [ @@ -5427,7 +5465,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 33 }, "id": 391, "panels": [ @@ -5895,7 +5933,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 34 }, "id": 416, "panels": [ @@ -6259,7 +6297,7 @@ "h": 1, "w": 24, "x": 0, - "y": 46 + "y": 35 }, "id": 523, "panels": [ @@ -6495,7 +6533,7 @@ "h": 1, "w": 24, "x": 0, - "y": 47 + "y": 36 }, "id": 567, "panels": [ @@ -6628,7 +6666,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 37 }, "id": 1106, "panels": [ @@ -6907,7 +6945,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 38 }, "id": 1174, "panels": [ @@ -7039,7 +7077,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 39 }, "id": 1245, "panels": [ @@ -7382,7 +7420,7 @@ "h": 1, "w": 24, "x": 0, - "y": 51 + "y": 40 }, "id": 1381, "panels": [ @@ -7700,7 +7738,7 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 41 }, "id": 1460, "panels": [ @@ -8465,11 +8503,7 @@ "templating": { "list": [ { - "current": { - "selected": false, - "text": "181ff9b0-e2be-4c71-897a-6575e3991a1c", - "value": "181ff9b0-e2be-4c71-897a-6575e3991a1c" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -8493,11 +8527,7 @@ "type": "query" }, { - "current": { - "selected": false, - "text": "Local Promethus", - "value": "prometheus" - }, + "current": {}, "description": "Prometheus datasource", "hide": 2, "includeAll": false, @@ -8513,11 +8543,7 @@ }, { "allValue": "", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -8574,6 +8600,6 @@ "timezone": "", "title": "Unit metrics", "uid": "W3S__804z", - "version": 11, + "version": 12, "weekStart": "" } From 89e8404f8654f704e342e930dd5f304ee5ad59d2 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 18 Sep 2024 12:22:17 +0200 Subject: [PATCH 29/42] grafana: vpn, add chart from prometheus --- imageroot/etc/dashboards/vpn.json | 323 ++++++++++++++++++++++++++++-- 1 file changed, 304 insertions(+), 19 deletions(-) diff --git a/imageroot/etc/dashboards/vpn.json b/imageroot/etc/dashboards/vpn.json index 577cfe0..917f93c 100644 --- a/imageroot/etc/dashboards/vpn.json +++ b/imageroot/etc/dashboards/vpn.json @@ -7,6 +7,14 @@ "type": "datasource", "pluginId": "grafana-postgresql-datasource", "pluginName": "PostgreSQL" + }, + { + "name": "DS_LOCAL_PROMETHUS", + "label": "Local Promethus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" } ], "__elements": {}, @@ -35,6 +43,12 @@ "name": "PostgreSQL", "version": "1.0.0" }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, { "type": "panel", "id": "stat", @@ -82,7 +96,7 @@ "panels": [], "repeat": "instance", "repeatDirection": "h", - "title": "OpenVPN Road Warrior instance: $instance", + "title": "OpenVPN Road Warrior instance: $rwinstance", "type": "row" }, { @@ -147,7 +161,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = '$instance'", + "rawSql": "SELECT count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = $rwinstance", "refId": "A", "sql": { "columns": [ @@ -260,7 +274,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT time, count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = '$instance'\nGROUP BY time", + "rawSql": "SELECT time, count(*) FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = $rwinstance\nGROUP BY time", "refId": "A", "sql": { "columns": [ @@ -343,7 +357,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = '$instance'\n", + "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = $rwinstance\n", "refId": "A", "sql": { "columns": [ @@ -451,7 +465,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes, common_name as user FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = '$instance'\nGROUP BY common_name\nORDER by bytes DESC\nLIMIT 20", + "rawSql": "SELECT SUM(bytes_sent) + SUM(bytes_received) as bytes, common_name as user FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = $rwinstance\nGROUP BY common_name\nORDER by bytes DESC\nLIMIT 20", "refId": "A", "sql": { "columns": [ @@ -475,6 +489,104 @@ "title": "Top 20 user by traffic", "type": "barchart" }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 0, + "y": 17 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_LOCAL_PROMETHUS}" + }, + "editorMode": "code", + "expr": "netdata_net_net_kilobits_persec_average{unit=~'$uuid',chart=~'net.$rwinterface'}", + "instant": false, + "legendFormat": "{{dimension}}", + "range": true, + "refId": "A" + } + ], + "title": "Interface $rwinterface traffic", + "type": "timeseries" + }, { "datasource": { "default": false, @@ -514,7 +626,7 @@ "h": 9, "w": 19, "x": 0, - "y": 17 + "y": 25 }, "id": 4, "options": { @@ -597,7 +709,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) AS count, country AS lookup FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = '$instance'\nGROUP BY lookup", + "rawSql": "SELECT count(*) AS count, country AS lookup FROM ovpnrw_connections\nWHERE $__timeFilter(time) AND uuid = '$uuid' AND instance = $rwinstance\nGROUP BY lookup", "refId": "A", "sql": { "columns": [ @@ -670,7 +782,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -686,7 +799,7 @@ "h": 8, "w": 19, "x": 0, - "y": 26 + "y": 34 }, "id": 14, "options": { @@ -711,7 +824,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = '$instance'\nGROUP BY bucket\n", + "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = $rwinstance\nGROUP BY bucket\n", "refId": "A", "sql": { "columns": [ @@ -773,7 +886,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -783,9 +897,9 @@ }, "gridPos": { "h": 8, - "w": 24, + "w": 8, "x": 0, - "y": 34 + "y": 42 }, "id": 7, "options": { @@ -821,7 +935,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = '$instance' AND common_name like $user\n AND (bytes_sent + bytes_received) > 0\nGROUP BY bucket\nORDER BY bucket", + "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = $rwinstance AND common_name like $user\n AND (bytes_sent + bytes_received) > 0\nGROUP BY bucket\nORDER BY bucket", "refId": "A", "sql": { "columns": [ @@ -844,6 +958,117 @@ ], "title": "Traffic by user: $user", "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 17, + "panels": [], + "title": "OpenVPN tunnels", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 0, + "y": 51 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_LOCAL_PROMETHUS}" + }, + "editorMode": "code", + "expr": "netdata_net_net_kilobits_persec_average{unit=~'$uuid',chart=~'net.$tuninterface'}", + "instant": false, + "legendFormat": "{{dimension}}", + "range": true, + "refId": "A" + } + ], + "title": "Interface $tuninterface traffic", + "type": "timeseries" } ], "refresh": "", @@ -895,14 +1120,15 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_LOCAL_TIMESCALE}" }, - "definition": "SELECT instance FROM ovpnrw_connections WHERE uuid = '$uuid'", + "definition": "SELECT instance AS __value, CONCAT(name,' (',instance,')') AS __text FROM openvpn_config WHERE uuid = '$uuid' AND type = 'rw'", "description": "", "hide": 0, - "includeAll": false, + "includeAll": true, + "label": "Road Warrior server", "multi": false, - "name": "instance", + "name": "rwinstance", "options": [], - "query": "SELECT instance FROM ovpnrw_connections WHERE uuid = '$uuid'", + "query": "SELECT instance AS __value, CONCAT(name,' (',instance,')') AS __text FROM openvpn_config WHERE uuid = '$uuid' AND type = 'rw'", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -943,6 +1169,65 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "definition": "SELECT device AS __value FROM openvpn_config WHERE uuid = '$uuid' AND instance = $rwinstance", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "rwinterface", + "options": [], + "query": "SELECT device AS __value FROM openvpn_config WHERE uuid = '$uuid' AND instance = $rwinstance", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "definition": "SELECT instance AS __value, CONCAT(name,' (',instance,')') AS __text FROM openvpn_config WHERE uuid = '$uuid' AND type IN ('server', 'client')", + "description": "", + "hide": 0, + "includeAll": true, + "label": "Tunnels", + "multi": false, + "name": "tunnel", + "options": [], + "query": "SELECT instance AS __value, CONCAT(name,' (',instance,')') AS __text FROM openvpn_config WHERE uuid = '$uuid' AND type IN ('server', 'client')", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "definition": "SELECT device AS __value FROM openvpn_config WHERE uuid = '$uuid' AND instance = '$tunnel'", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "tuninterface", + "options": [], + "query": "SELECT device AS __value FROM openvpn_config WHERE uuid = '$uuid' AND instance = '$tunnel'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, @@ -954,6 +1239,6 @@ "timezone": "browser", "title": "VPN", "uid": "fdxlb58zxvpj4f", - "version": 37, + "version": 56, "weekStart": "" } \ No newline at end of file From a4b6c295205466f7f8f6fee39a87848b3dc200be Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 18 Sep 2024 12:37:02 +0200 Subject: [PATCH 30/42] grafana: add promethus chart to connectivity --- imageroot/etc/dashboards/connectivity.json | 451 ++++++++++++++++++++- 1 file changed, 447 insertions(+), 4 deletions(-) diff --git a/imageroot/etc/dashboards/connectivity.json b/imageroot/etc/dashboards/connectivity.json index da3eb7f..41b5340 100644 --- a/imageroot/etc/dashboards/connectivity.json +++ b/imageroot/etc/dashboards/connectivity.json @@ -7,6 +7,14 @@ "type": "datasource", "pluginId": "grafana-postgresql-datasource", "pluginName": "PostgreSQL" + }, + { + "name": "DS_LOCAL_PROMETHUS", + "label": "Local Promethus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" } ], "__elements": {}, @@ -23,6 +31,12 @@ "name": "PostgreSQL", "version": "1.0.0" }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, { "type": "panel", "id": "table", @@ -121,7 +135,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT distinct(wan) as wan, interface FROM ca_mwan_events_hourly\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\n", + "rawSql": "SELECT interface, device FROM wan_config\nWHERE uuid = '$uuid'\n", "refId": "A", "sql": { "columns": [ @@ -444,10 +458,405 @@ ], "title": "MultiWAN events by hour", "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "repeat": "wan", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_LOCAL_PROMETHUS}" + }, + "editorMode": "code", + "expr": "netdata_net_net_kilobits_persec_average{unit=~'$uuid',chart='net.$wan'}", + "instant": false, + "legendFormat": "{{dimension}}", + "range": true, + "refId": "A" + } + ], + "title": "Interface $wan traffic", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_LOCAL_PROMETHUS}" + }, + "editorMode": "code", + "expr": "netdata_fping_latency_ms_average{unit=~'$uuid'}", + "instant": false, + "legendFormat": "{{dimension}}-{{family}}", + "range": true, + "refId": "A" + } + ], + "title": "Ping latency", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_LOCAL_PROMETHUS}" + }, + "editorMode": "code", + "expr": "netdata_fping_packets_packets_average{unit=~'$uuid'}", + "instant": false, + "legendFormat": "{{dimension}}-{{family}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_LOCAL_PROMETHUS}" + }, + "editorMode": "code", + "expr": "netdata_fping_quality_percentage_average{unit=~'$uuid'}", + "instant": false, + "legendFormat": "{{dimension}}-{{family}}", + "range": true, + "refId": "A" + } + ], + "title": "Quality", + "type": "timeseries" } ], "schemaVersion": 39, - "tags": [], + "tags": [ + "timescale", + "unit", + "monitoring" + ], "templating": { "list": [ { @@ -483,17 +892,51 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "current": {}, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "definition": "SELECT device AS __value, interface AS __text FROM wan_config WHERE uuid = '$uuid'", + "hide": 0, + "includeAll": true, + "label": "WAN interface", + "multi": false, + "name": "wan", + "options": [], + "query": "SELECT device AS __value, interface AS __text FROM wan_config WHERE uuid = '$uuid'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, "time": { - "from": "now-6h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Connectivity", "uid": "ady5vjxjqywowd", - "version": 10, + "version": 20, "weekStart": "" } \ No newline at end of file From cd04ba51b9286b0ebc1710fc4a92d1ac47838a52 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 18 Sep 2024 12:57:40 +0200 Subject: [PATCH 31/42] grafana: connectivity, add total traffic chart --- imageroot/etc/dashboards/connectivity.json | 87 ++++++++++++++++++++-- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/imageroot/etc/dashboards/connectivity.json b/imageroot/etc/dashboards/connectivity.json index 41b5340..735174e 100644 --- a/imageroot/etc/dashboards/connectivity.json +++ b/imageroot/etc/dashboards/connectivity.json @@ -37,6 +37,12 @@ "name": "Prometheus", "version": "1.0.0" }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, { "type": "panel", "id": "table", @@ -459,6 +465,72 @@ "title": "MultiWAN events by hour", "type": "timeseries" }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Kbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "repeat": "wan", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_LOCAL_PROMETHUS}" + }, + "editorMode": "code", + "expr": "sum(abs(netdata_net_net_kilobits_persec_average{unit=~'$uuid',chart='net.$wan'}))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total traffic: $wan", + "type": "stat" + }, { "datasource": { "default": false, @@ -523,9 +595,9 @@ }, "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 15 + "y": 23 }, "id": 5, "options": { @@ -624,7 +696,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 31 }, "id": 7, "options": { @@ -721,7 +793,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 31 }, "id": 8, "options": { @@ -802,8 +874,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -818,7 +889,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 39 }, "id": 9, "options": { @@ -937,6 +1008,6 @@ "timezone": "browser", "title": "Connectivity", "uid": "ady5vjxjqywowd", - "version": 20, + "version": 24, "weekStart": "" } \ No newline at end of file From bad7c4f519020f00afeee1c936dab990c94c65cb Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 19 Sep 2024 10:04:48 +0200 Subject: [PATCH 32/42] fix(grafana): using best effort method to format dates in dashboards --- imageroot/actions/configure-module/20configure | 1 + 1 file changed, 1 insertion(+) diff --git a/imageroot/actions/configure-module/20configure b/imageroot/actions/configure-module/20configure index a23d271..ddafec6 100755 --- a/imageroot/actions/configure-module/20configure +++ b/imageroot/actions/configure-module/20configure @@ -144,6 +144,7 @@ with open('grafana.env', 'w') as gfp: gfp.write("GF_SERVER_HTTP_ADDR=127.0.0.1\n") gfp.write(f'GF_SECURITY_ADMIN_USER={request["api_user"]}\n') gfp.write(f'GF_SECURITY_ADMIN_PASSWORD={request.get("api_password", config["api_password"])}\n') + gfp.write('GF_DATE_FORMATS_USE_BROWSER_LOCALE=true\n') with open('prometheus.env', 'w') as pfp: pfp.write(f"PROMETHEUS_PORT={ports[7]}\n") From 3db9d51dfa9f290ff3b83913eaea67e555ee9b16 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 19 Sep 2024 14:05:45 +0200 Subject: [PATCH 33/42] feat(grafana): added time range links for network traffic --- imageroot/etc/dashboards/network_traffic.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imageroot/etc/dashboards/network_traffic.json b/imageroot/etc/dashboards/network_traffic.json index 60a088a..55137ef 100644 --- a/imageroot/etc/dashboards/network_traffic.json +++ b/imageroot/etc/dashboards/network_traffic.json @@ -1036,7 +1036,7 @@ "value": [ { "title": "", - "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}" + "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}&${__url_time_range}" } ] } @@ -1143,7 +1143,7 @@ "value": [ { "title": "", - "url": "/grafana/d/c8d43e5f-068e-4cac-9283-2318d9e1911b/network-traffic-by-host?${__all_variables}&var-host=${__value.raw}" + "url": "/grafana/d/c8d43e5f-068e-4cac-9283-2318d9e1911b/network-traffic-by-host?${__all_variables}&var-host=${__value.raw}&${__url_time_range}" } ] } From c57b874c26830c21b80c2e5fbd6e259387e9bf6c Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 19 Sep 2024 15:01:40 +0200 Subject: [PATCH 34/42] grafana: add links to all dashboards --- imageroot/etc/dashboards/connectivity.json | 143 +++---- imageroot/etc/dashboards/logs.json | 104 +++-- imageroot/etc/dashboards/loki.json | 80 ++-- imageroot/etc/dashboards/nethsecurity.json | 91 ++-- imageroot/etc/dashboards/network_traffic.json | 397 ++++++++++-------- .../dashboards/network_traffic_by_client.json | 208 ++++----- .../dashboards/network_traffic_by_host.json | 57 +-- imageroot/etc/dashboards/security.json | 84 ++-- imageroot/etc/dashboards/vpn.json | 236 ++++++++--- 9 files changed, 812 insertions(+), 588 deletions(-) diff --git a/imageroot/etc/dashboards/connectivity.json b/imageroot/etc/dashboards/connectivity.json index 735174e..105a919 100644 --- a/imageroot/etc/dashboards/connectivity.json +++ b/imageroot/etc/dashboards/connectivity.json @@ -39,14 +39,14 @@ }, { "type": "panel", - "id": "stat", - "name": "Stat", + "id": "table", + "name": "Table", "version": "" }, { "type": "panel", - "id": "table", - "name": "Table", + "id": "text", + "name": "Text", "version": "" }, { @@ -76,7 +76,22 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, - "links": [], + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "unit" + ], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "panels": [ { "datasource": { @@ -228,7 +243,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \n time,\n wan,\n interface\nFROM mwan_events\nWHERE event = 'offline' AND $__timeFilter(time) AND uuid = '$uuid'\nORDER BY time DESC\nLIMIT 3;", + "rawSql": "SELECT \n time,\n wan,\n interface\nFROM mwan_events\nWHERE event = 'offline' AND uuid = '$uuid'\nORDER BY time DESC\nLIMIT 3;", "refId": "A", "sql": { "columns": [ @@ -328,7 +343,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT DISTINCT ON (uuid, wan, interface)\n wan,\n interface,\n event AS last_known_status,\n time AS last_update_time\nFROM mwan_events\nWHERE $__timeFilter(time) AND uuid = '$uuid'\nORDER BY uuid, wan, interface, time DESC;\n", + "rawSql": "SELECT DISTINCT ON (uuid, wan, interface)\n wan,\n interface,\n event AS last_known_status,\n time AS last_update_time\nFROM mwan_events\nWHERE uuid = '$uuid'\nORDER BY uuid, wan, interface, time DESC;\n", "refId": "A", "sql": { "columns": [ @@ -462,75 +477,9 @@ } } ], - "title": "MultiWAN events by hour", + "title": "MultiWAN events by time", "type": "timeseries" }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Kbits" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "repeat": "wan", - "repeatDirection": "h", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_LOCAL_PROMETHUS}" - }, - "editorMode": "code", - "expr": "sum(abs(netdata_net_net_kilobits_persec_average{unit=~'$uuid',chart='net.$wan'}))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Total traffic: $wan", - "type": "stat" - }, { "datasource": { "default": false, @@ -595,9 +544,9 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, - "y": 23 + "y": 15 }, "id": 5, "options": { @@ -633,10 +582,33 @@ }, { "datasource": { - "default": false, "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 10, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Line quality\n\nPing sent to configured hosts, the traffic\nmay use any interface.", + "mode": "markdown" + }, + "pluginVersion": "11.2.0", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_LOCAL_PROMETHUS}" + }, "fieldConfig": { "defaults": { "color": { @@ -680,8 +652,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -696,7 +667,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 26 }, "id": 7, "options": { @@ -777,8 +748,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -793,7 +763,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 26 }, "id": 8, "options": { @@ -889,7 +859,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 34 }, "id": 9, "options": { @@ -922,6 +892,7 @@ "type": "timeseries" } ], + "refresh": "", "schemaVersion": 39, "tags": [ "timescale", @@ -1001,13 +972,13 @@ ] }, "time": { - "from": "now-24h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Connectivity", "uid": "ady5vjxjqywowd", - "version": 24, + "version": 31, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/logs.json b/imageroot/etc/dashboards/logs.json index b9fd02e..83a4a9a 100644 --- a/imageroot/etc/dashboards/logs.json +++ b/imageroot/etc/dashboards/logs.json @@ -1,4 +1,41 @@ { + "__inputs": [ + { + "name": "DS_LOCAL_LOKI", + "label": "Local Loki", + "description": "", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "panel", + "id": "logs", + "name": "Logs", + "version": "" + }, + { + "type": "datasource", + "id": "loki", + "name": "Loki", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -20,14 +57,29 @@ "fiscalYearStartMonth": 0, "gnetId": 13186, "graphTooltip": 0, - "id": 3, - "links": [], + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "logs" + ], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOCAL_LOKI}" }, "fieldConfig": { "defaults": { @@ -41,6 +93,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", @@ -81,8 +134,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -114,7 +166,7 @@ { "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOCAL_LOKI}" }, "editorMode": "code", "expr": "sum(count_over_time({hostname=~\"$hostname\", application=~\"$application\", level=~\"$level\"} |~ \"$search\"[$__interval]))", @@ -129,7 +181,7 @@ { "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOCAL_LOKI}" }, "gridPos": { "h": 25, @@ -153,7 +205,7 @@ { "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOCAL_LOKI}" }, "editorMode": "code", "expr": "{hostname=~\"$hostname\", application=~\"$application\", level=~\"$level\"} |~ \"$search\"", @@ -168,19 +220,17 @@ ], "refresh": "", "schemaVersion": 39, - "tags": [], + "tags": [ + "logs" + ], "templating": { "list": [ { "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOCAL_LOKI}" }, "definition": "", "hide": 0, @@ -206,14 +256,10 @@ }, { "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOCAL_LOKI}" }, "definition": "", "hide": 0, @@ -236,18 +282,10 @@ }, { "allValue": ".*", - "current": { - "selected": true, - "text": [ - "error" - ], - "value": [ - "error" - ] - }, + "current": {}, "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOCAL_LOKI}" }, "definition": "", "hide": 0, @@ -309,6 +347,6 @@ "timezone": "", "title": "Logs", "uid": "liz0yRCZz", - "version": 17, + "version": 4, "weekStart": "" -} +} \ No newline at end of file diff --git a/imageroot/etc/dashboards/loki.json b/imageroot/etc/dashboards/loki.json index 9308c13..22657c1 100644 --- a/imageroot/etc/dashboards/loki.json +++ b/imageroot/etc/dashboards/loki.json @@ -21,8 +21,23 @@ "fiscalYearStartMonth": 0, "gnetId": 13407, "graphTooltip": 0, - "id": 4, - "links": [], + "id": 2, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "logs" + ], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { @@ -48,7 +63,7 @@ "content": "
\n \n Loki Global Metrics\n
", "mode": "html" }, - "pluginVersion": "10.3.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -85,8 +100,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -103,6 +117,7 @@ "graphMode": "none", "justifyMode": "center", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" @@ -114,7 +129,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.3.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -151,8 +166,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -169,6 +183,7 @@ "graphMode": "none", "justifyMode": "center", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" @@ -180,7 +195,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.3.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -217,8 +232,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -235,6 +249,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" @@ -246,7 +261,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.3.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -283,8 +298,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -301,6 +315,7 @@ "graphMode": "none", "justifyMode": "center", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" @@ -312,7 +327,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.3.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -346,6 +361,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -386,8 +402,7 @@ } ] }, - "unit": "s", - "unitScale": true + "unit": "s" }, "overrides": [] }, @@ -446,6 +461,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 50, "gradientMode": "none", @@ -486,8 +502,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -561,6 +576,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -601,8 +617,7 @@ } ] }, - "unit": "Bps", - "unitScale": true + "unit": "Bps" }, "overrides": [] }, @@ -661,6 +676,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -702,8 +718,7 @@ } ] }, - "unit": "s", - "unitScale": true + "unit": "s" }, "overrides": [] }, @@ -1017,6 +1032,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1056,8 +1072,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1145,6 +1160,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1184,8 +1200,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1243,6 +1258,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1282,8 +1298,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1436,7 +1451,8 @@ "schemaVersion": 39, "tags": [ "loki", - "prometheus" + "prometheus", + "logs" ], "templating": { "list": [] @@ -1462,6 +1478,6 @@ "timezone": "", "title": "Loki metrics", "uid": "MQHVDmtWk", - "version": 12, + "version": 2, "weekStart": "" -} +} \ No newline at end of file diff --git a/imageroot/etc/dashboards/nethsecurity.json b/imageroot/etc/dashboards/nethsecurity.json index 9b16490..0b5fa61 100644 --- a/imageroot/etc/dashboards/nethsecurity.json +++ b/imageroot/etc/dashboards/nethsecurity.json @@ -1,41 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Local Promethus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.2.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph (old)", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], "annotations": { "list": [ { @@ -63,8 +26,23 @@ "fiscalYearStartMonth": 0, "gnetId": 7107, "graphTooltip": 0, - "id": null, - "links": [], + "id": 4, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "unit" + ], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { @@ -851,7 +829,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "editorMode": "code", "expr": "netdata_net_net_kilobits_persec_average{unit=~\"$unit\",dimension=\"received\"}", @@ -948,7 +926,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "editorMode": "code", "expr": "abs(netdata_net_net_kilobits_persec_average{unit=~\"$unit\",dimension=\"sent\"})", @@ -1045,7 +1023,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "editorMode": "code", "expr": "netdata_net_packets_packets_persec_average{unit=~\"$unit\", dimension=~\"received\"}", @@ -1142,7 +1120,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "editorMode": "code", "expr": "abs(netdata_net_packets_packets_persec_average{unit=~\"$unit\", dimension=~\"sent\"})", @@ -4107,8 +4085,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -8503,7 +8480,11 @@ "templating": { "list": [ { - "current": {}, + "current": { + "selected": false, + "text": "181ff9b0-e2be-4c71-897a-6575e3991a1c", + "value": "181ff9b0-e2be-4c71-897a-6575e3991a1c" + }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -8527,7 +8508,11 @@ "type": "query" }, { - "current": {}, + "current": { + "selected": false, + "text": "Local Promethus", + "value": "prometheus" + }, "description": "Prometheus datasource", "hide": 2, "includeAll": false, @@ -8543,7 +8528,11 @@ }, { "allValue": "", - "current": {}, + "current": { + "selected": false, + "text": "net.tun_dedalo", + "value": "net.tun_dedalo" + }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -8598,8 +8587,8 @@ ] }, "timezone": "", - "title": "Unit metrics", + "title": "Operating system", "uid": "W3S__804z", - "version": 12, + "version": 14, "weekStart": "" -} +} \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic.json b/imageroot/etc/dashboards/network_traffic.json index 55137ef..4d0303b 100644 --- a/imageroot/etc/dashboards/network_traffic.json +++ b/imageroot/etc/dashboards/network_traffic.json @@ -68,7 +68,22 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, - "links": [], + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "unit" + ], + "targetBlank": false, + "title": "Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { @@ -87,12 +102,8 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "#73BF69", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -227,14 +238,14 @@ "showLegend": true }, "orientation": "auto", - "showValue": "auto", + "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "11.2.0", "targets": [ @@ -295,8 +306,8 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 6, + "h": 10, + "w": 12, "x": 0, "y": 8 }, @@ -368,55 +379,51 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 6, - "x": 6, + "h": 10, + "w": 12, + "x": 12, "y": 8 }, - "id": 16, + "id": 13, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "show": true + "fields": "", + "limit": 10, + "values": true }, - "showHeader": true + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -426,7 +433,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT replace(application, 'netify.', '') AS application, sum(bytes) as bytes FROM ca_dpi_stats_hourly_application\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY application\nORDER BY bytes DESC", + "rawSql": "SELECT client_name, sum(bytes) FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY client_name", "refId": "A", "sql": { "columns": [ @@ -447,8 +454,8 @@ } } ], - "title": "By application", - "type": "table" + "title": "Top 10 clients", + "type": "piechart" }, { "datasource": { @@ -475,12 +482,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 8 + "h": 10, + "w": 12, + "x": 0, + "y": 18 }, - "id": 13, + "id": 12, "options": { "displayLabels": [], "legend": { @@ -514,7 +521,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT client_name, sum(bytes) FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY client_name", + "rawSql": "SELECT\n CASE\n WHEN protocol_rank <= 10 THEN protocol\n ELSE 'Other'\n END AS protocol,\n SUM(bytes) AS total_bytes\nFROM (\n SELECT\n protocol,\n SUM(bytes) AS bytes,\n ROW_NUMBER() OVER (ORDER BY SUM(bytes) DESC) AS protocol_rank\n FROM\n ca_dpi_stats_hourly_protocol\n WHERE\n $__timeFilter(bucket) AND uuid = '$uuid'\n GROUP BY\n protocol\n) AS ranked_protocols\nGROUP BY\n CASE\n WHEN protocol_rank <= 10 THEN protocol\n ELSE 'Other'\n END\nORDER BY\n total_bytes DESC;\n", "refId": "A", "sql": { "columns": [ @@ -535,7 +542,7 @@ } } ], - "title": "By client", + "title": "Top 10 protocols", "type": "piechart" }, { @@ -548,55 +555,51 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 8 + "h": 10, + "w": 12, + "x": 12, + "y": 18 }, - "id": 17, + "id": 14, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "show": true + "fields": "", + "limit": 10, + "values": true }, - "showHeader": true + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -606,7 +609,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT client_name, sum(bytes) as bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY client_name\nORDER BY bytes DESC", + "rawSql": "SELECT\n CASE\n WHEN host_rank <= 10 THEN host\n ELSE 'Other'\n END AS host,\n SUM(bytes) AS total_bytes\nFROM (\n SELECT\n host,\n SUM(bytes) AS bytes,\n ROW_NUMBER() OVER (ORDER BY SUM(bytes) DESC) AS host_rank\n FROM\n ca_dpi_stats_hourly_host\n WHERE\n $__timeFilter(bucket) AND uuid = '$uuid'\n GROUP BY\n host\n) AS ranked_hosts\nGROUP BY\n CASE\n WHEN host_rank <= 10 THEN host\n ELSE 'Other'\n END\nORDER BY\n total_bytes DESC;\n", "refId": "A", "sql": { "columns": [ @@ -627,8 +630,8 @@ } } ], - "title": "By client", - "type": "table" + "title": "Top 10 remote hosts", + "type": "piechart" }, { "datasource": { @@ -636,55 +639,76 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "description": "", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false }, + "links": [], "mappings": [], - "unit": "decbytes" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "client_address" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}" + } + ] + } + ] + } + ] }, "gridPos": { - "h": 8, - "w": 6, + "h": 15, + "w": 7, "x": 0, - "y": 16 + "y": 28 }, - "id": 12, + "id": 5, "options": { - "displayLabels": [], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, "fields": "", - "limit": 10, - "values": true + "reducer": [ + "sum" + ], + "show": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showHeader": true }, + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -694,7 +718,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN protocol_rank <= 10 THEN protocol\n ELSE 'Other'\n END AS protocol,\n SUM(bytes) AS total_bytes\nFROM (\n SELECT\n protocol,\n SUM(bytes) AS bytes,\n ROW_NUMBER() OVER (ORDER BY SUM(bytes) DESC) AS protocol_rank\n FROM\n ca_dpi_stats_hourly_protocol\n WHERE\n $__timeFilter(bucket) AND uuid = '$uuid'\n GROUP BY\n protocol\n) AS ranked_protocols\nGROUP BY\n CASE\n WHEN protocol_rank <= 10 THEN protocol\n ELSE 'Other'\n END\nORDER BY\n total_bytes DESC;\n", + "rawSql": "SELECT client_address, client_name, bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nORDER BY bytes DESC\n", "refId": "A", "sql": { "columns": [ @@ -715,8 +739,8 @@ } } ], - "title": "Top 10 protocols", - "type": "piechart" + "title": "Active clients", + "type": "table" }, { "datasource": { @@ -743,8 +767,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -757,12 +780,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 16 + "h": 15, + "w": 8, + "x": 7, + "y": 28 }, - "id": 18, + "id": 16, "options": { "cellHeight": "sm", "footer": { @@ -786,7 +809,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT protocol, sum(bytes) as bytes FROM ca_dpi_stats_hourly_protocol\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY protocol\nORDER BY bytes DESC", + "rawSql": "SELECT replace(application, 'netify.', '') AS application, sum(bytes) as bytes FROM ca_dpi_stats_hourly_application\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY application\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -820,16 +843,29 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "decbytes" }, "overrides": [] @@ -837,34 +873,24 @@ "gridPos": { "h": 8, "w": 6, - "x": 12, - "y": 16 + "x": 18, + "y": 28 }, - "id": 14, + "id": 17, "options": { - "displayLabels": [], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, "fields": "", - "limit": 10, - "values": true + "reducer": [ + "sum" + ], + "show": true }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showHeader": true }, + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -874,7 +900,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN host_rank <= 10 THEN host\n ELSE 'Other'\n END AS host,\n SUM(bytes) AS total_bytes\nFROM (\n SELECT\n host,\n SUM(bytes) AS bytes,\n ROW_NUMBER() OVER (ORDER BY SUM(bytes) DESC) AS host_rank\n FROM\n ca_dpi_stats_hourly_host\n WHERE\n $__timeFilter(bucket) AND uuid = '$uuid'\n GROUP BY\n host\n) AS ranked_hosts\nGROUP BY\n CASE\n WHEN host_rank <= 10 THEN host\n ELSE 'Other'\n END\nORDER BY\n total_bytes DESC;\n", + "rawSql": "SELECT client_name, sum(bytes) as bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY client_name\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -895,8 +921,8 @@ } } ], - "title": "Top 10 hosts", - "type": "piechart" + "title": "By client", + "type": "table" }, { "datasource": { @@ -923,8 +949,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -940,7 +965,7 @@ "h": 8, "w": 6, "x": 18, - "y": 16 + "y": 36 }, "id": 19, "options": { @@ -987,7 +1012,7 @@ } } ], - "title": "By host", + "title": "By remote host", "type": "table" }, { @@ -996,6 +1021,7 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1006,9 +1032,9 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, - "links": [], "mappings": [], "thresholds": { "mode": "absolute", @@ -1022,8 +1048,9 @@ } ] }, - "unit": "bytes" + "unit": "decbytes" }, +<<<<<<< HEAD "overrides": [ { "matcher": { @@ -1043,18 +1070,22 @@ ] } ] +======= + "overrides": [] +>>>>>>> a24fefd (grafana: add links to all dashboards) }, "gridPos": { - "h": 8, + "h": 13, "w": 12, "x": 0, - "y": 24 + "y": 43 }, - "id": 5, + "id": 18, "options": { "cellHeight": "sm", "footer": { "countRows": false, + "enablePagination": true, "fields": "", "reducer": [ "sum" @@ -1073,7 +1104,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT client_address, client_name, bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nORDER BY bytes DESC\nLIMIT 20", + "rawSql": "SELECT protocol, sum(bytes) as bytes FROM ca_dpi_stats_hourly_protocol\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY protocol\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -1094,7 +1125,7 @@ } } ], - "title": "Top 20 active clients", + "title": "By protocol", "type": "table" }, { @@ -1155,7 +1186,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 44 }, "id": 15, "options": { @@ -1201,7 +1232,7 @@ } } ], - "title": "Top 20 contacted hosts", + "title": "Top 20 contacted remote hosts", "type": "table" } ], @@ -1253,13 +1284,13 @@ ] }, "time": { - "from": "now-24h", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Network traffic", "uid": "fe0af3cb-9be0-4b2a-8ccb-86704956cf2e", - "version": 19, + "version": 28, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_client.json b/imageroot/etc/dashboards/network_traffic_by_client.json index c11a06b..97a816e 100644 --- a/imageroot/etc/dashboards/network_traffic_by_client.json +++ b/imageroot/etc/dashboards/network_traffic_by_client.json @@ -62,7 +62,22 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, - "links": [], + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "unit" + ], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { @@ -391,12 +406,12 @@ } ] }, - "unit": "short" + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 11, "w": 12, "x": 0, "y": 9 @@ -414,14 +429,14 @@ "showLegend": true }, "orientation": "horizontal", - "showValue": "never", + "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "10.3.3", "targets": [ @@ -466,15 +481,28 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "filterable": true, - "inspect": false + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -495,26 +523,34 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 11, "w": 12, "x": 12, "y": 9 }, - "id": 8, + "id": 7, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": true + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true }, - "showHeader": true + "orientation": "horizontal", + "showValue": "never", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 }, - "pluginVersion": "11.2.0", + "pluginVersion": "10.3.3", "targets": [ { "datasource": { @@ -524,7 +560,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT REPLACE(application, 'netify.', '') as application, SUM(bytes) as bytes FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY application\nORDER BY bytes DESC", + "rawSql": "SELECT protocol, SUM(bytes) FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY protocol", "refId": "A", "sql": { "columns": [ @@ -545,8 +581,8 @@ } } ], - "title": "Traffic by application", - "type": "table" + "title": "Traffic by protocol (top 10)", + "type": "barchart" }, { "datasource": { @@ -594,39 +630,39 @@ } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 12, + "w": 24, "x": 0, - "y": 17 + "y": 20 }, - "id": 7, + "id": 10, "options": { "barRadius": 0, - "barWidth": 0.97, + "barWidth": 0.88, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "orientation": "horizontal", - "showValue": "never", + "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, - "pluginVersion": "10.3.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -636,7 +672,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT protocol, SUM(bytes) FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY protocol", + "rawSql": "SELECT host, SUM(bytes) as bytes FROM dpi_stats \nWHERE host!='' AND $__timeFilter(time) AND uuid = '$uuid' AND client_address like '$client_address'\nGROUP BY host\nORDER BY bytes DESC\nLIMIT 10", "refId": "A", "sql": { "columns": [ @@ -657,7 +693,7 @@ } } ], - "title": "Traffic by protocol (top 10)", + "title": "Traffic by host (top 10)", "type": "barchart" }, { @@ -684,8 +720,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -698,12 +733,12 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 13, "w": 12, - "x": 12, - "y": 17 + "x": 0, + "y": 32 }, - "id": 9, + "id": 8, "options": { "cellHeight": "sm", "footer": { @@ -727,7 +762,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT protocol, SUM(bytes) as bytes FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY protocol\nORDER BY bytes DESC", + "rawSql": "SELECT REPLACE(application, 'netify.', '') as application, SUM(bytes) as bytes FROM dpi_stats \nWHERE application!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY application\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -748,7 +783,7 @@ } } ], - "title": "Traffic by protocol", + "title": "Traffic by application", "type": "table" }, { @@ -760,28 +795,15 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" + "align": "auto", + "cellOptions": { + "type": "auto" }, - "thresholdsStyle": { - "mode": "off" - } + "filterable": true, + "inspect": false }, "mappings": [], "thresholds": { @@ -796,37 +818,29 @@ } ] }, - "unit": "bytes" + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 12, + "h": 13, "w": 12, - "x": 0, - "y": 25 + "x": 12, + "y": 32 }, - "id": 10, + "id": 9, "options": { - "barRadius": 0, - "barWidth": 0.88, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "showHeader": true }, "pluginVersion": "11.2.0", "targets": [ @@ -838,7 +852,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT host, SUM(bytes) as bytes FROM dpi_stats \nWHERE host!='' AND $__timeFilter(time) AND uuid = '$uuid' AND client_address like '$client_address'\nGROUP BY host\nORDER BY bytes DESC\nLIMIT 10", + "rawSql": "SELECT protocol, SUM(bytes) as bytes FROM dpi_stats \nWHERE protocol!='' AND $__timeFilter(time) AND client_address like '$client_address' AND uuid = '$uuid'\nGROUP BY protocol\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -859,8 +873,8 @@ } } ], - "title": "Traffic by host (top 10)", - "type": "barchart" + "title": "Traffic by protocol", + "type": "table" }, { "datasource": { @@ -898,8 +912,8 @@ "gridPos": { "h": 12, "w": 12, - "x": 12, - "y": 25 + "x": 0, + "y": 45 }, "id": 1, "options": { @@ -995,7 +1009,7 @@ "value": "" } ], - "query": "", + "query": "%", "skipUrlSync": false, "type": "textbox" }, @@ -1016,13 +1030,13 @@ ] }, "time": { - "from": "now-12h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Network traffic by client", "uid": "b14a1181-a2ee-4df4-a732-888e0190037f", - "version": 17, + "version": 22, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_host.json b/imageroot/etc/dashboards/network_traffic_by_host.json index 53d630e..04e38e9 100644 --- a/imageroot/etc/dashboards/network_traffic_by_host.json +++ b/imageroot/etc/dashboards/network_traffic_by_host.json @@ -62,7 +62,22 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, - "links": [], + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "unit" + ], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { @@ -92,7 +107,7 @@ }, "gridPos": { "h": 8, - "w": 7, + "w": 8, "x": 0, "y": 0 }, @@ -200,8 +215,8 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 7, + "w": 16, + "x": 8, "y": 0 }, "id": 3, @@ -217,14 +232,14 @@ "showLegend": true }, "orientation": "auto", - "showValue": "auto", + "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "11.2.0", "targets": [ @@ -312,7 +327,7 @@ }, "gridPos": { "h": 9, - "w": 19, + "w": 24, "x": 0, "y": 8 }, @@ -329,14 +344,14 @@ "showLegend": true }, "orientation": "auto", - "showValue": "auto", + "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "10.3.3", "targets": [ @@ -420,8 +435,8 @@ "overrides": [] }, "gridPos": { - "h": 12, - "w": 8, + "h": 14, + "w": 24, "x": 0, "y": 17 }, @@ -470,7 +485,7 @@ } } ], - "title": "Visited by client", + "title": "Total traffic by client", "type": "table" } ], @@ -506,18 +521,12 @@ { "current": { "selected": false, - "text": "%", - "value": "%" + "text": "pod-031-2010-09.backblaze.com", + "value": "pod-031-2010-09.backblaze.com" }, "hide": 0, "name": "host", - "options": [ - { - "selected": true, - "text": "%", - "value": "%" - } - ], + "options": [], "query": "%", "skipUrlSync": false, "type": "textbox" @@ -539,13 +548,13 @@ ] }, "time": { - "from": "now-24h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", - "title": "Network traffic by host", + "title": "Network traffic by remote host", "uid": "c8d43e5f-068e-4cac-9283-2318d9e1911b", - "version": 15, + "version": 18, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/security.json b/imageroot/etc/dashboards/security.json index 7051aa4..284d35d 100644 --- a/imageroot/etc/dashboards/security.json +++ b/imageroot/etc/dashboards/security.json @@ -86,7 +86,22 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, - "links": [], + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "unit" + ], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { @@ -204,6 +219,7 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_LOCAL_TIMESCALE}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -266,14 +282,14 @@ "showLegend": true }, "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", + "showValue": "always", + "stacking": "normal", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "10.3.3", "targets": [ @@ -784,8 +800,8 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 6, + "h": 11, + "w": 5, "x": 0, "y": 27 }, @@ -817,7 +833,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT count(*) FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND uuid = '$uuid'", + "rawSql": "SELECT sum(count) FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND uuid = '$uuid'", "refId": "A", "sql": { "columns": [ @@ -847,6 +863,7 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -885,38 +902,37 @@ "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 11, "w": 9, - "x": 6, + "x": 5, "y": 27 }, "id": 7, "options": { "barRadius": 0, - "barWidth": 0.8, + "barWidth": 0.7, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", + "showValue": "always", + "stacking": "normal", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "10.3.3", "targets": [ @@ -928,7 +944,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT country, SUM(count) as count FROM ca_ts_attacks_hourly\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND country != ''\nGROUP BY country\nORDER BY count DESC\nLIMIT 20", + "rawSql": "SELECT country, count(*) as count FROM ts_attacks \nWHERE $__timeFilter(time) AND uuid = '$uuid'\nGROUP by country\nORDER by count DESC\nLIMIT 10", "refId": "A", "sql": { "columns": [ @@ -949,7 +965,7 @@ } } ], - "title": "Top 20 blocked attacks by countries", + "title": "Top 10 blocked attack by country", "type": "barchart" }, { @@ -1002,9 +1018,9 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 9, - "x": 15, + "h": 11, + "w": 10, + "x": 14, "y": 27 }, "id": 16, @@ -1016,18 +1032,18 @@ "legend": { "calcs": [], "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "orientation": "horizontal", - "showValue": "auto", + "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "10.3.3", "targets": [ @@ -1039,7 +1055,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ip, count(*) as count FROM ts_attacks\nWHERE $__timeFilter(time) AND uuid = '$uuid'\nGROUP BY ip\nORDER BY count DESC\nLIMIT 20", + "rawSql": "SELECT ip, count(*) as count FROM ts_attacks\nWHERE $__timeFilter(time) AND uuid = '$uuid'\nGROUP BY ip\nORDER BY count DESC\nLIMIT 10", "refId": "A", "sql": { "columns": [ @@ -1060,7 +1076,7 @@ } } ], - "title": "Top 20 blocked attacks by IP", + "title": "Top 10 blocked attacks by IP", "type": "barchart" }, { @@ -1102,7 +1118,7 @@ "h": 13, "w": 12, "x": 0, - "y": 35 + "y": 38 }, "id": 11, "options": { @@ -1275,7 +1291,7 @@ "h": 13, "w": 12, "x": 12, - "y": 35 + "y": 38 }, "id": 12, "options": { @@ -1360,10 +1376,10 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 12, "w": 12, "x": 0, - "y": 48 + "y": 51 }, "id": 13, "options": { @@ -1450,10 +1466,10 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 12, "w": 12, "x": 12, - "y": 48 + "y": 51 }, "id": 17, "options": { @@ -1551,13 +1567,13 @@ ] }, "time": { - "from": "now-24h", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Security", "uid": "dd395331-5dc0-4172-b243-8646c0ca3ccd", - "version": 21, + "version": 25, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/vpn.json b/imageroot/etc/dashboards/vpn.json index 917f93c..4bc1e98 100644 --- a/imageroot/etc/dashboards/vpn.json +++ b/imageroot/etc/dashboards/vpn.json @@ -55,6 +55,12 @@ "name": "Stat", "version": "" }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, { "type": "panel", "id": "timeseries", @@ -82,7 +88,22 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, - "links": [], + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "unit" + ], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "panels": [ { "collapsed": false, @@ -248,7 +269,7 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 17, "x": 7, "y": 1 }, @@ -429,7 +450,7 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 17, "x": 7, "y": 9 }, @@ -446,14 +467,14 @@ "showLegend": true }, "orientation": "horizontal", - "showValue": "auto", + "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "11.2.0", "targets": [ @@ -553,7 +574,7 @@ }, "gridPos": { "h": 8, - "w": 19, + "w": 24, "x": 0, "y": 17 }, @@ -610,8 +631,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -623,8 +643,8 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 19, + "h": 10, + "w": 11, "x": 0, "y": 25 }, @@ -750,29 +770,17 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "scheme", + "fillOpacity": 80, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, + "lineWidth": 1, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, "thresholdsStyle": { "mode": "off" } @@ -782,8 +790,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -796,23 +803,32 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 19, - "x": 0, - "y": 34 + "h": 10, + "w": 13, + "x": 11, + "y": 25 }, "id": 14, "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, "legend": { "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - } + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, "pluginVersion": "11.2.0", "targets": [ @@ -846,7 +862,7 @@ } ], "title": "Traffic by hour", - "type": "timeseries" + "type": "barchart" }, { "datasource": { @@ -886,8 +902,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -897,9 +912,9 @@ }, "gridPos": { "h": 8, - "w": 8, + "w": 24, "x": 0, - "y": 42 + "y": 35 }, "id": 7, "options": { @@ -959,13 +974,139 @@ "title": "Traffic by user: $user", "type": "barchart" }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${DS_TIMESCALE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "bytes_received" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "bytes_sent" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 23, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_LOCAL_TIMESCALE}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT common_name, TO_TIMESTAMP(start_time) AS start_time, duration, TO_TIMESTAMP(start_time + duration) AS end_time, bytes_received, bytes_sent, virtual_ip_addr, remote_ip_addr\nFROM ovpnrw_connections\nWHERE uuid = '$uuid' AND instance = $rwinstance AND $__timeFilter(time) ", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Connection details: $rwinstance", + "type": "table" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 51 }, "id": 17, "panels": [], @@ -1021,8 +1162,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1035,10 +1175,10 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 19, + "h": 9, + "w": 24, "x": 0, - "y": 51 + "y": 52 }, "id": 18, "options": { @@ -1232,13 +1372,13 @@ ] }, "time": { - "from": "now-24h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "VPN", "uid": "fdxlb58zxvpj4f", - "version": 56, + "version": 65, "weekStart": "" } \ No newline at end of file From 44fe6abd2c4c5234dc05a5ebe74d89a211cb34f4 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 19 Sep 2024 15:03:23 +0200 Subject: [PATCH 35/42] grafana: fix network_traffic --- imageroot/etc/dashboards/network_traffic.json | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/imageroot/etc/dashboards/network_traffic.json b/imageroot/etc/dashboards/network_traffic.json index 4d0303b..4008a8f 100644 --- a/imageroot/etc/dashboards/network_traffic.json +++ b/imageroot/etc/dashboards/network_traffic.json @@ -1050,29 +1050,7 @@ }, "unit": "decbytes" }, -<<<<<<< HEAD - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "client_address" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}&${__url_time_range}" - } - ] - } - ] - } - ] -======= "overrides": [] ->>>>>>> a24fefd (grafana: add links to all dashboards) }, "gridPos": { "h": 13, @@ -1174,7 +1152,7 @@ "value": [ { "title": "", - "url": "/grafana/d/c8d43e5f-068e-4cac-9283-2318d9e1911b/network-traffic-by-host?${__all_variables}&var-host=${__value.raw}&${__url_time_range}" + "url": "/grafana/d/c8d43e5f-068e-4cac-9283-2318d9e1911b/network-traffic-by-host?${__all_variables}&var-host=${__value.raw}" } ] } From 279a2767cfe5bfe4abb8abf0c6c4c0a85da034dd Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 19 Sep 2024 15:52:29 +0200 Subject: [PATCH 36/42] grafana: dashboard multiple fixes --- imageroot/etc/dashboards/nethsecurity.json | 216 ++++++++------ imageroot/etc/dashboards/network_traffic.json | 276 ++++-------------- .../dashboards/network_traffic_by_client.json | 27 +- imageroot/etc/dashboards/security.json | 37 ++- 4 files changed, 208 insertions(+), 348 deletions(-) diff --git a/imageroot/etc/dashboards/nethsecurity.json b/imageroot/etc/dashboards/nethsecurity.json index 0b5fa61..1162068 100644 --- a/imageroot/etc/dashboards/nethsecurity.json +++ b/imageroot/etc/dashboards/nethsecurity.json @@ -1,4 +1,41 @@ { + "__inputs": [ + { + "name": "DS_LOCAL_PROMETHUS", + "label": "Local Promethus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph (old)", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -26,7 +63,7 @@ "fiscalYearStartMonth": 0, "gnetId": 7107, "graphTooltip": 0, - "id": 4, + "id": null, "links": [ { "asDropdown": false, @@ -163,7 +200,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_load_load_average{dimension!=\"load15\", unit=~\"$unit\"}", @@ -220,7 +257,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "description": "Total CPU utilization (all cores). 100% here means there is no CPU idle time at all. You can get per core usage at the _CPUs_ section and per application usage at the _Applications monitoring_ section.\n\nKeep an eye on **iowait**. If it is constantly high, your disks are a bottleneck and they slow your system down.\n\nAn important metric worth monitoring, is **softirq**. A constantly high percentage of softirq may indicate network driver issues.", "fieldConfig": { @@ -363,7 +400,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "description": "Total IPv4 Traffic.", "fieldConfig": { @@ -471,7 +508,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "description": "Total IPv6 Traffic.", "fieldConfig": { @@ -604,7 +641,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "description": "System Random Access Memory (i.e. physical memory) usage.", "fieldConfig": { @@ -704,7 +741,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_ram_MiB_average{dimension=\"free\", unit=~\"$unit\"}", @@ -829,7 +866,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_net_net_kilobits_persec_average{unit=~\"$unit\",dimension=\"received\"}", @@ -926,7 +963,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "abs(netdata_net_net_kilobits_persec_average{unit=~\"$unit\",dimension=\"sent\"})", @@ -1023,7 +1060,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_net_packets_packets_persec_average{unit=~\"$unit\", dimension=~\"received\"}", @@ -1120,7 +1157,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "abs(netdata_net_packets_packets_persec_average{unit=~\"$unit\", dimension=~\"sent\"})", @@ -1255,7 +1292,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_io_KiB_persec_average{unit=~\"$unit\"}", @@ -1362,7 +1399,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "exemplar": true, @@ -1495,7 +1532,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_processes_processes_average{unit=~\"$unit\"}", @@ -1596,7 +1633,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_forks_processes_persec_average{unit=~\"$unit\"}", @@ -1697,7 +1734,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_active_processes_processes_average{unit=~\"$unit\"}", @@ -1798,7 +1835,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_ctxt_context_switches_persec_average{unit=~\"$unit\"}", @@ -1931,7 +1968,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_idlejitter_microseconds_lost_persec_average{unit=~\"$unit\"}", @@ -2060,7 +2097,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_intr_interrupts_persec_average{unit=~\"$unit\"}", @@ -2185,7 +2222,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_interrupts_interrupts_persec_average{unit=~\"$unit\"}", @@ -2336,7 +2373,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_softirqs_softirqs_persec_average{unit=~\"$unit\"}", @@ -2465,7 +2502,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "disableTextWrap": false, "editorMode": "code", @@ -2601,7 +2638,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_system_swap_MiB_average{unit=~\"$unit\"}", @@ -2736,7 +2773,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "exemplar": false, @@ -2844,7 +2881,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_mem_committed_MiB_average{unit=~\"$unit\"}", @@ -2950,7 +2987,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_mem_pgfaults_faults_persec_average{unit=~\"$unit\"}", @@ -3105,7 +3142,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_mem_writeback_MiB_average{unit=~\"$unit\"}", @@ -3176,7 +3213,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_mem_kernel_MiB_average{unit=~\"$unit\"}", @@ -3337,7 +3374,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_disk_ops_operations_persec_average{unit=~\"$unit\"}", @@ -3442,7 +3479,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_disk_util___of_time_working_average{unit=~\"$unit\"}", @@ -3547,7 +3584,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_disk_backlog_milliseconds_average{unit=~\"$unit\"}", @@ -3652,7 +3689,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_disk_mops_merged_operations_persec_average{unit=~\"$unit\"}", @@ -3756,7 +3793,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_disk_await_milliseconds_operation_average{unit=~\"$unit\"}", @@ -3861,7 +3898,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_disk_space_GiB_average{unit=~\"$unit\"}", @@ -3996,7 +4033,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_disk_inodes_inodes_average{unit=~\"$unit\"}", @@ -4085,7 +4122,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4098,7 +4136,7 @@ }, "gridPos": { "h": 11, - "w": 8, + "w": 6, "x": 0, "y": 29 }, @@ -4121,13 +4159,13 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "exemplar": false, "expr": "netdata_net_net_kilobits_persec_average{unit=~\"$unit\",chart=~\"$vpninterface\"}", "instant": true, - "legendFormat": " {{dimension}}", + "legendFormat": "{{dimension}}", "range": false, "refId": "A" } @@ -4244,7 +4282,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_sockstat_sockets_sockets_average{unit=~\"$unit\"}", @@ -4397,7 +4435,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_packets_packets_persec_average{unit=~\"$unit\"}", @@ -4530,7 +4568,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_errors_packets_persec_average{unit=~\"$unit\"}", @@ -4662,7 +4700,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_sockstat_tcp_sockets_sockets_average{unit=~\"$unit\"}", @@ -4768,7 +4806,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_tcpsock_active_connections_average{unit=~\"$unit\"}", @@ -4874,7 +4912,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_tcpopens_connections_persec_average{unit=~\"$unit\"}", @@ -4979,7 +5017,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_tcppackets_packets_persec_average{unit=~\"$unit\"}", @@ -5086,7 +5124,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_tcppackets_packets_persec_average{unit=~\"$unit\"}", @@ -5193,7 +5231,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ip_tcpconnaborts_connections_persec_average{unit=~\"$unit\"}", @@ -5300,7 +5338,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_tcphandshake_events_persec_average{unit=~\"$unit\"}", @@ -5405,7 +5443,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_sockstat_tcp_mem_KiB_average{unit=~\"$unit\"}", @@ -5537,7 +5575,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_sockstat_udp_sockets_sockets_average{unit=~\"$unit\"}", @@ -5642,7 +5680,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_udppackets_packets_persec_average{unit=~\"$unit\"}", @@ -5747,7 +5785,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_sockstat_udp_mem_KiB_average{unit=~\"$unit\"}", @@ -5873,7 +5911,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_udperrors_events_persec_average{unit=~\"$unit\"}", @@ -6005,7 +6043,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_icmp_packets_persec_average{unit=~\"$unit\"}", @@ -6111,7 +6149,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv4_icmp_errors_packets_persec_average{unit=~\"$unit\"}", @@ -6237,7 +6275,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_icmp_messages_persec_average{unit=~\"$unit\"}", @@ -6368,7 +6406,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ip_bcast_kilobits_persec_average{unit=~\"$unit\"}", @@ -6473,7 +6511,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ip_bcastpkts_packets_persec_average{unit=~\"$unit\"}", @@ -6606,7 +6644,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ip_ecnpkts_packets_persec_average{unit=~\"$unit\"}", @@ -6759,7 +6797,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_packets_packets_persec_average{unit=~\"$unit\"}", @@ -6885,7 +6923,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_ect_packets_persec_average{unit=~\"$unit\"}", @@ -7017,7 +7055,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_sockstat6_tcp_sockets_sockets_average{unit=~\"$unit\"}", @@ -7149,7 +7187,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_sockstat6_udp_sockets_sockets_average{unit=~\"$unit\"}", @@ -7254,7 +7292,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_udppackets_packets_persec_average{unit=~\"$unit\"}", @@ -7360,7 +7398,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_udperrors_events_persec_average{unit=~\"$unit\"}", @@ -7532,7 +7570,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_mcast_kilobits_persec_average{unit=~\"$unit\"}", @@ -7678,7 +7716,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_mcastpkts_packets_persec_average{unit=~\"$unit\"}", @@ -7810,7 +7848,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_icmp_messages_persec_average{unit=~\"$unit\"}", @@ -7916,7 +7954,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_icmperrors_errors_persec_average{unit=~\"$unit\"}", @@ -8021,7 +8059,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_icmpechos_messages_persec_average{unit=~\"$unit\"}", @@ -8126,7 +8164,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_icmprouter_messages_persec_average{unit=~\"$unit\"}", @@ -8231,7 +8269,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_icmpneighbor_messages_persec_average{unit=~\"$unit\"}", @@ -8336,7 +8374,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_icmpmldv2_reports_persec_average{unit=~\"$unit\"}", @@ -8441,7 +8479,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "netdata_ipv6_icmprouter_messages_persec_average{unit=~\"$unit\"}", @@ -8480,11 +8518,7 @@ "templating": { "list": [ { - "current": { - "selected": false, - "text": "181ff9b0-e2be-4c71-897a-6575e3991a1c", - "value": "181ff9b0-e2be-4c71-897a-6575e3991a1c" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -8508,11 +8542,7 @@ "type": "query" }, { - "current": { - "selected": false, - "text": "Local Promethus", - "value": "prometheus" - }, + "current": {}, "description": "Prometheus datasource", "hide": 2, "includeAll": false, @@ -8528,16 +8558,12 @@ }, { "allValue": "", - "current": { - "selected": false, - "text": "net.tun_dedalo", - "value": "net.tun_dedalo" - }, + "current": {}, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${DS_LOCAL_PROMETHUS}" }, - "definition": "label_values(netdata_net_net_kilobits_persec_average{chart=~\"^net.tun.*|^net.tap.*\"},chart)", + "definition": "label_values(netdata_net_net_kilobits_persec_average{chart=~\"^net.tun.*|^net.tap.*|^net.ipsec.*\"},chart)", "hide": 0, "includeAll": true, "label": "VPN interface", @@ -8546,7 +8572,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(netdata_net_net_kilobits_persec_average{chart=~\"^net.tun.*|^net.tap.*\"},chart)", + "query": "label_values(netdata_net_net_kilobits_persec_average{chart=~\"^net.tun.*|^net.tap.*|^net.ipsec.*\"},chart)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -8558,7 +8584,7 @@ ] }, "time": { - "from": "now-5m", + "from": "now-6h", "to": "now" }, "timepicker": { @@ -8589,6 +8615,6 @@ "timezone": "", "title": "Operating system", "uid": "W3S__804z", - "version": 14, + "version": 17, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic.json b/imageroot/etc/dashboards/network_traffic.json index 4008a8f..cfdcade 100644 --- a/imageroot/etc/dashboards/network_traffic.json +++ b/imageroot/etc/dashboards/network_traffic.json @@ -639,6 +639,7 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -652,13 +653,13 @@ "filterable": true, "inspect": false }, - "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -666,35 +667,17 @@ } ] }, - "unit": "bytes" + "unit": "decbytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "client_address" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}" - } - ] - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 15, - "w": 7, + "w": 12, "x": 0, "y": 28 }, - "id": 5, + "id": 18, "options": { "cellHeight": "sm", "footer": { @@ -704,7 +687,7 @@ "reducer": [ "sum" ], - "show": false + "show": true }, "showHeader": true }, @@ -718,7 +701,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT client_address, client_name, bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nORDER BY bytes DESC\n", + "rawSql": "SELECT protocol, sum(bytes) as bytes FROM ca_dpi_stats_hourly_protocol\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY protocol\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -739,7 +722,7 @@ } } ], - "title": "Active clients", + "title": "Protocols", "type": "table" }, { @@ -767,7 +750,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -781,8 +765,8 @@ }, "gridPos": { "h": 15, - "w": 8, - "x": 7, + "w": 12, + "x": 12, "y": 28 }, "id": 16, @@ -830,7 +814,7 @@ } } ], - "title": "By application", + "title": "Applications", "type": "table" }, { @@ -839,7 +823,6 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -853,12 +836,14 @@ "filterable": true, "inspect": false }, + "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -866,199 +851,35 @@ } ] }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 28 - }, - "id": 17, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": true + "unit": "bytes" }, - "showHeader": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT client_name, sum(bytes) as bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY client_name\nORDER BY bytes DESC", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "By client", - "type": "table" - }, - { - "datasource": { - "default": false, - "type": "grafana-postgresql-datasource", - "uid": "${DS_TIMESCALE}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "client_address" }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, + "properties": [ { - "color": "red", - "value": 80 + "id": "links", + "value": [ + { + "title": "Client details", + "url": "/grafana/d/b14a1181-a2ee-4df4-a732-888e0190037f/network-traffic-by-host?${__all_variables}&var-client_address=${__value.raw}" + } + ] } ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 36 - }, - "id": 19, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT host, sum(bytes) as bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY host\nORDER BY bytes DESC", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 } - } - ], - "title": "By remote host", - "type": "table" - }, - { - "datasource": { - "default": false, - "type": "grafana-postgresql-datasource", - "uid": "${DS_TIMESCALE}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] + ] }, "gridPos": { - "h": 13, + "h": 17, "w": 12, "x": 0, "y": 43 }, - "id": 18, + "id": 5, "options": { "cellHeight": "sm", "footer": { @@ -1082,7 +903,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT protocol, sum(bytes) as bytes FROM ca_dpi_stats_hourly_protocol\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY protocol\nORDER BY bytes DESC", + "rawSql": "SELECT client_address, client_name, bytes FROM ca_dpi_stats_hourly_client\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nORDER BY bytes DESC\n", "refId": "A", "sql": { "columns": [ @@ -1103,7 +924,7 @@ } } ], - "title": "By protocol", + "title": "Clients", "type": "table" }, { @@ -1112,6 +933,7 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_TIMESCALE}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1122,15 +944,16 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, - "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1138,7 +961,7 @@ } ] }, - "unit": "bytes" + "unit": "decbytes" }, "overrides": [ { @@ -1151,7 +974,7 @@ "id": "links", "value": [ { - "title": "", + "title": "Remote host details", "url": "/grafana/d/c8d43e5f-068e-4cac-9283-2318d9e1911b/network-traffic-by-host?${__all_variables}&var-host=${__value.raw}" } ] @@ -1161,16 +984,17 @@ ] }, "gridPos": { - "h": 8, + "h": 17, "w": 12, "x": 12, - "y": 44 + "y": 43 }, - "id": 15, + "id": 19, "options": { "cellHeight": "sm", "footer": { "countRows": false, + "enablePagination": true, "fields": "", "reducer": [ "sum" @@ -1189,7 +1013,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT host, bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nORDER BY bytes DESC\nLIMIT 20", + "rawSql": "SELECT host, sum(bytes) as bytes FROM ca_dpi_stats_hourly_host\nWHERE $__timeFilter(bucket) AND uuid = '$uuid'\nGROUP BY host\nORDER BY bytes DESC", "refId": "A", "sql": { "columns": [ @@ -1210,7 +1034,7 @@ } } ], - "title": "Top 20 contacted remote hosts", + "title": "Remote hosts", "type": "table" } ], @@ -1269,6 +1093,6 @@ "timezone": "", "title": "Network traffic", "uid": "fe0af3cb-9be0-4b2a-8ccb-86704956cf2e", - "version": 28, + "version": 31, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_client.json b/imageroot/etc/dashboards/network_traffic_by_client.json index 97a816e..4bf3b51 100644 --- a/imageroot/etc/dashboards/network_traffic_by_client.json +++ b/imageroot/etc/dashboards/network_traffic_by_client.json @@ -469,7 +469,7 @@ } } ], - "title": "Traffic by application (top 10)", + "title": "Top 10 applications", "type": "barchart" }, { @@ -541,7 +541,7 @@ "showLegend": true }, "orientation": "horizontal", - "showValue": "never", + "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", @@ -581,7 +581,7 @@ } } ], - "title": "Traffic by protocol (top 10)", + "title": "Top 10 protocols", "type": "barchart" }, { @@ -693,7 +693,7 @@ } } ], - "title": "Traffic by host (top 10)", + "title": "Top 10 remote hosts", "type": "barchart" }, { @@ -720,7 +720,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -810,7 +811,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -897,7 +899,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -997,16 +1000,16 @@ { "current": { "selected": false, - "text": "", - "value": "" + "text": "%", + "value": "%" }, "hide": 0, "name": "client_address", "options": [ { "selected": true, - "text": "", - "value": "" + "text": "%", + "value": "%" } ], "query": "%", @@ -1037,6 +1040,6 @@ "timezone": "", "title": "Network traffic by client", "uid": "b14a1181-a2ee-4df4-a732-888e0190037f", - "version": 22, + "version": 24, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/security.json b/imageroot/etc/dashboards/security.json index 284d35d..9af6c95 100644 --- a/imageroot/etc/dashboards/security.json +++ b/imageroot/etc/dashboards/security.json @@ -788,7 +788,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -833,7 +834,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT sum(count) FROM ca_ts_attacks_hourly \nWHERE $__timeFilter(bucket) AND uuid = '$uuid'", + "rawSql": "SELECT count(*) FROM ts_attacks \nWHERE $__timeFilter(time) AND uuid = '$uuid'", "refId": "A", "sql": { "columns": [ @@ -867,7 +868,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { "axisBorderShow": false, @@ -895,7 +896,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -915,7 +917,7 @@ "id": 7, "options": { "barRadius": 0, - "barWidth": 0.7, + "barWidth": 0.88, "fullHighlight": false, "groupWidth": 0.7, "legend": { @@ -926,7 +928,7 @@ }, "orientation": "horizontal", "showValue": "always", - "stacking": "normal", + "stacking": "none", "tooltip": { "mode": "single", "sort": "none" @@ -934,7 +936,7 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 100 }, - "pluginVersion": "10.3.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -944,7 +946,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT country, count(*) as count FROM ts_attacks \nWHERE $__timeFilter(time) AND uuid = '$uuid'\nGROUP by country\nORDER by count DESC\nLIMIT 10", + "rawSql": "SELECT country, count(*) as count FROM ts_attacks \nWHERE $__timeFilter(time) AND uuid = '$uuid'\nGROUP BY country\nORDER BY count DESC\nLIMIT 10\n", "refId": "A", "sql": { "columns": [ @@ -1005,7 +1007,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1103,7 +1106,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1276,7 +1280,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1363,7 +1368,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1453,7 +1459,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1567,13 +1574,13 @@ ] }, "time": { - "from": "now-1h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Security", "uid": "dd395331-5dc0-4172-b243-8646c0ca3ccd", - "version": 25, + "version": 26, "weekStart": "" } \ No newline at end of file From e6d42d9fa49d581fc2679389a409707a7e8c9f6d Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 19 Sep 2024 16:10:16 +0200 Subject: [PATCH 37/42] grafana: make dashboard read-only --- imageroot/etc/dashboards/connectivity.json | 4 +- imageroot/etc/dashboards/logs.json | 4 +- imageroot/etc/dashboards/loki.json | 119 ++++++++++++------ imageroot/etc/dashboards/nethsecurity.json | 9 +- imageroot/etc/dashboards/network_traffic.json | 16 +-- .../dashboards/network_traffic_by_client.json | 21 ++-- .../dashboards/network_traffic_by_host.json | 4 +- imageroot/etc/dashboards/security.json | 32 +++-- imageroot/etc/dashboards/vpn.json | 4 +- 9 files changed, 122 insertions(+), 91 deletions(-) diff --git a/imageroot/etc/dashboards/connectivity.json b/imageroot/etc/dashboards/connectivity.json index 105a919..57e1aa5 100644 --- a/imageroot/etc/dashboards/connectivity.json +++ b/imageroot/etc/dashboards/connectivity.json @@ -72,7 +72,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -979,6 +979,6 @@ "timezone": "browser", "title": "Connectivity", "uid": "ady5vjxjqywowd", - "version": 31, + "version": 32, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/logs.json b/imageroot/etc/dashboards/logs.json index 83a4a9a..f0904f3 100644 --- a/imageroot/etc/dashboards/logs.json +++ b/imageroot/etc/dashboards/logs.json @@ -53,7 +53,7 @@ ] }, "description": "Loki dashboard with quick search and timeline.", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "gnetId": 13186, "graphTooltip": 0, @@ -347,6 +347,6 @@ "timezone": "", "title": "Logs", "uid": "liz0yRCZz", - "version": 4, + "version": 5, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/loki.json b/imageroot/etc/dashboards/loki.json index 22657c1..5fc7149 100644 --- a/imageroot/etc/dashboards/loki.json +++ b/imageroot/etc/dashboards/loki.json @@ -1,4 +1,47 @@ { + "__inputs": [ + { + "name": "DS_LOCAL_PROMETHUS", + "label": "Local Promethus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -17,11 +60,11 @@ ] }, "description": "Loki metrics via 2.0", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "gnetId": 13407, "graphTooltip": 0, - "id": 2, + "id": null, "links": [ { "asDropdown": false, @@ -43,7 +86,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "description": "", "gridPos": { @@ -68,7 +111,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "loki_build_info", "format": "table", @@ -84,7 +127,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -134,7 +177,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "loki_build_info", "format": "table", @@ -150,7 +193,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -200,7 +243,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "sum(loki_log_messages_total)", @@ -216,7 +259,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -266,7 +309,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "loki_ingester_streams_created_total", @@ -282,7 +325,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -332,7 +375,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "loki_panic_total", "instant": false, @@ -347,7 +390,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -433,7 +476,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "histogram_quantile(0.95, sum(rate(loki_request_duration_seconds_bucket[5m])) by (le,route))", "interval": "", @@ -447,7 +490,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -533,7 +576,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "editorMode": "code", "expr": "sum(irate(loki_log_messages_total[5m])) by (level)", @@ -562,7 +605,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -648,7 +691,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_bytes_processed_per_seconds_bucket[5m])) by (le,type))", "interval": "", @@ -662,7 +705,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -749,7 +792,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_latency_seconds_bucket[5m])) by (le,type))", "interval": "", @@ -777,7 +820,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -858,7 +901,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "sum by (tenant) (rate(loki_distributor_lines_received_total[5m]))", "interval": "", @@ -872,7 +915,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -955,7 +998,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "rate(loki_distributor_ingester_appends_total[5m])", "interval": "", @@ -965,7 +1008,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "loki_distributor_ingester_append_failures_total", "interval": "", @@ -1018,7 +1061,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -1102,7 +1145,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "rate(loki_ingester_chunks_created_total[5m])", "interval": "", @@ -1112,7 +1155,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total[5m]))", "interval": "", @@ -1122,7 +1165,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "sum by (fake) (rate(loki_ingester_chunks_stored_total[5m]))", "interval": "", @@ -1132,7 +1175,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "", "interval": "", @@ -1146,7 +1189,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -1230,7 +1273,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "loki_ingester_memory_chunks", "interval": "", @@ -1244,7 +1287,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -1328,7 +1371,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "sum by (tenan) (loki_ingester_memory_streams)", "interval": "", @@ -1342,7 +1385,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "fieldConfig": { "defaults": { @@ -1425,7 +1468,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "rate(loki_ingester_streams_created_total[5m])", "interval": "", @@ -1435,7 +1478,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_LOCAL_PROMETHUS}" }, "expr": "rate(loki_ingester_streams_removed_total[5m])", "interval": "", @@ -1478,6 +1521,6 @@ "timezone": "", "title": "Loki metrics", "uid": "MQHVDmtWk", - "version": 2, + "version": 3, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/nethsecurity.json b/imageroot/etc/dashboards/nethsecurity.json index 1162068..0b41086 100644 --- a/imageroot/etc/dashboards/nethsecurity.json +++ b/imageroot/etc/dashboards/nethsecurity.json @@ -59,7 +59,7 @@ ] }, "description": "Netdata dashboards for Grafana via Prometheus.", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "gnetId": 7107, "graphTooltip": 0, @@ -4122,8 +4122,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -8584,7 +8583,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-5m", "to": "now" }, "timepicker": { @@ -8615,6 +8614,6 @@ "timezone": "", "title": "Operating system", "uid": "W3S__804z", - "version": 17, + "version": 18, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic.json b/imageroot/etc/dashboards/network_traffic.json index cfdcade..afde728 100644 --- a/imageroot/etc/dashboards/network_traffic.json +++ b/imageroot/etc/dashboards/network_traffic.json @@ -64,7 +64,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -658,8 +658,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -750,8 +749,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -842,8 +840,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -952,8 +949,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1093,6 +1089,6 @@ "timezone": "", "title": "Network traffic", "uid": "fe0af3cb-9be0-4b2a-8ccb-86704956cf2e", - "version": 31, + "version": 32, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_client.json b/imageroot/etc/dashboards/network_traffic_by_client.json index 4bf3b51..4dffb96 100644 --- a/imageroot/etc/dashboards/network_traffic_by_client.json +++ b/imageroot/etc/dashboards/network_traffic_by_client.json @@ -58,7 +58,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -720,8 +720,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -811,8 +810,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -899,8 +897,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1000,16 +997,16 @@ { "current": { "selected": false, - "text": "%", - "value": "%" + "text": "", + "value": "" }, "hide": 0, "name": "client_address", "options": [ { "selected": true, - "text": "%", - "value": "%" + "text": "", + "value": "" } ], "query": "%", @@ -1040,6 +1037,6 @@ "timezone": "", "title": "Network traffic by client", "uid": "b14a1181-a2ee-4df4-a732-888e0190037f", - "version": 24, + "version": 25, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/network_traffic_by_host.json b/imageroot/etc/dashboards/network_traffic_by_host.json index 04e38e9..82c7c8c 100644 --- a/imageroot/etc/dashboards/network_traffic_by_host.json +++ b/imageroot/etc/dashboards/network_traffic_by_host.json @@ -58,7 +58,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -555,6 +555,6 @@ "timezone": "", "title": "Network traffic by remote host", "uid": "c8d43e5f-068e-4cac-9283-2318d9e1911b", - "version": 18, + "version": 19, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/security.json b/imageroot/etc/dashboards/security.json index 9af6c95..7835444 100644 --- a/imageroot/etc/dashboards/security.json +++ b/imageroot/etc/dashboards/security.json @@ -82,7 +82,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -357,7 +357,10 @@ "legend": { "displayMode": "list", "placement": "right", - "showLegend": true + "showLegend": true, + "values": [ + "value" + ] }, "pieType": "pie", "reduceOptions": { @@ -788,8 +791,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -896,8 +898,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1007,8 +1008,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1106,8 +1106,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1280,8 +1279,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1368,8 +1366,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1459,8 +1456,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1574,13 +1570,13 @@ ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Security", "uid": "dd395331-5dc0-4172-b243-8646c0ca3ccd", - "version": 26, + "version": 28, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/vpn.json b/imageroot/etc/dashboards/vpn.json index 4bc1e98..6b0a17c 100644 --- a/imageroot/etc/dashboards/vpn.json +++ b/imageroot/etc/dashboards/vpn.json @@ -84,7 +84,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -1379,6 +1379,6 @@ "timezone": "browser", "title": "VPN", "uid": "fdxlb58zxvpj4f", - "version": 65, + "version": 66, "weekStart": "" } \ No newline at end of file From 99239514526f653453e2abc834cc5bbee97e0c4f Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 19 Sep 2024 18:01:37 +0200 Subject: [PATCH 38/42] grafana: connectivity, add wan staus --- imageroot/etc/dashboards/connectivity.json | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/imageroot/etc/dashboards/connectivity.json b/imageroot/etc/dashboards/connectivity.json index 57e1aa5..cbba1cd 100644 --- a/imageroot/etc/dashboards/connectivity.json +++ b/imageroot/etc/dashboards/connectivity.json @@ -156,7 +156,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT interface, device FROM wan_config\nWHERE uuid = '$uuid'\n", + "rawSql": "SELECT interface, device FROM wan_config\nWHERE uuid = '$uuid'\nORDER By interface", "refId": "A", "sql": { "columns": [ @@ -283,6 +283,7 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, "mappings": [ @@ -325,6 +326,7 @@ "cellHeight": "sm", "footer": { "countRows": false, + "enablePagination": true, "fields": "", "reducer": [ "sum" @@ -343,7 +345,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT DISTINCT ON (uuid, wan, interface)\n wan,\n interface,\n event AS last_known_status,\n time AS last_update_time\nFROM mwan_events\nWHERE uuid = '$uuid'\nORDER BY uuid, wan, interface, time DESC;\n", + "rawSql": "SELECT\n interface,\n device,\n status,\n created_at AS last_update_time\nFROM wan_config\nWHERE uuid = '$uuid'\nORDER BY interface\n", "refId": "A", "sql": { "columns": [ @@ -589,7 +591,7 @@ "h": 3, "w": 24, "x": 0, - "y": 23 + "y": 31 }, "id": 10, "options": { @@ -667,7 +669,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 34 }, "id": 7, "options": { @@ -763,7 +765,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 34 }, "id": 8, "options": { @@ -859,7 +861,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 42 }, "id": 9, "options": { @@ -972,13 +974,13 @@ ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Connectivity", "uid": "ady5vjxjqywowd", - "version": 32, + "version": 35, "weekStart": "" } \ No newline at end of file From 488f45db32bfb1f5b47bcc53e602c0b8d6e9c8e5 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Mon, 23 Sep 2024 11:44:30 +0200 Subject: [PATCH 39/42] grafana: dashboards minor fixes --- imageroot/etc/dashboards/connectivity.json | 160 +++++---------------- imageroot/etc/dashboards/vpn.json | 20 +-- 2 files changed, 50 insertions(+), 130 deletions(-) diff --git a/imageroot/etc/dashboards/connectivity.json b/imageroot/etc/dashboards/connectivity.json index cbba1cd..683c139 100644 --- a/imageroot/etc/dashboards/connectivity.json +++ b/imageroot/etc/dashboards/connectivity.json @@ -98,6 +98,7 @@ "type": "grafana-postgresql-datasource", "uid": "${DS_LOCAL_TIMESCALE}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -108,19 +109,32 @@ "cellOptions": { "type": "auto" }, + "filterable": true, "inspect": false }, - "mappings": [], + "mappings": [ + { + "options": { + "offline": { + "color": "dark-red", + "index": 0, + "text": "Offline" + }, + "online": { + "color": "dark-green", + "index": 1, + "text": "Online" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -129,15 +143,16 @@ }, "gridPos": { "h": 6, - "w": 5, + "w": 11, "x": 0, "y": 0 }, - "id": 2, + "id": 3, "options": { "cellHeight": "sm", "footer": { "countRows": false, + "enablePagination": true, "fields": "", "reducer": [ "sum" @@ -156,7 +171,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT interface, device FROM wan_config\nWHERE uuid = '$uuid'\nORDER By interface", + "rawSql": "SELECT\n interface,\n device,\n status,\n created_at AS last_update_time\nFROM wan_config\nWHERE uuid = '$uuid'\nORDER BY interface\n", "refId": "A", "sql": { "columns": [ @@ -177,7 +192,7 @@ } } ], - "title": "MultiWAN list", + "title": "MultiWAN status", "type": "table" }, { @@ -216,8 +231,8 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 5, + "w": 13, + "x": 11, "y": 0 }, "id": 4, @@ -267,108 +282,6 @@ "title": "Last 3 downtime events", "type": "table" }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [ - { - "options": { - "offline": { - "color": "dark-red", - "index": 0, - "text": "Offline" - }, - "online": { - "color": "dark-green", - "index": 1, - "text": "Online" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 11, - "x": 13, - "y": 0 - }, - "id": 3, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "${DS_LOCAL_TIMESCALE}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT\n interface,\n device,\n status,\n created_at AS last_update_time\nFROM wan_config\nWHERE uuid = '$uuid'\nORDER BY interface\n", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "MultiWAN status", - "type": "table" - }, { "datasource": { "type": "grafana-postgresql-datasource", @@ -591,7 +504,7 @@ "h": 3, "w": 24, "x": 0, - "y": 31 + "y": 23 }, "id": 10, "options": { @@ -654,7 +567,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -669,7 +583,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 26 }, "id": 7, "options": { @@ -750,7 +664,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -765,7 +680,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 26 }, "id": 8, "options": { @@ -846,7 +761,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -861,7 +777,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 34 }, "id": 9, "options": { @@ -974,13 +890,13 @@ ] }, "time": { - "from": "now-1h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Connectivity", "uid": "ady5vjxjqywowd", - "version": 35, + "version": 37, "weekStart": "" } \ No newline at end of file diff --git a/imageroot/etc/dashboards/vpn.json b/imageroot/etc/dashboards/vpn.json index 6b0a17c..3330814 100644 --- a/imageroot/etc/dashboards/vpn.json +++ b/imageroot/etc/dashboards/vpn.json @@ -631,7 +631,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -790,7 +791,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -828,7 +830,7 @@ "sort": "none" }, "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 }, "pluginVersion": "11.2.0", "targets": [ @@ -840,7 +842,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = $rwinstance\nGROUP BY bucket\n", + "rawSql": "SELECT bucket, SUM(bytes_sent) + SUM(bytes_received) as bytes FROM ca_ovpnrw_connections_hourly_bytes\nWHERE $__timeFilter(bucket) AND uuid = '$uuid' AND instance = $rwinstance\nGROUP BY bucket\nORDER BY bucket \n", "refId": "A", "sql": { "columns": [ @@ -902,7 +904,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -998,7 +1001,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1372,13 +1376,13 @@ ] }, "time": { - "from": "now-6h", + "from": "now-7d", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "VPN", "uid": "fdxlb58zxvpj4f", - "version": 66, + "version": 69, "weekStart": "" } \ No newline at end of file From fa4ad12c98c79a56dd114a5031bf54f2172dfe62 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Mon, 23 Sep 2024 17:18:12 +0200 Subject: [PATCH 40/42] build: update controller to 1.1.0 --- build-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-images.sh b/build-images.sh index f6a3b17..07e216c 100755 --- a/build-images.sh +++ b/build-images.sh @@ -9,7 +9,7 @@ images=() repobase="${REPOBASE:-ghcr.io/nethserver}" # Configure the image name reponame="nethsecurity-controller" -controller_version="1.0.1" +controller_version="1.1.0" promtail_version=2.7.1 loki_version=2.9.4 prometheus_version=2.50.1 From 015363fedf2fd1e746a0ab0e09a5bec3967e9681 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Mon, 23 Sep 2024 17:37:50 +0200 Subject: [PATCH 41/42] build: use timescale fixed version Co-authored-by: Bailetti Tommaso --- build-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-images.sh b/build-images.sh index 07e216c..d70a941 100755 --- a/build-images.sh +++ b/build-images.sh @@ -15,7 +15,7 @@ loki_version=2.9.4 prometheus_version=2.50.1 grafana_version=11.2.0 webssh_version=1.6.2 -timescale_version="latest-pg16" +timescale_version="2.16.1-pg16" # Create a new empty container for webssh echo "Build webssh container" # from https://github.com/huashengdun/webssh From 5e276359f3227adb2752e9afbb2e35bc598d7e32 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Mon, 23 Sep 2024 17:42:49 +0200 Subject: [PATCH 42/42] README: improve timescale connection info --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1c2b139..7b8cfb3 100644 --- a/README.md +++ b/README.md @@ -180,10 +180,10 @@ Access to WebSSH is protected using a random generated URL, you can find it insi [Timescale](https://docs.timescale.com/latest/main) is a time-series database for storing metrics. It's configured via environment variables and the configuration is available at `/home/nethsecurity-controller1/.config/state/db.env`. -If Postgres client is installed, you can connect to the database with the following command: +You can connect to the database with the following command: ``` runagent -m nethsecurity-controller1 -source db.env; psql -d $REPORT_DB_URI +source db.env; podman exec -it timescale psql -U "${POSTGRES_USER}" -p "${POSTGRES_PORT}" ``` ## Uninstall