diff --git a/test/performance/monitoring/deploy.sh b/test/performance/monitoring/deploy.sh index 865fdee190b..7837d6d2087 100644 --- a/test/performance/monitoring/deploy.sh +++ b/test/performance/monitoring/deploy.sh @@ -8,7 +8,6 @@ _usage="Usage: $0 [--help|-h] --localhost Enable port-forwarding to localhost for Grafana access" WORKDIR=$(dirname "$0") -source $WORKDIR/../tests/util.sh DESTROY=false DEFAULT_WORKDIR=$(dirname "$0") @@ -86,6 +85,9 @@ function destroy() { # ====== Delete kube-state-metrics ====== kubectl delete -f ${DEFAULT_WORKDIR}/kube-state-metrics/kube-state-metrics.yml || true + # ===== Delete node-exporter ===== + kubectl delete -f ${DEFAULT_WORKDIR}/node_exporter/node-exporter.yml || true + # ====== Delete Prometheus ====== kubectl delete -f ${DEFAULT_WORKDIR}/prometheus/prometheus-all-in-one.yml || true diff --git a/test/performance/monitoring/grafana/dashboards/base.json b/test/performance/monitoring/grafana/dashboards/base.json index 12a1dc8d260..d41c95199a1 100644 --- a/test/performance/monitoring/grafana/dashboards/base.json +++ b/test/performance/monitoring/grafana/dashboards/base.json @@ -3,10 +3,7 @@ "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource":"Prometheus-Nodes", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -15,307 +12,504 @@ } ] }, + "description": "Monitors Kubernetes cluster using Prometheus. Shows overall cluster CPU / Memory / Filesystem usage as well as individual pod, containers, systemd services statistics. Uses cAdvisor metrics only.", "editable": true, "fiscalYearStartMonth": 0, + "gnetId": 315, "graphTooltip": 0, - "id": 3, + "id": 2, "links": [], "liveNow": false, "panels": [ { "collapsed": false, + "datasource":"Prometheus-Nodes", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 4, + "id": 33, + "panels": [], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "Network I/O pressure", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource":"Prometheus-Nodes", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 1 + }, + "height": "200px", + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_network_receive_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "- sum (rate (container_network_transmit_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "format": "Bps", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 34, "panels": [], - "title": "Node", + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "Total usage", "type": "row" }, { - "datasource": "Prometheus", + "datasource":"Prometheus-Nodes", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "rgba(50, 172, 45, 0.97)", "value": null }, { - "color": "red", - "value": 80 + "color": "rgba(237, 129, 40, 0.89)", + "value": 65 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 } ] - } + }, + "unit": "percent", + "unitScale": true }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 5, + "w": 8, "x": 0, - "y": 1 + "y": 8 }, - "id": 5, + "id": 4, + "links": [], + "maxDataPoints": 100, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, + "pluginVersion": "10.3.3", "targets": [ { - "datasource": "Prometheus", - "editorMode": "code", - "exemplar": false, - "expr": "kube_node_status_capacity{resource=\"memory\",unit=\"byte\"} /1024/1024 ", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{node}}", - "range": true, - "refId": "A" - }, - { - "datasource": "Prometheus", - "editorMode": "code", - "expr": "(sum(kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\"}) by (node) ) /1024/1024 ", - "hide": false, - "instant": false, - "legendFormat": "all_pods_on_{{node}}", - "range": true, - "refId": "B" + "datasource":"Prometheus-Nodes", + "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 } ], - "title": "Memory", - "type": "timeseries" + "title": "Cluster memory usage", + "type": "gauge" }, { - "datasource": "Prometheus", + "datasource":"Prometheus-Nodes", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, - "mappings": [], + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "rgba(50, 172, 45, 0.97)", "value": null }, { - "color": "red", - "value": 80 + "color": "rgba(237, 129, 40, 0.89)", + "value": 65 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 } ] - } + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) / sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "title": "Cluster CPU usage (1m avg)", + "type": "gauge" + }, + { + "datasource":"Prometheus-Nodes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { "options": { - "mode": "exclude", - "names": [ - "Pods CPU Core" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true + "match": "null", + "result": { + "text": "N/A" } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 65 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 } ] - } - ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 + "h": 5, + "w": 8, + "x": 16, + "y": 8 }, - "id": 6, + "id": 7, + "links": [], + "maxDataPoints": 100, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, + "pluginVersion": "10.3.3", "targets": [ { - "datasource": "Prometheus", - "editorMode": "code", - "expr": "kube_node_status_capacity{resource=\"cpu\",unit=\"core\"}", - "hide": false, - "instant": false, - "format": "time_series", - "legendFormat": "{{node}}", - "range": true, - "refId": "A" - }, - { - "datasource": "Prometheus", - "editorMode": "code", - "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\",unit=\"core\"}) by (node)", - "hide": false, - "instant": false, - "legendFormat": "pods_on_{{node}}", - "range": true, - "refId": "B" + "datasource":"Prometheus-Nodes", + "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (container_fs_limit_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10 } ], - "title": "CPU", - "type": "timeseries" + "title": "Cluster filesystem usage", + "type": "gauge" }, { - "collapsed": false, + "datasource":"Prometheus-Nodes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, + "h": 3, + "w": 4, "x": 0, - "y": 9 + "y": 13 }, - "id": 2, - "panels": [], - "title": "Antrea", - "type": "row" + "id": 9, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "title": "Used", + "type": "stat" }, { - "datasource": "Prometheus", + "datasource":"Prometheus-Nodes", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, - "mappings": [], + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -328,84 +522,70 @@ "value": 80 } ] - } + }, + "unit": "bytes", + "unitScale": true }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 + "h": 3, + "w": 4, + "x": 4, + "y": 13 }, - "id": 1, + "id": 10, + "links": [], + "maxDataPoints": 100, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "10.3.3", "targets": [ { - "datasource": "Prometheus", - "editorMode": "code", - "expr": "(sum(container_memory_usage_bytes{container=~\"antrea-agent|antrea-controller|antrea-ovs\"}) by (instance))/ 1024 / 1024", - "instant": true, - "legendFormat": "__auto", - "range": true, - "refId": "A" + "datasource":"Prometheus-Nodes", + "expr": "sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 } ], - "title": "Memory Usage", - "type": "timeseries" + "title": "Total", + "type": "stat" }, { - "datasource": "Prometheus", - "description": "", + "datasource":"Prometheus-Nodes", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, - "mappings": [], + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -418,99 +598,146 @@ "value": 80 } ] - } + }, + "unit": "none", + "unitScale": true }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 + "h": 3, + "w": 4, + "x": 8, + "y": 13 }, - "id": 0, + "id": 11, + "links": [], + "maxDataPoints": 100, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "10.3.3", "targets": [ { - "datasource": { - "type": "prometheus", - "name": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(rate(container_cpu_user_seconds_total{container=~\"antrea-agent|antrea-controller|antrea-ovs\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 } ], - "title": "CPU Usage", - "type": "timeseries" + "title": "Used", + "type": "stat" }, { - "collapsed": false, + "datasource":"Prometheus-Nodes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 + "h": 3, + "w": 4, + "x": 12, + "y": 13 }, - "id": 7, - "panels": [], - "title": "Kubernetes", - "type": "row" + "id": 12, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "title": "Total", + "type": "stat" }, { - "datasource": "Prometheus", + "datasource":"Prometheus-Nodes", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, - "mappings": [], + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -523,84 +750,70 @@ "value": 80 } ] - } + }, + "unit": "bytes", + "unitScale": true }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 + "h": 3, + "w": 4, + "x": 16, + "y": 13 }, - "id": 8, + "id": 13, + "links": [], + "maxDataPoints": 100, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "10.3.3", "targets": [ { - "datasource": "Prometheus", - "editorMode": "code", - "expr": "count(kube_pod_info)", - "instant": true, - "legendFormat": "__auto", - "range": true, - "refId": "A" + "datasource":"Prometheus-Nodes", + "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 } ], - "title": "Total Pods", - "type": "timeseries" + "title": "Used", + "type": "stat" }, { - "datasource": "Prometheus", - "description": "", + "datasource":"Prometheus-Nodes", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, - "mappings": [], + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -613,79 +826,1415 @@ "value": 80 } ] - } + }, + "unit": "bytes", + "unitScale": true }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 + "h": 3, + "w": 4, + "x": 20, + "y": 13 }, - "id": 3, + "id": 14, + "links": [], + "maxDataPoints": 100, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "10.3.3", "targets": [ { - "datasource": { - "type": "prometheus", - "name": "Prometheus" - }, - "editorMode": "code", - "expr": "count(kube_service_info)", - "instant": false, - "legendFormat": "__auto", - "range": true, + "datasource":"Prometheus-Nodes", + "expr": "sum (container_fs_limit_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "title": "Total", + "type": "stat" + }, + { + "collapsed": false, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 35, + "panels": [], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "Pods CPU usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource":"Prometheus-Nodes", + "decimals": 3, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 17 + }, + "height": "", + "hiddenSeries": false, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Pods CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 36, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource":"Prometheus-Nodes", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "", + "id": 23, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_cpu_usage_seconds_total{systemd_service_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (systemd_service_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ systemd_service_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "title": "System services CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ] + } + ], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "System services CPU usage", + "type": "row" + }, + { + "collapsed": true, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 37, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource":"Prometheus-Nodes", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 25 + }, + "height": "", + "id": 24, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",container_name!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "pod: {{ pod_name }} | {{ container_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "container_cpu", + "refId": "B", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_cpu_usage_seconds_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "container_cpu", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "title": "Containers CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ] + } + ], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "Containers CPU usage", + "type": "row" + }, + { + "collapsed": false, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 38, + "panels": [], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "All processes CPU usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource":"Prometheus-Nodes", + "decimals": 3, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_cpu_usage_seconds_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ id }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "All processes CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 39, + "panels": [], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "Pods memory usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource":"Prometheus-Nodes", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Pods memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 40, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource":"Prometheus-Nodes", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 26, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (container_memory_working_set_bytes{systemd_service_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (systemd_service_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ systemd_service_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "title": "System services memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ] + } + ], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "System services memory usage", + "type": "row" + }, + { + "collapsed": true, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 41, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource":"Prometheus-Nodes", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 27, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",container_name!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}) by (container_name, pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "pod: {{ pod_name }} | {{ container_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, name, image)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "container_memory_usage:sort_desc", + "refId": "B", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "sum (container_memory_working_set_bytes{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, rkt_container_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "title": "Containers memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ] + } + ], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "Containers memory usage", + "type": "row" + }, + { + "collapsed": true, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 42, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource":"Prometheus-Nodes", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 28, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (container_memory_working_set_bytes{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ id }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "title": "All processes memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ] + } + ], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "All processes memory usage", + "type": "row" + }, + { + "collapsed": false, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 43, + "panels": [], + "targets": [ + { + "datasource":"Prometheus-Nodes", "refId": "A" } ], - "title": "Total Services", - "type": "timeseries" + "title": "Pods network I/O", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource":"Prometheus-Nodes", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 53 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ pod_name }}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ pod_name }}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Pods network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 44, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource":"Prometheus-Nodes", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 30, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> pod: {{ pod_name }} | {{ container_name }}", + "metric": "network", + "refId": "B", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- pod: {{ pod_name }} | {{ container_name }}", + "metric": "network", + "refId": "D", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "network", + "refId": "C", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "network", + "refId": "E", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "- sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "network", + "refId": "F", + "step": 10 + } + ], + "thresholds": [], + "title": "Containers network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ] + } + ], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "Containers network I/O", + "type": "row" + }, + { + "collapsed": true, + "datasource":"Prometheus-Nodes", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 45, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource":"Prometheus-Nodes", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 29, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource":"Prometheus-Nodes", + "expr": "sum (rate (container_network_receive_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ id }}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "datasource":"Prometheus-Nodes", + "expr": "- sum (rate (container_network_transmit_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ id }}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "title": "All processes network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ] + } + ], + "targets": [ + { + "datasource":"Prometheus-Nodes", + "refId": "A" + } + ], + "title": "All processes network I/O", + "type": "row" } ], - "refresh": "", - "schemaVersion": 38, - "tags": [], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "kubernetes" + ], "templating": { "list": [ { + "allValue": ".*", "current": { "selected": false, - "text": "No data sources found", - "value": "" + "text": "All", + "value": "$__all" }, - "hide": 2, - "includeAll": false, - "label": "", + "datasource":"Prometheus-Nodes", + "definition": "", + "hide": 0, + "includeAll": true, "multi": false, - "name": "datasource", + "name": "Node", "options": [], - "query": "prometheus", + "query": "label_values(kubernetes_io_hostname)", "refresh": 1, - "regex": "/$ds/", + "regex": "", "skipUrlSync": false, - "type": "datasource" + "sort": 0, + "type": "query" } ] }, "time": { - "from": "now-5m", + "from": "now-6h", "to": "now" }, - "timepicker": {}, - "timezone": "", - "title": "Benchmark Metrics", - "version": 11, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kubernetes cluster monitoring (via Prometheus)", + "uid": "f751e973-1de4-4419-b7d5-1722e542595d", + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/test/performance/monitoring/grafana/datasources.yml b/test/performance/monitoring/grafana/datasources.yml index 44fb75b8d78..24921b9ea33 100644 --- a/test/performance/monitoring/grafana/datasources.yml +++ b/test/performance/monitoring/grafana/datasources.yml @@ -3,14 +3,14 @@ apiVersion: 1 # list of datasources that should be deleted from the database deleteDatasources: - - name: Prometheus - orgId: 1 + - name: Prometheus-Nodes + orgId: 2 # list of datasources to insert/update depending # whats available in the database datasources: # name of the datasource. Required - - name: Prometheus + - name: Prometheus-Nodes # datasource type. Required type: prometheus # access mode. direct or proxy. Required @@ -20,5 +20,5 @@ datasources: # url url: "http://prometheus-service.monitoring.svc.cluster.local:8080" # mark as default datasource. Max one per org - isDefault: true + isDefault: false # fields that will be converted to json and stored in json_data diff --git a/test/performance/monitoring/grafana/grafana.yml b/test/performance/monitoring/grafana/grafana.yml index aae89b48ff3..ab4d24df613 100644 --- a/test/performance/monitoring/grafana/grafana.yml +++ b/test/performance/monitoring/grafana/grafana.yml @@ -22,14 +22,14 @@ data: # list of datasources that should be deleted from the database deleteDatasources: - - name: Prometheus - orgId: 1 + - name: Prometheus-Nodes + orgId: 2 # list of datasources to insert/update depending # whats available in the database datasources: # name of the datasource. Required - - name: Prometheus + - name: Prometheus-Nodes # datasource type. Required type: prometheus # access mode. direct or proxy. Required @@ -39,7 +39,7 @@ data: # url url: "http://prometheus-service.monitoring.svc.cluster.local:8080" # mark as default datasource. Max one per org - isDefault: true + isDefault: false # fields that will be converted to json and stored in json_data kind: ConfigMap metadata: