diff --git a/charts/spiderpool/files/grafana-rdma-pod.json b/charts/spiderpool/files/grafana-rdma-pod.json index 59689f3f4..faef3fde4 100644 --- a/charts/spiderpool/files/grafana-rdma-pod.json +++ b/charts/spiderpool/files/grafana-rdma-pod.json @@ -3943,100 +3943,6 @@ ], "title": "rdma_roce_adp_retrans", "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 89 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(rdma_rp_cnp_handled_total{pod_name!=\"\", pod_namespace=~\"$namespace\", pod_name=~\"$pod\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{ifname}} - {{pod_namespace}}/{{pod_name}}", - "range": true, - "refId": "A" - } - ], - "title": "rdma_rp_cnp_handled", - "type": "timeseries" } ], "refresh": "5s", @@ -4119,8 +4025,8 @@ { "current": { "selected": false, - "text": "rdma-test-gpu-tool-jgt9t", - "value": "rdma-test-gpu-tool-jgt9t" + "text": "pytorch-sample-master-0", + "value": "pytorch-sample-master-0" }, "datasource": { "type": "prometheus", @@ -4146,13 +4052,13 @@ ] }, "time": { - "from": "now-1h", + "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Spiderpool RDMA | Pod", "uid": "DenUibiNk", - "version": 16, + "version": 17, "weekStart": "" } \ No newline at end of file diff --git a/charts/spiderpool/files/grafana-rdma-workload.json b/charts/spiderpool/files/grafana-rdma-workload.json new file mode 100644 index 000000000..0d6b78cdc --- /dev/null +++ b/charts/spiderpool/files/grafana-rdma-workload.json @@ -0,0 +1,1012 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 12, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (owner_name) (\n rate(rdma_rx_vport_rdma_unicast_bytes_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\"}[1m])\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth per $kind | Read ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (owner_name) (\n rate(rdma_tx_vport_rdma_unicast_bytes_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\"}[1m])\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth per $kind | Write", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 30, + "panels": [], + "title": "Pods", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rdma-test-gpu-tool-n792j" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod_name) (\n rate(rdma_rx_vport_rdma_unicast_bytes_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth per Pod | Read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rdma-test-gpu-tool-n792j" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod_name) (\n rate(rdma_tx_vport_rdma_unicast_bytes_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth per Pod | Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rdma-test-gpu-tool-n792j" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.14", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod_name) (\n rate(rdma_rx_vport_rdma_unicast_packets_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets per Pod | unicast | Read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rdma-test-gpu-tool-n792j" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.14", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod_name) (\n rate(rdma_tx_vport_rdma_unicast_packets_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets per Pod | unicast | Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rdma-test-gpu-tool-n792j" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.14", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod_name) (\n rate(rdma_tx_vport_rdma_multicast_packets_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets per Pod | multicast | Read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rdma-test-gpu-tool-n792j" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.14", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod_name) (\n rate(rdma_tx_vport_rdma_multicast_packets_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets per Pod | multicast | Write", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 2, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "DaemonSet", + "value": "DaemonSet" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rdma_vport_speed_mbps_total{}, owner_kind)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "kind", + "options": [], + "query": { + "query": "label_values(rdma_vport_speed_mbps_total{}, owner_kind)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "huailou", + "value": "huailou" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rdma_vport_speed_mbps_total{owner_kind=~\"$kind\"}, owner_namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(rdma_vport_speed_mbps_total{owner_kind=~\"$kind\"}, owner_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "rdma-test-gpu-tool", + "value": "rdma-test-gpu-tool" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rdma_vport_speed_mbps_total{owner_kind=~\"$kind\", owner_namespace=~\"$namespace\"}, owner_name)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "name", + "options": [], + "query": { + "query": "label_values(rdma_vport_speed_mbps_total{owner_kind=~\"$kind\", owner_namespace=~\"$namespace\"}, owner_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Spiderpool RDMA | AI Workload", + "uid": "AAT6f2ZNz", + "version": 40, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/spiderpool/templates/daemonset.yaml b/charts/spiderpool/templates/daemonset.yaml index 2fbaad673..af672a82b 100644 --- a/charts/spiderpool/templates/daemonset.yaml +++ b/charts/spiderpool/templates/daemonset.yaml @@ -205,7 +205,7 @@ spec: {{- with .Values.spiderpoolAgent.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} - {{- if or .Values.spiderpoolAgent.tuneSysctlConfig .Values.spiderpoolAgent.securityContext }} + {{- if or .Values.spiderpoolAgent.tuneSysctlConfig .Values.spiderpoolAgent.securityContext .Values.spiderpoolAgent.prometheus.enabledRdmaMetric }} securityContext: privileged: true {{- with .Values.spiderpoolAgent.securityContext }} @@ -216,6 +216,7 @@ spec: {{- if .Values.spiderpoolAgent.prometheus.enabledRdmaMetric }} - name: host-ns mountPath: /var/run/netns + mountPropagation: Bidirectional {{- end }} - name: config-path mountPath: /tmp/spiderpool/config-map diff --git a/charts/spiderpool/templates/grafanaDashboardRdmaWorkload.yaml b/charts/spiderpool/templates/grafanaDashboardRdmaWorkload.yaml new file mode 100644 index 000000000..ca226760a --- /dev/null +++ b/charts/spiderpool/templates/grafanaDashboardRdmaWorkload.yaml @@ -0,0 +1,25 @@ +{{- if .Values.grafanaDashboard.install -}} +apiVersion: integreatly.org/v1alpha1 +kind: GrafanaDashboard +metadata: + name: {{ default "spiderpool" .Values.global.nameOverride }}-rdma-workload + namespace: {{ default .Release.Namespace .Values.grafanaDashboard.namespace }} + labels: + {{- if .Values.global.commonLabels }} + {{- include "tplvalues.render" ( dict "value" .Values.global.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.grafanaDashboard.labels }} + {{- include "tplvalues.render" ( dict "value" .Values.grafanaDashboard.labels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.global.commonAnnotations .Values.grafanaDashboard.annotations }} + annotations: + {{- if .Values.global.commonAnnotations }} + {{- include "tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.grafanaDashboard.annotations }} + {{- include "tplvalues.render" ( dict "value" .Values.grafanaDashboard.annotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + json: {{ .Files.Get "files/grafana-rdma-workload.json" | toJson }} +{{- end }} diff --git a/charts/spiderpool/templates/role_binding.yaml b/charts/spiderpool/templates/role_binding.yaml index f18317670..7ca687f78 100644 --- a/charts/spiderpool/templates/role_binding.yaml +++ b/charts/spiderpool/templates/role_binding.yaml @@ -19,4 +19,4 @@ subjects: name: {{ .Values.spiderpoolInit.name | trunc 63 | trimSuffix "-" }} namespace: {{ .Release.Namespace }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/spiderpool/templates/role_rdma-exporter.yaml b/charts/spiderpool/templates/role_rdma-exporter.yaml new file mode 100644 index 000000000..7fc4a90cc --- /dev/null +++ b/charts/spiderpool/templates/role_rdma-exporter.yaml @@ -0,0 +1,37 @@ +# for rdma metrics exporter, read rdma pod owner's info +# for example, the rdma pod owner is a job, the job's owner is a cronjob +{{- if .Values.spiderpoolAgent.prometheus.enabledRdmaMetric }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spiderpool-rdma-metrics-exporter +rules: + - apiGroups: ["kubeflow.org"] + resources: ["mpijobs", "mxjobs", "notebooks", "paddlejobs", "pytorchjobs", "tfjobs", "xgboostjobs"] + verbs: ["get"] + - apiGroups: ["serving.baize.io"] + resources: ["inferences"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods", "pods/status"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spiderpool-rdma-metrics-exporter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: spiderpool-rdma-metrics-exporter +subjects: + - kind: ServiceAccount + name: {{ .Values.spiderpoolAgent.name | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/cmd/spiderpool-agent/cmd/config.go b/cmd/spiderpool-agent/cmd/config.go index 45d981ae8..4ca927e4e 100644 --- a/cmd/spiderpool-agent/cmd/config.go +++ b/cmd/spiderpool-agent/cmd/config.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/pflag" "go.uber.org/atomic" "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "github.com/spidernet-io/spiderpool/api/v1/agent/client" @@ -121,6 +122,9 @@ type AgentContext struct { SubnetManager subnetmanager.SubnetManager KubevirtManager kubevirtmanager.KubevirtManager + // k8s client + ClientSet *kubernetes.Clientset + // handler HttpServer *server.Server UnixServer *server.Server diff --git a/cmd/spiderpool-agent/cmd/daemon.go b/cmd/spiderpool-agent/cmd/daemon.go index 3eeb485c8..8196bfaef 100644 --- a/cmd/spiderpool-agent/cmd/daemon.go +++ b/cmd/spiderpool-agent/cmd/daemon.go @@ -20,6 +20,7 @@ import ( "go.uber.org/automaxprocs/maxprocs" "go.uber.org/zap" apiruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -147,6 +148,13 @@ func DaemonMain() { } agentContext.CRDManager = mgr + logger.Info("Begin to initialize k8s clientSet") + clientSet, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) + if nil != err { + logger.Sugar().Fatalf("failed to init K8s clientset: %v", err) + } + agentContext.ClientSet = clientSet + logger.Info("Begin to initialize spiderpool-agent metrics HTTP server") initAgentMetricsServer(agentContext.InnerCtx) diff --git a/cmd/spiderpool-agent/cmd/metrics_server.go b/cmd/spiderpool-agent/cmd/metrics_server.go index 9d263baa1..186d338af 100644 --- a/cmd/spiderpool-agent/cmd/metrics_server.go +++ b/cmd/spiderpool-agent/cmd/metrics_server.go @@ -8,8 +8,11 @@ import ( "fmt" "net/http" + "k8s.io/client-go/informers" + "github.com/spidernet-io/spiderpool/pkg/constant" "github.com/spidernet-io/spiderpool/pkg/metric" + "github.com/spidernet-io/spiderpool/pkg/podownercache" ) // initAgentMetricsServer will start an opentelemetry http server for spiderpool agent. @@ -23,7 +26,23 @@ func initAgentMetricsServer(ctx context.Context) { logger.Fatal(err.Error()) } - err = metric.InitSpiderpoolAgentMetrics(ctx, agentContext.Cfg.EnableRDMAMetric, agentContext.CRDManager.GetClient()) + var cache podownercache.CacheInterface + // nolint is used to disable the golint warning for the following line. + if agentContext.Cfg.EnableRDMAMetric { //nolint:golint + logger.Info("enable rdma metric exporter") + informerFactory := informers.NewSharedInformerFactory(agentContext.ClientSet, 0) + podInformer := informerFactory.Core().V1().Pods().Informer() + informerFactory.Start(ctx.Done()) + + cache, err = podownercache.New(ctx, podInformer, agentContext.CRDManager.GetClient()) + if err != nil { + logger.Fatal(err.Error()) + } + } else { + logger.Info("disable rdma metric exporter") + } + + err = metric.InitSpiderpoolAgentMetrics(ctx, cache) if nil != err { logger.Fatal(err.Error()) } diff --git a/docs/images/rdma/rdma-cluster.png b/docs/images/rdma/rdma-cluster.png new file mode 100644 index 000000000..58e74e2c4 Binary files /dev/null and b/docs/images/rdma/rdma-cluster.png differ diff --git a/docs/images/rdma/rdma-node.png b/docs/images/rdma/rdma-node.png new file mode 100644 index 000000000..71cb5cec3 Binary files /dev/null and b/docs/images/rdma/rdma-node.png differ diff --git a/docs/images/rdma/rdma-pod.png b/docs/images/rdma/rdma-pod.png new file mode 100644 index 000000000..524253594 Binary files /dev/null and b/docs/images/rdma/rdma-pod.png differ diff --git a/docs/images/rdma/rdma-workload.png b/docs/images/rdma/rdma-workload.png new file mode 100644 index 000000000..c18b6e78d Binary files /dev/null and b/docs/images/rdma/rdma-workload.png differ diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md index 99c34c52c..8ae860606 100644 --- a/docs/reference/metrics.md +++ b/docs/reference/metrics.md @@ -99,3 +99,50 @@ Spiderpool controller exports some metrics related with SpiderIPPool IP garbage | spiderpool_debug_subnet_total_ip_counts | Number of Spiderpool Subnet corresponding total IPs (per-Subnet), prometheus type: gauge. (debug level metric) | | spiderpool_debug_subnet_available_ip_counts | Number of Spiderpool Subnet corresponding availbale IPs (per-Subnet), prometheus type: gauge. (debug level metric) | | spiderpool_debug_auto_pool_waited_for_available_counts | Number of waiting for auto-created IPPool available, prometheus type: couter. (debug level metric) | + + +### RDMA exporter + +Spiderpool also provides RDMA exporter to export RDMA metrics. The RDMA metrics include: + +| Metric Name | Type | Description | Remarks | +|--------------------------------------|---------|---------------------------------------------------------------------------------|-----------------------| +| rdma_rx_write_requests | Counter | Number of received write requests | | +| rdma_rx_read_requests | Counter | Number of received read requests | | +| rdma_rx_atomic_requests | Counter | Number of received atomic requests | | +| rdma_rx_dct_connect | Counter | Number of received DCT connection requests | | +| rdma_out_of_buffer | Counter | Number of buffer insufficiency errors | | +| rdma_out_of_sequence | Counter | Number of out-of-sequence packets received | | +| rdma_duplicate_request | Counter | Number of duplicate requests | | +| rdma_rnr_nak_retry_err | Counter | Count of RNR NAK packets not exceeding QP retry limit | | +| rdma_packet_seq_err | Counter | Number of packet sequence errors | | +| rdma_implied_nak_seq_err | Counter | Number of implied NAK sequence errors | | +| rdma_local_ack_timeout_err | Counter | Number of times the sender's QP ack timer expired | RC, XRC, DCT QPs only | +| rdma_resp_local_length_error | Counter | Number of times a respondent detected a local length error | | +| rdma_resp_cqe_error | Counter | Number of response CQE errors | | +| rdma_req_cqe_error | Counter | Number of times a requester detected CQE completion with errors | | +| rdma_req_remote_invalid_request | Counter | Number of remote invalid request errors detected by requester | | +| rdma_req_remote_access_errors | Counter | Number of requested remote access errors | | +| rdma_resp_remote_access_errors | Counter | Number of response remote access errors | | +| rdma_resp_cqe_flush_error | Counter | Number of response CQE flush errors | | +| rdma_req_cqe_flush_error | Counter | Number of request CQE flush errors | | +| rdma_roce_adp_retrans | Counter | Number of RoCE adaptive retransmissions | | +| rdma_roce_adp_retrans_to | Counter | Number of RoCE adaptive retransmission timeouts | | +| rdma_roce_slow_restart | Counter | Number of RoCE slow restarts | | +| rdma_roce_slow_restart_cnps | Counter | Number of CNP packets generated during RoCE slow restart | | +| rdma_roce_slow_restart_trans | Counter | Number of times state transitioned to slow restart | | +| rdma_rp_cnp_ignored | Counter | Number of CNP packets received and ignored by Reaction Point HCA | | +| rdma_rp_cnp_handled | Counter | Number of CNP packets handled by Reaction Point HCA to reduce transmission rate | | +| rdma_np_ecn_marked_roce_packets | Counter | Number of ECN-marked RoCE packets indicating path congestion | | +| rdma_np_cnp_sent | Counter | Number of CNP packets sent when congestion is experienced in RoCEv2 IP header | | +| rdma_rx_icrc_encapsulated | Counter | Number of RoCE packets with ICRC errors | | +| rdma_rx_vport_rdma_unicast_packets | Counter | Number of received unicast RDMA packets | | +| rdma_tx_vport_rdma_unicast_packets | Counter | Number of transmitted unicast RDMA packets | | +| rdma_rx_vport_rdma_multicast_packets | Counter | Number of received multicast RDMA packets | | +| rdma_tx_vport_rdma_multicast_packets | Counter | Number of transmitted multicast RDMA packets | | +| rdma_rx_vport_rdma_unicast_bytes | Counter | Number of bytes received in unicast RDMA packets | | +| rdma_tx_vport_rdma_unicast_bytes | Counter | Number of bytes transmitted in unicast RDMA packets | | +| rdma_rx_vport_rdma_multicast_bytes | Counter | Number of bytes received in multicast RDMA packets | | +| rdma_tx_vport_rdma_multicast_bytes | Counter | Number of bytes transmitted in multicast RDMA packets | | +| rdma_vport_speed_mbps | Speed | Speed of the port in Mbps | | + diff --git a/docs/usage/rdma-metrics-zh_CN.md b/docs/usage/rdma-metrics-zh_CN.md index 833ea6377..654d91965 100644 --- a/docs/usage/rdma-metrics-zh_CN.md +++ b/docs/usage/rdma-metrics-zh_CN.md @@ -39,47 +39,22 @@ helm upgrade --install spiderpool spiderpool/spiderpool --reuse-values --wait -- - 通过设置 `--set spiderpoolAgent.prometheus.enabledRdmaMetric=true`,可以启用 RDMA 指标 exporter - 通过设置 `--set grafanaDashboard.install=true`,可以启用 GrafanaDashboard CR 看板 -## 指标列表 - -以下是经过整理后的表格,包含了"指标名称"、"指标类型"、"指标含义"和"备注"四列: - -| 指标名称 | 指标类型 | 指标含义 | 备注 | -|---------------------------------|---------|---------------------------------------------|----| -| rx_write_requests | Counter | 接收到的写请求的数量 | | -| rx_read_requests | Counter | 接收到的读请求的数量 | | -| rx_atomic_requests | Counter | 接收到的原子请求的数量 | | -| rx_dct_connect | Counter | 接收到的 DCT 连接请求的数量 | | -| out_of_buffer | Counter | 缓冲区不足错误的数量 | | -| out_of_sequence | Counter | 收到的乱序包数量 | | -| duplicate_request | Counter | 重复请求的数量 | | -| rnr_nak_retry_err | Counter | 收到的 RNR NAK 包未超过 QP 重试限制的数量 | | -| packet_seq_err | Counter | 包序列错误的数量 | | -| implied_nak_seq_err | Counter | 隐含 NAK 序列错误的数量 | | -| local_ack_timeout_err | Counter | 发送端 QP 的 ack 计时器过期的次数(适用于 RC, XRC, DCT QPs) | | -| resp_local_length_error | Counter | 响应者检测到本地长度错误的次数 | | -| resp_cqe_error | Counter | 响应 CQE 错误的数量 | | -| req_cqe_error | Counter | 请求者检测到 CQE 完成且带错误的次数 | | -| req_remote_invalid_request | Counter | 请求者检测到远程无效请求错误的次数 | | -| req_remote_access_errors | Counter | 请求的远程访问错误的数量 | | -| resp_remote_access_errors | Counter | 响应的远程访问错误的数量 | | -| resp_cqe_flush_error | Counter | 响应 CQE 刷新错误的数量 | | -| req_cqe_flush_error | Counter | 请求 CQE 刷新错误的数量 | | -| roce_adp_retrans | Counter | RoCE 自适应重传的次数 | | -| roce_adp_retrans_to | Counter | RoCE 自适应重传超时的次数 | | -| roce_slow_restart | Counter | RoCE 缓慢重启的次数 | | -| roce_slow_restart_cnps | Counter | RoCE 缓慢重启产生的CNP包数 | | -| roce_slow_restart_trans | Counter | RoCE 缓慢重启状态转换为缓慢重启的次数 | | -| rp_cnp_ignored | Counter | Reaction Point HCA 接收到并忽略的 CNP 包数量 | | -| rp_cnp_handled | Counter | Reaction Point HCA 处理以降低传输速率的 CNP 包数量 | | -| np_ecn_marked_roce_packets | Counter | 进入 Pod/Node 的方向收到的 ECN,表示路径拥塞 | | -| np_cnp_sent | Counter | 通知点在 RoCEv2 IP 头部注意到拥塞体验时发送的 CNP 包数 | | -| rx_icrc_encapsulated | Counter | 具有 ICRC 错误的 RoCE 包数量 | | -| rx_vport_rdma_unicast_packets | Counter | 单播 RDMA 包数量 | | -| tx_vport_rdma_unicast_packets | Counter | 发送的单播 RDMA 包数量 | | -| rx_vport_rdma_multicast_packets | Counter | 接收到的多播 RDMA 包数量 | | -| tx_vport_rdma_multicast_packets | Counter | 发送的多播 RDMA 包数量 | | -| rx_vport_rdma_unicast_bytes | Counter | 接收到的单播 RDMA 包的字节数 | | -| tx_vport_rdma_unicast_bytes | Counter | 发送的单播 RDMA 包的字节数 | | -| rx_vport_rdma_multicast_bytes | Counter | 接收到的多播 RDMA 包的字节数 | | -| tx_vport_rdma_multicast_bytes | Counter | 发送的多播 RDMA 包的字节数 | | -| vport_speed_mbps | 速度 | 端口的速度,以兆位每秒(Mbps)表示 | | +## 指标参考 + +访问 [Metrics 参考](../reference/metrics.md) 查看指标的详细信息。 + +## Grafana 监控面板 + +以下四个监控面板中,RDMA Pod 监控面板仅展示来自 RDMA 隔离子系统中 SR-IOV Pod 的监控数据。而对于采用共享方式的 macVLAN Pod,其 RDMA 网卡数据未纳入该面板。 + +Grafana RDMA Cluster 监控面板,可以查看当前集群每个节点的 RDMA 监控。 +![RDMA Dashboard](../images/rdma/rdma-cluster.png) + +Grafana RDMA Node 监控面板,可以查看每个物理网卡的 RDMA 监控,以及该物理网卡的带宽利用率。同时提供了该节点宿主机的 vf 网卡统计,以及该节点 Pod 使用 RDMA 网卡的监控。 +![RDMA Dashboard](../images/rdma/rdma-node.png) + +Grafana RDMA Pod 监控面板,可以查看 Pod 里面每张网卡的 RDMA 监控,同时提供了网卡错误统计信息,根据这些信息的统计可以排查问题。 +![RDMA Dashboard](../images/rdma/rdma-pod.png) + +Grafana RDMA Workload 监控面板。在进行 AI 推理和训练时,往往使用 Job, Deployment, KServer 等顶层资源下发 CR 启动一组 Pod 进行训练,可以查看每个顶层资源的 RDMA 监控。 +![RDMA Dashboard](../images/rdma/rdma-workload.png) diff --git a/docs/usage/rdma-metrics.md b/docs/usage/rdma-metrics.md index 5e54ddd0d..86073891f 100644 --- a/docs/usage/rdma-metrics.md +++ b/docs/usage/rdma-metrics.md @@ -39,47 +39,22 @@ helm upgrade --install spiderpool spiderpool/spiderpool --reuse-values --wait -- - Use `--set spiderpoolAgent.prometheus.enabledRdmaMetric=true` to enable the RDMA metric exporter. - Use `--set grafanaDashboard.install=true` to enable GrafanaDashboard CR. -## Metrics List - -Below is a table containing "Metric Name," "Metric Type," "Metric Meaning," and "Remarks": - -| Metric Name | Type | Meaning | Remarks | -|---------------------------------|---------|---------------------------------------------------------------------------------|-----------------------| -| rx_write_requests | Counter | Number of received write requests | | -| rx_read_requests | Counter | Number of received read requests | | -| rx_atomic_requests | Counter | Number of received atomic requests | | -| rx_dct_connect | Counter | Number of received DCT connection requests | | -| out_of_buffer | Counter | Number of buffer insufficiency errors | | -| out_of_sequence | Counter | Number of out-of-sequence packets received | | -| duplicate_request | Counter | Number of duplicate requests | | -| rnr_nak_retry_err | Counter | Count of RNR NAK packets not exceeding QP retry limit | | -| packet_seq_err | Counter | Number of packet sequence errors | | -| implied_nak_seq_err | Counter | Number of implied NAK sequence errors | | -| local_ack_timeout_err | Counter | Number of times the sender's QP ack timer expired | RC, XRC, DCT QPs only | -| resp_local_length_error | Counter | Number of times a respondent detected a local length error | | -| resp_cqe_error | Counter | Number of response CQE errors | | -| req_cqe_error | Counter | Number of times a requester detected CQE completion with errors | | -| req_remote_invalid_request | Counter | Number of remote invalid request errors detected by requester | | -| req_remote_access_errors | Counter | Number of requested remote access errors | | -| resp_remote_access_errors | Counter | Number of response remote access errors | | -| resp_cqe_flush_error | Counter | Number of response CQE flush errors | | -| req_cqe_flush_error | Counter | Number of request CQE flush errors | | -| roce_adp_retrans | Counter | Number of RoCE adaptive retransmissions | | -| roce_adp_retrans_to | Counter | Number of RoCE adaptive retransmission timeouts | | -| roce_slow_restart | Counter | Number of RoCE slow restarts | | -| roce_slow_restart_cnps | Counter | Number of CNP packets generated during RoCE slow restart | | -| roce_slow_restart_trans | Counter | Number of times state transitioned to slow restart | | -| rp_cnp_ignored | Counter | Number of CNP packets received and ignored by Reaction Point HCA | | -| rp_cnp_handled | Counter | Number of CNP packets handled by Reaction Point HCA to reduce transmission rate | | -| np_ecn_marked_roce_packets | Counter | Number of ECN-marked RoCE packets indicating path congestion | | -| np_cnp_sent | Counter | Number of CNP packets sent when congestion is experienced in RoCEv2 IP header | | -| rx_icrc_encapsulated | Counter | Number of RoCE packets with ICRC errors | | -| rx_vport_rdma_unicast_packets | Counter | Number of received unicast RDMA packets | | -| tx_vport_rdma_unicast_packets | Counter | Number of transmitted unicast RDMA packets | | -| rx_vport_rdma_multicast_packets | Counter | Number of received multicast RDMA packets | | -| tx_vport_rdma_multicast_packets | Counter | Number of transmitted multicast RDMA packets | | -| rx_vport_rdma_unicast_bytes | Counter | Number of bytes received in unicast RDMA packets | | -| tx_vport_rdma_unicast_bytes | Counter | Number of bytes transmitted in unicast RDMA packets | | -| rx_vport_rdma_multicast_bytes | Counter | Number of bytes received in multicast RDMA packets | | -| tx_vport_rdma_multicast_bytes | Counter | Number of bytes transmitted in multicast RDMA packets | | -| vport_speed_mbps | Speed | Speed of the port in Mbps | | +## Metric Reference + +Visit [Metrics Reference](../reference/metrics.md) to view detailed information about the metrics. + +## Grafana Monitoring Dashboard + +Among the following four monitoring dashboards, the RDMA Pod monitoring dashboard only displays monitoring data from SR-IOV Pods in the RDMA-isolated subsystem. As for macVLAN Pods, which use a shared mode, their RDMA network card data is not included in this dashboard. + +The Grafana RDMA Cluster monitoring dashboard provides a view of the RDMA metrics for each node in the current cluster. +![RDMA Dashboard](../images/rdma/rdma-cluster.png) + +The Grafana RDMA Node monitoring dashboard displays RDMA metrics for each physical NIC (Network Interface Card) and the bandwidth utilization of those NICs. It also includes statistics for VF NICs on the host node and monitoring metrics for Pods using RDMA NICs on that node. +![RDMA Dashboard](../images/rdma/rdma-node.png) + +The Grafana RDMA Pod monitoring dashboard provides RDMA metrics for each NIC within a Pod, along with NIC error statistics. These metrics help in troubleshooting issues. +![RDMA Dashboard](../images/rdma/rdma-pod.png) + +The Grafana RDMA Workload monitoring dashboard is designed for monitoring RDMA metrics for top-level resources such as Jobs, Deployments, and KServers. These resources typically initiate a set of Pods for AI inference and training tasks. +![RDMA Dashboard](../images/rdma/rdma-workload.png) diff --git a/pkg/ippoolmanager/ippool_manager_suite_test.go b/pkg/ippoolmanager/ippool_manager_suite_test.go index 0e9f0aa6d..94fbc82a8 100644 --- a/pkg/ippoolmanager/ippool_manager_suite_test.go +++ b/pkg/ippoolmanager/ippool_manager_suite_test.go @@ -71,7 +71,7 @@ var _ = BeforeSuite(func() { Build() _, err = metric.InitMetric(context.TODO(), constant.SpiderpoolAgent, false, false) Expect(err).NotTo(HaveOccurred()) - err = metric.InitSpiderpoolAgentMetrics(context.TODO(), false, fakeClient) + err = metric.InitSpiderpoolAgentMetrics(context.TODO(), nil) Expect(err).NotTo(HaveOccurred()) tracker = k8stesting.NewObjectTracker(scheme, k8sscheme.Codecs.UniversalDecoder()) diff --git a/pkg/metric/metrics_instance.go b/pkg/metric/metrics_instance.go index 6c6e53532..6bf503d5f 100644 --- a/pkg/metric/metrics_instance.go +++ b/pkg/metric/metrics_instance.go @@ -9,9 +9,9 @@ import ( "go.opentelemetry.io/otel/attribute" api "go.opentelemetry.io/otel/metric" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/spidernet-io/spiderpool/pkg/lock" + "github.com/spidernet-io/spiderpool/pkg/podownercache" "github.com/spidernet-io/spiderpool/pkg/rdmametrics" ) @@ -221,10 +221,10 @@ func (a *asyncInt64Gauge) Record(value int64, attrs ...attribute.KeyValue) { } // InitSpiderpoolAgentMetrics serves for spiderpool agent metrics initialization -func InitSpiderpoolAgentMetrics(ctx context.Context, enableRDMAMetric bool, client client.Client) error { +func InitSpiderpoolAgentMetrics(ctx context.Context, cache podownercache.CacheInterface) error { // for rdma - if enableRDMAMetric { - err := rdmametrics.Register(ctx, meter, client) + if cache != nil { + err := rdmametrics.Register(ctx, meter, cache) if err != nil { return err } diff --git a/pkg/podownercache/pod_owner_cache.go b/pkg/podownercache/pod_owner_cache.go new file mode 100644 index 000000000..7a58cf924 --- /dev/null +++ b/pkg/podownercache/pod_owner_cache.go @@ -0,0 +1,177 @@ +// Copyright 2024 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package podownercache + +import ( + "context" + "fmt" + "github.com/spidernet-io/spiderpool/pkg/lock" + "github.com/spidernet-io/spiderpool/pkg/logutils" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type PodOwnerCache struct { + ctx context.Context + apiReader client.Reader + + cacheLock lock.RWMutex + pods map[types.NamespacedName]Pod + ipToPod map[string]types.NamespacedName +} + +type Pod struct { + types.NamespacedName + OwnerInfo OwnerInfo + IPs []string +} + +type OwnerInfo struct { + APIVersion string + Kind string + Namespace string + Name string +} + +type CacheInterface interface { + GetPodByIP(ip string) *Pod +} + +var logger *zap.Logger + +func New(ctx context.Context, podInformer cache.SharedIndexInformer, apiReader client.Reader) (CacheInterface, error) { + logger = logutils.Logger.Named("PodOwnerCache") + logger.Info("create PodOwnerCache informer") + + res := &PodOwnerCache{ + ctx: ctx, + apiReader: apiReader, + cacheLock: lock.RWMutex{}, + pods: make(map[types.NamespacedName]Pod), + ipToPod: make(map[string]types.NamespacedName), + } + + _, err := podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: res.onPodAdd, + UpdateFunc: res.onPodUpdate, + DeleteFunc: res.onPodDel, + }) + if nil != err { + logger.Error(err.Error()) + return nil, err + } + + return res, nil +} + +func (s *PodOwnerCache) onPodAdd(obj interface{}) { + if pod, ok := obj.(*corev1.Pod); ok { + if pod.Spec.HostNetwork { + return + } + if len(pod.Status.PodIPs) > 0 { + ips := make([]string, 0, len(pod.Status.PodIPs)) + for _, p := range pod.Status.PodIPs { + ips = append(ips, p.IP) + } + owner, err := s.getFinalOwner(pod) + if err != nil { + if errors.IsForbidden(err) { + logger.Sugar().Debugf("forbidden to get owner of pod %s/%s", pod.Namespace, pod.Name) + return + } + logger.Warn("", zap.Error(err)) + return + } + s.cacheLock.Lock() + defer s.cacheLock.Unlock() + key := types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} + s.pods[key] = Pod{ + NamespacedName: key, + OwnerInfo: *owner, + IPs: ips, + } + for _, ip := range ips { + s.ipToPod[ip] = key + } + } + } +} + +func (s *PodOwnerCache) onPodUpdate(oldObj, newObj interface{}) { + s.onPodAdd(newObj) +} + +func (s *PodOwnerCache) onPodDel(obj interface{}) { + if pod, ok := obj.(*corev1.Pod); ok { + s.cacheLock.Lock() + defer s.cacheLock.Unlock() + + key := types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} + if _, ok := s.pods[key]; !ok { + return + } + for _, ip := range s.pods[key].IPs { + delete(s.ipToPod, ip) + } + delete(s.pods, key) + } +} + +func (s *PodOwnerCache) getFinalOwner(obj metav1.Object) (*OwnerInfo, error) { + var finalOwner *OwnerInfo + + for { + ownerRefs := obj.GetOwnerReferences() + if len(ownerRefs) == 0 { + break + } + + ownerRef := ownerRefs[0] // Assuming the first owner reference + finalOwner = &OwnerInfo{ + APIVersion: ownerRef.APIVersion, + Kind: ownerRef.Kind, + Namespace: obj.GetNamespace(), + Name: ownerRef.Name, + } + + // Prepare an empty object of the owner kind + ownerObj := &unstructured.Unstructured{} + ownerObj.SetAPIVersion(ownerRef.APIVersion) + ownerObj.SetKind(ownerRef.Kind) + + err := s.apiReader.Get(s.ctx, client.ObjectKey{ + Namespace: obj.GetNamespace(), + Name: ownerRef.Name, + }, ownerObj) + if err != nil { + return nil, fmt.Errorf("error fetching owner: %v", err) + } + + // Set obj to the current owner to continue the loop + obj = ownerObj + } + + return finalOwner, nil +} + +func (s *PodOwnerCache) GetPodByIP(ip string) *Pod { + s.cacheLock.RLock() + defer s.cacheLock.RUnlock() + item, exists := s.ipToPod[ip] + if !exists { + return nil + } + pod, exists := s.pods[item] + if !exists { + return nil + } + return &pod +} diff --git a/pkg/podownercache/pod_owner_cache_test.go b/pkg/podownercache/pod_owner_cache_test.go new file mode 100644 index 000000000..b024ca84c --- /dev/null +++ b/pkg/podownercache/pod_owner_cache_test.go @@ -0,0 +1,157 @@ +// Copyright 2024 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package podownercache + +import ( + "context" + "fmt" + appv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + kruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client" + k8sfakecli "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" + "time" +) + +// Label(K00002) + +func TestPodOwnerCache(t *testing.T) { + fakeCli := fake.NewSimpleClientset() + factory := informers.NewSharedInformerFactory(fakeCli, 0*time.Second) + informer := factory.Core().V1().Pods().Informer() + //indexer := informer.GetIndexer() + + pod := &corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-pod", + Namespace: "test-ns", + OwnerReferences: []v1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "ReplicaSet", + Name: "test-rs", + }, + }, + }, + Status: corev1.PodStatus{ + PodIPs: []corev1.PodIP{ + { + IP: "10.6.1.20", + }, + }, + }, + } + + //err := indexer.Add() + //if err != nil { + // t.Fatal(err) + //} + + stopCh := make(chan struct{}) + defer close(stopCh) + go factory.Start(stopCh) + + scheme := kruntime.NewScheme() + err := corev1.AddToScheme(scheme) + if err != nil { + t.Fatal(err) + } + err = appv1.AddToScheme(scheme) + if err != nil { + t.Fatal(err) + } + + objs := getMockObjs() + cli := k8sfakecli.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + + cache, err := New(context.Background(), informer, cli) + if err != nil { + t.Fatal(err) + } + + // case add + _, err = fakeCli.CoreV1().Pods("test-ns").Create(context.Background(), pod, v1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Second * 6) + + res := cache.GetPodByIP("10.6.1.20") + if res == nil { + t.Fatal("res is nil") + } + if res.OwnerInfo.Namespace != "test-ns" && res.OwnerInfo.Name != "test-deployment" { + t.Fatal(fmt.Println("res is not equal to test-ns and test-deployment")) + } + + // case update + _, err = fakeCli.CoreV1().Pods("test-ns").Update(context.Background(), pod, v1.UpdateOptions{}) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Second * 6) + res = cache.GetPodByIP("10.6.1.20") + if res == nil { + t.Fatal("res is nil") + } + if res.OwnerInfo.Namespace != "test-ns" && res.OwnerInfo.Name != "test-deployment" { + t.Fatal(fmt.Println("res is not equal to test-ns and test-deployment")) + } + + // case delete + err = fakeCli.CoreV1().Pods("test-ns").Delete(context.Background(), "test-pod", v1.DeleteOptions{}) + if err != nil { + t.Fatal("res is nil") + } + time.Sleep(time.Second * 6) + res = cache.GetPodByIP("10.6.1.20") + if res != nil { + t.Fatal("res is not nil") + } + + // case for not exist ip + res = cache.GetPodByIP("10.6.1.29") + if res != nil { + t.Fatal("res is not nil") + } +} + +func getMockObjs() []client.Object { + return []client.Object{ + &appv1.ReplicaSet{ + TypeMeta: v1.TypeMeta{ + Kind: "ReplicaSet", + APIVersion: "apps/v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "test-rs", + Namespace: "test-ns", + OwnerReferences: []v1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "test-deployment", + UID: "test-uid", + }, + }, + }, + }, + + &appv1.Deployment{ + TypeMeta: v1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "test-deployment", + Namespace: "test-ns", + }, + }, + } +} diff --git a/pkg/rdmametrics/metrics.go b/pkg/rdmametrics/metrics.go index 10268b989..17d1143fa 100644 --- a/pkg/rdmametrics/metrics.go +++ b/pkg/rdmametrics/metrics.go @@ -25,11 +25,10 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/spidernet-io/spiderpool/pkg/lock" "github.com/spidernet-io/spiderpool/pkg/logutils" + "github.com/spidernet-io/spiderpool/pkg/podownercache" "github.com/spidernet-io/spiderpool/pkg/rdmametrics/ethtool" ) -var cli client.Client - const netnsPath = "/var/run/netns" var ( @@ -110,8 +109,7 @@ type RDMADevice struct { IsRoot bool } -func Register(ctx context.Context, meter metric.Meter, client client.Client) error { - cli = client +func Register(ctx context.Context, meter metric.Meter, cache podownercache.CacheInterface) error { log := logutils.Logger.Named("rdma-metrics-exporter") nodeName, err := os.Hostname() if err != nil { @@ -143,6 +141,7 @@ func Register(ctx context.Context, meter metric.Meter, client client.Client) err RdmaLinkList: netlink.RdmaLinkList, LinkList: netlink.LinkList, }, + cache: cache, } err = e.registerMetrics(meter) if err != nil { @@ -166,6 +165,7 @@ type exporter struct { registration metric.Registration waitToRegisterMetrics map[string]struct{} observableMap map[string]metric.Int64ObservableCounter + cache podownercache.CacheInterface } func (e *exporter) registerMetrics(meter metric.Meter) error { @@ -241,11 +241,6 @@ func (e *exporter) Callback(ctx context.Context, observer metric.Observer) error } list = append(list, "") - podIPMap, err := getIPToPodMap(ctx, cli) - if err != nil { - e.log.Error("failed to get IP to pod map", zap.Error(err)) - return fmt.Errorf("failed to get ip map pod: %w", err) - } unRegistrationMetric := make([]string, 0) getObservable := func(key string) (metric.Int64ObservableCounter, bool) { if val, ok := e.observableMap[rdmaMetricsPrefix+key]; ok { @@ -260,7 +255,7 @@ func (e *exporter) Callback(ctx context.Context, observer metric.Observer) error return fmt.Errorf("failed to get node guid net device name map: %w", err) } for _, netNsID := range list { - if err := e.processNetNS(netNsID, podIPMap, nodeGuidNetDeviceNameMap, observer, getObservable); err != nil { + if err := e.processNetNS(netNsID, nodeGuidNetDeviceNameMap, observer, getObservable); err != nil { e.log.Error("failed to process net ns", zap.String("net_ns_id", netNsID), zap.Error(err)) continue } @@ -287,7 +282,7 @@ func (e *exporter) updateUnregisteredMetrics(unRegistrationMetric []string) { } } -func (e *exporter) processNetNS(netNsID string, ipMapPod map[string]types.NamespacedName, +func (e *exporter) processNetNS(netNsID string, nodeGuidNetDeviceNameMap map[string]string, observer metric.Observer, getObservable GetObservable) error { podPrimaryIP, statsList, err := getRDMAStats(netNsID, e.netns, nodeGuidNetDeviceNameMap, e.netlinkImpl, e.exec, e.ethtool) @@ -299,12 +294,21 @@ func (e *exporter) processNetNS(netNsID string, ipMapPod map[string]types.Namesp return nil } - var attributeNamespace, attributeName *attribute.KeyValue - if item, ok := ipMapPod[podPrimaryIP]; ok { - attributeNamespace, attributeName = getPodAttributes(item) + attributes := []*attribute.KeyValue{ + e.nodeName, + } + + if pod := e.cache.GetPodByIP(podPrimaryIP); pod != nil { + namespace := attribute.String("pod_namespace", pod.Namespace) + name := attribute.String("pod_name", pod.Name) + ownerAPIVersion := attribute.String("owner_api_version", pod.OwnerInfo.APIVersion) + ownerKind := attribute.String("owner_kind", pod.OwnerInfo.Kind) + ownerNamespace := attribute.String("owner_namespace", pod.OwnerInfo.Namespace) + ownerName := attribute.String("owner_name", pod.OwnerInfo.Name) + attributes = append(attributes, &namespace, &name, &ownerAPIVersion, &ownerKind, &ownerNamespace, &ownerName) } for _, stats := range statsList { - processStats(stats, observer, getObservable, attributeNamespace, attributeName, e.nodeName) + processStats(stats, observer, getObservable, attributes...) } return nil } diff --git a/pkg/rdmametrics/metrics_test.go b/pkg/rdmametrics/metrics_test.go index 9672ffafb..05350119c 100644 --- a/pkg/rdmametrics/metrics_test.go +++ b/pkg/rdmametrics/metrics_test.go @@ -29,16 +29,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/spidernet-io/spiderpool/pkg/logutils" + "github.com/spidernet-io/spiderpool/pkg/podownercache" ) // Label(K00002) +type FakeCache struct { + IPToPodMap map[string]podownercache.Pod +} + +func (f *FakeCache) GetPodByIP(ip string) *podownercache.Pod { + if val, ok := f.IPToPodMap[ip]; ok { + return &val + } + return nil +} + func TestRegister(t *testing.T) { ctx := context.Background() meter := noop.NewMeterProvider().Meter("test") - cli := fake.NewClientBuilder().Build() - err := Register(ctx, meter, cli) + err := Register(ctx, meter, &FakeCache{IPToPodMap: map[string]podownercache.Pod{}}) if err != nil { t.Errorf("Unexpected error: %v", err) } @@ -972,13 +983,13 @@ func TestProcessNetNS(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { meter := noop.NewMeterProvider().Meter("test") - cli = fake.NewClientBuilder().Build() fakeExec := &testexec.FakeExec{ CommandScript: tt.commandScript, } e := &exporter{ + cache: &FakeCache{}, meter: meter, netlinkImpl: NetlinkImpl{ RdmaLinkList: func() ([]*netlink.RdmaLink, error) { @@ -998,7 +1009,7 @@ func TestProcessNetNS(t *testing.T) { guidMapNetDeviceName := map[string]string{ "b6:65:05:0c:9c:5c:f6:00": "ib1", } - err := e.processNetNS(tt.netnsID, tt.ipPodMap, guidMapNetDeviceName, observer, tt.getObservable) + err := e.processNetNS(tt.netnsID, guidMapNetDeviceName, observer, tt.getObservable) if (err != nil) != tt.expectError { t.Errorf("Expected error: %v, but got: %v", tt.expectError, err) } @@ -1010,7 +1021,6 @@ func TestUpdateUnregisteredMetrics(t *testing.T) { var commandScript []testexec.FakeCommandAction meter := noop.NewMeterProvider().Meter("test") - cli = fake.NewClientBuilder().Build() fakeExec := &testexec.FakeExec{ CommandScript: commandScript, @@ -1066,7 +1076,6 @@ func TestCallback(t *testing.T) { } meter := noop.NewMeterProvider().Meter("test") - cli = fake.NewClientBuilder().Build() fakeExec := &testexec.FakeExec{ CommandScript: commandScript, @@ -1087,8 +1096,9 @@ func TestCallback(t *testing.T) { netns: func(netnsID string, toRun func() error) error { return toRun() }, - exec: fakeExec, - log: logutils.Logger.Named("rdma-metrics-exporter"), + exec: fakeExec, + log: logutils.Logger.Named("rdma-metrics-exporter"), + cache: &FakeCache{}, } err := e.registerMetrics(e.meter) if err != nil {