diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 00ddc1727..1599edf84 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,10 +21,9 @@ limitations under the License. package v1alpha1 import ( + "github.com/oceanbase/ob-operator/api/types" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - - "github.com/oceanbase/ob-operator/api/types" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/charts/oceanbase-dashboard/Chart.yaml b/charts/oceanbase-dashboard/Chart.yaml index 70bee3913..3faf7cd18 100644 --- a/charts/oceanbase-dashboard/Chart.yaml +++ b/charts/oceanbase-dashboard/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.1 +version: 0.3.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.3.1" +appVersion: "0.3.2" diff --git a/charts/oceanbase-dashboard/templates/cluster-role.yaml b/charts/oceanbase-dashboard/templates/cluster-role.yaml index d118e0a8f..18736c11a 100644 --- a/charts/oceanbase-dashboard/templates/cluster-role.yaml +++ b/charts/oceanbase-dashboard/templates/cluster-role.yaml @@ -42,6 +42,14 @@ rules: - endpoints - pods verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] - apiGroups: - extensions resources: diff --git a/go.mod b/go.mod index a2dc1c158..83a40067e 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( k8s.io/apimachinery v0.29.3 k8s.io/client-go v0.29.3 k8s.io/kubernetes v1.27.2 + k8s.io/metrics v0.0.0 sigs.k8s.io/controller-runtime v0.15.0 ) diff --git a/go.sum b/go.sum index 4190467db..e45c74497 100644 --- a/go.sum +++ b/go.sum @@ -1004,6 +1004,8 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/kubernetes v1.27.2 h1:g4v9oY6u7vBUDEuq4FvC50Bbw2K7GZuvM00IIESWVf4= k8s.io/kubernetes v1.27.2/go.mod h1:U8ZXeKBAPxeb4J4/HOaxjw1A9K6WfSH+fY2SS7CR6IM= +k8s.io/metrics v0.29.3 h1:nN+eavbMQ7Kuif2tIdTr2/F2ec2E/SIAWSruTZ+Ye6U= +k8s.io/metrics v0.29.3/go.mod h1:kb3tGGC4ZcIDIuvXyUE291RwJ5WmDu0tB4wAVZM6h2I= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/internal/dashboard/business/k8s/k8s.go b/internal/dashboard/business/k8s/k8s.go index 043f5ec5b..2eddaf0b4 100644 --- a/internal/dashboard/business/k8s/k8s.go +++ b/internal/dashboard/business/k8s/k8s.go @@ -22,6 +22,7 @@ import ( storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" "github.com/oceanbase/ob-operator/internal/dashboard/business/common" "github.com/oceanbase/ob-operator/internal/dashboard/business/constant" @@ -105,34 +106,21 @@ func extractNodeConditions(node *corev1.Node) []response.K8sNodeCondition { return conditions } -func extractNodeResource(ctx context.Context, node *corev1.Node) *response.K8sNodeResource { +func extractNodeResource(metricsMap map[string]metricsv1beta1.NodeMetrics, node *corev1.Node) *response.K8sNodeResource { nodeResource := &response.K8sNodeResource{} + metrics, ok := metricsMap[node.Name] nodeResource.CpuTotal = node.Status.Capacity.Cpu().AsApproximateFloat64() nodeResource.MemoryTotal = node.Status.Capacity.Memory().AsApproximateFloat64() / constant.GB - podList, err := resource.ListAllPods(ctx) - if err == nil { - cpuRequested := 0.0 - memoryRequested := 0.0 - for _, pod := range podList.Items { - if !strings.Contains(pod.Spec.NodeName, node.Name) { - continue - } - for _, container := range pod.Spec.Containers { - cpuRequest, found := container.Resources.Requests[ResourceCpu] - if found { - cpuRequested += cpuRequest.AsApproximateFloat64() - } - memoryRequest, found := container.Resources.Requests[ResourceMemory] - if found { - memoryRequested += memoryRequest.AsApproximateFloat64() / constant.GB - } - } + if ok { + if cpuUsed, ok := metrics.Usage[corev1.ResourceCPU]; ok { + nodeResource.CpuUsed = cpuUsed.AsApproximateFloat64() + } + if memoryUsed, ok := metrics.Usage[corev1.ResourceMemory]; ok { + nodeResource.MemoryUsed = memoryUsed.AsApproximateFloat64() / constant.GB } - nodeResource.CpuUsed = cpuRequested - nodeResource.MemoryUsed = memoryRequested + nodeResource.CpuFree = nodeResource.CpuTotal - nodeResource.CpuUsed + nodeResource.MemoryFree = nodeResource.MemoryTotal - nodeResource.MemoryUsed } - nodeResource.CpuFree = nodeResource.CpuTotal - nodeResource.CpuUsed - nodeResource.MemoryFree = nodeResource.MemoryTotal - nodeResource.MemoryUsed return nodeResource } @@ -222,6 +210,7 @@ func ListEvents(ctx context.Context, queryEventParam *param.QueryEventParam) ([] func ListNodes(ctx context.Context) ([]response.K8sNode, error) { nodes := make([]response.K8sNode, 0) nodeList, err := resource.ListNodes(ctx) + nodeMetricsMap, metricsErr := resource.ListNodeMetrics(ctx) if err == nil { for _, node := range nodeList.Items { internalAddress, externalAddress := extractNodeAddress(&node) @@ -240,28 +229,21 @@ func ListNodes(ctx context.Context) ([]response.K8sNode, error) { CRI: node.Status.NodeInfo.ContainerRuntimeVersion, } + nodeResource := &response.K8sNodeResource{} + if metricsErr == nil { + nodeResource = extractNodeResource(nodeMetricsMap, &node) + } else { + logger.Errorf("Got error when list node metrics, err: %v", metricsErr) + } nodes = append(nodes, response.K8sNode{ Info: nodeInfo, - Resource: extractNodeResource(ctx, &node), + Resource: nodeResource, }) } } return nodes, err } -func ListNodeResources(ctx context.Context) ([]response.K8sNodeResource, error) { - nodeList, err := resource.ListNodes(ctx) - if err != nil { - return nil, err - } - nodeResources := make([]response.K8sNodeResource, 0, len(nodeList.Items)) - for _, node := range nodeList.Items { - nodeResource := extractNodeResource(ctx, &node) - nodeResources = append(nodeResources, *nodeResource) - } - return nodeResources, nil -} - func ListStorageClasses(ctx context.Context) ([]response.StorageClass, error) { storageClasses := make([]response.StorageClass, 0) storageClassList, err := resource.ListStorageClasses(ctx) diff --git a/internal/dashboard/business/metric/constant/metric_constant.go b/internal/dashboard/business/metric/constant/metric_constant.go index 501a80aba..b3017df46 100644 --- a/internal/dashboard/business/metric/constant/metric_constant.go +++ b/internal/dashboard/business/metric/constant/metric_constant.go @@ -13,10 +13,10 @@ See the Mulan PSL v2 for more details. package constant const ( - MetricConfigFile = "internal/assets/metric.yaml" - MetricConfigFileEnUS = "internal/assets/metric_en_US.yaml" - MetricConfigFileZhCN = "internal/assets/metric_zh_CN.yaml" - MetricExprConfigFile = "internal/assets/metric_expr.yaml" + MetricConfigFile = "internal/assets/dashboard/metric.yaml" + MetricConfigFileEnUS = "internal/assets/dashboard/metric_en_US.yaml" + MetricConfigFileZhCN = "internal/assets/dashboard/metric_zh_CN.yaml" + MetricExprConfigFile = "internal/assets/dashboard/metric_expr.yaml" ) const ( diff --git a/internal/dashboard/business/oceanbase/obcluster.go b/internal/dashboard/business/oceanbase/obcluster.go index ef53080e5..7a711fe89 100644 --- a/internal/dashboard/business/oceanbase/obcluster.go +++ b/internal/dashboard/business/oceanbase/obcluster.go @@ -203,12 +203,28 @@ func buildOBClusterTopologyResp(ctx context.Context, obcluster *v1alpha1.OBClust affinities := make([]modelcommon.AffinitySpec, 0) if obzone.Spec.Topology.Affinity != nil { zoneAffinity := obzone.Spec.Topology.Affinity - switch { - case zoneAffinity.NodeAffinity != nil: - for _, term := range zoneAffinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { - for _, req := range term.MatchExpressions { + if zoneAffinity.NodeAffinity != nil { + zn := zoneAffinity.NodeAffinity + if zn.RequiredDuringSchedulingIgnoredDuringExecution != nil { + for _, term := range zn.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + for _, req := range term.MatchExpressions { + affinities = append(affinities, modelcommon.AffinitySpec{ + Type: modelcommon.NodeAffinityType, + SelectorExpression: modelcommon.SelectorExpression{ + Key: req.Key, + Operator: string(req.Operator), + Values: req.Values, + }, + }) + } + } + } + for _, term := range zn.PreferredDuringSchedulingIgnoredDuringExecution { + for _, req := range term.Preference.MatchExpressions { affinities = append(affinities, modelcommon.AffinitySpec{ - Type: modelcommon.NodeAffinityType, + Type: modelcommon.NodeAffinityType, + Weight: term.Weight, + Preferred: true, SelectorExpression: modelcommon.SelectorExpression{ Key: req.Key, Operator: string(req.Operator), @@ -217,8 +233,10 @@ func buildOBClusterTopologyResp(ctx context.Context, obcluster *v1alpha1.OBClust }) } } - case zoneAffinity.PodAffinity != nil: - for _, term := range zoneAffinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + } + if zoneAffinity.PodAffinity != nil { + zp := zoneAffinity.PodAffinity + for _, term := range zp.RequiredDuringSchedulingIgnoredDuringExecution { for _, req := range term.LabelSelector.MatchExpressions { affinities = append(affinities, modelcommon.AffinitySpec{ Type: modelcommon.PodAffinityType, @@ -230,8 +248,24 @@ func buildOBClusterTopologyResp(ctx context.Context, obcluster *v1alpha1.OBClust }) } } - case zoneAffinity.PodAntiAffinity != nil: - for _, term := range zoneAffinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + for _, term := range zp.PreferredDuringSchedulingIgnoredDuringExecution { + for _, req := range term.PodAffinityTerm.LabelSelector.MatchExpressions { + affinities = append(affinities, modelcommon.AffinitySpec{ + Type: modelcommon.PodAffinityType, + Weight: term.Weight, + Preferred: true, + SelectorExpression: modelcommon.SelectorExpression{ + Key: req.Key, + Operator: string(req.Operator), + Values: req.Values, + }, + }) + } + } + } + if zoneAffinity.PodAntiAffinity != nil { + zpa := zoneAffinity.PodAntiAffinity + for _, term := range zpa.RequiredDuringSchedulingIgnoredDuringExecution { for _, req := range term.LabelSelector.MatchExpressions { affinities = append(affinities, modelcommon.AffinitySpec{ Type: modelcommon.PodAntiAffinityType, @@ -243,14 +277,33 @@ func buildOBClusterTopologyResp(ctx context.Context, obcluster *v1alpha1.OBClust }) } } + for _, term := range zpa.PreferredDuringSchedulingIgnoredDuringExecution { + for _, req := range term.PodAffinityTerm.LabelSelector.MatchExpressions { + affinities = append(affinities, modelcommon.AffinitySpec{ + Type: modelcommon.PodAntiAffinityType, + Weight: term.Weight, + Preferred: true, + SelectorExpression: modelcommon.SelectorExpression{ + Key: req.Key, + Operator: string(req.Operator), + Values: req.Values, + }, + }) + } + } } } - tolerations := make([]modelcommon.KVPair, 0) + tolerations := make([]modelcommon.TolerationSpec, 0) for _, toleration := range obzone.Spec.Topology.Tolerations { - tolerations = append(tolerations, modelcommon.KVPair{ - Key: toleration.Key, - Value: toleration.Value, + tolerations = append(tolerations, modelcommon.TolerationSpec{ + KVPair: modelcommon.KVPair{ + Key: toleration.Key, + Value: toleration.Value, + }, + Operator: string(toleration.Operator), + Effect: string(toleration.Effect), + TolerationSeconds: toleration.TolerationSeconds, }) } respZone := response.OBZone{ diff --git a/internal/dashboard/model/common/common.go b/internal/dashboard/model/common/common.go index d324fb30f..827b89a57 100644 --- a/internal/dashboard/model/common/common.go +++ b/internal/dashboard/model/common/common.go @@ -38,6 +38,15 @@ type AffinityType string type AffinitySpec struct { SelectorExpression `json:",inline"` Type AffinityType `json:"type"` + Weight int32 `json:"weight,omitempty"` + Preferred bool `json:"preferred,omitempty"` +} + +type TolerationSpec struct { + KVPair `json:",inline"` + Operator string `json:"operator"` + Effect string `json:"effect"` + TolerationSeconds *int64 `json:"tolerationSeconds,omitempty"` } type ClusterMode string diff --git a/internal/dashboard/model/response/obcluster.go b/internal/dashboard/model/response/obcluster.go index fa9d9228d..5dc586753 100644 --- a/internal/dashboard/model/response/obcluster.go +++ b/internal/dashboard/model/response/obcluster.go @@ -41,8 +41,8 @@ type OBZone struct { OBServers []OBServer `json:"observers,omitempty"` NodeSelector []common.KVPair `json:"nodeSelector,omitempty"` - Tolerations []common.KVPair `json:"tolerations,omitempty"` - Affinities []common.AffinitySpec `json:"affinities,omitempty"` + Tolerations []common.TolerationSpec `json:"tolerations,omitempty"` + Affinities []common.AffinitySpec `json:"affinities,omitempty"` } type OBMetrics struct { diff --git a/make/dashboard.mk b/make/dashboard.mk index 0f87bb146..feb1cf975 100644 --- a/make/dashboard.mk +++ b/make/dashboard.mk @@ -4,7 +4,7 @@ PROJECT=oceanbase-dashboard PROCESSOR=4 PWD ?= $(shell pwd) -DASHBOARD_VERSION ?= 0.3.1 +DASHBOARD_VERSION ?= 0.3.2 DASHBOARD_IMG ?= quay.io/oceanbase/oceanbase-dashboard:${DASHBOARD_VERSION} COMMIT_HASH ?= $(shell git rev-parse --short HEAD) BUILD_TIMESTAMP ?= $(shell date '+%Y%m%d%H%M%S') @@ -31,7 +31,7 @@ dashboard-build: dashboard-bindata-gen dashboard-doc-gen ## Build oceanbase-dash .PHONY: dashboard-bindata-gen dashboard-bindata-gen: dashboard-dep-install ## Generate bindata - go-bindata -o internal/dashboard/generated/bindata/bindata.go -pkg bindata internal/assets/... + go-bindata -o internal/dashboard/generated/bindata/bindata.go -pkg bindata internal/assets/dashboard/... .PHONY: dashboard-clean dashboard-clean: ## Clean build diff --git a/pkg/k8s/client/client.go b/pkg/k8s/client/client.go index d69c10c6a..04c5e8208 100644 --- a/pkg/k8s/client/client.go +++ b/pkg/k8s/client/client.go @@ -21,13 +21,15 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/metadata" "k8s.io/client-go/rest" + "k8s.io/metrics/pkg/client/clientset/versioned" ) type Client struct { - ClientSet *kubernetes.Clientset - DynamicClient dynamic.Interface - MetaClient metadata.Interface - config *rest.Config + ClientSet *kubernetes.Clientset + DynamicClient dynamic.Interface + MetaClient metadata.Interface + MetricsClientset *versioned.Clientset + config *rest.Config } var client *Client @@ -75,11 +77,16 @@ func getClientFromConfig(config *rest.Config) (*Client, error) { if err != nil { return nil, errors.Wrap(err, "failed to create meta client") } + metricsClientset, err := versioned.NewForConfig(config) + if err != nil { + return nil, errors.Wrap(err, "failed to create metrics client") + } return &Client{ - ClientSet: clientset, - DynamicClient: dynamicClient, - MetaClient: metaClient, - config: config, + ClientSet: clientset, + DynamicClient: dynamicClient, + MetaClient: metaClient, + MetricsClientset: metricsClientset, + config: config, }, nil } diff --git a/pkg/k8s/resource/node.go b/pkg/k8s/resource/node.go index 7787844ed..69144d184 100644 --- a/pkg/k8s/resource/node.go +++ b/pkg/k8s/resource/node.go @@ -17,6 +17,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" "github.com/oceanbase/ob-operator/pkg/k8s/client" k8sconst "github.com/oceanbase/ob-operator/pkg/k8s/constants" @@ -30,3 +31,17 @@ func ListNodes(ctx context.Context) (*corev1.NodeList, error) { TimeoutSeconds: &timeout, }) } + +func ListNodeMetrics(ctx context.Context) (map[string]metricsv1beta1.NodeMetrics, error) { + client := client.GetClient() + nodeMetricsMap := make(map[string]metricsv1beta1.NodeMetrics) + metricsList, err := client.MetricsClientset.MetricsV1beta1().NodeMetricses().List(ctx, metav1.ListOptions{ + TimeoutSeconds: &timeout, + }) + if err == nil { + for _, metrics := range metricsList.Items { + nodeMetricsMap[metrics.Name] = metrics + } + } + return nodeMetricsMap, err +} diff --git a/pkg/k8s/resource/pod.go b/pkg/k8s/resource/pod.go index b9447eef1..a73309dc4 100644 --- a/pkg/k8s/resource/pod.go +++ b/pkg/k8s/resource/pod.go @@ -21,13 +21,6 @@ import ( "github.com/oceanbase/ob-operator/pkg/k8s/client" ) -func ListAllPods(ctx context.Context) (*corev1.PodList, error) { - client := client.GetClient() - return client.ClientSet.CoreV1().Pods(corev1.NamespaceAll).List(ctx, metav1.ListOptions{ - TimeoutSeconds: &timeout, - }) -} - func ListPods(ctx context.Context, namespace string) (*corev1.PodList, error) { client := client.GetClient() return client.ClientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{