Some optimizations for dashboard handlers (#251)

internal/assets/metric_zh_CN.yaml

@@ -648,6 +648,7 @@ OBTENANT:
   name: "\u5B58\u50A8\u4E0E\u7F13\u5B58"
 OBCLUSTER_OVERVIEW:
 - description: Cluster Overview
+  name: "\u96c6\u7fa4\u6027\u80fd\u6982\u89c8"
   metricGroups:
   - description: "\u5E73\u5747\u6BCF\u79D2\u5904\u7406 SQL \u8BED\u53E5\u6570"
     metrics:

internal/oceanbase/clients.go → internal/clients/clients.go

@@ -10,11 +10,11 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 See the Mulan PSL v2 for more details.
 */
 
-package oceanbase
+package clients
 
 import (
 	"github.com/oceanbase/ob-operator/api/v1alpha1"
-	"github.com/oceanbase/ob-operator/internal/oceanbase/schema"
+	"github.com/oceanbase/ob-operator/internal/clients/schema"
 	"github.com/oceanbase/ob-operator/pkg/k8s/client"
 )
 

internal/oceanbase/clients_test.go → internal/clients/clients_test.go

@@ -10,7 +10,7 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 See the Mulan PSL v2 for more details.
 */
 
-package oceanbase
+package clients
 
 import (
 	"context"

internal/oceanbase/obcluster.go → internal/clients/obcluster.go

@@ -10,7 +10,7 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 See the Mulan PSL v2 for more details.
 */
 
-package oceanbase
+package clients
 
 import (
 	"context"
@@ -24,8 +24,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 
 	"github.com/oceanbase/ob-operator/api/v1alpha1"
+	"github.com/oceanbase/ob-operator/internal/clients/schema"
 	oceanbaseconst "github.com/oceanbase/ob-operator/internal/const/oceanbase"
-	"github.com/oceanbase/ob-operator/internal/oceanbase/schema"
 	"github.com/oceanbase/ob-operator/pkg/k8s/client"
 )
 
@@ -156,7 +156,6 @@ func ListOBZonesOfOBCluster(ctx context.Context, obcluster *v1alpha1.OBCluster)
 func ListOBServersOfOBZone(ctx context.Context, obzone *v1alpha1.OBZone) (*v1alpha1.OBServerList, error) {
 	client := client.GetClient()
 	var observerList v1alpha1.OBServerList
-	logger.Infof("get observer list of obzone %s", obzone.Name)
 	obj, err := client.DynamicClient.Resource(schema.OBServerRes).Namespace(obzone.Namespace).List(ctx, metav1.ListOptions{
 		LabelSelector: fmt.Sprintf("%s=%s", oceanbaseconst.LabelRefOBZone, obzone.Name),
 	})

internal/oceanbase/obtenant.go → internal/clients/obtenant.go

@@ -10,7 +10,7 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 See the Mulan PSL v2 for more details.
 */
 
-package oceanbase
+package clients
 
 import (
 	"context"
@@ -20,8 +20,8 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/oceanbase/ob-operator/api/v1alpha1"
+	"github.com/oceanbase/ob-operator/internal/clients/schema"
 	oceanbaseconst "github.com/oceanbase/ob-operator/internal/const/oceanbase"
-	"github.com/oceanbase/ob-operator/internal/oceanbase/schema"
 )
 
 func CreateOBTenant(ctx context.Context, tenant *v1alpha1.OBTenant) (*v1alpha1.OBTenant, error) {

internal/oceanbase/oceanbase_suite_test.go → internal/clients/oceanbase_suite_test.go

@@ -10,7 +10,7 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 See the Mulan PSL v2 for more details.
 */
 
-package oceanbase_test
+package clients
 
 import (
 	"testing"

internal/dashboard/business/k8s/k8s.go

@@ -17,16 +17,15 @@ import (
 	"strings"
 
 	logger "github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/oceanbase/ob-operator/internal/dashboard/business/common"
 	"github.com/oceanbase/ob-operator/internal/dashboard/business/constant"
 	"github.com/oceanbase/ob-operator/internal/dashboard/model/param"
 	"github.com/oceanbase/ob-operator/internal/dashboard/model/response"
 	"github.com/oceanbase/ob-operator/pkg/k8s/resource"
-
-	corev1 "k8s.io/api/core/v1"
-	storagev1 "k8s.io/api/storage/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
@@ -147,6 +146,8 @@ func ListEvents(queryEventParam *param.QueryEventParam) ([]response.K8sEvent, er
 			kind = "OBCluster"
 		case "OBTENANT":
 			kind = "OBTenant"
+		case "OBBACKUPPOLICY":
+			kind = "OBTenantBackupPolicy"
 		default:
 			kind = queryEventParam.ObjectType
 		}
@@ -170,15 +171,15 @@ func ListEvents(queryEventParam *param.QueryEventParam) ([]response.K8sEvent, er
 		listOptions.FieldSelector = strings.Join(selectors, ",")
 	}
 	eventList, err := resource.ListEvents(ns, listOptions)
-	logger.Infof("query events with param: %v", queryEventParam)
+	logger.Infof("Query events with param: %+v", queryEventParam)
 	if err == nil {
 		for _, event := range eventList.Items {
 			events = append(events, response.K8sEvent{
 				Namespace:  event.Namespace,
 				Type:       event.Type,
 				Count:      event.Count,
-				FirstOccur: float64(event.FirstTimestamp.UnixMilli()) / 1000,
-				LastSeen:   float64(event.LastTimestamp.UnixMilli()) / 1000,
+				FirstOccur: event.FirstTimestamp.Unix(),
+				LastSeen:   event.LastTimestamp.Unix(),
 				Reason:     event.Reason,
 				Message:    event.Message,
 				Object:     fmt.Sprintf("%s/%s", event.InvolvedObject.Kind, event.InvolvedObject.Name),
@@ -200,7 +201,7 @@ func ListNodes() ([]response.K8sNode, error) {
 				Roles:      extractNodeRoles(&node),
 				Labels:     common.MapToKVs(node.Labels),
 				Conditions: extractNodeConditions(&node),
-				Uptime:     float64(node.CreationTimestamp.UnixMilli()) / 1000,
+				Uptime:     node.CreationTimestamp.Unix(),
 				InternalIP: internalAddress,
 				ExternalIP: externalAddress,
 				Version:    node.Status.NodeInfo.KubeletVersion,

internal/dashboard/business/metric/metric.go

@@ -68,6 +68,7 @@ func ListMetricClasses(scope, language string) ([]response.MetricClass, error) {
 	}
 	logger.Debugf("metric configs contents: %s", string(metricConfigContent))
 	metricConfigMap := make(map[string][]response.MetricClass)
+	// TODO: Do not unmarshal the file every time, cache the result
 	err = yaml.Unmarshal(metricConfigContent, &metricConfigMap)
 	if err != nil {
 		return metricClasses, err
@@ -99,20 +100,52 @@ func extractMetricData(name string, resp *external.PrometheusQueryRangeResponse)
 			Name:   name,
 			Labels: bizcommon.MapToKVs(result.Metric),
 		}
+		lastValid := math.NaN()
+		invalidTimestamps := make([]float64, 0)
+		// one loop to handle invalid timestamps interpolation
 		for _, value := range result.Values {
 			t := value[0].(float64)
 			v, err := strconv.ParseFloat(value[1].(string), 64)
 			if err != nil {
 				logger.Warnf("failed to parse value %v", value)
+				invalidTimestamps = append(invalidTimestamps, t)
 			} else if math.IsNaN(v) {
-				logger.Debugf("skip NaN value at timestamp %f", t)
+				logger.Debugf("value at timestamp %f is NaN", t)
+				invalidTimestamps = append(invalidTimestamps, t)
 			} else {
+				// if there are invalid timestamps, interpolate them
+				if len(invalidTimestamps) > 0 {
+					var interpolated float64
+					if math.IsNaN(lastValid) {
+						interpolated = v
+					} else {
+						interpolated = (lastValid + v) / 2
+					}
+					// interpolate invalid slots with last valid value
+					for _, it := range invalidTimestamps {
+						values = append(values, response.MetricValue{
+							Timestamp: it,
+							Value:     interpolated,
+						})
+					}
+					invalidTimestamps = invalidTimestamps[:0]
+				}
 				values = append(values, response.MetricValue{
 					Timestamp: t,
 					Value:     v,
 				})
+				lastValid = v
 			}
 		}
+		if math.IsNaN(lastValid) {
+			lastValid = 0.0
+		}
+		for _, it := range invalidTimestamps {
+			values = append(values, response.MetricValue{
+				Timestamp: it,
+				Value:     lastValid,
+			})
+		}
 		metricDatas = append(metricDatas, response.MetricData{
 			Metric: metric,
 			Values: values,

internal/dashboard/business/oceanbase/obcluster.go

@@ -26,14 +26,14 @@ import (
 
 	apitypes "github.com/oceanbase/ob-operator/api/types"
 	"github.com/oceanbase/ob-operator/api/v1alpha1"
+	"github.com/oceanbase/ob-operator/internal/clients"
 	oceanbaseconst "github.com/oceanbase/ob-operator/internal/const/oceanbase"
 	clusterstatus "github.com/oceanbase/ob-operator/internal/const/status/obcluster"
 	"github.com/oceanbase/ob-operator/internal/dashboard/business/common"
 	"github.com/oceanbase/ob-operator/internal/dashboard/business/constant"
 	modelcommon "github.com/oceanbase/ob-operator/internal/dashboard/model/common"
 	"github.com/oceanbase/ob-operator/internal/dashboard/model/param"
 	"github.com/oceanbase/ob-operator/internal/dashboard/model/response"
-	"github.com/oceanbase/ob-operator/internal/oceanbase"
 )
 
 const (
@@ -158,20 +158,19 @@ func buildOBClusterResponse(ctx context.Context, obcluster *v1alpha1.OBCluster)
 }
 
 func buildOBClusterTopologyResp(ctx context.Context, obcluster *v1alpha1.OBCluster) ([]response.OBZone, error) {
-	obzoneList, err := oceanbase.ListOBZonesOfOBCluster(ctx, obcluster)
+	obzoneList, err := clients.ListOBZonesOfOBCluster(ctx, obcluster)
 	if err != nil {
 		return nil, errors.Wrapf(err, "List obzone of obcluster %s %s", obcluster.Namespace, obcluster.Name)
 	}
 	topology := make([]response.OBZone, 0, len(obzoneList.Items))
 	for _, obzone := range obzoneList.Items {
 		observers := make([]response.OBServer, 0)
-		observerList, err := oceanbase.ListOBServersOfOBZone(ctx, &obzone)
+		observerList, err := clients.ListOBServersOfOBZone(ctx, &obzone)
 		if err != nil {
 			return nil, errors.Wrapf(err, "List observers of obzone %s %s", obzone.Namespace, obzone.Name)
 		}
-		logger.Infof("found %d observer", len(observerList.Items))
 		for _, observer := range observerList.Items {
-			logger.Infof("add observer %s to result", observer.Name)
+			logger.Debugf("add observer %s to result", observer.Name)
 			observers = append(observers, response.OBServer{
 				Namespace:    observer.Namespace,
 				Name:         observer.Name,
@@ -266,7 +265,7 @@ func buildOBClusterTopologyResp(ctx context.Context, obcluster *v1alpha1.OBClust
 
 func ListOBClusters(ctx context.Context) ([]response.OBClusterOverview, error) {
 	obclusters := make([]response.OBClusterOverview, 0)
-	obclusterList, err := oceanbase.ListAllOBClusters(ctx)
+	obclusterList, err := clients.ListAllOBClusters(ctx)
 	if err != nil {
 		return obclusters, errors.Wrap(err, "failed to list obclusters")
 	}
@@ -481,28 +480,28 @@ func generateOBClusterInstance(param *param.CreateOBClusterParam) *v1alpha1.OBCl
 
 func CreateOBCluster(ctx context.Context, param *param.CreateOBClusterParam) error {
 	obcluster := generateOBClusterInstance(param)
-	err := oceanbase.CreateSecretsForOBCluster(ctx, obcluster, param.RootPassword)
+	err := clients.CreateSecretsForOBCluster(ctx, obcluster, param.RootPassword)
 	if err != nil {
 		return errors.Wrap(err, "Create secrets for obcluster")
 	}
 	logger.Infof("Generated obcluster instance:%v", obcluster)
-	return oceanbase.CreateOBCluster(ctx, obcluster)
+	return clients.CreateOBCluster(ctx, obcluster)
 }
 
 func UpgradeObCluster(ctx context.Context, obclusterIdentity *param.K8sObjectIdentity, updateParam *param.UpgradeOBClusterParam) error {
-	obcluster, err := oceanbase.GetOBCluster(ctx, obclusterIdentity.Namespace, obclusterIdentity.Name)
+	obcluster, err := clients.GetOBCluster(ctx, obclusterIdentity.Namespace, obclusterIdentity.Name)
 	if err != nil {
 		return errors.Wrapf(err, "Get obcluster %s %s", obclusterIdentity.Namespace, obclusterIdentity.Name)
 	}
 	if obcluster.Status.Status != clusterstatus.Running {
 		return errors.Errorf("Obcluster status invalid %s", obcluster.Status.Status)
 	}
 	obcluster.Spec.OBServerTemplate.Image = updateParam.Image
-	return oceanbase.UpdateOBCluster(ctx, obcluster)
+	return clients.UpdateOBCluster(ctx, obcluster)
 }
 
 func ScaleOBServer(ctx context.Context, obzoneIdentity *param.OBZoneIdentity, scaleParam *param.ScaleOBServerParam) error {
-	obcluster, err := oceanbase.GetOBCluster(ctx, obzoneIdentity.Namespace, obzoneIdentity.Name)
+	obcluster, err := clients.GetOBCluster(ctx, obzoneIdentity.Namespace, obzoneIdentity.Name)
 	if err != nil {
 		return errors.Wrapf(err, "Get obcluster %s %s", obzoneIdentity.Namespace, obzoneIdentity.Name)
 	}
@@ -527,11 +526,11 @@ func ScaleOBServer(ctx context.Context, obzoneIdentity *param.OBZoneIdentity, sc
 	if !replicaChanged {
 		return errors.Errorf("obzone %s replica already satisfied in obcluster %s %s", obzoneIdentity.OBZoneName, obzoneIdentity.Namespace, obzoneIdentity.Name)
 	}
-	return oceanbase.UpdateOBCluster(ctx, obcluster)
+	return clients.UpdateOBCluster(ctx, obcluster)
 }
 
 func DeleteOBZone(ctx context.Context, obzoneIdentity *param.OBZoneIdentity) error {
-	obcluster, err := oceanbase.GetOBCluster(ctx, obzoneIdentity.Namespace, obzoneIdentity.Name)
+	obcluster, err := clients.GetOBCluster(ctx, obzoneIdentity.Namespace, obzoneIdentity.Name)
 	if err != nil {
 		return errors.Wrapf(err, "Get obcluster %s %s", obzoneIdentity.Namespace, obzoneIdentity.Name)
 	}
@@ -551,11 +550,11 @@ func DeleteOBZone(ctx context.Context, obzoneIdentity *param.OBZoneIdentity) err
 		return errors.Errorf("obzone %s not found in obcluster %s %s", obzoneIdentity.OBZoneName, obzoneIdentity.Namespace, obzoneIdentity.Name)
 	}
 	obcluster.Spec.Topology = newTopology
-	return oceanbase.UpdateOBCluster(ctx, obcluster)
+	return clients.UpdateOBCluster(ctx, obcluster)
 }
 
 func AddOBZone(ctx context.Context, obclusterIdentity *param.K8sObjectIdentity, zone *param.ZoneTopology) error {
-	obcluster, err := oceanbase.GetOBCluster(ctx, obclusterIdentity.Namespace, obclusterIdentity.Name)
+	obcluster, err := clients.GetOBCluster(ctx, obclusterIdentity.Namespace, obclusterIdentity.Name)
 	if err != nil {
 		return errors.Wrapf(err, "Get obcluster %s %s", obclusterIdentity.Namespace, obclusterIdentity.Name)
 	}
@@ -572,24 +571,24 @@ func AddOBZone(ctx context.Context, obclusterIdentity *param.K8sObjectIdentity,
 		NodeSelector: common.KVsToMap(zone.NodeSelector),
 		Replica:      zone.Replicas,
 	})
-	return oceanbase.UpdateOBCluster(ctx, obcluster)
+	return clients.UpdateOBCluster(ctx, obcluster)
 }
 
 func GetOBCluster(ctx context.Context, obclusterIdentity *param.K8sObjectIdentity) (*response.OBCluster, error) {
-	obcluster, err := oceanbase.GetOBCluster(ctx, obclusterIdentity.Namespace, obclusterIdentity.Name)
+	obcluster, err := clients.GetOBCluster(ctx, obclusterIdentity.Namespace, obclusterIdentity.Name)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Get obcluster %s %s", obclusterIdentity.Namespace, obclusterIdentity.Name)
 	}
 	return buildOBClusterResponse(ctx, obcluster)
 }
 
 func DeleteOBCluster(ctx context.Context, obclusterIdentity *param.K8sObjectIdentity) error {
-	return oceanbase.DeleteOBCluster(ctx, obclusterIdentity.Namespace, obclusterIdentity.Name)
+	return clients.DeleteOBCluster(ctx, obclusterIdentity.Namespace, obclusterIdentity.Name)
 }
 
 func GetOBClusterStatistic(ctx context.Context) ([]response.OBClusterStastistic, error) {
 	statisticResult := make([]response.OBClusterStastistic, 0)
-	obclusterList, err := oceanbase.ListAllOBClusters(ctx)
+	obclusterList, err := clients.ListAllOBClusters(ctx)
 	if err != nil {
 		return statisticResult, errors.Wrap(err, "failed to list obclusters")
 	}

internal/dashboard/business/oceanbase/obcluster_usage.go

@@ -19,25 +19,25 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/oceanbase/ob-operator/api/v1alpha1"
+	"github.com/oceanbase/ob-operator/internal/clients"
 	oceanbaseconst "github.com/oceanbase/ob-operator/internal/const/oceanbase"
 	"github.com/oceanbase/ob-operator/internal/dashboard/model/param"
 	"github.com/oceanbase/ob-operator/internal/dashboard/model/response"
-	"github.com/oceanbase/ob-operator/internal/oceanbase"
 	httpErr "github.com/oceanbase/ob-operator/pkg/errors"
 	"github.com/oceanbase/ob-operator/pkg/k8s/client"
 	"github.com/oceanbase/ob-operator/pkg/oceanbase-sdk/connector"
 	"github.com/oceanbase/ob-operator/pkg/oceanbase-sdk/model"
 	"github.com/oceanbase/ob-operator/pkg/oceanbase-sdk/operation"
 )
 
-func GetOBClusterEssentialParameters(ctx context.Context, nn *param.K8sObjectIdentity) (*response.OBClusterResources, error) {
-	obcluster, err := oceanbase.GetOBCluster(ctx, nn.Namespace, nn.Name)
+func GetOBClusterUsages(ctx context.Context, nn *param.K8sObjectIdentity) (*response.OBClusterResources, error) {
+	obcluster, err := clients.GetOBCluster(ctx, nn.Namespace, nn.Name)
 	if err != nil {
 		return nil, err
 	}
 	clt := client.GetClient()
 	serverList := &v1alpha1.OBServerList{}
-	err = oceanbase.ServerClient.List(ctx, nn.Namespace, serverList, metav1.ListOptions{})
+	err = clients.ServerClient.List(ctx, nn.Namespace, serverList, metav1.ListOptions{})
 	if err != nil {
 		return nil, httpErr.NewInternal(err.Error())
 	}