diff --git a/charts/charts/grafana/dashboards/0.json b/charts/charts/grafana/dashboards/0.json index 83fcdc5..3958ab7 100644 --- a/charts/charts/grafana/dashboards/0.json +++ b/charts/charts/grafana/dashboards/0.json @@ -17,7 +17,7 @@ "gnetId": 3070, "graphTooltip": 0, "id": 1, - "iteration": 1642043560459, + "iteration": 1652444252935, "links": [], "panels": [ { @@ -674,6 +674,7 @@ "steppedLine": true, "targets": [ { + "exemplar": true, "expr": "histogram_quantile(1, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$job\"}[5m])) by (job,instance,endpoint,le))", "format": "time_series", "hide": false, @@ -2359,9 +2360,10 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "go_goroutines{job=\"$job\"}", "interval": "", - "legendFormat": "{{remark}}_{{etcdResourceName}}_{{instance}}_{{endpoint}}", + "legendFormat": "{{job}}_{{instance}}_{{endpoint}}", "refId": "A" } ], @@ -2892,6 +2894,7 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "", "fieldConfig": { "defaults": { "links": [] @@ -2907,7 +2910,7 @@ "y": 112 }, "hiddenSeries": false, - "id": 50, + "id": 86, "legend": { "avg": false, "current": false, @@ -2927,7 +2930,7 @@ "paceLength": 10, "percentage": false, "pluginVersion": "8.0.3", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -2936,18 +2939,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(etcd_http_received_total{job=\"$job\"}[5m])) by (method,instance,job)", + "exemplar": true, + "expr": "etcd_debugging_lease_granted_total{job=\"$job\"} - etcd_debugging_lease_revoked_total{job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{job}}_{{instance}}_{{endpoint}}_{{method}}", - "refId": "A" + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}_{{instance}}_{{endpoint}}", + "refId": "A", + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "ETCD V2 HTTP QPS", + "title": "current lease count", "tooltip": { "shared": true, "sort": 2, @@ -2963,6 +2969,7 @@ }, "yaxes": [ { + "$$hashKey": "object:231", "format": "short", "label": null, "logBase": 1, @@ -2971,6 +2978,7 @@ "show": true }, { + "$$hashKey": "object:232", "format": "short", "label": null, "logBase": 1, @@ -2990,6 +2998,7 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "", "fieldConfig": { "defaults": { "links": [] @@ -3005,6 +3014,213 @@ "y": 112 }, "hiddenSeries": false, + "id": 87, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "8.0.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_bucket{job=\"$job\"}[5m])) by (job,instance,endpoint,le))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}_{{instance}}_{{endpoint}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "compaction_pause_duration P99", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:231", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:232", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 121 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "8.0.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(etcd_debugging_mvcc_index_compaction_pause_duration_milliseconds_bucket{job=\"$job\"}[5m])) by (job,instance,endpoint,le))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}_{{instance}}_{{endpoint}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "mvcc_index_compaction_pause_duration P99", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:231", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:232", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 121 + }, + "hiddenSeries": false, "id": 8, "legend": { "avg": false, @@ -3109,7 +3325,105 @@ "h": 9, "w": 12, "x": 0, - "y": 121 + "y": 130 + }, + "hiddenSeries": false, + "id": 50, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "8.0.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_http_received_total{job=\"$job\"}[5m])) by (method,instance,job)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{job}}_{{instance}}_{{endpoint}}_{{method}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ETCD V2 HTTP QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 130 }, "hiddenSeries": false, "id": 52, @@ -3196,7 +3510,7 @@ "h": 1, "w": 24, "x": 0, - "y": 130 + "y": 139 }, "id": 78, "panels": [], @@ -3302,7 +3616,7 @@ "h": 9, "w": 12, "x": 0, - "y": 131 + "y": 140 }, "id": 63, "interval": null, @@ -3369,7 +3683,7 @@ "h": 9, "w": 12, "x": 12, - "y": 131 + "y": 140 }, "hiddenSeries": false, "id": 56, @@ -3471,7 +3785,7 @@ "h": 9, "w": 12, "x": 0, - "y": 140 + "y": 149 }, "hiddenSeries": false, "id": 58, @@ -3574,7 +3888,7 @@ "h": 9, "w": 12, "x": 12, - "y": 140 + "y": 149 }, "hiddenSeries": false, "id": 54, @@ -3677,7 +3991,7 @@ "h": 9, "w": 12, "x": 0, - "y": 149 + "y": 158 }, "hiddenSeries": false, "id": 81, @@ -3850,7 +4164,7 @@ "h": 9, "w": 12, "x": 12, - "y": 149 + "y": 158 }, "id": 85, "options": { @@ -3939,7 +4253,7 @@ "h": 9, "w": 12, "x": 0, - "y": 158 + "y": 167 }, "id": 84, "options": { @@ -3965,7 +4279,7 @@ "type": "timeseries" } ], - "refresh": false, + "refresh": "", "schemaVersion": 30, "style": "dark", "tags": [], @@ -3974,8 +4288,8 @@ { "current": { "selected": false, - "text": "KSTONE-PROM", - "value": "KSTONE-PROM" + "text": "", + "value": "" }, "description": null, "error": null, @@ -4002,7 +4316,9 @@ "datasource": "$datasource", "definition": "query_result(sum by (job) (increase(etcd_server_has_leader{}[$__range])))", "description": null, - "error": null, + "error": { + "message": "Datasource named was not found" + }, "hide": 0, "includeAll": false, "label": "etcd", @@ -4055,5 +4371,5 @@ "timezone": "browser", "title": "Kstone", "uid": "Hw7tu7aZz123123", - "version": 3 + "version": 12 } \ No newline at end of file diff --git a/go.mod b/go.mod index adf57d1..4700c8f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/aws/aws-sdk-go v1.13.8 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect + github.com/coreos/etcd v3.3.13+incompatible github.com/coreos/etcd-operator v0.9.4 github.com/gin-gonic/gin v1.7.2 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab diff --git a/go.sum b/go.sum index dd8c974..6bcfb14 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,7 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= diff --git a/pkg/apis/kstone/v1alpha2/types.go b/pkg/apis/kstone/v1alpha2/types.go index f07318b..19158d6 100644 --- a/pkg/apis/kstone/v1alpha2/types.go +++ b/pkg/apis/kstone/v1alpha2/types.go @@ -84,6 +84,13 @@ const ( EtcdClusterImported EtcdClusterType = "imported" ) +type EtcdStorageBackend string + +const ( + EtcdStorageV2 EtcdStorageBackend = "v2" + EtcdStorageV3 EtcdStorageBackend = "v3" +) + // EtcdClusterSpec defines the desired state of EtcdCluster type EtcdClusterSpec struct { Name string `json:"name" protobuf:"bytes,1,opt,name=name"` // etcd cluster name,uniqueKey @@ -95,11 +102,12 @@ type EtcdClusterSpec struct { DiskSize uint `json:"diskSize" protobuf:"varint,5,opt,name=diskSize"` // single node's disk size, unit: GB Size uint `json:"size" protobuf:"varint,6,opt,name=size"` // etcd cluster member count: support 1, 3, 5, 7 - Affinity corev1.Affinity `json:"affinity,omitempty" protobuf:"bytes,7,opt,name=affinity"` - Args []string `json:"args,omitempty" protobuf:"bytes,8,rep,name=args"` - Env []corev1.EnvVar `json:"env,omitempty" protobuf:"bytes,9,rep,name=env"` // etcd environment variables - Version string `json:"version" protobuf:"bytes,10,opt,name=version"` // etcd version - Repository string `json:"repository,omitempty" protobuf:"bytes,11,opt,name=repository"` // etcd image + Affinity corev1.Affinity `json:"affinity,omitempty" protobuf:"bytes,7,opt,name=affinity"` + Args []string `json:"args,omitempty" protobuf:"bytes,8,rep,name=args"` + Env []corev1.EnvVar `json:"env,omitempty" protobuf:"bytes,9,rep,name=env"` // etcd environment variables + StorageBackend string `json:"storageBackend" protobuf:"bytes,7,opt,name=storageBackend"` + Version string `json:"version" protobuf:"bytes,10,opt,name=version"` // etcd version + Repository string `json:"repository,omitempty" protobuf:"bytes,11,opt,name=repository"` // etcd image ClusterType EtcdClusterType `json:"clusterType" protobuf:"bytes,12,opt,name=clusterType,casttype=EtcdClusterType"` // ClusterType specifies the etcd cluster provider. diff --git a/pkg/clusterprovider/helper.go b/pkg/clusterprovider/helper.go index 84d0b4f..5edddd1 100644 --- a/pkg/clusterprovider/helper.go +++ b/pkg/clusterprovider/helper.go @@ -20,13 +20,14 @@ package clusterprovider import ( "fmt" - "strconv" "strings" "k8s.io/klog/v2" kstonev1alpha2 "tkestack.io/kstone/pkg/apis/kstone/v1alpha2" "tkestack.io/kstone/pkg/etcd" + versionClient "tkestack.io/kstone/pkg/etcd/client" + _ "tkestack.io/kstone/pkg/etcd/client/versions" // import etcd client including v2 and v3 ) type EtcdAlarm struct { @@ -68,22 +69,23 @@ func populateExtensionCientURLMap(extensionClientURLs string) (map[string]string // GetRuntimeEtcdMembers get members of etcd func GetRuntimeEtcdMembers( + storageBackend string, endpoints []string, extensionClientURLs string, config *etcd.ClientConfig) ([]kstonev1alpha2.MemberStatus, error) { etcdMembers := make([]kstonev1alpha2.MemberStatus, 0) config.Endpoints = endpoints - - // GetMemberList - client, err := etcd.NewClientv3(config) + versioned, err := versionClient.GetEtcdClientProvider(kstonev1alpha2.EtcdStorageBackend(storageBackend), + &versionClient.VersionContext{Config: config}) if err != nil { - klog.Errorf("failed to get new etcd clientv3,err is %v ", err) + klog.Errorf("failed get etcd version, err is %v", err) return etcdMembers, err } - defer client.Close() - memberRsp, err := etcd.MemberList(client) + defer versioned.Close() + + memberRsp, err := versioned.MemberList() if err != nil { klog.Errorf("failed to get member list, endpoints is %s,err is %v", endpoints, err) return etcdMembers, err @@ -95,7 +97,7 @@ func GetRuntimeEtcdMembers( return etcdMembers, err } - for _, m := range memberRsp.Members { + for _, m := range memberRsp { // parse url if m.ClientURLs == nil { continue @@ -122,7 +124,7 @@ func GetRuntimeEtcdMembers( // default info memberVersion, memberStatus, memberRole := "", kstonev1alpha2.MemberPhaseUnStarted, kstonev1alpha2.EtcdMemberUnKnown var errors []string - statusRsp, err := etcd.Status(extensionClientURL, client) + statusRsp, err := versioned.Status(extensionClientURL) if err == nil && statusRsp != nil { memberStatus = kstonev1alpha2.MemberPhaseRunning memberVersion = statusRsp.Version @@ -133,7 +135,6 @@ func GetRuntimeEtcdMembers( } else { memberRole = kstonev1alpha2.EtcdMemberFollower } - errors = statusRsp.Errors } else { klog.Errorf("failed to get member %s status,err is %v", extensionClientURL, err) errors = append(errors, err.Error()) @@ -141,7 +142,7 @@ func GetRuntimeEtcdMembers( etcdMembers = append(etcdMembers, kstonev1alpha2.MemberStatus{ Name: m.Name, - MemberId: strconv.FormatUint(m.ID, 10), + MemberId: m.ID, ClientUrl: m.ClientURLs[0], ExtensionClientUrl: extensionClientURL, Role: memberRole, diff --git a/pkg/clusterprovider/providers/imported/cluster.go b/pkg/clusterprovider/providers/imported/cluster.go index ac5c150..512cca4 100644 --- a/pkg/clusterprovider/providers/imported/cluster.go +++ b/pkg/clusterprovider/providers/imported/cluster.go @@ -127,6 +127,7 @@ func (c *EtcdClusterImported) Status(config *etcd.ClientConfig, cluster *kstonev } members, err := clusterprovider.GetRuntimeEtcdMembers( + cluster.Spec.StorageBackend, endpoints, cluster.Annotations[util.ClusterExtensionClientURL], config, diff --git a/pkg/clusterprovider/providers/kstone/kstone.go b/pkg/clusterprovider/providers/kstone/kstone.go index 7392941..cb61dce 100644 --- a/pkg/clusterprovider/providers/kstone/kstone.go +++ b/pkg/clusterprovider/providers/kstone/kstone.go @@ -289,6 +289,7 @@ func (c *EtcdClusterKstone) Status(config *etcd.ClientConfig, cluster *kstonev1a } members, err := clusterprovider.GetRuntimeEtcdMembers( + cluster.Spec.StorageBackend, endpoints, cluster.Annotations[util.ClusterExtensionClientURL], config, @@ -422,13 +423,19 @@ func (c *EtcdClusterKstone) generateEtcdSpec(cluster *kstonev1alpha2.EtcdCluster spec["repository"] = cluster.Annotations["repository"] } - affinity := &cluster.Spec.Affinity + // TODO: Use struct to replace + affinity := make(map[string]interface{}) + affinityBytes, _ := json.Marshal(cluster.Spec.Affinity) + _ = json.Unmarshal(affinityBytes, &affinity) if affinity != nil { spec["template"].(map[string]interface{})["affinity"] = affinity } if cluster.Spec.Tolerations != nil && len(cluster.Spec.Tolerations) > 0 { - spec["template"].(map[string]interface{})["tolerations"] = cluster.Spec.Tolerations + tolerations := make([]interface{}, 0) + tolerationsBytes, _ := json.Marshal(cluster.Spec.Tolerations) + _ = json.Unmarshal(tolerationsBytes, &tolerations) + spec["template"].(map[string]interface{})["tolerations"] = tolerations } return spec diff --git a/pkg/etcd/client/cleint.go b/pkg/etcd/client/cleint.go new file mode 100644 index 0000000..acc64ca --- /dev/null +++ b/pkg/etcd/client/cleint.go @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2023 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package client + +import ( + "tkestack.io/kstone/pkg/etcd" +) + +// Member contains member info including v2 and v3 +type Member struct { + ID string + Name string + PeerURLs []string + ClientURLs []string + Version string + IsLearner bool + Leader string +} + +type VersionClient interface { + MemberList() ([]Member, error) + Status(endpoint string) (*Member, error) + Close() +} + +type VersionContext struct { + Config *etcd.ClientConfig +} diff --git a/pkg/etcd/client/version.go b/pkg/etcd/client/version.go new file mode 100644 index 0000000..fefb7ad --- /dev/null +++ b/pkg/etcd/client/version.go @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2023 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package client + +import ( + "errors" + "sync" + + "k8s.io/klog/v2" + kstonev1alpha2 "tkestack.io/kstone/pkg/apis/kstone/v1alpha2" +) + +type Factory func(cluster *VersionContext) (VersionClient, error) + +var ( + mutex sync.Mutex + providers = make(map[kstonev1alpha2.EtcdStorageBackend]Factory) +) + +// RegisterEtcdClientFactory registers the specified etcd client +func RegisterEtcdClientFactory(name kstonev1alpha2.EtcdStorageBackend, factory Factory) { + mutex.Lock() + defer mutex.Unlock() + + if _, found := providers[name]; found { + klog.V(2).Infof("etcdcluster provider %s was registered twice", name) + } + + klog.V(2).Infof("register etcdCluster provider %s", name) + providers[name] = factory +} + +// GetEtcdClientProvider gets the specified etcd client +func GetEtcdClientProvider( + name kstonev1alpha2.EtcdStorageBackend, + ctx *VersionContext, +) (VersionClient, error) { + mutex.Lock() + defer mutex.Unlock() + + // compatible with existing clusters + if name == "" { + name = kstonev1alpha2.EtcdStorageV3 + } + f, found := providers[name] + + klog.V(1).Infof("get provider name %s,status:%t", name, found) + if !found { + return nil, errors.New("fatal error,etcd cluster provider not found") + } + return f(ctx) +} diff --git a/pkg/etcd/client/versions/providers.go b/pkg/etcd/client/versions/providers.go new file mode 100644 index 0000000..6830139 --- /dev/null +++ b/pkg/etcd/client/versions/providers.go @@ -0,0 +1,24 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2023 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package versions + +import ( + _ "tkestack.io/kstone/pkg/etcd/client/versions/v2" // import etcd client of v2 + _ "tkestack.io/kstone/pkg/etcd/client/versions/v3" // import etcd client of v3 +) diff --git a/pkg/etcd/client/versions/v2/client.go b/pkg/etcd/client/versions/v2/client.go new file mode 100644 index 0000000..2c1871c --- /dev/null +++ b/pkg/etcd/client/versions/v2/client.go @@ -0,0 +1,165 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2023 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package v2 + +import ( + "context" + "fmt" + "net" + "net/http" + + "github.com/coreos/etcd/pkg/transport" + clientv2 "go.etcd.io/etcd/client/v2" + klog "k8s.io/klog/v2" + + kstonev1alpha2 "tkestack.io/kstone/pkg/apis/kstone/v1alpha2" + "tkestack.io/kstone/pkg/etcd" + "tkestack.io/kstone/pkg/etcd/client" +) + +type V2 struct { + ctx *client.VersionContext + cli *clientv2.Client +} + +func (c *V2) MemberList() ([]client.Member, error) { + API := clientv2.NewMembersAPI(*c.cli) + rsp, err := API.List(context.Background()) + if err != nil { + return nil, fmt.Errorf("load members err of endpoints:%s err:%s", + c.ctx.Config.Endpoints, err.Error()) + } + members := make([]client.Member, 0) + for _, m := range rsp { + members = append(members, client.Member{ + ID: m.ID, + Name: m.Name, + PeerURLs: m.PeerURLs, + ClientURLs: m.ClientURLs, + IsLearner: false, + }) + } + return members, nil +} + +func (c *V2) Status(endpoint string) (*client.Member, error) { + backend, err := etcd.NewEtcdHealthCheckBackend(etcd.HealthCheckHTTP) + if err != nil { + klog.Errorf("failed to get version backend,method %s,err is %v", etcd.HealthCheckHTTP, err) + return nil, err + } + config := c.ctx.Config + err = backend.Init(config.CaCert, config.Cert, config.Key, endpoint) + if err != nil { + klog.Errorf("failed to init version client,endpoint is %s,err is %v", endpoint, err) + return nil, err + } + defer backend.Close() + var version string + version, err = backend.Version() + if err != nil { + klog.Errorf("failed to version,endpoint is %s,err is %v", endpoint, err) + return nil, err + } + + //get leader & memberID + stats, err := backend.Stats() + if err != nil { + return nil, err + } + return &client.Member{ + ID: stats.ID, + Name: stats.Name, + Version: version, + Leader: stats.LeaderInfo.Leader, + }, nil +} + +func (c *V2) Close() {} + +func init() { + client.RegisterEtcdClientFactory(kstonev1alpha2.EtcdStorageV2, + func(ctx *client.VersionContext) (client.VersionClient, error) { + return initClient(ctx) + }) +} + +func newClientCfg(ctx *client.VersionContext) (*clientv2.Config, error) { + config := ctx.Config + cfg := &clientv2.Config{ + Endpoints: config.Endpoints, + Username: config.Username, + Password: config.Password, + } + var cfgtls *transport.TLSInfo + tlsinfo := transport.TLSInfo{} + if ctx.Config.Cert != "" { + tlsinfo.CertFile = config.Cert + cfgtls = &tlsinfo + } + + if config.Key != "" { + tlsinfo.KeyFile = config.Key + cfgtls = &tlsinfo + } + + if config.CaCert != "" { + tlsinfo.TrustedCAFile = config.CaCert + cfgtls = &tlsinfo + } + + if cfgtls != nil { + clientTLS, err := cfgtls.ClientConfig() + if err != nil { + return nil, err + } + cfg.Transport = &http.Transport{ + Dial: (&net.Dialer{ + Timeout: config.DialKeepAliveTimeout, + KeepAlive: config.DialKeepAliveTime, + }).Dial, + TLSHandshakeTimeout: config.DialTimeout, + TLSClientConfig: clientTLS, + MaxIdleConnsPerHost: 1, + DisableKeepAlives: true, + } + } + return cfg, nil +} + +func initClient(ctx *client.VersionContext) (client.VersionClient, error) { + client := &V2{ + ctx: ctx, + cli: nil, + } + cfg, err := newClientCfg(ctx) + if err != nil { + klog.Errorf("get new clientv2 cfg failed:%s", err) + return nil, err + } + + cli, err := clientv2.New(*cfg) + if err != nil { + klog.Errorf("create new clientv2 failed:%s", err) + return nil, err + } + klog.V(2).Infof("init client ready of:%s", ctx.Config.Endpoints) + client.cli = &cli + return client, nil +} diff --git a/pkg/etcd/client/versions/v3/client.go b/pkg/etcd/client/versions/v3/client.go new file mode 100644 index 0000000..d5a2a68 --- /dev/null +++ b/pkg/etcd/client/versions/v3/client.go @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2023 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package v3 + +import ( + "strconv" + + clientv3 "go.etcd.io/etcd/client/v3" + "k8s.io/klog/v2" + + kstonev1alpha2 "tkestack.io/kstone/pkg/apis/kstone/v1alpha2" + "tkestack.io/kstone/pkg/etcd" + "tkestack.io/kstone/pkg/etcd/client" +) + +type V3 struct { + ctx *client.VersionContext + cli *clientv3.Client +} + +func (c *V3) MemberList() ([]client.Member, error) { + members := make([]client.Member, 0) + memberRsp, err := etcd.MemberList(c.cli) + if err != nil { + klog.Errorf("failed to get member list, endpoints is %s,err is %v", c.ctx.Config.Endpoints, err) + return members, err + } + for _, m := range memberRsp.Members { + members = append(members, client.Member{ + ID: strconv.FormatUint(m.ID, 10), + Name: m.Name, + PeerURLs: m.PeerURLs, + ClientURLs: m.ClientURLs, + IsLearner: m.IsLearner, + }) + } + return members, nil +} + +func (c *V3) Status(endpoint string) (*client.Member, error) { + statusRsp, err := etcd.Status(c.ctx.Config.Endpoints[0], c.cli) + if err != nil { + return nil, err + } + return &client.Member{ + Version: statusRsp.Version, + IsLearner: statusRsp.IsLearner, + Leader: strconv.FormatUint(statusRsp.Leader, 10), + }, nil +} + +func (c *V3) Close() { + c.cli.Close() +} + +func init() { + client.RegisterEtcdClientFactory(kstonev1alpha2.EtcdStorageV3, + func(ctx *client.VersionContext) (client.VersionClient, error) { + return initClient(ctx) + }) +} + +func initClient(ctx *client.VersionContext) (client.VersionClient, error) { + client := &V3{ + ctx: ctx, + cli: nil, + } + var err error + client.cli, err = etcd.NewClientv3(ctx.Config) + if err != nil { + klog.Errorf("failed to get new etcd clientv3,err is %v ", err) + return nil, err + } + return client, nil +} diff --git a/pkg/etcd/health.go b/pkg/etcd/health.go index d1e6311..3221c16 100644 --- a/pkg/etcd/health.go +++ b/pkg/etcd/health.go @@ -39,6 +39,12 @@ type Health interface { // IsHealthy checks etcd health info IsHealthy() error + // Version returns etcd version + Version() (string, error) + + // Stats returns etcd status + Stats() (*Stats, error) + // Close closes etcd healthcheck client Close() error } @@ -127,6 +133,55 @@ func (c *HealthCheckHTTPClient) IsHealthy() error { return c.etcdHealthCheck(body) } +func (c *HealthCheckHTTPClient) GetByAPI(path string) ([]byte, error) { + target := fmt.Sprintf("%s/%s", c.endpoint, path) + resp, err := c.cli.Get(target) + if err != nil { + klog.Errorf("failed to check etcd healthy,err is %v", err) + return make([]byte, 0), err + } + defer resp.Body.Close() + return ioutil.ReadAll(resp.Body) +} + +type Version struct { + EtcdServer string `json:"etcdserver"` +} + +func (c *HealthCheckHTTPClient) Version() (string, error) { + body, err := c.GetByAPI("version") + if err != nil { + return "", fmt.Errorf("send request failed:%s", err.Error()) + } + var version Version + err = json.Unmarshal(body, &version) + if err != nil { + return "", fmt.Errorf("version result json failed:%s, body:%s", err.Error(), string(body)) + } + return version.EtcdServer, nil +} + +type Stats struct { + Name string `json:"name"` + ID string `json:"id"` + LeaderInfo struct { + Leader string `json:"leader"` + } `json:"leaderInfo"` +} + +func (c *HealthCheckHTTPClient) Stats() (*Stats, error) { + body, err := c.GetByAPI("v2/stats/self") + if err != nil { + return nil, fmt.Errorf("send request failed:%s", err.Error()) + } + var stats Stats + err = json.Unmarshal(body, &stats) + if err != nil { + return nil, fmt.Errorf("version result json failed:%s, body:%s", err.Error(), string(body)) + } + return &stats, nil +} + // Close closes etcd healthcheck client func (c *HealthCheckHTTPClient) Close() error { return nil diff --git a/test/fixtures/fixtures.go b/test/fixtures/fixtures.go index 101adfe..6db86ea 100644 --- a/test/fixtures/fixtures.go +++ b/test/fixtures/fixtures.go @@ -80,10 +80,11 @@ func NewEtcdCluster( }, }, Spec: kstonev1alpha2.EtcdClusterSpec{ - ClusterType: clusterType, - Size: replicas, - DiskSize: 1, - Version: "3.4.13", + ClusterType: clusterType, + Size: replicas, + DiskSize: 1, + Version: "3.4.13", + StorageBackend: "v3", }, } switch clusterType {