Merge pull request #136 from tangcong/0.2

 cherry-pick feature and bug fixed pr to release-0.2

go.mod

@@ -5,6 +5,7 @@ go 1.16
 require (
 	github.com/aws/aws-sdk-go v1.13.8
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
+	github.com/coreos/etcd v3.3.13+incompatible
 	github.com/coreos/etcd-operator v0.9.4
 	github.com/gin-gonic/gin v1.7.2
 	github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab

go.sum

@@ -111,6 +111,7 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=

pkg/apis/kstone/v1alpha2/types.go

@@ -84,6 +84,13 @@ const (
 	EtcdClusterImported EtcdClusterType = "imported"
 )
 
+type EtcdStorageBackend string
+
+const (
+	EtcdStorageV2 EtcdStorageBackend = "v2"
+	EtcdStorageV3 EtcdStorageBackend = "v3"
+)
+
 // EtcdClusterSpec defines the desired state of EtcdCluster
 type EtcdClusterSpec struct {
 	Name        string `json:"name" protobuf:"bytes,1,opt,name=name"`               // etcd cluster name，uniqueKey
@@ -95,11 +102,12 @@ type EtcdClusterSpec struct {
 	DiskSize uint   `json:"diskSize" protobuf:"varint,5,opt,name=diskSize"` // single node's disk size, unit: GB
 	Size     uint   `json:"size"  protobuf:"varint,6,opt,name=size"`        // etcd cluster member count: support 1, 3, 5, 7
 
-	Affinity   corev1.Affinity `json:"affinity,omitempty" protobuf:"bytes,7,opt,name=affinity"`
-	Args       []string        `json:"args,omitempty" protobuf:"bytes,8,rep,name=args"`
-	Env        []corev1.EnvVar `json:"env,omitempty" protobuf:"bytes,9,rep,name=env"`                // etcd environment variables
-	Version    string          `json:"version" protobuf:"bytes,10,opt,name=version"`                 // etcd version
-	Repository string          `json:"repository,omitempty" protobuf:"bytes,11,opt,name=repository"` // etcd image
+	Affinity       corev1.Affinity `json:"affinity,omitempty" protobuf:"bytes,7,opt,name=affinity"`
+	Args           []string        `json:"args,omitempty" protobuf:"bytes,8,rep,name=args"`
+	Env            []corev1.EnvVar `json:"env,omitempty" protobuf:"bytes,9,rep,name=env"` // etcd environment variables
+	StorageBackend string          `json:"storageBackend" protobuf:"bytes,7,opt,name=storageBackend"`
+	Version        string          `json:"version" protobuf:"bytes,10,opt,name=version"`                 // etcd version
+	Repository     string          `json:"repository,omitempty" protobuf:"bytes,11,opt,name=repository"` // etcd image
 
 	ClusterType EtcdClusterType `json:"clusterType" protobuf:"bytes,12,opt,name=clusterType,casttype=EtcdClusterType"` // ClusterType specifies the etcd cluster provider.
 

pkg/clusterprovider/helper.go

@@ -20,13 +20,14 @@ package clusterprovider
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 
 	"k8s.io/klog/v2"
 
 	kstonev1alpha2 "tkestack.io/kstone/pkg/apis/kstone/v1alpha2"
 	"tkestack.io/kstone/pkg/etcd"
+	versionClient "tkestack.io/kstone/pkg/etcd/client"
+	_ "tkestack.io/kstone/pkg/etcd/client/versions" // import etcd client including v2 and v3
 )
 
 type EtcdAlarm struct {
@@ -68,22 +69,23 @@ func populateExtensionCientURLMap(extensionClientURLs string) (map[string]string
 
 // GetRuntimeEtcdMembers get members of etcd
 func GetRuntimeEtcdMembers(
+	storageBackend string,
 	endpoints []string,
 	extensionClientURLs string,
 	config *etcd.ClientConfig) ([]kstonev1alpha2.MemberStatus, error) {
 	etcdMembers := make([]kstonev1alpha2.MemberStatus, 0)
 
 	config.Endpoints = endpoints
-
-	// GetMemberList
-	client, err := etcd.NewClientv3(config)
+	versioned, err := versionClient.GetEtcdClientProvider(kstonev1alpha2.EtcdStorageBackend(storageBackend),
+		&versionClient.VersionContext{Config: config})
 	if err != nil {
-		klog.Errorf("failed to get new etcd clientv3,err is %v ", err)
+		klog.Errorf("failed get etcd version, err is %v", err)
 		return etcdMembers, err
 	}
-	defer client.Close()
 
-	memberRsp, err := etcd.MemberList(client)
+	defer versioned.Close()
+
+	memberRsp, err := versioned.MemberList()
 	if err != nil {
 		klog.Errorf("failed to get member list, endpoints is %s,err is %v", endpoints, err)
 		return etcdMembers, err
@@ -95,7 +97,7 @@ func GetRuntimeEtcdMembers(
 		return etcdMembers, err
 	}
 
-	for _, m := range memberRsp.Members {
+	for _, m := range memberRsp {
 		// parse url
 		if m.ClientURLs == nil {
 			continue
@@ -122,7 +124,7 @@ func GetRuntimeEtcdMembers(
 		// default info
 		memberVersion, memberStatus, memberRole := "", kstonev1alpha2.MemberPhaseUnStarted, kstonev1alpha2.EtcdMemberUnKnown
 		var errors []string
-		statusRsp, err := etcd.Status(extensionClientURL, client)
+		statusRsp, err := versioned.Status(extensionClientURL)
 		if err == nil && statusRsp != nil {
 			memberStatus = kstonev1alpha2.MemberPhaseRunning
 			memberVersion = statusRsp.Version
@@ -133,15 +135,14 @@ func GetRuntimeEtcdMembers(
 			} else {
 				memberRole = kstonev1alpha2.EtcdMemberFollower
 			}
-			errors = statusRsp.Errors
 		} else {
 			klog.Errorf("failed to get member %s status,err is %v", extensionClientURL, err)
 			errors = append(errors, err.Error())
 		}
 
 		etcdMembers = append(etcdMembers, kstonev1alpha2.MemberStatus{
 			Name:               m.Name,
-			MemberId:           strconv.FormatUint(m.ID, 10),
+			MemberId:           m.ID,
 			ClientUrl:          m.ClientURLs[0],
 			ExtensionClientUrl: extensionClientURL,
 			Role:               memberRole,

pkg/clusterprovider/providers/imported/cluster.go

@@ -127,6 +127,7 @@ func (c *EtcdClusterImported) Status(config *etcd.ClientConfig, cluster *kstonev
 	}
 
 	members, err := clusterprovider.GetRuntimeEtcdMembers(
+		cluster.Spec.StorageBackend,
 		endpoints,
 		cluster.Annotations[util.ClusterExtensionClientURL],
 		config,

pkg/clusterprovider/providers/kstone/kstone.go

@@ -289,6 +289,7 @@ func (c *EtcdClusterKstone) Status(config *etcd.ClientConfig, cluster *kstonev1a
 	}
 
 	members, err := clusterprovider.GetRuntimeEtcdMembers(
+		cluster.Spec.StorageBackend,
 		endpoints,
 		cluster.Annotations[util.ClusterExtensionClientURL],
 		config,
@@ -422,13 +423,19 @@ func (c *EtcdClusterKstone) generateEtcdSpec(cluster *kstonev1alpha2.EtcdCluster
 		spec["repository"] = cluster.Annotations["repository"]
 	}
 
-	affinity := &cluster.Spec.Affinity
+	// TODO: Use struct to replace
+	affinity := make(map[string]interface{})
+	affinityBytes, _ := json.Marshal(cluster.Spec.Affinity)
+	_ = json.Unmarshal(affinityBytes, &affinity)
 	if affinity != nil {
 		spec["template"].(map[string]interface{})["affinity"] = affinity
 	}
 
 	if cluster.Spec.Tolerations != nil && len(cluster.Spec.Tolerations) > 0 {
-		spec["template"].(map[string]interface{})["tolerations"] = cluster.Spec.Tolerations
+		tolerations := make([]interface{}, 0)
+		tolerationsBytes, _ := json.Marshal(cluster.Spec.Tolerations)
+		_ = json.Unmarshal(tolerationsBytes, &tolerations)
+		spec["template"].(map[string]interface{})["tolerations"] = tolerations
 	}
 
 	return spec

pkg/etcd/client/cleint.go

@@ -0,0 +1,44 @@
+/*
+ * Tencent is pleased to support the open source community by making TKEStack
+ * available.
+ *
+ * Copyright (C) 2012-2023 Tencent. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+ * this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * https://opensource.org/licenses/Apache-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package client
+
+import (
+	"tkestack.io/kstone/pkg/etcd"
+)
+
+// Member contains member info including v2 and v3
+type Member struct {
+	ID         string
+	Name       string
+	PeerURLs   []string
+	ClientURLs []string
+	Version    string
+	IsLearner  bool
+	Leader     string
+}
+
+type VersionClient interface {
+	MemberList() ([]Member, error)
+	Status(endpoint string) (*Member, error)
+	Close()
+}
+
+type VersionContext struct {
+	Config *etcd.ClientConfig
+}

pkg/etcd/client/version.go

@@ -0,0 +1,68 @@
+/*
+ * Tencent is pleased to support the open source community by making TKEStack
+ * available.
+ *
+ * Copyright (C) 2012-2023 Tencent. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+ * this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * https://opensource.org/licenses/Apache-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package client
+
+import (
+	"errors"
+	"sync"
+
+	"k8s.io/klog/v2"
+	kstonev1alpha2 "tkestack.io/kstone/pkg/apis/kstone/v1alpha2"
+)
+
+type Factory func(cluster *VersionContext) (VersionClient, error)
+
+var (
+	mutex     sync.Mutex
+	providers = make(map[kstonev1alpha2.EtcdStorageBackend]Factory)
+)
+
+// RegisterEtcdClientFactory registers the specified etcd client
+func RegisterEtcdClientFactory(name kstonev1alpha2.EtcdStorageBackend, factory Factory) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	if _, found := providers[name]; found {
+		klog.V(2).Infof("etcdcluster provider %s was registered twice", name)
+	}
+
+	klog.V(2).Infof("register etcdCluster provider %s", name)
+	providers[name] = factory
+}
+
+// GetEtcdClientProvider gets the specified etcd client
+func GetEtcdClientProvider(
+	name kstonev1alpha2.EtcdStorageBackend,
+	ctx *VersionContext,
+) (VersionClient, error) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	// compatible with existing clusters
+	if name == "" {
+		name = kstonev1alpha2.EtcdStorageV3
+	}
+	f, found := providers[name]
+
+	klog.V(1).Infof("get provider name %s,status:%t", name, found)
+	if !found {
+		return nil, errors.New("fatal error,etcd cluster provider not found")
+	}
+	return f(ctx)
+}

pkg/etcd/client/versions/providers.go

@@ -0,0 +1,24 @@
+/*
+ * Tencent is pleased to support the open source community by making TKEStack
+ * available.
+ *
+ * Copyright (C) 2012-2023 Tencent. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+ * this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * https://opensource.org/licenses/Apache-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package versions
+
+import (
+	_ "tkestack.io/kstone/pkg/etcd/client/versions/v2" // import etcd client of v2
+	_ "tkestack.io/kstone/pkg/etcd/client/versions/v3" // import etcd client of v3
+)