From ad78efec8f3d5b11cd57e249be73fdd51c971c0b Mon Sep 17 00:00:00 2001 From: abdurrehman107 Date: Sun, 19 Jan 2025 22:30:33 +0500 Subject: [PATCH 1/6] initial scaffold --- e2e-test/e2e_suite_test.go | 152 +++++++++++++++++++++ e2e-test/e2e_test.go | 269 +++++++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+) create mode 100644 e2e-test/e2e_suite_test.go create mode 100644 e2e-test/e2e_test.go diff --git a/e2e-test/e2e_suite_test.go b/e2e-test/e2e_suite_test.go new file mode 100644 index 0000000..9946ecb --- /dev/null +++ b/e2e-test/e2e_suite_test.go @@ -0,0 +1,152 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "log" + "os" + "testing" + "time" + + // "sigs.k8s.io/e2e-framework/klient" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" + "sigs.k8s.io/e2e-framework/support/kind" + "sigs.k8s.io/e2e-framework/support/utils" + // "go.etcd.io/etcd-operator/test/utils" +) + +var ( + testenv env.Environment + namespace = "etcd-operator-system" + kindClusterName string + kustomizeVer = "v5.5.0" + ctrlGenVer = "v0.16.4" + dockerImage = "razashahid107/etcd-operator:v0.1" + +) + +func TestMain(m *testing.M) { + testenv = env.New() + kindClusterName = "etcd-cluster" + kindCluster := kind.NewCluster(kindClusterName) + log.Println("Creating Kind cluster...") + + // Use pre-defined environment funcs to create a kind cluster prior to test run + testenv.Setup( + envfuncs.CreateCluster(kindCluster, kindClusterName), + envfuncs.CreateNamespace(namespace), + // install tool dependencies + func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + log.Println("Installing bin tools...") + if p := utils.RunCommand(fmt.Sprintf("go install sigs.k8s.io/kustomize/kustomize/v5@%s", kustomizeVer)); p.Err() != nil { + log.Printf("Failed to install kustomize binary: %s: %s", p.Err(), p.Out()) + return ctx, p.Err() + } + if p := utils.RunCommand(fmt.Sprintf("go install sigs.k8s.io/controller-tools/cmd/controller-gen@%s", ctrlGenVer)); p.Err() != nil { + log.Printf("Failed to install controller-gen binary: %s: %s", p.Err(), p.Out()) + return ctx, p.Err() + } + return ctx, nil + }, + // generate and deploy resource configurations + func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + log.Println("Building source components...") + origWd, _ := os.Getwd() + + // change dir for Make file or it will fail + if err := os.Chdir("../"); err != nil { + log.Printf("Unable to set working directory: %s", err) + return ctx, err + } + + // gen manifest files + log.Println("Generate manifests...") + if p := utils.RunCommand(`controller-gen rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases`); p.Err() != nil { + log.Printf("Failed to generate manifests: %s: %s", p.Err(), p.Out()) + return ctx, p.Err() + } + + // gen api objects + log.Println("Generate API objects...") + if p := utils.RunCommand(`controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..."`); p.Err() != nil { + log.Printf("Failed to generate API objects: %s: %s", p.Err(), p.Out()) + return ctx, p.Err() + } + + // Build docker image + log.Println("Building docker image...") + if p := utils.RunCommand(fmt.Sprintf("docker build -t %s .", dockerImage)); p.Err() != nil { + log.Printf("Failed to build docker image: %s: %s", p.Err(), p.Out()) + return ctx, p.Err() + } + + // Load docker image into kind + log.Println("Loading docker image into kind cluster...") + if err := kindCluster.LoadImage(ctx, dockerImage); err != nil { + log.Printf("Failed to load image into kind: %s", err) + return ctx, err + } + + // Deploy components + log.Println("Deploying controller-manager resources...") + if p := utils.RunCommand(`bash -c "kustomize build config/default | kubectl apply --server-side -f -"`); p.Err() != nil { + log.Printf("Failed to deploy resource configurations: %s: %s", p.Err(), p.Out()) + return ctx, p.Err() + } + + // wait for controller-manager to be ready + log.Println("Waiting for controller-manager deployment to be available...") + client := cfg.Client() + if err := wait.For( + conditions.New(client.Resources()).DeploymentAvailable("etcd-operator-controller-manager", "etcd-operator-system"), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(10*time.Second), + ); err != nil { + log.Printf("Timed out while waiting for etcd-operator-controller-manager deployment: %s", err) + return ctx, err + } + + if err := os.Chdir(origWd); err != nil { + log.Printf("Unable to set working directory: %s", err) + return ctx, err + } + log.Println("Finished setting up controller-manager") + return ctx, nil + }, + + ) + + // Use pre-defined environment funcs to teardown kind cluster after tests + testenv.Finish( + func(ctx context.Context, c *envconf.Config) (context.Context, error) { + log.Println("Finishing tests, cleaning cluster ...") + return ctx, nil + }, + envfuncs.DeleteNamespace(namespace), + envfuncs.TeardownCRDs("./testdata/crd", "*"), + envfuncs.DestroyCluster(kindClusterName), + ) + + // launch package tests + os.Exit(testenv.Run(m)) +} diff --git a/e2e-test/e2e_test.go b/e2e-test/e2e_test.go new file mode 100644 index 0000000..f8e12cd --- /dev/null +++ b/e2e-test/e2e_test.go @@ -0,0 +1,269 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "log" + + // "os" + "testing" + "time" + + // "sigs.k8s.io/e2e-framework/klient" + apiextensionsV1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + // corev1 "k8s.io/api/core/v1" + appsv1 "k8s.io/api/apps/v1" + + // "k8s.io/klog/v2" + // "sigs.k8s.io/e2e-framework/klient/decoder" + // "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/e2e-framework/klient/k8s" + // "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + // "k8s.io/apimachinery/pkg/api/errors" + + + // "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + // "sigs.k8s.io/e2e-framework/pkg/envfuncs" + + // "sigs.k8s.io/e2e-framework/pkg/envfuncs" + "sigs.k8s.io/e2e-framework/pkg/features" + // "sigs.k8s.io/e2e-framework/support/kind" + + // "go.etcd.io/etcd-operator/test/utils" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +var ( + etcdClusterName = "etcdcluster-sample" + size = 1 + etcdCluster = &ecv1alpha1.EtcdCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "operator.etcd.io/v1alpha1", + Kind: "EtcdCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: etcdClusterName, + Namespace: namespace, + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: size, + Version: "v3.5.0", + }, + } +) + + +func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { + feature := features.New("etcd-operator controller") + size = 1 + + // feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + // r, err := resources.New(c.Client().RESTConfig()) + // if err != nil { + // t.Fail() + // } + // ecv1alpha1.AddToScheme(r.GetScheme()) + // r.WithNamespace(namespace) + // err = decoder.DecodeEachFile( + // ctx, os.DirFS("./testdata/etcd_cluster"), "*", + // decoder.CreateHandler(r), + // decoder.MutateNamespace(namespace), + // ) + // if err != nil { + // t.Fatalf("Failed due to error: %s", err) + // } + // return ctx + // }) + + feature.Assess("Check if CRD created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + apiextensionsV1.AddToScheme(client.Resources().GetScheme()) + name := "etcdclusters.operator.etcd.io" + var crd apiextensionsV1.CustomResourceDefinition + if err := client.Resources().Get(ctx, name, "", &crd); err != nil { + t.Fatalf("Failed due to error: %s", err) + } + + if crd.Spec.Group != "operator.etcd.io" { + t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) + } + + // klog.InfoS("CRD Details", "cr", crd) + return ctx + }) + + feature.Assess("Check if etcd cluster resource created with size=1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) + if err := client.Resources().Create(ctx, etcdCluster); err != nil { + t.Fatalf("Failed to create an etcd cluster resource: %s", err) + } + // wait for resource to be created + if err := wait.For( + conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + return true + }), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(30*time.Second), + ); err != nil { + t.Fatal(err) + } + + var ec ecv1alpha1.EtcdCluster + if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + if ec.Spec.Size != 1 { + log.Fatalf("etcd cluster size is not equal to 1: %s", err) + } + } + + return ctx + }) + + feature.Assess("Check if the sts exists and is of size 1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + appsv1.AddToScheme(client.Resources(namespace).GetScheme()) + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("sts does not exist: %s", err) + } + + if *sts.Spec.Replicas != int32(etcdCluster.Spec.Size) { + t.Fatalf("sts does not have the same number of replicas as the required etcd cluster size") + } + return ctx + }) + + // feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + // client := c.Client() + // var ec ecv1alpha1.EtcdCluster + // if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + // log.Println("no resource to teardown...") + // log.Println(err) + // } + // log.Println("deleting etcd cluster resource") + // if err := client.Resources().Delete(ctx, &ec); err != nil { + // t.Fatalf("unable to delete etcd cluster: %s", err) + // } + + // // sts must not exist + // var sts appsv1.StatefulSet + // if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + // if !errors.IsNotFound(err) { + // t.Fatalf("there was a problem with the statefulset: %s", err) + // } + // } + // return ctx + // }) + + _ = testenv.Test(t, feature.Feature()) + +} + +func TestScaleOutExistingEtcdCluster(t *testing.T) { + feature := features.New("etcd-operator controller") + feature.Assess("Check if CRD created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + apiextensionsV1.AddToScheme(client.Resources().GetScheme()) + name := "etcdclusters.operator.etcd.io" + var crd apiextensionsV1.CustomResourceDefinition + if err := client.Resources().Get(ctx, name, "", &crd); err != nil { + t.Fatalf("Failed due to error: %s", err) + } + + if crd.Spec.Group != "operator.etcd.io" { + t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) + } + + // klog.InfoS("CRD Details", "cr", crd) + return ctx + }) + + feature.Assess("Scale out etcd cluster to size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) + // if err := client.Resources().Create(ctx, etcdCluster); err != nil { + // t.Fatalf("Failed to create an etcd cluster resource: %s", err) + // } + // // wait for resource to be created + // if err := wait.For( + // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + // return true + // }), + // wait.WithTimeout(3*time.Minute), + // wait.WithInterval(30*time.Second), + // ); err != nil { + // t.Fatal(err) + // } + + etcdCluster.Spec.Size = 3 + var ec ecv1alpha1.EtcdCluster + log.Println("beginning to scale to 3...") + if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &ec); err == nil { + if ec.Spec.Size != 3 { + if err := c.Client().Resources().Update(ctx, etcdCluster); err != nil { + t.Fatalf("failed to update etcd cluster size: %v", err) + } + + var sts appsv1.StatefulSet + c.Client().Resources().Get(ctx, etcdClusterName, namespace, &sts) + // Wait for StatefulSet to scale to 3 replicas + if err := wait.For( + conditions.New(c.Client().Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 { + // Fetch the current replica count from the StatefulSet + if s, ok := object.(*appsv1.StatefulSet); ok { + return s.Status.Replicas + } + return 0 + },3, // Desired replica count + ), + wait.WithTimeout(3*time.Minute), // Timeout for the wait + wait.WithInterval(30*time.Second), // Polling interval + ); err != nil { + t.Fatalf("failed to scale StatefulSet to 3 replicas: %v", err) + } + + log.Println("etcd cluster successfully scaled to size 3...") + } + } + + return ctx + }) + + feature.Assess("Check if the sts exists and is of size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + appsv1.AddToScheme(client.Resources(namespace).GetScheme()) + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("sts does not exist: %s", err) + } + + if *sts.Spec.Replicas != int32(etcdCluster.Spec.Size) { + t.Fatalf("sts does not have the same number of replicas as the required etcd cluster size") + } + return ctx + }) + + _ = testenv.Test(t, feature.Feature()) + +} From a0caf7483225b65044daed3ba849d5423cc0cf72 Mon Sep 17 00:00:00 2001 From: abdurrehman107 Date: Mon, 20 Jan 2025 00:29:59 +0500 Subject: [PATCH 2/6] add testdata folder --- .../crd/operator.etcd.io_etcdclusters.yaml | 60 +++++++++++++++++++ .../operator_v1alpha1_etcdcluster.yaml | 10 ++++ 2 files changed, 70 insertions(+) create mode 100644 e2e-test/testdata/crd/operator.etcd.io_etcdclusters.yaml create mode 100644 e2e-test/testdata/etcd_cluster/operator_v1alpha1_etcdcluster.yaml diff --git a/e2e-test/testdata/crd/operator.etcd.io_etcdclusters.yaml b/e2e-test/testdata/crd/operator.etcd.io_etcdclusters.yaml new file mode 100644 index 0000000..0cb092a --- /dev/null +++ b/e2e-test/testdata/crd/operator.etcd.io_etcdclusters.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: etcdclusters.operator.etcd.io +spec: + group: operator.etcd.io + names: + kind: EtcdCluster + listKind: EtcdClusterList + plural: etcdclusters + singular: etcdcluster + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: EtcdCluster is the Schema for the etcdclusters API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EtcdClusterSpec defines the desired state of EtcdCluster. + properties: + size: + description: Size is the expected size of the etcd cluster. + type: integer + version: + description: Version is the expected version of the etcd container + image. + type: string + required: + - size + - version + type: object + status: + description: EtcdClusterStatus defines the observed state of EtcdCluster. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/e2e-test/testdata/etcd_cluster/operator_v1alpha1_etcdcluster.yaml b/e2e-test/testdata/etcd_cluster/operator_v1alpha1_etcdcluster.yaml new file mode 100644 index 0000000..ec558e9 --- /dev/null +++ b/e2e-test/testdata/etcd_cluster/operator_v1alpha1_etcdcluster.yaml @@ -0,0 +1,10 @@ +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdCluster +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdcluster-sample +spec: + size: 1 + version: "v3.5.0" From 77203b126f246c961b8b9b0b3b8313618c6e60a7 Mon Sep 17 00:00:00 2001 From: abdurrehman107 Date: Tue, 21 Jan 2025 23:10:03 +0500 Subject: [PATCH 3/6] clean the branch --- e2e-test/e2e_suite_test.go | 2 -- e2e-test/e2e_test.go | 73 ++------------------------------------ go.mod | 6 +++- go.sum | 10 ++++++ 4 files changed, 17 insertions(+), 74 deletions(-) diff --git a/e2e-test/e2e_suite_test.go b/e2e-test/e2e_suite_test.go index 9946ecb..7e31481 100644 --- a/e2e-test/e2e_suite_test.go +++ b/e2e-test/e2e_suite_test.go @@ -24,7 +24,6 @@ import ( "testing" "time" - // "sigs.k8s.io/e2e-framework/klient" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" "sigs.k8s.io/e2e-framework/pkg/env" @@ -32,7 +31,6 @@ import ( "sigs.k8s.io/e2e-framework/pkg/envfuncs" "sigs.k8s.io/e2e-framework/support/kind" "sigs.k8s.io/e2e-framework/support/utils" - // "go.etcd.io/etcd-operator/test/utils" ) var ( diff --git a/e2e-test/e2e_test.go b/e2e-test/e2e_test.go index f8e12cd..197642a 100644 --- a/e2e-test/e2e_test.go +++ b/e2e-test/e2e_test.go @@ -19,37 +19,23 @@ package e2e import ( "context" "log" - - // "os" "testing" "time" - // "sigs.k8s.io/e2e-framework/klient" apiextensionsV1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - // corev1 "k8s.io/api/core/v1" appsv1 "k8s.io/api/apps/v1" - // "k8s.io/klog/v2" - // "sigs.k8s.io/e2e-framework/klient/decoder" - // "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/e2e-framework/klient/k8s" - // "sigs.k8s.io/e2e-framework/klient/k8s/resources" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" - // "k8s.io/apimachinery/pkg/api/errors" - // "sigs.k8s.io/e2e-framework/pkg/env" "sigs.k8s.io/e2e-framework/pkg/envconf" - // "sigs.k8s.io/e2e-framework/pkg/envfuncs" - // "sigs.k8s.io/e2e-framework/pkg/envfuncs" "sigs.k8s.io/e2e-framework/pkg/features" - // "sigs.k8s.io/e2e-framework/support/kind" - // "go.etcd.io/etcd-operator/test/utils" ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" ) @@ -78,24 +64,6 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { feature := features.New("etcd-operator controller") size = 1 - // feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - // r, err := resources.New(c.Client().RESTConfig()) - // if err != nil { - // t.Fail() - // } - // ecv1alpha1.AddToScheme(r.GetScheme()) - // r.WithNamespace(namespace) - // err = decoder.DecodeEachFile( - // ctx, os.DirFS("./testdata/etcd_cluster"), "*", - // decoder.CreateHandler(r), - // decoder.MutateNamespace(namespace), - // ) - // if err != nil { - // t.Fatalf("Failed due to error: %s", err) - // } - // return ctx - // }) - feature.Assess("Check if CRD created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() apiextensionsV1.AddToScheme(client.Resources().GetScheme()) @@ -109,7 +77,6 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) } - // klog.InfoS("CRD Details", "cr", crd) return ctx }) @@ -154,34 +121,13 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { return ctx }) - // feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - // client := c.Client() - // var ec ecv1alpha1.EtcdCluster - // if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { - // log.Println("no resource to teardown...") - // log.Println(err) - // } - // log.Println("deleting etcd cluster resource") - // if err := client.Resources().Delete(ctx, &ec); err != nil { - // t.Fatalf("unable to delete etcd cluster: %s", err) - // } - - // // sts must not exist - // var sts appsv1.StatefulSet - // if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { - // if !errors.IsNotFound(err) { - // t.Fatalf("there was a problem with the statefulset: %s", err) - // } - // } - // return ctx - // }) - _ = testenv.Test(t, feature.Feature()) } func TestScaleOutExistingEtcdCluster(t *testing.T) { feature := features.New("etcd-operator controller") + etcdCluster.Spec.Size = 3 feature.Assess("Check if CRD created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() apiextensionsV1.AddToScheme(client.Resources().GetScheme()) @@ -202,21 +148,7 @@ func TestScaleOutExistingEtcdCluster(t *testing.T) { feature.Assess("Scale out etcd cluster to size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) - // if err := client.Resources().Create(ctx, etcdCluster); err != nil { - // t.Fatalf("Failed to create an etcd cluster resource: %s", err) - // } - // // wait for resource to be created - // if err := wait.For( - // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { - // return true - // }), - // wait.WithTimeout(3*time.Minute), - // wait.WithInterval(30*time.Second), - // ); err != nil { - // t.Fatal(err) - // } - - etcdCluster.Spec.Size = 3 + var ec ecv1alpha1.EtcdCluster log.Println("beginning to scale to 3...") if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &ec); err == nil { @@ -224,7 +156,6 @@ func TestScaleOutExistingEtcdCluster(t *testing.T) { if err := c.Client().Resources().Update(ctx, etcdCluster); err != nil { t.Fatalf("failed to update etcd cluster size: %v", err) } - var sts appsv1.StatefulSet c.Client().Resources().Get(ctx, etcdClusterName, namespace, &sts) // Wait for StatefulSet to scale to 3 replicas diff --git a/go.mod b/go.mod index f36173d..84ffdc6 100644 --- a/go.mod +++ b/go.mod @@ -64,9 +64,11 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.19.1 // indirect @@ -79,6 +81,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect + github.com/vladimirvivien/gexe v0.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.9 // indirect @@ -118,9 +121,10 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/apiserver v0.31.0 // indirect - k8s.io/component-base v0.31.0 // indirect + k8s.io/component-base v0.31.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect + sigs.k8s.io/e2e-framework v0.5.0 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 4053312..1eae1e0 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -138,6 +140,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= @@ -190,6 +194,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/vladimirvivien/gexe v0.3.0 h1:4xwiOwGrDob5OMR6E92B9olDXYDglXdHhzR1ggYtWJM= +github.com/vladimirvivien/gexe v0.3.0/go.mod h1:fp7cy60ON1xjhtEI/+bfSEIXX35qgmI+iRYlGOqbBFM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -369,6 +375,8 @@ k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs= k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= +k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= +k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= @@ -379,6 +387,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsA sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.4 h1:SUmheabttt0nx8uJtoII4oIP27BVVvAKFvdvGFwV/Qo= sigs.k8s.io/controller-runtime v0.19.4/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/e2e-framework v0.5.0 h1:YLhk8R7EHuTFQAe6Fxy5eBzn5Vb+yamR5u8MH1Rq3cE= +sigs.k8s.io/e2e-framework v0.5.0/go.mod h1:jJSH8u2RNmruekUZgHAtmRjb5Wj67GErli9UjLSY7Zc= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= From 985edfbf43d302cfafaf3222ff14b18f5e1b8341 Mon Sep 17 00:00:00 2001 From: abdurrehman107 Date: Fri, 24 Jan 2025 16:06:08 +0500 Subject: [PATCH 4/6] push e2e test scaling --- e2e-test/e2e_test.go | 404 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 383 insertions(+), 21 deletions(-) diff --git a/e2e-test/e2e_test.go b/e2e-test/e2e_test.go index 197642a..0b21853 100644 --- a/e2e-test/e2e_test.go +++ b/e2e-test/e2e_test.go @@ -27,23 +27,26 @@ import ( appsv1 "k8s.io/api/apps/v1" + // "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" - "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" ) -var ( - etcdClusterName = "etcdcluster-sample" - size = 1 - etcdCluster = &ecv1alpha1.EtcdCluster{ + +func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { + feature := features.New("etcd-operator controller") + + const etcdClusterName = "etcd-cluster-test-1" + const size = 1 + + var etcdCluster = &ecv1alpha1.EtcdCluster{ TypeMeta: metav1.TypeMeta{ APIVersion: "operator.etcd.io/v1alpha1", Kind: "EtcdCluster", @@ -57,14 +60,131 @@ var ( Version: "v3.5.0", }, } -) + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + // Check if the CRD exists + client := c.Client() + apiextensionsV1.AddToScheme(client.Resources().GetScheme()) + name := "etcdclusters.operator.etcd.io" + var crd apiextensionsV1.CustomResourceDefinition + if err := client.Resources().Get(ctx, name, "", &crd); err != nil { + t.Fatalf("Failed due to error: %s", err) + } + + if crd.Spec.Group != "operator.etcd.io" { + t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) + } + + // Create the etcd cluster resource + ecv1alpha1.AddToScheme(client.Resources().GetScheme()) + if err := client.Resources().Create(ctx, etcdCluster); err != nil { + t.Fatalf("unable to create an etcd cluster resource of size 1: %s", err) + } -func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { + if err := wait.For( + conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + return true + }), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + return ctx + }) + + feature.Assess("Check if etcd cluster resource created with size=1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + // ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) + // if err := client.Resources().Create(ctx, etcdCluster); err != nil { + // t.Fatalf("Failed to create an etcd cluster resource: %s", err) + // } + // // wait for resource to be created + // if err := wait.For( + // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + // return true + // }), + // wait.WithTimeout(3*time.Minute), + // wait.WithInterval(30*time.Second), + // ); err != nil { + // t.Fatal(err) + // } + + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + if ec.Spec.Size != 1 { + log.Fatalf("etcd cluster size is not equal to 1: %s", err) + } + } + + return ctx + }) + + feature.Assess("Check if the sts exists and is of size 1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + appsv1.AddToScheme(client.Resources(namespace).GetScheme()) + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("sts does not exist: %s", err) + } + + if *sts.Spec.Replicas != int32(etcdCluster.Spec.Size) { + t.Fatalf("sts does not have the same number of replicas as the required etcd cluster size") + } + return ctx + }) + + // Remove all the resources created + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + t.Fatalf("unable to fetch etcd cluster while tearing down feature: %s", err) + } + // delete etcd cluster + if err := client.Resources().Delete(ctx, etcdCluster); err != nil { + t.Fatalf("error in deleting etcd cluster while tearing down feature: %s", err) + } + // verify that no sts for the etcd cluster exists + if err := wait.For( + conditions.New(client.Resources()).ResourceDeleted(etcdCluster), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + + log.Println("tore down etcd cluster test 1 resource successfully...") + + return ctx + }) + + _ = testenv.Test(t, feature.Feature()) +} + +func TestCreateAndDeleteThreeMemeberEtcdCluster(t *testing.T) { feature := features.New("etcd-operator controller") - size = 1 + + const etcdClusterName = "etcd-cluster-test-2" + const size = 3 - feature.Assess("Check if CRD created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + var etcdCluster = &ecv1alpha1.EtcdCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "operator.etcd.io/v1alpha1", + Kind: "EtcdCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: etcdClusterName, + Namespace: namespace, + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: size, + Version: "v3.5.0", + }, + } + + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + // Check if the CRD exists client := c.Client() apiextensionsV1.AddToScheme(client.Resources().GetScheme()) name := "etcdclusters.operator.etcd.io" @@ -76,29 +196,202 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { if crd.Spec.Group != "operator.etcd.io" { t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) } + + // Create the etcd cluster resource + ecv1alpha1.AddToScheme(client.Resources().GetScheme()) + if err := client.Resources().Create(ctx, etcdCluster); err != nil { + t.Fatalf("unable to create an etcd cluster resource of size 3: %s", err) + } + + if err := wait.For( + conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + return true + }), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + + // fetch sts created via etcd cluster + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("unable to fetch sts resource: %s", err) + } + + // wait for sts to scale to 3 + if err := wait.For( + conditions.New(client.Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 { + return sts.Status.ReadyReplicas + }, 3), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } return ctx }) - feature.Assess("Check if etcd cluster resource created with size=1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("Check if etcd cluster resource created with size=3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() - ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) + // ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) + // if err := client.Resources().Create(ctx, etcdCluster); err != nil { + // t.Fatalf("Failed to create an etcd cluster resource: %s", err) + // } + // // wait for resource to be created + // if err := wait.For( + // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + // return true + // }), + // wait.WithTimeout(3*time.Minute), + // wait.WithInterval(30*time.Second), + // ); err != nil { + // t.Fatal(err) + // } + + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + if ec.Spec.Size != 3 { + log.Fatalf("etcd cluster size is not equal to 3: %s", err) + } + } + + return ctx + }) + + feature.Assess("Check if the sts exists and is of size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + appsv1.AddToScheme(client.Resources(namespace).GetScheme()) + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("sts does not exist: %s", err) + } + + if *sts.Spec.Replicas != int32(etcdCluster.Spec.Size) { + t.Fatalf("sts does not have the same number of replicas as the required etcd cluster size") + } + return ctx + }) + + // Remove all the resources created + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + t.Fatalf("unable to fetch etcd cluster while tearing down feature: %s", err) + } + // delete etcd cluster + if err := client.Resources().Delete(ctx, etcdCluster); err != nil { + t.Fatalf("error in deleting etcd cluster while tearing down feature: %s", err) + } + // verify that no sts for the etcd cluster exists + if err := wait.For( + conditions.New(client.Resources()).ResourceDeleted(&ec), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + + log.Println("tore down etcd cluster test 2 resource successfully...") + + return ctx + }) + + + _ = testenv.Test(t, feature.Feature()) +} + +func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { + feature := features.New("etcd-operator controller") + + const etcdClusterName = "etcd-cluster-test-3" + + var size = 1 + + var etcdCluster = &ecv1alpha1.EtcdCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "operator.etcd.io/v1alpha1", + Kind: "EtcdCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: etcdClusterName, + Namespace: namespace, + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: size, + Version: "v3.5.0", + }, + } + + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + // Check if the CRD exists + client := c.Client() + apiextensionsV1.AddToScheme(client.Resources().GetScheme()) + name := "etcdclusters.operator.etcd.io" + var crd apiextensionsV1.CustomResourceDefinition + if err := client.Resources().Get(ctx, name, "", &crd); err != nil { + t.Fatalf("Failed due to error: %s", err) + } + + if crd.Spec.Group != "operator.etcd.io" { + t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) + } + + // Create the etcd cluster resource + ecv1alpha1.AddToScheme(client.Resources().GetScheme()) if err := client.Resources().Create(ctx, etcdCluster); err != nil { - t.Fatalf("Failed to create an etcd cluster resource: %s", err) + t.Fatalf("unable to create an etcd cluster resource of size 1: %s", err) } - // wait for resource to be created if err := wait.For( conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { return true - }), - wait.WithTimeout(3*time.Minute), - wait.WithInterval(30*time.Second), + }), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), ); err != nil { t.Fatal(err) } + // ensure that sts gets its pod in ready state + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("sts does not exist: %s", err) + } + + if err := wait.For( + conditions.New(client.Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 { + return sts.Status.ReadyReplicas + }, 1), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + + return ctx + }) + + feature.Assess("Check if etcd cluster resource created with size=1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + // ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) + // if err := client.Resources().Create(ctx, etcdCluster); err != nil { + // t.Fatalf("Failed to create an etcd cluster resource: %s", err) + // } + // // wait for resource to be created + // if err := wait.For( + // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + // return true + // }), + // wait.WithTimeout(3*time.Minute), + // wait.WithInterval(30*time.Second), + // ); err != nil { + // t.Fatal(err) + // } + var ec ecv1alpha1.EtcdCluster - if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { if ec.Spec.Size != 1 { log.Fatalf("etcd cluster size is not equal to 1: %s", err) } @@ -115,16 +408,85 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { t.Fatalf("sts does not exist: %s", err) } - if *sts.Spec.Replicas != int32(etcdCluster.Spec.Size) { + if int(*sts.Spec.Replicas) != etcdCluster.Spec.Size { t.Fatalf("sts does not have the same number of replicas as the required etcd cluster size") } return ctx }) + feature.Assess("Scale etcd cluster size to 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + + // get etcd cluster resource + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + t.Fatalf("unable to get etcd cluster: %s", err) + } + + // scale etcd cluster + etcdCluster.Spec.Size = 3 + + if err := client.Resources().Update(ctx, etcdCluster); err != nil { + t.Fatalf("failed to update etcd cluster size to 3: %s", err) + } + + return ctx + }) + + feature.Assess("Check if the sts exists and is of the size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + + // get the sts + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("unable to fetch sts resource: %s", err) + } + + // check the sts size + // if not 3 then wait for it to scale + if err := wait.For( + conditions.New(client.Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 { + return sts.Status.ReadyReplicas + }, 3), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatalf("unable to scale sts to 3: %s", err) + } + + return ctx + }) + + // Remove all the resources created + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + t.Fatalf("unable to fetch etcd cluster while tearing down feature: %s", err) + } + // delete etcd cluster + if err := client.Resources().Delete(ctx, etcdCluster); err != nil { + t.Fatalf("error in deleting etcd cluster while tearing down feature: %s", err) + } + // verify that no sts for the etcd cluster exists + if err := wait.For( + conditions.New(client.Resources()).ResourceDeleted(&ec), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + + log.Println("tore down etcd cluster test 3 resource successfully...") + + return ctx + }) + + _ = testenv.Test(t, feature.Feature()) } - +/* func TestScaleOutExistingEtcdCluster(t *testing.T) { feature := features.New("etcd-operator controller") etcdCluster.Spec.Size = 3 @@ -196,5 +558,5 @@ func TestScaleOutExistingEtcdCluster(t *testing.T) { }) _ = testenv.Test(t, feature.Feature()) - } +*/ From 961e945201accb82be7aab1271c5151b07984fdf Mon Sep 17 00:00:00 2001 From: Abdur Rehman Date: Tue, 28 Jan 2025 14:31:53 +0500 Subject: [PATCH 5/6] push new cases --- e2e-test/e2e_test.go | 215 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 209 insertions(+), 6 deletions(-) diff --git a/e2e-test/e2e_test.go b/e2e-test/e2e_test.go index 0b21853..c86126d 100644 --- a/e2e-test/e2e_test.go +++ b/e2e-test/e2e_test.go @@ -24,6 +24,7 @@ import ( apiextensionsV1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + klog "k8s.io/klog/v2" appsv1 "k8s.io/api/apps/v1" @@ -37,6 +38,7 @@ import ( "sigs.k8s.io/e2e-framework/pkg/features" ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + controller "go.etcd.io/etcd-operator/internal/controller" ) @@ -155,7 +157,7 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { } log.Println("tore down etcd cluster test 1 resource successfully...") - + return ctx }) @@ -299,7 +301,6 @@ func TestCreateAndDeleteThreeMemeberEtcdCluster(t *testing.T) { return ctx }) - _ = testenv.Test(t, feature.Feature()) } @@ -360,10 +361,20 @@ func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { t.Fatalf("sts does not exist: %s", err) } + // if err := wait.For( + // conditions.New(client.Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 { + // return sts.Status.ReadyReplicas + // }, 1), + // wait.WithTimeout(3 * time.Minute), + // wait.WithInterval(30 * time.Second), + // ); err != nil { + // t.Fatal(err) + // } + if err := wait.For( - conditions.New(client.Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 { - return sts.Status.ReadyReplicas - }, 1), + conditions.New(client.Resources()).ResourceMatch(&sts, func(object k8s.Object) bool { + return sts.Status.ReadyReplicas == 1 + }), wait.WithTimeout(3 * time.Minute), wait.WithInterval(30 * time.Second), ); err != nil { @@ -457,7 +468,7 @@ func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { return ctx }) - // Remove all the resources created + // Remove all the resources created feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() var ec ecv1alpha1.EtcdCluster @@ -482,10 +493,202 @@ func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { return ctx }) + _ = testenv.Test(t, feature.Feature()) +} + +func TestHealthCheck(t *testing.T) { + feature := features.New("etcd-operator-controller") + + const etcdClusterName = "etcd-cluster-test-4" + + var size = 3 + + var etcdCluster = &ecv1alpha1.EtcdCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "operator.etcd.io/v1alpha1", + Kind: "EtcdCluster", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: etcdClusterName, + Namespace: namespace, + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: size, + Version: "v3.5.0", + }, + } + + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + // Check if the CRD exists + client := c.Client() + apiextensionsV1.AddToScheme(client.Resources().GetScheme()) + name := "etcdclusters.operator.etcd.io" + var crd apiextensionsV1.CustomResourceDefinition + if err := client.Resources().Get(ctx, name, "", &crd); err != nil { + t.Fatalf("Failed due to error: %s", err) + } + + if crd.Spec.Group != "operator.etcd.io" { + t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) + } + + // Create the etcd cluster resource + ecv1alpha1.AddToScheme(client.Resources().GetScheme()) + if err := client.Resources().Create(ctx, etcdCluster); err != nil { + t.Fatalf("unable to create an etcd cluster resource of size 1: %s", err) + } + if err := wait.For( + conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + return true + }), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + + // ensure that sts gets its pod in ready state + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("sts does not exist: %s", err) + } + + if err := wait.For( + conditions.New(client.Resources()).ResourceMatch(&sts, func(object k8s.Object) bool { + return sts.Status.ReadyReplicas == 3 + }), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + + return ctx + }) + + feature.Assess("Check if etcd cluster resource created with size=3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + // ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) + // if err := client.Resources().Create(ctx, etcdCluster); err != nil { + // t.Fatalf("Failed to create an etcd cluster resource: %s", err) + // } + // // wait for resource to be created + // if err := wait.For( + // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { + // return true + // }), + // wait.WithTimeout(3*time.Minute), + // wait.WithInterval(30*time.Second), + // ); err != nil { + // t.Fatal(err) + // } + + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + if ec.Spec.Size != 3 { + log.Fatalf("etcd cluster size is not equal to 3: %s", err) + } + } + + return ctx + }) + + feature.Assess("Check if the sts exists and is of size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + appsv1.AddToScheme(client.Resources(namespace).GetScheme()) + var sts appsv1.StatefulSet + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + t.Fatalf("sts does not exist: %s", err) + } + + if *sts.Spec.Replicas != int32(etcdCluster.Spec.Size) { + t.Fatalf("sts does not have the same number of replicas as the required etcd cluster size") + } + return ctx + }) + + // feature.Assess("Hit URL for the sts pod", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + // client := c.Client() + // log := klog.FromContext(ctx) + + // // fetch the sts resource + // var sts appsv1.StatefulSet + // if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { + // t.Fatalf("unable to fetch sts resource: %s", err) + // } + + + // if err := wait.For( + // conditions.New(client.Resources()).ResourceMatch( + // &sts, + // func(obj k8s.Object) bool { + // // Cast to the just-refetched resource + // statefulSet, ok := obj.(*appsv1.StatefulSet) + // if !ok { + // // If for some reason it isn't a StatefulSet, not healthy + // return false + // } + + // memberListResp, healthInfos, err := controller.HealthCheck(statefulSet, log) + // if err != nil { + // // Not healthy yet, keep waiting + // return false + // } + + // memberCount := 0 + // if memberListResp != nil { + // memberCount = len(memberListResp.Members) + // } + // if memberCount != len(healthInfos) { + // // t.Fatalf( + // // "memberCnt (%d) isn't equal to healthy member count (%d)", + // // memberCount, len(healthInfos), + // // ) + // return false + // } + + // log.Info("health check successful...") + // return true + // }, + // ), + // wait.WithTimeout(3*time.Minute), + // wait.WithInterval(30*time.Second), + // ); err != nil { + // t.Fatalf("timed out waiting for health check to succeed: %v", err) + // } + + // return ctx + // }) + + // Remove all the resources created + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client := c.Client() + var ec ecv1alpha1.EtcdCluster + if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { + t.Fatalf("unable to fetch etcd cluster while tearing down feature: %s", err) + } + // delete etcd cluster + if err := client.Resources().Delete(ctx, etcdCluster); err != nil { + t.Fatalf("error in deleting etcd cluster while tearing down feature: %s", err) + } + // verify that no sts for the etcd cluster exists + if err := wait.For( + conditions.New(client.Resources()).ResourceDeleted(&ec), + wait.WithTimeout(3 * time.Minute), + wait.WithInterval(30 * time.Second), + ); err != nil { + t.Fatal(err) + } + + log.Println("tore down etcd cluster test 4 resource successfully...") + + return ctx + }) _ = testenv.Test(t, feature.Feature()) } + /* func TestScaleOutExistingEtcdCluster(t *testing.T) { feature := features.New("etcd-operator controller") From c641fd86823fe9760178f4ce047bc390e869787d Mon Sep 17 00:00:00 2001 From: Abdur Rehman Date: Sun, 2 Feb 2025 18:41:05 +0500 Subject: [PATCH 6/6] update suite test --- e2e-test/e2e_suite_test.go | 34 ++++ e2e-test/e2e_test.go | 329 +------------------------------------ 2 files changed, 41 insertions(+), 322 deletions(-) diff --git a/e2e-test/e2e_suite_test.go b/e2e-test/e2e_suite_test.go index 7e31481..275d81a 100644 --- a/e2e-test/e2e_suite_test.go +++ b/e2e-test/e2e_suite_test.go @@ -41,6 +41,11 @@ var ( ctrlGenVer = "v0.16.4" dockerImage = "razashahid107/etcd-operator:v0.1" + certMgrVer = "v1.13.1" + certMgrUrl = fmt.Sprintf("https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml", certMgrVer) + promVer = "v0.60.0" + promUrl = fmt.Sprintf("https://github.com/prometheus-operator/prometheus-operator/releases/download/%s/bundle.yaml", promVer) + ) func TestMain(m *testing.M) { @@ -53,6 +58,35 @@ func TestMain(m *testing.M) { testenv.Setup( envfuncs.CreateCluster(kindCluster, kindClusterName), envfuncs.CreateNamespace(namespace), + // install cluster dependencies mgr + func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + log.Println("Installing prometheus operator...") + if p := utils.RunCommand(fmt.Sprintf("kubectl apply -f %s --server-side", promUrl)); p.Err() != nil { + log.Printf("Failed to deploy prometheus: %s: %s", p.Err(), p.Out()) + return ctx, p.Err() + } + + log.Println("Installing cert-manager...") + client := cfg.Client() + + if p := utils.RunCommand(fmt.Sprintf("kubectl apply -f %s", certMgrUrl)); p.Err() != nil { + log.Printf("Failed to deploy cert-manager: %s: %s", p.Err(), p.Out()) + return ctx, p.Err() + } + + // wait for CertManager to be ready + log.Println("Waiting for cert-manager deployment to be available...") + if err := wait.For( + conditions.New(client.Resources()).DeploymentAvailable("cert-manager-webhook", "cert-manager"), + wait.WithTimeout(5*time.Minute), + wait.WithInterval(10*time.Second), + ); err != nil { + log.Printf("Timedout while waiting for cert-manager deployment: %s", err) + return ctx, err + } + return ctx, nil + }, + // install tool dependencies func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { log.Println("Installing bin tools...") diff --git a/e2e-test/e2e_test.go b/e2e-test/e2e_test.go index c86126d..dd4a2f1 100644 --- a/e2e-test/e2e_test.go +++ b/e2e-test/e2e_test.go @@ -24,11 +24,9 @@ import ( apiextensionsV1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - klog "k8s.io/klog/v2" appsv1 "k8s.io/api/apps/v1" - // "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" @@ -38,7 +36,6 @@ import ( "sigs.k8s.io/e2e-framework/pkg/features" ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" - controller "go.etcd.io/etcd-operator/internal/controller" ) @@ -97,20 +94,6 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { feature.Assess("Check if etcd cluster resource created with size=1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() - // ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) - // if err := client.Resources().Create(ctx, etcdCluster); err != nil { - // t.Fatalf("Failed to create an etcd cluster resource: %s", err) - // } - // // wait for resource to be created - // if err := wait.For( - // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { - // return true - // }), - // wait.WithTimeout(3*time.Minute), - // wait.WithInterval(30*time.Second), - // ); err != nil { - // t.Fatal(err) - // } var ec ecv1alpha1.EtcdCluster if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { @@ -124,6 +107,7 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { feature.Assess("Check if the sts exists and is of size 1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() + appsv1.AddToScheme(client.Resources(namespace).GetScheme()) var sts appsv1.StatefulSet if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { @@ -139,6 +123,7 @@ func TestCreateAndDeleteOneMemberEtcdCluster(t *testing.T) { // Remove all the resources created feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() + var ec ecv1alpha1.EtcdCluster if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { t.Fatalf("unable to fetch etcd cluster while tearing down feature: %s", err) @@ -188,6 +173,7 @@ func TestCreateAndDeleteThreeMemeberEtcdCluster(t *testing.T) { feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { // Check if the CRD exists client := c.Client() + apiextensionsV1.AddToScheme(client.Resources().GetScheme()) name := "etcdclusters.operator.etcd.io" var crd apiextensionsV1.CustomResourceDefinition @@ -237,20 +223,6 @@ func TestCreateAndDeleteThreeMemeberEtcdCluster(t *testing.T) { feature.Assess("Check if etcd cluster resource created with size=3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() - // ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) - // if err := client.Resources().Create(ctx, etcdCluster); err != nil { - // t.Fatalf("Failed to create an etcd cluster resource: %s", err) - // } - // // wait for resource to be created - // if err := wait.For( - // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { - // return true - // }), - // wait.WithTimeout(3*time.Minute), - // wait.WithInterval(30*time.Second), - // ); err != nil { - // t.Fatal(err) - // } var ec ecv1alpha1.EtcdCluster if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { @@ -264,6 +236,7 @@ func TestCreateAndDeleteThreeMemeberEtcdCluster(t *testing.T) { feature.Assess("Check if the sts exists and is of size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() + appsv1.AddToScheme(client.Resources(namespace).GetScheme()) var sts appsv1.StatefulSet if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { @@ -329,6 +302,7 @@ func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { // Check if the CRD exists client := c.Client() + apiextensionsV1.AddToScheme(client.Resources().GetScheme()) name := "etcdclusters.operator.etcd.io" var crd apiextensionsV1.CustomResourceDefinition @@ -361,16 +335,6 @@ func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { t.Fatalf("sts does not exist: %s", err) } - // if err := wait.For( - // conditions.New(client.Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 { - // return sts.Status.ReadyReplicas - // }, 1), - // wait.WithTimeout(3 * time.Minute), - // wait.WithInterval(30 * time.Second), - // ); err != nil { - // t.Fatal(err) - // } - if err := wait.For( conditions.New(client.Resources()).ResourceMatch(&sts, func(object k8s.Object) bool { return sts.Status.ReadyReplicas == 1 @@ -386,20 +350,6 @@ func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { feature.Assess("Check if etcd cluster resource created with size=1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() - // ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) - // if err := client.Resources().Create(ctx, etcdCluster); err != nil { - // t.Fatalf("Failed to create an etcd cluster resource: %s", err) - // } - // // wait for resource to be created - // if err := wait.For( - // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { - // return true - // }), - // wait.WithTimeout(3*time.Minute), - // wait.WithInterval(30*time.Second), - // ); err != nil { - // t.Fatal(err) - // } var ec ecv1alpha1.EtcdCluster if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { @@ -413,6 +363,7 @@ func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { feature.Assess("Check if the sts exists and is of size 1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() + appsv1.AddToScheme(client.Resources(namespace).GetScheme()) var sts appsv1.StatefulSet if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { @@ -471,198 +422,7 @@ func TestCreateAndScaleOneMemberEtcdClusterToThree(t *testing.T) { // Remove all the resources created feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client := c.Client() - var ec ecv1alpha1.EtcdCluster - if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { - t.Fatalf("unable to fetch etcd cluster while tearing down feature: %s", err) - } - // delete etcd cluster - if err := client.Resources().Delete(ctx, etcdCluster); err != nil { - t.Fatalf("error in deleting etcd cluster while tearing down feature: %s", err) - } - // verify that no sts for the etcd cluster exists - if err := wait.For( - conditions.New(client.Resources()).ResourceDeleted(&ec), - wait.WithTimeout(3 * time.Minute), - wait.WithInterval(30 * time.Second), - ); err != nil { - t.Fatal(err) - } - - log.Println("tore down etcd cluster test 3 resource successfully...") - - return ctx - }) - - _ = testenv.Test(t, feature.Feature()) -} - -func TestHealthCheck(t *testing.T) { - feature := features.New("etcd-operator-controller") - - const etcdClusterName = "etcd-cluster-test-4" - - var size = 3 - - var etcdCluster = &ecv1alpha1.EtcdCluster{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "operator.etcd.io/v1alpha1", - Kind: "EtcdCluster", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: etcdClusterName, - Namespace: namespace, - }, - Spec: ecv1alpha1.EtcdClusterSpec{ - Size: size, - Version: "v3.5.0", - }, - } - - feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - // Check if the CRD exists - client := c.Client() - apiextensionsV1.AddToScheme(client.Resources().GetScheme()) - name := "etcdclusters.operator.etcd.io" - var crd apiextensionsV1.CustomResourceDefinition - if err := client.Resources().Get(ctx, name, "", &crd); err != nil { - t.Fatalf("Failed due to error: %s", err) - } - - if crd.Spec.Group != "operator.etcd.io" { - t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) - } - - // Create the etcd cluster resource - ecv1alpha1.AddToScheme(client.Resources().GetScheme()) - if err := client.Resources().Create(ctx, etcdCluster); err != nil { - t.Fatalf("unable to create an etcd cluster resource of size 1: %s", err) - } - if err := wait.For( - conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { - return true - }), - wait.WithTimeout(3 * time.Minute), - wait.WithInterval(30 * time.Second), - ); err != nil { - t.Fatal(err) - } - - // ensure that sts gets its pod in ready state - var sts appsv1.StatefulSet - if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { - t.Fatalf("sts does not exist: %s", err) - } - - if err := wait.For( - conditions.New(client.Resources()).ResourceMatch(&sts, func(object k8s.Object) bool { - return sts.Status.ReadyReplicas == 3 - }), - wait.WithTimeout(3 * time.Minute), - wait.WithInterval(30 * time.Second), - ); err != nil { - t.Fatal(err) - } - - return ctx - }) - - feature.Assess("Check if etcd cluster resource created with size=3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - client := c.Client() - // ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) - // if err := client.Resources().Create(ctx, etcdCluster); err != nil { - // t.Fatalf("Failed to create an etcd cluster resource: %s", err) - // } - // // wait for resource to be created - // if err := wait.For( - // conditions.New(client.Resources()).ResourceMatch(etcdCluster, func(object k8s.Object) bool { - // return true - // }), - // wait.WithTimeout(3*time.Minute), - // wait.WithInterval(30*time.Second), - // ); err != nil { - // t.Fatal(err) - // } - var ec ecv1alpha1.EtcdCluster - if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { - if ec.Spec.Size != 3 { - log.Fatalf("etcd cluster size is not equal to 3: %s", err) - } - } - - return ctx - }) - - feature.Assess("Check if the sts exists and is of size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - client := c.Client() - appsv1.AddToScheme(client.Resources(namespace).GetScheme()) - var sts appsv1.StatefulSet - if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { - t.Fatalf("sts does not exist: %s", err) - } - - if *sts.Spec.Replicas != int32(etcdCluster.Spec.Size) { - t.Fatalf("sts does not have the same number of replicas as the required etcd cluster size") - } - return ctx - }) - - // feature.Assess("Hit URL for the sts pod", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - // client := c.Client() - // log := klog.FromContext(ctx) - - // // fetch the sts resource - // var sts appsv1.StatefulSet - // if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { - // t.Fatalf("unable to fetch sts resource: %s", err) - // } - - - // if err := wait.For( - // conditions.New(client.Resources()).ResourceMatch( - // &sts, - // func(obj k8s.Object) bool { - // // Cast to the just-refetched resource - // statefulSet, ok := obj.(*appsv1.StatefulSet) - // if !ok { - // // If for some reason it isn't a StatefulSet, not healthy - // return false - // } - - // memberListResp, healthInfos, err := controller.HealthCheck(statefulSet, log) - // if err != nil { - // // Not healthy yet, keep waiting - // return false - // } - - // memberCount := 0 - // if memberListResp != nil { - // memberCount = len(memberListResp.Members) - // } - // if memberCount != len(healthInfos) { - // // t.Fatalf( - // // "memberCnt (%d) isn't equal to healthy member count (%d)", - // // memberCount, len(healthInfos), - // // ) - // return false - // } - - // log.Info("health check successful...") - // return true - // }, - // ), - // wait.WithTimeout(3*time.Minute), - // wait.WithInterval(30*time.Second), - // ); err != nil { - // t.Fatalf("timed out waiting for health check to succeed: %v", err) - // } - - // return ctx - // }) - - // Remove all the resources created - feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - client := c.Client() var ec ecv1alpha1.EtcdCluster if err := client.Resources().Get(ctx, etcdClusterName, namespace, &ec); err != nil { t.Fatalf("unable to fetch etcd cluster while tearing down feature: %s", err) @@ -680,86 +440,11 @@ func TestHealthCheck(t *testing.T) { t.Fatal(err) } - log.Println("tore down etcd cluster test 4 resource successfully...") + log.Println("tore down etcd cluster test 3 resource successfully...") return ctx }) _ = testenv.Test(t, feature.Feature()) - } -/* -func TestScaleOutExistingEtcdCluster(t *testing.T) { - feature := features.New("etcd-operator controller") - etcdCluster.Spec.Size = 3 - feature.Assess("Check if CRD created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - client := c.Client() - apiextensionsV1.AddToScheme(client.Resources().GetScheme()) - name := "etcdclusters.operator.etcd.io" - var crd apiextensionsV1.CustomResourceDefinition - if err := client.Resources().Get(ctx, name, "", &crd); err != nil { - t.Fatalf("Failed due to error: %s", err) - } - - if crd.Spec.Group != "operator.etcd.io" { - t.Fatalf("Expected crd group to be operator.etcd.io, got %s", crd.Spec.Group) - } - - // klog.InfoS("CRD Details", "cr", crd) - return ctx - }) - - feature.Assess("Scale out etcd cluster to size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - client := c.Client() - ecv1alpha1.AddToScheme(client.Resources(namespace).GetScheme()) - - var ec ecv1alpha1.EtcdCluster - log.Println("beginning to scale to 3...") - if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &ec); err == nil { - if ec.Spec.Size != 3 { - if err := c.Client().Resources().Update(ctx, etcdCluster); err != nil { - t.Fatalf("failed to update etcd cluster size: %v", err) - } - var sts appsv1.StatefulSet - c.Client().Resources().Get(ctx, etcdClusterName, namespace, &sts) - // Wait for StatefulSet to scale to 3 replicas - if err := wait.For( - conditions.New(c.Client().Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 { - // Fetch the current replica count from the StatefulSet - if s, ok := object.(*appsv1.StatefulSet); ok { - return s.Status.Replicas - } - return 0 - },3, // Desired replica count - ), - wait.WithTimeout(3*time.Minute), // Timeout for the wait - wait.WithInterval(30*time.Second), // Polling interval - ); err != nil { - t.Fatalf("failed to scale StatefulSet to 3 replicas: %v", err) - } - - log.Println("etcd cluster successfully scaled to size 3...") - } - } - - return ctx - }) - - feature.Assess("Check if the sts exists and is of size 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - client := c.Client() - appsv1.AddToScheme(client.Resources(namespace).GetScheme()) - var sts appsv1.StatefulSet - if err := client.Resources().Get(ctx, etcdClusterName, namespace, &sts); err != nil { - t.Fatalf("sts does not exist: %s", err) - } - - if *sts.Spec.Replicas != int32(etcdCluster.Spec.Size) { - t.Fatalf("sts does not have the same number of replicas as the required etcd cluster size") - } - return ctx - }) - - _ = testenv.Test(t, feature.Feature()) -} -*/