Merge pull request #441 from harshanarayana/feature/k3d-provider-support

provider: added k3d provider and node lifecycle handlers

examples/k3d/k3d_test.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k3d
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"sigs.k8s.io/e2e-framework/pkg/stepfuncs"
+	"sigs.k8s.io/e2e-framework/support"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func newDeployment(namespace string, name string, replicaCount int32) *appsv1.Deployment {
+	podSpec := corev1.PodSpec{
+		Containers: []corev1.Container{
+			{
+				Name:  "my-container",
+				Image: "nginx",
+			},
+		},
+	}
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app": "test-app"}},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &replicaCount,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"app": "test-app"},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "test-app"}},
+				Spec:       podSpec,
+			},
+		},
+	}
+}
+
+func TestK3DCluster(t *testing.T) {
+	deploymentFeature := features.New("Should be able to create a new deployment in the k3d cluster").
+		Assess("Create a new deployment", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+			deployment := newDeployment(c.Namespace(), "test-deployment", 1)
+			if err := c.Client().Resources().Create(ctx, deployment); err != nil {
+				t.Fatal(err)
+			}
+			var dep appsv1.Deployment
+			if err := c.Client().Resources().Get(ctx, "test-deployment", c.Namespace(), &dep); err != nil {
+				t.Fatal(err)
+			}
+			err := wait.For(conditions.New(c.Client().Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, corev1.ConditionTrue), wait.WithTimeout(time.Minute*3))
+			if err != nil {
+				t.Fatal(err)
+			}
+			return context.WithValue(ctx, "test-deployment", &dep)
+		}).
+		Feature()
+
+	nodeAddFeature := features.New("Should be able to add a new node to the k3d cluster").
+		Setup(stepfuncs.PerformNodeOperation(support.AddNode, &support.Node{
+			Name:    fmt.Sprintf("%s-agent", clusterName),
+			Cluster: clusterName,
+			Role:    "agent",
+		})).
+		Assess("Check if the node is added to the cluster", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+			var node corev1.Node
+			if err := c.Client().Resources().Get(ctx, fmt.Sprintf("k3d-%s-agent-0", clusterName), c.Namespace(), &node); err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		}).Feature()
+
+	testEnv.Test(t, deploymentFeature, nodeAddFeature)
+}

examples/k3d/main_test.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k3d
+
+import (
+	"os"
+	"testing"
+
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+	"sigs.k8s.io/e2e-framework/support/k3d"
+)
+
+var (
+	testEnv     env.Environment
+	clusterName string
+)
+
+func TestMain(m *testing.M) {
+	testEnv = env.New()
+	clusterName = envconf.RandomName("test", 16)
+	namespace := envconf.RandomName("k3d-ns", 16)
+
+	testEnv.Setup(
+		envfuncs.CreateClusterWithOpts(k3d.NewProvider(), clusterName, k3d.WithImage("rancher/k3s:v1.29.6-k3s1")),
+		envfuncs.CreateNamespace(namespace),
+		envfuncs.LoadImageToCluster(clusterName, "rancher/k3s:v1.29.6-k3s1", "--verbose", "--mode", "direct"),
+	)
+
+	testEnv.Finish(
+		envfuncs.DeleteNamespace(namespace),
+		envfuncs.DestroyCluster(clusterName),
+	)
+
+	os.Exit(testEnv.Run(m))
+}

pkg/envfuncs/provider_funcs.go

@@ -20,19 +20,19 @@ import (
 	"context"
 	"fmt"
 
+	"sigs.k8s.io/e2e-framework/pkg/utils"
+
 	"sigs.k8s.io/e2e-framework/pkg/env"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/support"
 )
 
-type clusterNameContextKey string
-
 var LoadDockerImageToCluster = LoadImageToCluster
 
 // GetClusterFromContext helps extract the E2EClusterProvider object from the context.
 // This can be used to setup and run tests of multi cluster e2e Prioviders.
 func GetClusterFromContext(ctx context.Context, clusterName string) (support.E2EClusterProvider, bool) {
-	c := ctx.Value(clusterNameContextKey(clusterName))
+	c := ctx.Value(support.ClusterNameContextKey(clusterName))
 	if c == nil {
 		return nil, false
 	}
@@ -47,8 +47,19 @@ func GetClusterFromContext(ctx context.Context, clusterName string) (support.E2E
 // NOTE: the returned function will update its env config with the
 // kubeconfig file for the config client.
 func CreateCluster(p support.E2EClusterProvider, clusterName string) env.Func {
+	return CreateClusterWithOpts(p, clusterName)
+}
+
+// CreateClusterWithOpts returns an env.Func that is used to
+// create an E2E provider cluster that is then injected in the context
+// using the name as a key. This can be provided with additional opts to extend the create
+// workflow of the cluster.
+//
+// NOTE: the returned function will update its env config with the
+// kubeconfig file for the config client.
+func CreateClusterWithOpts(p support.E2EClusterProvider, clusterName string, opts ...support.ClusterOpts) env.Func {
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
-		k := p.SetDefaults().WithName(clusterName)
+		k := p.SetDefaults().WithName(clusterName).WithOpts(opts...)
 		kubecfg, err := k.Create(ctx)
 		if err != nil {
 			return ctx, err
@@ -63,7 +74,7 @@ func CreateCluster(p support.E2EClusterProvider, clusterName string) env.Func {
 		}
 
 		// store entire cluster value in ctx for future access using the cluster name
-		return context.WithValue(ctx, clusterNameContextKey(clusterName), k), nil
+		return context.WithValue(ctx, support.ClusterNameContextKey(clusterName), k), nil
 	}
 }
 
@@ -90,7 +101,7 @@ func CreateClusterWithConfig(p support.E2EClusterProvider, clusterName, configFi
 		}
 
 		// store entire cluster value in ctx for future access using the cluster name
-		return context.WithValue(ctx, clusterNameContextKey(clusterName), k), nil
+		return context.WithValue(ctx, support.ClusterNameContextKey(clusterName), k), nil
 	}
 }
 
@@ -100,7 +111,7 @@ func CreateClusterWithConfig(p support.E2EClusterProvider, clusterName, configFi
 // NOTE: this should be used in a Environment.Finish step.
 func DestroyCluster(name string) env.Func {
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
-		clusterVal := ctx.Value(clusterNameContextKey(name))
+		clusterVal := ctx.Value(support.ClusterNameContextKey(name))
 		if clusterVal == nil {
 			return ctx, fmt.Errorf("destroy e2e provider cluster func: context cluster is nil")
 		}
@@ -121,9 +132,9 @@ func DestroyCluster(name string) env.Func {
 // LoadImageToCluster returns an EnvFunc that
 // retrieves a previously saved e2e provider Cluster in the context (using the name), and then loads a container image
 // from the host into the cluster.
-func LoadImageToCluster(name, image string) env.Func {
+func LoadImageToCluster(name, image string, args ...string) env.Func {
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
-		clusterVal := ctx.Value(clusterNameContextKey(name))
+		clusterVal := ctx.Value(support.ClusterNameContextKey(name))
 		if clusterVal == nil {
 			return ctx, fmt.Errorf("load image func: context cluster is nil")
 		}
@@ -133,7 +144,7 @@ func LoadImageToCluster(name, image string) env.Func {
 			return ctx, fmt.Errorf("load image archive func: cluster provider does not support LoadImage helper")
 		}
 
-		if err := cluster.LoadImage(ctx, image); err != nil {
+		if err := cluster.LoadImage(ctx, image, args...); err != nil {
 			return ctx, fmt.Errorf("load image: %w", err)
 		}
 
@@ -144,9 +155,9 @@ func LoadImageToCluster(name, image string) env.Func {
 // LoadImageArchiveToCluster returns an EnvFunc that
 // retrieves a previously saved e2e provider Cluster in the context (using the name), and then loads a container image TAR archive
 // from the host into the cluster.
-func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
+func LoadImageArchiveToCluster(name, imageArchive string, args ...string) env.Func {
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
-		clusterVal := ctx.Value(clusterNameContextKey(name))
+		clusterVal := ctx.Value(support.ClusterNameContextKey(name))
 		if clusterVal == nil {
 			return ctx, fmt.Errorf("load image archive func: context cluster is nil")
 		}
@@ -156,7 +167,7 @@ func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
 			return ctx, fmt.Errorf("load image archive func: cluster provider does not support LoadImageArchive helper")
 		}
 
-		if err := cluster.LoadImageArchive(ctx, imageArchive); err != nil {
+		if err := cluster.LoadImageArchive(ctx, imageArchive, args...); err != nil {
 			return ctx, fmt.Errorf("load image archive: %w", err)
 		}
 
@@ -169,7 +180,7 @@ func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
 // in the provided destination.
 func ExportClusterLogs(name, dest string) env.Func {
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
-		clusterVal := ctx.Value(clusterNameContextKey(name))
+		clusterVal := ctx.Value(support.ClusterNameContextKey(name))
 		if clusterVal == nil {
 			return ctx, fmt.Errorf("export e2e provider cluster logs: context cluster is nil")
 		}
@@ -186,3 +197,12 @@ func ExportClusterLogs(name, dest string) env.Func {
 		return ctx, nil
 	}
 }
+
+// PerformNodeOperation returns an EnvFunc that can be used to perform some node lifecycle operations.
+// This can be used to add/remove/start/stop nodes in the cluster.
+func PerformNodeOperation(clusterName string, action support.NodeOperation, node *support.Node, args ...string) env.Func {
+	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+		err := utils.PerformNodeLifecycleOperation(ctx, action, node, args...)
+		return ctx, err
+	}
+}

pkg/stepfuncs/nodelifecycle_funcs.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stepfuncs
+
+import (
+	"context"
+	"testing"
+
+	"sigs.k8s.io/e2e-framework/pkg/utils"
+
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/types"
+	"sigs.k8s.io/e2e-framework/support"
+)
+
+// PerformNodeOperation returns a step function that performs a node operation on a cluster.
+// This can be integrated as a setup function for a feature in question before the feature
+// is tested.
+func PerformNodeOperation(action support.NodeOperation, node *support.Node, args ...string) types.StepFunc {
+	return func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
+		t.Helper()
+
+		err := utils.PerformNodeLifecycleOperation(ctx, action, node, args...)
+		if err != nil {
+			t.Fatalf("failed to perform node operation: %v", err)
+		}
+		return ctx
+	}
+}