Support multiple Kubernetes clusters by supporting kubeconfig in test…

… steps - Follow up for #266 (#291)

* Support multiple Kubernetes clusters by supporting kubeconfig in test steps

Adds a `kubeconfig` setting to the TestStep API allowing an alternative
kubeconfig path to be used for asserts and applies in a test step.

Signed-off-by: [email protected] <[email protected]>

* Resolve relative kubeconfig path against test step dir.

Signed-off-by: Tarun Gupta Akirala <[email protected]>

* fix linter errors

Signed-off-by: Tarun Gupta Akirala <[email protected]>

* fix variable name collision with package imports

Signed-off-by: Tarun Gupta Akirala <[email protected]>

Co-authored-by: [email protected] <[email protected]>
Co-authored-by: chhsia0 <[email protected]>

pkg/apis/testharness/v1beta1/test_types.go

@@ -99,6 +99,9 @@ type TestStep struct {
 
 	// Allowed environment labels
 	// Disallowed environment labels
+
+	// Kubeconfig to use when applying and asserting for this step.
+	Kubeconfig string `json:"kubeconfig,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/test/case.go

@@ -20,6 +20,8 @@ import (
 	"k8s.io/client-go/discovery"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"k8s.io/client-go/tools/clientcmd"
+
 	"github.com/kudobuilder/kuttl/pkg/report"
 	testutils "github.com/kudobuilder/kuttl/pkg/test/utils"
 )
@@ -51,19 +53,14 @@ type namespace struct {
 }
 
 // DeleteNamespace deletes a namespace in Kubernetes after we are done using it.
-func (t *Case) DeleteNamespace(ns *namespace) error {
+func (t *Case) DeleteNamespace(cl client.Client, ns *namespace) error {
 	if !ns.AutoCreated {
 		t.Logger.Log("Skipping deletion of user-supplied namespace:", ns.Name)
 		return nil
 	}
 
 	t.Logger.Log("Deleting namespace:", ns.Name)
 
-	cl, err := t.Client(false)
-	if err != nil {
-		return err
-	}
-
 	ctx := context.Background()
 	if t.Timeout > 0 {
 		var cancel context.CancelFunc
@@ -82,18 +79,13 @@ func (t *Case) DeleteNamespace(ns *namespace) error {
 }
 
 // CreateNamespace creates a namespace in Kubernetes to use for a test.
-func (t *Case) CreateNamespace(ns *namespace) error {
+func (t *Case) CreateNamespace(cl client.Client, ns *namespace) error {
 	if !ns.AutoCreated {
 		t.Logger.Log("Skipping creation of user-supplied namespace:", ns.Name)
 		return nil
 	}
 	t.Logger.Log("Creating namespace:", ns.Name)
 
-	cl, err := t.Client(false)
-	if err != nil {
-		return err
-	}
-
 	ctx := context.Background()
 	if t.Timeout > 0 {
 		var cancel context.CancelFunc
@@ -194,22 +186,55 @@ func shortString(obj *corev1.ObjectReference) string {
 // Run runs a test case including all of its steps.
 func (t *Case) Run(test *testing.T, tc *report.Testcase) {
 	ns := t.determineNamespace()
-	if err := t.CreateNamespace(ns); err != nil {
+
+	cl, err := t.Client(false)
+	if err != nil {
 		tc.Failure = report.NewFailure(err.Error(), nil)
 		test.Fatal(err)
 	}
 
+	clients := map[string]client.Client{"": cl}
+
+	for _, testStep := range t.Steps {
+		if clients[testStep.Kubeconfig] != nil {
+			continue
+		}
+
+		cl, err := newClient(testStep.Kubeconfig)(false)
+		if err != nil {
+			tc.Failure = report.NewFailure(err.Error(), nil)
+			test.Fatal(err)
+		}
+
+		clients[testStep.Kubeconfig] = cl
+	}
+
+	for _, c := range clients {
+		if err := t.CreateNamespace(c, ns); err != nil {
+			tc.Failure = report.NewFailure(err.Error(), nil)
+			test.Fatal(err)
+		}
+	}
+
 	if !t.SkipDelete {
 		defer func() {
-			if err := t.DeleteNamespace(ns); err != nil {
-				test.Error(err)
+			for _, c := range clients {
+				if err := t.DeleteNamespace(c, ns); err != nil {
+					test.Error(err)
+				}
 			}
 		}()
 	}
 
 	for _, testStep := range t.Steps {
 		testStep.Client = t.Client
+		if testStep.Kubeconfig != "" {
+			testStep.Client = newClient(testStep.Kubeconfig)
+		}
 		testStep.DiscoveryClient = t.DiscoveryClient
+		if testStep.Kubeconfig != "" {
+			testStep.DiscoveryClient = newDiscoveryClient(testStep.Kubeconfig)
+		}
 		testStep.Logger = t.Logger.WithPrefix(testStep.String())
 		tc.Assertions += len(testStep.Asserts)
 		tc.Assertions += len(testStep.Errors)
@@ -347,3 +372,27 @@ func (t *Case) LoadTestSteps() error {
 	t.Steps = testSteps
 	return nil
 }
+
+func newClient(kubeconfig string) func(bool) (client.Client, error) {
+	return func(bool) (client.Client, error) {
+		config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+		if err != nil {
+			return nil, err
+		}
+
+		return testutils.NewRetryClient(config, client.Options{
+			Scheme: testutils.Scheme(),
+		})
+	}
+}
+
+func newDiscoveryClient(kubeconfig string) func() (discovery.DiscoveryInterface, error) {
+	return func() (discovery.DiscoveryInterface, error) {
+		config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+		if err != nil {
+			return nil, err
+		}
+
+		return discovery.NewDiscoveryClientForConfig(config)
+	}
+}

pkg/test/case_integration_test.go

@@ -0,0 +1,105 @@
+// +build integration
+
+package test
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"k8s.io/client-go/discovery"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/kudobuilder/kuttl/pkg/report"
+	testutils "github.com/kudobuilder/kuttl/pkg/test/utils"
+)
+
+// Create two test environments, ensure that the second environment is used when
+// Kubeconfig is set on a Step.
+func TestMultiClusterCase(t *testing.T) {
+	testenv, err := testutils.StartTestEnvironment(testutils.APIServerDefaultArgs, false)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer testenv.Environment.Stop()
+
+	testenv2, err := testutils.StartTestEnvironment(testutils.APIServerDefaultArgs, false)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer testenv2.Environment.Stop()
+
+	podSpec := map[string]interface{}{
+		"restartPolicy": "Never",
+		"containers": []map[string]interface{}{
+			{
+				"name":  "nginx",
+				"image": "nginx:1.7.9",
+			},
+		},
+	}
+
+	tmpfile, err := ioutil.TempFile("", "kubeconfig")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer os.Remove(tmpfile.Name())
+	if err := testutils.Kubeconfig(testenv2.Config, tmpfile); err != nil {
+		t.Error(err)
+		return
+	}
+
+	c := Case{
+		Logger: testutils.NewTestLogger(t, ""),
+		Steps: []*Step{
+			{
+				Name:  "initialize-testenv",
+				Index: 0,
+				Apply: []client.Object{
+					testutils.WithSpec(t, testutils.NewPod("hello", ""), podSpec),
+				},
+				Asserts: []client.Object{
+					testutils.WithSpec(t, testutils.NewPod("hello", ""), podSpec),
+				},
+				Timeout: 2,
+			},
+			{
+				Name:  "use-testenv2",
+				Index: 1,
+				Apply: []client.Object{
+					testutils.WithSpec(t, testutils.NewPod("hello2", ""), podSpec),
+				},
+				Asserts: []client.Object{
+					testutils.WithSpec(t, testutils.NewPod("hello2", ""), podSpec),
+				},
+				Errors: []client.Object{
+					testutils.WithSpec(t, testutils.NewPod("hello", ""), podSpec),
+				},
+				Timeout:    2,
+				Kubeconfig: tmpfile.Name(),
+			},
+			{
+				Name:  "verify-testenv-does-not-have-testenv2-resources",
+				Index: 2,
+				Asserts: []client.Object{
+					testutils.WithSpec(t, testutils.NewPod("hello", ""), podSpec),
+				},
+				Errors: []client.Object{
+					testutils.WithSpec(t, testutils.NewPod("hello2", ""), podSpec),
+				},
+				Timeout: 2,
+			},
+		},
+		Client: func(bool) (client.Client, error) {
+			return testenv.Client, nil
+		},
+		DiscoveryClient: func() (discovery.DiscoveryInterface, error) {
+			return testenv.DiscoveryClient, nil
+		},
+	}
+
+	c.Run(t, &report.Testcase{})
+}

pkg/test/harness.go

@@ -495,7 +495,7 @@ func (h *Harness) Setup() {
 			h.fatal(fmt.Errorf("fatal error installing manifests: %v", err))
 		}
 	}
-	bgs, err := testutils.RunCommands(context.TODO(), h.GetLogger(), "default", h.TestSuite.Commands, "", h.TestSuite.Timeout)
+	bgs, err := testutils.RunCommands(context.TODO(), h.GetLogger(), "default", h.TestSuite.Commands, "", h.TestSuite.Timeout, "")
 	// assign any background processes first for cleanup in case of any errors
 	h.bgProcesses = append(h.bgProcesses, bgs...)
 	if err != nil {

pkg/test/step.go

@@ -45,6 +45,7 @@ type Step struct {
 
 	Timeout int
 
+	Kubeconfig      string
 	Client          func(forceNew bool) (client.Client, error)
 	DiscoveryClient func() (discovery.DiscoveryInterface, error)
 
@@ -367,7 +368,7 @@ func (s *Step) CheckResourceAbsent(expected runtime.Object, namespace string) er
 // the errors returned can be a a failure of executing the command or the failure of the command executed.
 func (s *Step) CheckAssertCommands(ctx context.Context, namespace string, commands []harness.TestAssertCommand, timeout int) []error {
 	testErrors := []error{}
-	if _, err := testutils.RunAssertCommands(ctx, s.Logger, namespace, commands, "", timeout); err != nil {
+	if _, err := testutils.RunAssertCommands(ctx, s.Logger, namespace, commands, "", timeout, s.Kubeconfig); err != nil {
 		testErrors = append(testErrors, err)
 	}
 	return testErrors
@@ -413,7 +414,7 @@ func (s *Step) Run(namespace string) []error {
 				command.Background = false
 			}
 		}
-		if _, err := testutils.RunCommands(context.TODO(), s.Logger, namespace, s.Step.Commands, s.Dir, s.Timeout); err != nil {
+		if _, err := testutils.RunCommands(context.TODO(), s.Logger, namespace, s.Step.Commands, s.Dir, s.Timeout, s.Kubeconfig); err != nil {
 			testErrors = append(testErrors, err)
 		}
 	}
@@ -455,7 +456,7 @@ func (s *Step) Run(namespace string) []error {
 			s.Logger.Log("skipping invalid assertion collector")
 			continue
 		}
-		_, err := testutils.RunCommand(context.TODO(), namespace, *collector.Command(), s.Dir, s.Logger, s.Logger, s.Logger, s.Timeout)
+		_, err := testutils.RunCommand(context.TODO(), namespace, *collector.Command(), s.Dir, s.Logger, s.Logger, s.Logger, s.Timeout, s.Kubeconfig)
 		if err != nil {
 			s.Logger.Log("post assert collector failure: %s", err)
 		}
@@ -515,6 +516,10 @@ func (s *Step) LoadYAML(file string) error {
 			if s.Step.Name != "" {
 				s.Name = s.Step.Name
 			}
+			if s.Step.Kubeconfig != "" {
+				exKubeconfig := env.Expand(s.Step.Kubeconfig)
+				s.Kubeconfig = cleanPath(exKubeconfig, s.Dir)
+			}
 		} else {
 			applies = append(applies, obj)
 		}

pkg/test/utils/kubernetes.go

@@ -1006,7 +1006,7 @@ func GetArgs(ctx context.Context, cmd harness.Command, namespace string, envMap
 // RunCommand runs a command with args.
 // args gets split on spaces (respecting quoted strings).
 // if the command is run in the background a reference to the process is returned for later cleanup
-func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd string, stdout io.Writer, stderr io.Writer, logger Logger, timeout int) (*exec.Cmd, error) {
+func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd string, stdout io.Writer, stderr io.Writer, logger Logger, timeout int, kubeconfigOverride string) (*exec.Cmd, error) {
 	actualDir, err := os.Getwd()
 	if err != nil {
 		return nil, fmt.Errorf("command %q with %w", cmd.Command, err)
@@ -1015,6 +1015,9 @@ func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd
 	kuttlENV := make(map[string]string)
 	kuttlENV["NAMESPACE"] = namespace
 	kuttlENV["KUBECONFIG"] = fmt.Sprintf("%s/kubeconfig", actualDir)
+	if kubeconfigOverride != "" {
+		kuttlENV["KUBECONFIG"] = filepath.Join(actualDir, kubeconfigOverride)
+	}
 	kuttlENV["PATH"] = fmt.Sprintf("%s/bin/:%s", actualDir, os.Getenv("PATH"))
 
 	// by default testsuite timeout is the command timeout
@@ -1102,14 +1105,14 @@ func convertAssertCommand(assertCommands []harness.TestAssertCommand, timeout in
 }
 
 // RunAssertCommands runs a set of commands specified as TestAssertCommand
-func RunAssertCommands(ctx context.Context, logger Logger, namespace string, commands []harness.TestAssertCommand, workdir string, timeout int) ([]*exec.Cmd, error) {
-	return RunCommands(ctx, logger, namespace, convertAssertCommand(commands, timeout), workdir, timeout)
+func RunAssertCommands(ctx context.Context, logger Logger, namespace string, commands []harness.TestAssertCommand, workdir string, timeout int, kubeconfigOverride string) ([]*exec.Cmd, error) {
+	return RunCommands(ctx, logger, namespace, convertAssertCommand(commands, timeout), workdir, timeout, kubeconfigOverride)
 }
 
 // RunCommands runs a set of commands, returning any errors.
 // If any (non-background) command fails, the following commands are skipped
 // commands running in the background are returned
-func RunCommands(ctx context.Context, logger Logger, namespace string, commands []harness.Command, workdir string, timeout int) ([]*exec.Cmd, error) {
+func RunCommands(ctx context.Context, logger Logger, namespace string, commands []harness.Command, workdir string, timeout int, kubeconfigOverride string) ([]*exec.Cmd, error) {
 	bgs := []*exec.Cmd{}
 
 	if commands == nil {
@@ -1118,7 +1121,7 @@ func RunCommands(ctx context.Context, logger Logger, namespace string, commands
 
 	for i, cmd := range commands {
 
-		bg, err := RunCommand(ctx, namespace, cmd, workdir, logger, logger, logger, timeout)
+		bg, err := RunCommand(ctx, namespace, cmd, workdir, logger, logger, logger, timeout, kubeconfigOverride)
 		if err != nil {
 			cmdListSize := len(commands)
 			if i+1 < cmdListSize {