From fc8c0f2b2f338dc39de1ff346077eee175ed4172 Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Fri, 14 May 2021 23:44:37 +0530 Subject: [PATCH] Support multiple Kubernetes clusters by supporting kubeconfig in test steps - Follow up for #266 (#291) * Support multiple Kubernetes clusters by supporting kubeconfig in test steps Adds a `kubeconfig` setting to the TestStep API allowing an alternative kubeconfig path to be used for asserts and applies in a test step. Signed-off-by: jbarrick@mesosphere.com * Resolve relative kubeconfig path against test step dir. Signed-off-by: Tarun Gupta Akirala * fix linter errors Signed-off-by: Tarun Gupta Akirala * fix variable name collision with package imports Signed-off-by: Tarun Gupta Akirala Co-authored-by: jbarrick@mesosphere.com Co-authored-by: chhsia0 --- pkg/apis/testharness/v1beta1/test_types.go | 3 + pkg/test/case.go | 79 ++++++++++--- pkg/test/case_integration_test.go | 105 ++++++++++++++++++ pkg/test/harness.go | 2 +- pkg/test/step.go | 11 +- pkg/test/utils/kubernetes.go | 13 ++- pkg/test/utils/kubernetes_integration_test.go | 18 +-- pkg/test/utils/kubernetes_test.go | 2 +- 8 files changed, 199 insertions(+), 34 deletions(-) create mode 100644 pkg/test/case_integration_test.go diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index 975d2cec..7907eaf7 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -99,6 +99,9 @@ type TestStep struct { // Allowed environment labels // Disallowed environment labels + + // Kubeconfig to use when applying and asserting for this step. + Kubeconfig string `json:"kubeconfig,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/test/case.go b/pkg/test/case.go index edaccbd3..bff8616d 100644 --- a/pkg/test/case.go +++ b/pkg/test/case.go @@ -20,6 +20,8 @@ import ( "k8s.io/client-go/discovery" "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/tools/clientcmd" + "github.com/kudobuilder/kuttl/pkg/report" testutils "github.com/kudobuilder/kuttl/pkg/test/utils" ) @@ -51,7 +53,7 @@ type namespace struct { } // DeleteNamespace deletes a namespace in Kubernetes after we are done using it. -func (t *Case) DeleteNamespace(ns *namespace) error { +func (t *Case) DeleteNamespace(cl client.Client, ns *namespace) error { if !ns.AutoCreated { t.Logger.Log("Skipping deletion of user-supplied namespace:", ns.Name) return nil @@ -59,11 +61,6 @@ func (t *Case) DeleteNamespace(ns *namespace) error { t.Logger.Log("Deleting namespace:", ns.Name) - cl, err := t.Client(false) - if err != nil { - return err - } - ctx := context.Background() if t.Timeout > 0 { var cancel context.CancelFunc @@ -82,18 +79,13 @@ func (t *Case) DeleteNamespace(ns *namespace) error { } // CreateNamespace creates a namespace in Kubernetes to use for a test. -func (t *Case) CreateNamespace(ns *namespace) error { +func (t *Case) CreateNamespace(cl client.Client, ns *namespace) error { if !ns.AutoCreated { t.Logger.Log("Skipping creation of user-supplied namespace:", ns.Name) return nil } t.Logger.Log("Creating namespace:", ns.Name) - cl, err := t.Client(false) - if err != nil { - return err - } - ctx := context.Background() if t.Timeout > 0 { var cancel context.CancelFunc @@ -194,22 +186,55 @@ func shortString(obj *corev1.ObjectReference) string { // Run runs a test case including all of its steps. func (t *Case) Run(test *testing.T, tc *report.Testcase) { ns := t.determineNamespace() - if err := t.CreateNamespace(ns); err != nil { + + cl, err := t.Client(false) + if err != nil { tc.Failure = report.NewFailure(err.Error(), nil) test.Fatal(err) } + clients := map[string]client.Client{"": cl} + + for _, testStep := range t.Steps { + if clients[testStep.Kubeconfig] != nil { + continue + } + + cl, err := newClient(testStep.Kubeconfig)(false) + if err != nil { + tc.Failure = report.NewFailure(err.Error(), nil) + test.Fatal(err) + } + + clients[testStep.Kubeconfig] = cl + } + + for _, c := range clients { + if err := t.CreateNamespace(c, ns); err != nil { + tc.Failure = report.NewFailure(err.Error(), nil) + test.Fatal(err) + } + } + if !t.SkipDelete { defer func() { - if err := t.DeleteNamespace(ns); err != nil { - test.Error(err) + for _, c := range clients { + if err := t.DeleteNamespace(c, ns); err != nil { + test.Error(err) + } } }() } for _, testStep := range t.Steps { testStep.Client = t.Client + if testStep.Kubeconfig != "" { + testStep.Client = newClient(testStep.Kubeconfig) + } testStep.DiscoveryClient = t.DiscoveryClient + if testStep.Kubeconfig != "" { + testStep.DiscoveryClient = newDiscoveryClient(testStep.Kubeconfig) + } testStep.Logger = t.Logger.WithPrefix(testStep.String()) tc.Assertions += len(testStep.Asserts) tc.Assertions += len(testStep.Errors) @@ -347,3 +372,27 @@ func (t *Case) LoadTestSteps() error { t.Steps = testSteps return nil } + +func newClient(kubeconfig string) func(bool) (client.Client, error) { + return func(bool) (client.Client, error) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, err + } + + return testutils.NewRetryClient(config, client.Options{ + Scheme: testutils.Scheme(), + }) + } +} + +func newDiscoveryClient(kubeconfig string) func() (discovery.DiscoveryInterface, error) { + return func() (discovery.DiscoveryInterface, error) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, err + } + + return discovery.NewDiscoveryClientForConfig(config) + } +} diff --git a/pkg/test/case_integration_test.go b/pkg/test/case_integration_test.go new file mode 100644 index 00000000..882b67b3 --- /dev/null +++ b/pkg/test/case_integration_test.go @@ -0,0 +1,105 @@ +// +build integration + +package test + +import ( + "io/ioutil" + "os" + "testing" + + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kudobuilder/kuttl/pkg/report" + testutils "github.com/kudobuilder/kuttl/pkg/test/utils" +) + +// Create two test environments, ensure that the second environment is used when +// Kubeconfig is set on a Step. +func TestMultiClusterCase(t *testing.T) { + testenv, err := testutils.StartTestEnvironment(testutils.APIServerDefaultArgs, false) + if err != nil { + t.Error(err) + return + } + defer testenv.Environment.Stop() + + testenv2, err := testutils.StartTestEnvironment(testutils.APIServerDefaultArgs, false) + if err != nil { + t.Error(err) + return + } + defer testenv2.Environment.Stop() + + podSpec := map[string]interface{}{ + "restartPolicy": "Never", + "containers": []map[string]interface{}{ + { + "name": "nginx", + "image": "nginx:1.7.9", + }, + }, + } + + tmpfile, err := ioutil.TempFile("", "kubeconfig") + if err != nil { + t.Error(err) + return + } + defer os.Remove(tmpfile.Name()) + if err := testutils.Kubeconfig(testenv2.Config, tmpfile); err != nil { + t.Error(err) + return + } + + c := Case{ + Logger: testutils.NewTestLogger(t, ""), + Steps: []*Step{ + { + Name: "initialize-testenv", + Index: 0, + Apply: []client.Object{ + testutils.WithSpec(t, testutils.NewPod("hello", ""), podSpec), + }, + Asserts: []client.Object{ + testutils.WithSpec(t, testutils.NewPod("hello", ""), podSpec), + }, + Timeout: 2, + }, + { + Name: "use-testenv2", + Index: 1, + Apply: []client.Object{ + testutils.WithSpec(t, testutils.NewPod("hello2", ""), podSpec), + }, + Asserts: []client.Object{ + testutils.WithSpec(t, testutils.NewPod("hello2", ""), podSpec), + }, + Errors: []client.Object{ + testutils.WithSpec(t, testutils.NewPod("hello", ""), podSpec), + }, + Timeout: 2, + Kubeconfig: tmpfile.Name(), + }, + { + Name: "verify-testenv-does-not-have-testenv2-resources", + Index: 2, + Asserts: []client.Object{ + testutils.WithSpec(t, testutils.NewPod("hello", ""), podSpec), + }, + Errors: []client.Object{ + testutils.WithSpec(t, testutils.NewPod("hello2", ""), podSpec), + }, + Timeout: 2, + }, + }, + Client: func(bool) (client.Client, error) { + return testenv.Client, nil + }, + DiscoveryClient: func() (discovery.DiscoveryInterface, error) { + return testenv.DiscoveryClient, nil + }, + } + + c.Run(t, &report.Testcase{}) +} diff --git a/pkg/test/harness.go b/pkg/test/harness.go index aceae480..1e972295 100644 --- a/pkg/test/harness.go +++ b/pkg/test/harness.go @@ -495,7 +495,7 @@ func (h *Harness) Setup() { h.fatal(fmt.Errorf("fatal error installing manifests: %v", err)) } } - bgs, err := testutils.RunCommands(context.TODO(), h.GetLogger(), "default", h.TestSuite.Commands, "", h.TestSuite.Timeout) + bgs, err := testutils.RunCommands(context.TODO(), h.GetLogger(), "default", h.TestSuite.Commands, "", h.TestSuite.Timeout, "") // assign any background processes first for cleanup in case of any errors h.bgProcesses = append(h.bgProcesses, bgs...) if err != nil { diff --git a/pkg/test/step.go b/pkg/test/step.go index a4d5dda5..29857d05 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -45,6 +45,7 @@ type Step struct { Timeout int + Kubeconfig string Client func(forceNew bool) (client.Client, error) DiscoveryClient func() (discovery.DiscoveryInterface, error) @@ -367,7 +368,7 @@ func (s *Step) CheckResourceAbsent(expected runtime.Object, namespace string) er // the errors returned can be a a failure of executing the command or the failure of the command executed. func (s *Step) CheckAssertCommands(ctx context.Context, namespace string, commands []harness.TestAssertCommand, timeout int) []error { testErrors := []error{} - if _, err := testutils.RunAssertCommands(ctx, s.Logger, namespace, commands, "", timeout); err != nil { + if _, err := testutils.RunAssertCommands(ctx, s.Logger, namespace, commands, "", timeout, s.Kubeconfig); err != nil { testErrors = append(testErrors, err) } return testErrors @@ -413,7 +414,7 @@ func (s *Step) Run(namespace string) []error { command.Background = false } } - if _, err := testutils.RunCommands(context.TODO(), s.Logger, namespace, s.Step.Commands, s.Dir, s.Timeout); err != nil { + if _, err := testutils.RunCommands(context.TODO(), s.Logger, namespace, s.Step.Commands, s.Dir, s.Timeout, s.Kubeconfig); err != nil { testErrors = append(testErrors, err) } } @@ -455,7 +456,7 @@ func (s *Step) Run(namespace string) []error { s.Logger.Log("skipping invalid assertion collector") continue } - _, err := testutils.RunCommand(context.TODO(), namespace, *collector.Command(), s.Dir, s.Logger, s.Logger, s.Logger, s.Timeout) + _, err := testutils.RunCommand(context.TODO(), namespace, *collector.Command(), s.Dir, s.Logger, s.Logger, s.Logger, s.Timeout, s.Kubeconfig) if err != nil { s.Logger.Log("post assert collector failure: %s", err) } @@ -515,6 +516,10 @@ func (s *Step) LoadYAML(file string) error { if s.Step.Name != "" { s.Name = s.Step.Name } + if s.Step.Kubeconfig != "" { + exKubeconfig := env.Expand(s.Step.Kubeconfig) + s.Kubeconfig = cleanPath(exKubeconfig, s.Dir) + } } else { applies = append(applies, obj) } diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 007e8806..b02bd1d6 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -1006,7 +1006,7 @@ func GetArgs(ctx context.Context, cmd harness.Command, namespace string, envMap // RunCommand runs a command with args. // args gets split on spaces (respecting quoted strings). // if the command is run in the background a reference to the process is returned for later cleanup -func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd string, stdout io.Writer, stderr io.Writer, logger Logger, timeout int) (*exec.Cmd, error) { +func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd string, stdout io.Writer, stderr io.Writer, logger Logger, timeout int, kubeconfigOverride string) (*exec.Cmd, error) { actualDir, err := os.Getwd() if err != nil { return nil, fmt.Errorf("command %q with %w", cmd.Command, err) @@ -1015,6 +1015,9 @@ func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd kuttlENV := make(map[string]string) kuttlENV["NAMESPACE"] = namespace kuttlENV["KUBECONFIG"] = fmt.Sprintf("%s/kubeconfig", actualDir) + if kubeconfigOverride != "" { + kuttlENV["KUBECONFIG"] = filepath.Join(actualDir, kubeconfigOverride) + } kuttlENV["PATH"] = fmt.Sprintf("%s/bin/:%s", actualDir, os.Getenv("PATH")) // by default testsuite timeout is the command timeout @@ -1102,14 +1105,14 @@ func convertAssertCommand(assertCommands []harness.TestAssertCommand, timeout in } // RunAssertCommands runs a set of commands specified as TestAssertCommand -func RunAssertCommands(ctx context.Context, logger Logger, namespace string, commands []harness.TestAssertCommand, workdir string, timeout int) ([]*exec.Cmd, error) { - return RunCommands(ctx, logger, namespace, convertAssertCommand(commands, timeout), workdir, timeout) +func RunAssertCommands(ctx context.Context, logger Logger, namespace string, commands []harness.TestAssertCommand, workdir string, timeout int, kubeconfigOverride string) ([]*exec.Cmd, error) { + return RunCommands(ctx, logger, namespace, convertAssertCommand(commands, timeout), workdir, timeout, kubeconfigOverride) } // RunCommands runs a set of commands, returning any errors. // If any (non-background) command fails, the following commands are skipped // commands running in the background are returned -func RunCommands(ctx context.Context, logger Logger, namespace string, commands []harness.Command, workdir string, timeout int) ([]*exec.Cmd, error) { +func RunCommands(ctx context.Context, logger Logger, namespace string, commands []harness.Command, workdir string, timeout int, kubeconfigOverride string) ([]*exec.Cmd, error) { bgs := []*exec.Cmd{} if commands == nil { @@ -1118,7 +1121,7 @@ func RunCommands(ctx context.Context, logger Logger, namespace string, commands for i, cmd := range commands { - bg, err := RunCommand(ctx, namespace, cmd, workdir, logger, logger, logger, timeout) + bg, err := RunCommand(ctx, namespace, cmd, workdir, logger, logger, logger, timeout, kubeconfigOverride) if err != nil { cmdListSize := len(commands) if i+1 < cmdListSize { diff --git a/pkg/test/utils/kubernetes_integration_test.go b/pkg/test/utils/kubernetes_integration_test.go index 08c66a45..3723cdab 100644 --- a/pkg/test/utils/kubernetes_integration_test.go +++ b/pkg/test/utils/kubernetes_integration_test.go @@ -121,7 +121,7 @@ func TestRunCommand(t *testing.T) { logger := NewTestLogger(t, "") // assert foreground cmd returns nil - cmd, err := RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + cmd, err := RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") assert.NoError(t, err) assert.Nil(t, cmd) // foreground processes should have stdout @@ -131,7 +131,7 @@ func TestRunCommand(t *testing.T) { stdout = &bytes.Buffer{} // assert background cmd returns process - cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") assert.NoError(t, err) assert.NotNil(t, cmd) // no stdout for background processes @@ -142,7 +142,7 @@ func TestRunCommand(t *testing.T) { hcmd.Command = "sleep 42" // assert foreground cmd times out - cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 2) + cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 2, "") assert.Error(t, err) assert.True(t, strings.Contains(err.Error(), "timeout")) assert.Nil(t, cmd) @@ -153,7 +153,7 @@ func TestRunCommand(t *testing.T) { hcmd.Timeout = 2 // assert foreground cmd times out with command timeout - cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") assert.Error(t, err) assert.True(t, strings.Contains(err.Error(), "timeout")) assert.Nil(t, cmd) @@ -170,12 +170,12 @@ func TestRunCommandIgnoreErrors(t *testing.T) { logger := NewTestLogger(t, "") // assert foreground cmd returns nil - cmd, err := RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + cmd, err := RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") assert.NoError(t, err) assert.Nil(t, cmd) hcmd.IgnoreFailure = false - cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") assert.Error(t, err) assert.Nil(t, cmd) @@ -184,7 +184,7 @@ func TestRunCommandIgnoreErrors(t *testing.T) { Command: "bad-command", IgnoreFailure: true, } - cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") assert.Error(t, err) assert.Nil(t, cmd) } @@ -198,7 +198,7 @@ func TestRunCommandSkipLogOutput(t *testing.T) { logger := NewTestLogger(t, "") // test there is a stdout - cmd, err := RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + cmd, err := RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") assert.NoError(t, err) assert.Nil(t, cmd) assert.True(t, stdout.Len() > 0) @@ -207,7 +207,7 @@ func TestRunCommandSkipLogOutput(t *testing.T) { stdout = &bytes.Buffer{} stderr = &bytes.Buffer{} // test there is no stdout - cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + cmd, err = RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") assert.NoError(t, err) assert.Nil(t, cmd) assert.True(t, stdout.Len() == 0) diff --git a/pkg/test/utils/kubernetes_test.go b/pkg/test/utils/kubernetes_test.go index c0f698a4..dba9ea86 100644 --- a/pkg/test/utils/kubernetes_test.go +++ b/pkg/test/utils/kubernetes_test.go @@ -482,7 +482,7 @@ func TestRunScript(t *testing.T) { logger := NewTestLogger(t, "") // script runs with output - _, err := RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0) + _, err := RunCommand(context.TODO(), "", hcmd, "", stdout, stderr, logger, 0, "") if tt.wantedErr { assert.Error(t, err)