From 47d7ce767003eefd73f721313d0b703234dcb93c Mon Sep 17 00:00:00 2001 From: Ethan Johnston Date: Tue, 14 Apr 2020 17:12:25 -0400 Subject: [PATCH] Fix autoscaling e2e tests on top of updates (#253) --- test/e2e/appsody_autoscaling.go | 181 +++++++++++++++++++---------- test/e2e/appsody_basic.go | 16 +-- test/e2e/appsody_servicemonitor.go | 91 +++++++-------- test/e2e/appsody_storage.go | 14 +-- test/util/util.go | 24 ++++ 5 files changed, 194 insertions(+), 132 deletions(-) diff --git a/test/e2e/appsody_autoscaling.go b/test/e2e/appsody_autoscaling.go index 9a35903..b4d013a 100644 --- a/test/e2e/appsody_autoscaling.go +++ b/test/e2e/appsody_autoscaling.go @@ -48,9 +48,11 @@ func AppsodyAutoScalingTest(t *testing.T) { t.Fatal(err) } + const name = "example-appsody-autoscaling" + // Make basic appsody application with 1 replica replicas := int32(1) - appsodyApplication := util.MakeBasicAppsodyApplication(t, f, "example-appsody-autoscaling", namespace, replicas) + appsodyApplication := util.MakeBasicAppsodyApplication(t, f, name, namespace, replicas) // use TestCtx's create helper to create the object and add a cleanup function for the new object err = f.Client.Create(goctx.TODO(), appsodyApplication, &framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) @@ -59,31 +61,27 @@ func AppsodyAutoScalingTest(t *testing.T) { } // wait for example-appsody-autoscaling to reach 1 replicas - err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-appsody-autoscaling", 1, retryInterval, timeout) + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, name, 1, retryInterval, timeout) if err != nil { util.FailureCleanup(t, f, namespace, err) } // Check the name field that matches - m := map[string]string{"metadata.name": "example-appsody-autoscaling"} + m := map[string]string{"metadata.name": name} l := fields.Set(m) selec := l.AsSelector() - err = f.Client.Get(goctx.TODO(), types.NamespacedName{Name: "example-appsody-autoscaling", Namespace: namespace}, appsodyApplication) - if err != nil { - t.Fatal(err) - } - - appsodyApplication.Spec.ResourceConstraints = setResources("0.2") - appsodyApplication.Spec.Autoscaling = setAutoScale(5, 50) - - err = f.Client.Update(goctx.TODO(), appsodyApplication) + target := types.NamespacedName{Name: name, Namespace: namespace} + err = util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + a.Spec.ResourceConstraints = setResources("0.2") + a.Spec.Autoscaling = setAutoScale(5, 50) + }) if err != nil { - t.Fatal(err) + util.FailureCleanup(t, f, namespace, err) } hpa := &autoscalingv1.HorizontalPodAutoscalerList{} - options := k.ListOptions{FieldSelector: selec} + options := k.ListOptions{FieldSelector: selec, Namespace: namespace} hpa = getHPA(hpa, t, f, options) timestamp = time.Now().UTC() @@ -98,6 +96,7 @@ func AppsodyAutoScalingTest(t *testing.T) { minMaxTest(t, f, appsodyApplication, options, namespace, hpa) minBoundaryTest(t, f, appsodyApplication, options, namespace, hpa) incorrectFieldsTest(t, f, ctx) + replicasTest(t, f, ctx) } func getHPA(hpa *autoscalingv1.HorizontalPodAutoscalerList, t *testing.T, f *framework.Framework, options k.ListOptions) *autoscalingv1.HorizontalPodAutoscalerList { @@ -109,7 +108,7 @@ func getHPA(hpa *autoscalingv1.HorizontalPodAutoscalerList, t *testing.T, f *fra func waitForHPA(hpa *autoscalingv1.HorizontalPodAutoscalerList, t *testing.T, minReplicas int32, maxReplicas int32, utiliz int32, f *framework.Framework, options k.ListOptions) error { for counter := 0; counter < 6; counter++ { - time.Sleep(4000 * time.Millisecond) + time.Sleep(6000 * time.Millisecond) hpa = getHPA(hpa, t, f, options) if checkValues(hpa, t, minReplicas, maxReplicas, utiliz) == nil { return nil @@ -169,18 +168,13 @@ func checkValues(hpa *autoscalingv1.HorizontalPodAutoscalerList, t *testing.T, m // Updates the values and checks they are changed func updateTest(t *testing.T, f *framework.Framework, appsodyApplication *appsodyv1beta1.AppsodyApplication, options k.ListOptions, namespace string, hpa *autoscalingv1.HorizontalPodAutoscalerList) { - - err := f.Client.Get(goctx.TODO(), types.NamespacedName{Name: "example-appsody-autoscaling", Namespace: namespace}, appsodyApplication) - if err != nil { - t.Fatal(err) - } - - appsodyApplication.Spec.ResourceConstraints = setResources("0.2") - appsodyApplication.Spec.Autoscaling = setAutoScale(3, 2, 30) - - err = f.Client.Update(goctx.TODO(), appsodyApplication) + target := types.NamespacedName{Name: "example-appsody-autoscaling", Namespace: namespace} + err := util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + a.Spec.ResourceConstraints = setResources("0.2") + a.Spec.Autoscaling = setAutoScale(3, 2, 30) + }) if err != nil { - t.Fatal(err) + util.FailureCleanup(t, f, namespace, err) } timestamp := time.Now().UTC() @@ -192,22 +186,22 @@ func updateTest(t *testing.T, f *framework.Framework, appsodyApplication *appsod if err != nil { t.Fatal(err) } -} -// Checks when max is less than min, there should be no update -func minMaxTest(t *testing.T, f *framework.Framework, appsodyApplication *appsodyv1beta1.AppsodyApplication, options k.ListOptions, namespace string, hpa *autoscalingv1.HorizontalPodAutoscalerList) { - - err := f.Client.Get(goctx.TODO(), types.NamespacedName{Name: "example-appsody-autoscaling", Namespace: namespace}, appsodyApplication) + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-appsody-autoscaling", 2, retryInterval, timeout) if err != nil { - t.Fatal(err) + util.FailureCleanup(t, f, namespace, err) } +} - appsodyApplication.Spec.ResourceConstraints = setResources("0.2") - appsodyApplication.Spec.Autoscaling = setAutoScale(1, 6, 10) - - err = f.Client.Update(goctx.TODO(), appsodyApplication) +// Checks when max is less than min, there should be no update +func minMaxTest(t *testing.T, f *framework.Framework, appsodyApplication *appsodyv1beta1.AppsodyApplication, options k.ListOptions, namespace string, hpa *autoscalingv1.HorizontalPodAutoscalerList) { + target := types.NamespacedName{Name: "example-appsody-autoscaling", Namespace: namespace} + err := util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + a.Spec.ResourceConstraints = setResources("0.2") + a.Spec.Autoscaling = setAutoScale(1, 6, 10) + }) if err != nil { - t.Fatal(err) + util.FailureCleanup(t, f, namespace, err) } timestamp := time.Now().UTC() @@ -219,22 +213,23 @@ func minMaxTest(t *testing.T, f *framework.Framework, appsodyApplication *appsod if err != nil { t.Fatal(err) } -} -// When min is set to less than 1, there should be no update since the minReplicas are updated to a value less than 1 -func minBoundaryTest(t *testing.T, f *framework.Framework, appsodyApplication *appsodyv1beta1.AppsodyApplication, options k.ListOptions, namespace string, hpa *autoscalingv1.HorizontalPodAutoscalerList) { - - err := f.Client.Get(goctx.TODO(), types.NamespacedName{Name: "example-appsody-autoscaling", Namespace: namespace}, appsodyApplication) + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-appsody-autoscaling", 2, retryInterval, timeout) if err != nil { - t.Fatal(err) + util.FailureCleanup(t, f, namespace, err) } +} - appsodyApplication.Spec.ResourceConstraints = setResources("0.5") - appsodyApplication.Spec.Autoscaling = setAutoScale(4, 0, 20) +// When min is set to less than 1, there should be no update since the minReplicas are updated to a value less than 1 +func minBoundaryTest(t *testing.T, f *framework.Framework, appsodyApplication *appsodyv1beta1.AppsodyApplication, options k.ListOptions, namespace string, hpa *autoscalingv1.HorizontalPodAutoscalerList) { - err = f.Client.Update(goctx.TODO(), appsodyApplication) + target := types.NamespacedName{Name: "example-appsody-autoscaling", Namespace: namespace} + err := util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + a.Spec.ResourceConstraints = setResources("0.5") + a.Spec.Autoscaling = setAutoScale(4, 0, 20) + }) if err != nil { - t.Fatal(err) + util.FailureCleanup(t, f, namespace, err) } timestamp := time.Now().UTC() @@ -246,11 +241,16 @@ func minBoundaryTest(t *testing.T, f *framework.Framework, appsodyApplication *a if err != nil { t.Fatal(err) } + + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-appsody-autoscaling", 2, retryInterval, timeout) + if err != nil { + util.FailureCleanup(t, f, namespace, err) + } } // When the mandatory fields for autoscaling are not set func incorrectFieldsTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) { - + const name = "example-appsody-autoscaling2" namespace, err := ctx.GetNamespace() if err != nil { t.Fatalf("could not get namespace: %v", err) @@ -261,7 +261,7 @@ func incorrectFieldsTest(t *testing.T, f *framework.Framework, ctx *framework.Te // Make basic appsody application with 1 replica replicas := int32(1) - appsodyApplication := util.MakeBasicAppsodyApplication(t, f, "example-appsody-autoscaling2", namespace, replicas) + appsodyApplication := util.MakeBasicAppsodyApplication(t, f, name, namespace, replicas) // use TestCtx's create helper to create the object and add a cleanup function for the new object err = f.Client.Create(goctx.TODO(), appsodyApplication, &framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) @@ -270,29 +270,25 @@ func incorrectFieldsTest(t *testing.T, f *framework.Framework, ctx *framework.Te } // wait for example-appsody-autoscaling to reach 1 replicas - err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-appsody-autoscaling2", 1, retryInterval, timeout) + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, name, 1, retryInterval, timeout) if err != nil { t.Fatal(err) } // Check the name field that matches - m := map[string]string{"metadata.name": "example-appsody-autoscaling2"} + m := map[string]string{"metadata.name": name} l := fields.Set(m) selec := l.AsSelector() - options := k.ListOptions{FieldSelector: selec} - - err = f.Client.Get(goctx.TODO(), types.NamespacedName{Name: "example-appsody-autoscaling2", Namespace: namespace}, appsodyApplication) - if err != nil { - t.Fatal(err) - } - - appsodyApplication.Spec.ResourceConstraints = setResources("0.3") - appsodyApplication.Spec.Autoscaling = setAutoScale(4) + options := k.ListOptions{FieldSelector: selec, Namespace: namespace} - err = f.Client.Update(goctx.TODO(), appsodyApplication) + target := types.NamespacedName{Name: name, Namespace: namespace} + err = util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + a.Spec.ResourceConstraints = setResources("0.3") + a.Spec.Autoscaling = setAutoScale(4) + }) if err != nil { - t.Fatal(err) + util.FailureCleanup(t, f, namespace, err) } timestamp = time.Now().UTC() @@ -307,3 +303,64 @@ func incorrectFieldsTest(t *testing.T, f *framework.Framework, ctx *framework.Te t.Fatal("Error: The mandatory fields were not set so autoscaling should not be enabled") } } + +// verify behaviour between spec replicas and HPA minReplicas +func replicasTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) { + const name = "appsody-autoscaling-replicas" + namespace, err := ctx.GetNamespace() + if err != nil { + t.Fatalf("could not get namespace: %v", err) + } + + timestamp := time.Now().UTC() + t.Logf("%s - Starting appsody autoscaling test...", timestamp) + + // Make basic appsody omponent with 1 replica + replicas := int32(2) + appsody := util.MakeBasicAppsodyApplication(t, f, name, namespace, replicas) + + err = f.Client.Create(goctx.TODO(), appsody, &framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) + if err != nil { + util.FailureCleanup(t, f, namespace, err) + } + + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, name, int(replicas), retryInterval, timeout) + if err != nil { + util.FailureCleanup(t, f, namespace, err) + } + + // check that it prioritizes the HPA's minimum number of replicas over spec replicas + target := types.NamespacedName{Namespace: namespace, Name: name} + err = util.UpdateApplication(f, target, func(r *appsodyv1beta1.AppsodyApplication) { + r.Spec.ResourceConstraints = setResources("0.5") + var cpu int32 = 50 + var min int32 = 3 + r.Spec.Autoscaling = &appsodyv1beta1.AppsodyApplicationAutoScaling{ + TargetCPUUtilizationPercentage: &cpu, + MaxReplicas: 5, + MinReplicas: &min, + } + }) + if err != nil { + util.FailureCleanup(t, f, namespace, err) + } + + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, name, 3, retryInterval, timeout) + if err != nil { + util.FailureCleanup(t, f, namespace, err) + } + + // check that it correctly returns to defined replica count after deleting HPA + err = util.UpdateApplication(f, target, func(r *appsodyv1beta1.AppsodyApplication) { + r.Spec.ResourceConstraints = nil + r.Spec.Autoscaling = nil + }) + if err != nil { + util.FailureCleanup(t, f, namespace, err) + } + + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, name, int(replicas), retryInterval, timeout) + if err != nil { + util.FailureCleanup(t, f, namespace, err) + } +} diff --git a/test/e2e/appsody_basic.go b/test/e2e/appsody_basic.go index 723b411..0ce4bec 100644 --- a/test/e2e/appsody_basic.go +++ b/test/e2e/appsody_basic.go @@ -82,17 +82,11 @@ func appsodyBasicScaleTest(t *testing.T, f *framework.Framework, ctx *framework. } func appsodyUpdateScaleTest(t *testing.T, f *framework.Framework, namespace string, exampleAppsody *appsodyv1beta1.AppsodyApplication) error { - err := f.Client.Get(goctx.TODO(), types.NamespacedName{Name: "example-appsody", Namespace: namespace}, exampleAppsody) - if err != nil { - return err - } - - helper2 := int32(2) - exampleAppsody.Spec.Replicas = &helper2 - err = f.Client.Update(goctx.TODO(), exampleAppsody) - if err != nil { - return err - } + target := types.NamespacedName{Name: "example-appsody", Namespace: namespace} + err := util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + helper2 := int32(2) + a.Spec.Replicas = &helper2 + }) // wait for example-memcached to reach 2 replicas err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-appsody", 2, retryInterval, timeout) diff --git a/test/e2e/appsody_servicemonitor.go b/test/e2e/appsody_servicemonitor.go index da58c77..c21fd06 100644 --- a/test/e2e/appsody_servicemonitor.go +++ b/test/e2e/appsody_servicemonitor.go @@ -77,21 +77,15 @@ func AppsodyServiceMonitorTest(t *testing.T) { util.FailureCleanup(t, f, namespace, errors.New("There is another service monitor running")) } - err = f.Client.Get(goctx.TODO(), types.NamespacedName{Name: "example-appsody-sm", Namespace: namespace}, appsody) - if err != nil { - util.FailureCleanup(t, f, namespace, err) - } - - // Adds the mandatory label to the application so it will be picked up by the prometheus operator - label := map[string]string{"apps-prometheus": ""} - monitor := &appsodyv1beta1.AppsodyApplicationMonitoring{Labels: label} - appsody.Spec.Monitoring = monitor - - // Updates the application so the operator is reconciled - helper = int32(2) - appsody.Spec.Replicas = &helper - - err = f.Client.Update(goctx.TODO(), appsody) + target := types.NamespacedName{Name: "example-appsody-sm", Namespace: namespace} + err = util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + // Adds the mandatory label to the application so it will be picked up by the prometheus operator + label := map[string]string{"apps-prometheus": ""} + monitor := &appsodyv1beta1.AppsodyApplicationMonitoring{Labels: label} + a.Spec.Monitoring = monitor + helper = int32(2) + a.Spec.Replicas = &helper + }) if err != nil { util.FailureCleanup(t, f, namespace, err) } @@ -154,45 +148,42 @@ func AppsodyServiceMonitorTest(t *testing.T) { } func testSettingAppsodyServiceMonitor(t *testing.T, f *framework.Framework, namespace string, appsody *appsodyv1beta1.AppsodyApplication) { - err := f.Client.Get(goctx.TODO(), types.NamespacedName{Name: "example-appsody-sm", Namespace: namespace}, appsody) - if err != nil { - t.Fatal(err) - } - - params := map[string][]string{ - "params": []string{"param1", "param2"}, - } - username := v1.SecretKeySelector{Key: "username"} - password := v1.SecretKeySelector{Key: "password"} - - // Creates the endpoint fields the user can customize - endpoint := prometheusv1.Endpoint{ - Path: "/path", - Scheme: "myScheme", - Params: params, - Interval: "30s", - ScrapeTimeout: "10s", - TLSConfig: &prometheusv1.TLSConfig{InsecureSkipVerify: true}, - BearerTokenFile: "myBTF", - BasicAuth: &prometheusv1.BasicAuth{Username: username, Password: password}, - } - - endpoints := []prometheusv1.Endpoint{endpoint} - - // Adds the mandatory label to the application so it will be picked up by the prometheus operator - label := map[string]string{"apps-prometheus": ""} - monitor := &appsodyv1beta1.AppsodyApplicationMonitoring{Labels: label, Endpoints: endpoints} - appsody.Spec.Monitoring = monitor - - // Updates the application so the operator is reconciled - helper := int32(3) - appsody.Spec.Replicas = &helper - - err = f.Client.Update(goctx.TODO(), appsody) + target := types.NamespacedName{Name: "example-appsody-sm", Namespace: namespace} + err := util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + params := map[string][]string{ + "params": []string{"param1", "param2"}, + } + username := v1.SecretKeySelector{Key: "username"} + password := v1.SecretKeySelector{Key: "password"} + + // Creates the endpoint fields the user can customize + endpoint := prometheusv1.Endpoint{ + Path: "/path", + Scheme: "myScheme", + Params: params, + Interval: "30s", + ScrapeTimeout: "10s", + TLSConfig: &prometheusv1.TLSConfig{InsecureSkipVerify: true}, + BearerTokenFile: "myBTF", + BasicAuth: &prometheusv1.BasicAuth{Username: username, Password: password}, + } + + endpoints := []prometheusv1.Endpoint{endpoint} + + // Adds the mandatory label to the application so it will be picked up by the prometheus operator + label := map[string]string{"apps-prometheus": ""} + monitor := &appsodyv1beta1.AppsodyApplicationMonitoring{Labels: label, Endpoints: endpoints} + a.Spec.Monitoring = monitor + + // Updates the application so the operator is reconciled + helper := int32(3) + a.Spec.Replicas = &helper + }) if err != nil { util.FailureCleanup(t, f, namespace, err) } + err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-appsody-sm", 3, retryInterval, timeout) if err != nil { util.FailureCleanup(t, f, namespace, err) diff --git a/test/e2e/appsody_storage.go b/test/e2e/appsody_storage.go index a7bff57..c1b4607 100644 --- a/test/e2e/appsody_storage.go +++ b/test/e2e/appsody_storage.go @@ -65,15 +65,11 @@ func updateStorageConfig(t *testing.T, f *framework.Framework, ctx *framework.Te return err } - err = f.Client.Get(goctx.TODO(), types.NamespacedName{Name: app.Name, Namespace: namespace}, app) - if err != nil { - return err - } - // remove storage definition to return it to a deployment - app.Spec.Storage = nil - app.Spec.VolumeMounts = nil - - err = f.Client.Update(goctx.TODO(), app) + target := types.NamespacedName{Name: app.Name, Namespace: namespace} + err = util.UpdateApplication(f, target, func(a *appsodyv1beta1.AppsodyApplication) { + a.Spec.Storage = nil + a.Spec.VolumeMounts = nil + }) if err != nil { return err } diff --git a/test/util/util.go b/test/util/util.go index 17c6c6f..e1d207a 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -193,3 +193,27 @@ func WaitForKnativeDeployment(t *testing.T, f *framework.Framework, ns, n string }) return err } + +// UpdateApplication updates target app using provided function, retrying in the case that status has changed +func UpdateApplication(f *framework.Framework, target types.NamespacedName, update func(r *appsodyv1beta1.AppsodyApplication)) error { + retryInterval := time.Second * 5 + timeout := time.Second * 30 + err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { + temp := &appsodyv1beta1.AppsodyApplication{} + err = f.Client.Get(goctx.TODO(), target, temp) + if err != nil { + return true, err + } + + update(temp) + + err = f.Client.Update(goctx.TODO(), temp) + if err != nil { + return false, err + } + + return true, nil + }) + + return err +}