Skip to content

Commit

Permalink
add EmergencyPhase test case (#57)
Browse files Browse the repository at this point in the history 
* add EmergencyPhase test case

* cleanup child vpa and hpa created by tortoise for every iteration

* improvements to cleanup() and fix test case for targetCPUUtilization value
  • Loading branch information
@harpratap
harpratap authored Apr 13, 2023
1 parent f4f350c commit 57e5a79
Showing 1 changed file with 246 additions and 1 deletion.
247 changes: 246 additions & 1 deletion controllers/tortoise_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package controllers
import (
"context"
"fmt"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/mercari/tortoise/api/v1alpha1"
Expand Down Expand Up @@ -33,6 +34,22 @@ import (
var _ = Describe("Test TortoiseController", func() {
ctx := context.Background()
var stopFunc func()
cleanUp := func() {
err := deleteObj(ctx, &v1alpha1.Tortoise{}, "mercari")
Expect(err).ShouldNot(HaveOccurred())

err = deleteObj(ctx, &v1.Deployment{}, "mercari-app")
Expect(err).ShouldNot(HaveOccurred())

err = deleteObj(ctx, &autoscalingv1.VerticalPodAutoscaler{}, "tortoise-updater-mercari")
Expect(err).ShouldNot(HaveOccurred())

err = deleteObj(ctx, &autoscalingv1.VerticalPodAutoscaler{}, "tortoise-monitor-mercari")
Expect(err).ShouldNot(HaveOccurred())

err = deleteObj(ctx, &v2.HorizontalPodAutoscaler{}, "hpa")
Expect(err).ShouldNot(HaveOccurred())
}

BeforeEach(func() {
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Expand Down Expand Up @@ -69,6 +86,7 @@ var _ = Describe("Test TortoiseController", func() {
})

AfterEach(func() {
cleanUp()
stopFunc()
time.Sleep(100 * time.Millisecond)
})
Expand Down Expand Up @@ -291,7 +309,222 @@ var _ = Describe("Test TortoiseController", func() {
}).Should(Succeed())
})
It("TortoisePhaseEmergency", func() {
// TODO: add
now := time.Now()
tc := testCase{
before: resources{
tortoise: utils.NewTortoiseBuilder().
SetName("mercari").
SetNamespace("default").
SetTargetRefs(v1alpha1.TargetRefs{
DeploymentName: "mercari-app",
HorizontalPodAutoscalerName: pointer.String("hpa"),
}).
SetUpdateMode(v1alpha1.UpdateModeEmergency).
AddResourcePolicy(v1alpha1.ContainerResourcePolicy{
ContainerName: "app",
AutoscalingPolicy: map[corev1.ResourceName]v1alpha1.AutoscalingType{
corev1.ResourceCPU: v1alpha1.AutoscalingTypeHorizontal,
corev1.ResourceMemory: v1alpha1.AutoscalingTypeVertical,
},
}).
SetTortoisePhase(v1alpha1.TortoisePhaseWorking). // will be updated.
SetRecommendations(v1alpha1.Recommendations{
Horizontal: &v1alpha1.HorizontalRecommendations{
TargetUtilizations: []v1alpha1.HPATargetUtilizationRecommendationPerContainer{
{
ContainerName: "app",
TargetUtilization: map[corev1.ResourceName]int32{
corev1.ResourceCPU: 50, // will be updated.
},
},
},
MaxReplicas: []v1alpha1.ReplicasRecommendation{
{
From: 0,
To: 24,
WeekDay: now.Weekday().String(),
TimeZone: now.Location().String(),
Value: 15, // will be updated
UpdatedAt: metav1.NewTime(now),
},
},
MinReplicas: []v1alpha1.ReplicasRecommendation{
{
From: 0,
To: 24,
WeekDay: now.Weekday().String(),
TimeZone: now.Location().String(),
Value: 3, // will be updated
UpdatedAt: metav1.NewTime(now),
},
},
},
}).
AddCondition(v1alpha1.ContainerRecommendationFromVPA{
ContainerName: "app",
Recommendation: map[corev1.ResourceName]v1alpha1.ResourceQuantity{
corev1.ResourceCPU: {},
corev1.ResourceMemory: {},
},
MaxRecommendation: map[corev1.ResourceName]v1alpha1.ResourceQuantity{
corev1.ResourceCPU: {},
corev1.ResourceMemory: {},
},
}).
Build(),
deployment: deploymentWithReplicaNum(10),
},
}

err := tc.createResources(ctx, k8sClient, cfg)
Expect(err).ShouldNot(HaveOccurred())

// create the desired HPA from the created definition.
wantHPA := tc.before.hpa.DeepCopy()
wantHPA.Spec.MinReplicas = pointer.Int32(20)
wantHPA.Spec.MaxReplicas = 20
for i, m := range wantHPA.Spec.Metrics {
if m.External != nil && m.External.Metric.Name == "datadogmetric@default:mercari-app-cpu-app" {
wantHPA.Spec.Metrics[i].External.Target.Value = resourceQuantityPtr(resource.MustParse("75"))
}
}

// create the desired VPA from the created definition.
wantVPA := map[v1alpha1.VerticalPodAutoscalerRole]*autoscalingv1.VerticalPodAutoscaler{}
wantUpdater := tc.before.vpa[v1alpha1.VerticalPodAutoscalerRoleUpdater].DeepCopy()
wantUpdater.Status.Recommendation = &autoscalingv1.RecommendedPodResources{}
wantUpdater.Status.Recommendation.ContainerRecommendations = []autoscalingv1.RecommendedContainerResources{
{
ContainerName: "app",
Target: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("4"),
corev1.ResourceMemory: resource.MustParse("3Gi"),
},
LowerBound: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("4"),
corev1.ResourceMemory: resource.MustParse("3Gi"),
},
UpperBound: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("4"),
corev1.ResourceMemory: resource.MustParse("3Gi"),
},
UncappedTarget: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("4"),
corev1.ResourceMemory: resource.MustParse("3Gi"),
},
},
}
wantVPA[v1alpha1.VerticalPodAutoscalerRoleUpdater] = wantUpdater
wantVPA[v1alpha1.VerticalPodAutoscalerRoleMonitor] = tc.before.vpa[v1alpha1.VerticalPodAutoscalerRoleMonitor].DeepCopy()

want := resources{
tortoise: utils.NewTortoiseBuilder().
SetName("mercari").
SetNamespace("default").
SetTargetRefs(v1alpha1.TargetRefs{
DeploymentName: "mercari-app",
HorizontalPodAutoscalerName: pointer.String("hpa"),
}).
SetUpdateMode(v1alpha1.UpdateModeEmergency).
AddResourcePolicy(v1alpha1.ContainerResourcePolicy{
ContainerName: "app",
AutoscalingPolicy: map[corev1.ResourceName]v1alpha1.AutoscalingType{
corev1.ResourceCPU: v1alpha1.AutoscalingTypeHorizontal,
corev1.ResourceMemory: v1alpha1.AutoscalingTypeVertical,
},
}).
SetTortoisePhase(v1alpha1.TortoisePhaseEmergency).
SetTargetsStatus(v1alpha1.TargetsStatus{
HorizontalPodAutoscaler: "hpa",
VerticalPodAutoscalers: []v1alpha1.TargetStatusVerticalPodAutoscaler{
{Name: "tortoise-updater-mercari", Role: "Updater"},
{Name: "tortoise-monitor-mercari", Role: "Monitor"},
},
}).
SetRecommendations(v1alpha1.Recommendations{
Vertical: &v1alpha1.VerticalRecommendations{
ContainerResourceRecommendation: []v1alpha1.RecommendedContainerResources{
{
ContainerName: "app",
RecommendedResource: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("4"),
corev1.ResourceMemory: resource.MustParse("3Gi"),
},
},
},
},
Horizontal: &v1alpha1.HorizontalRecommendations{
TargetUtilizations: []v1alpha1.HPATargetUtilizationRecommendationPerContainer{
{
ContainerName: "app",
TargetUtilization: map[corev1.ResourceName]int32{
corev1.ResourceCPU: 75,
corev1.ResourceMemory: 90,
},
},
},
MaxReplicas: []v1alpha1.ReplicasRecommendation{
{
From: 0,
To: 24,
WeekDay: now.Weekday().String(),
TimeZone: now.Location().String(),
Value: 20,
UpdatedAt: metav1.NewTime(now),
},
},
MinReplicas: []v1alpha1.ReplicasRecommendation{
{
From: 0,
To: 24,
WeekDay: now.Weekday().String(),
TimeZone: now.Location().String(),
Value: 5,
UpdatedAt: metav1.NewTime(now),
},
},
},
}).
AddCondition(v1alpha1.ContainerRecommendationFromVPA{
ContainerName: "app",
Recommendation: map[corev1.ResourceName]v1alpha1.ResourceQuantity{
corev1.ResourceCPU: {
Quantity: resource.MustParse("3"),
},
corev1.ResourceMemory: {
Quantity: resource.MustParse("3Gi"),
},
},
MaxRecommendation: map[corev1.ResourceName]v1alpha1.ResourceQuantity{
corev1.ResourceCPU: {
Quantity: resource.MustParse("3"),
},
corev1.ResourceMemory: {
Quantity: resource.MustParse("3Gi"),
},
},
}).
Build(),
hpa: wantHPA,
}
tc.want = want
Eventually(func(g Gomega) {
gotTortoise := &v1alpha1.Tortoise{}
err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: "mercari"}, gotTortoise)
g.Expect(err).ShouldNot(HaveOccurred())
gotHPA := &v2.HorizontalPodAutoscaler{}
err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: "hpa"}, gotHPA)
g.Expect(err).ShouldNot(HaveOccurred())
gotUpdaterVPA := &autoscalingv1.VerticalPodAutoscaler{}
err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: "tortoise-updater-mercari"}, gotUpdaterVPA)
g.Expect(err).ShouldNot(HaveOccurred())
gotMonitorVPA := &autoscalingv1.VerticalPodAutoscaler{}
err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: "tortoise-monitor-mercari"}, gotMonitorVPA)
g.Expect(err).ShouldNot(HaveOccurred())

err = tc.compare(resources{tortoise: gotTortoise, hpa: gotHPA})
g.Expect(err).ShouldNot(HaveOccurred())
}).Should(Succeed())
})
})
})
Expand Down Expand Up @@ -451,3 +684,15 @@ func deploymentWithReplicaNum(replica int32) *v1.Deployment {
func resourceQuantityPtr(quantity resource.Quantity) *resource.Quantity {
return &quantity
}

func deleteObj(ctx context.Context, deleteObj client.Object, name string) error {
err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: name}, deleteObj)
if err != nil {
return err
}
err = k8sClient.Delete(ctx, deleteObj)
if err != nil {
return err
}
return nil
}

0 comments on commit 57e5a79

Please sign in to comment.