From c79cdec58e0634889484556c60b7ef5ead0c8c99 Mon Sep 17 00:00:00 2001 From: karampok Date: Thu, 19 Sep 2024 10:13:37 +0200 Subject: [PATCH] E2E: make sure frr-k8s-webhook pod is Ready Frr-k8s-webhook is crashes at least 3 times and then it becomes ready, but sometimes it remains crashing. Current check is checking if pod is Running but this is not enough ``` containerStatuses: - containerID: containerd://fe46fd402ccb0ea512037e4fbc3c12c90de7c7e40ed4adb0ee5c51681f89b8d7 image: quay.io/metallb/frr-k8s:v0.0.14 imageID: quay.io/metallb/frr-k8s@sha256:00f8c40129fb1403760d2e846fc970dc11ca8d19068f012a26f2b683f98cb598 lastState: terminated: containerID: containerd://22f5f5ccbe3832019b15460f5a6947ce22bba838bc45cbaa6c60b0c0f94b4aaf exitCode: 0 finishedAt: "2024-09-19T08:11:19Z" reason: Completed startedAt: "2024-09-19T08:11:18Z" name: frr-k8s-webhook-server ready: false restartCount: 1 started: false state: terminated: containerID: containerd://fe46fd402ccb0ea512037e4fbc3c12c90de7c7e40ed4adb0ee5c51681f89b8d7 exitCode: 1 finishedAt: "2024-09-19T08:11:21Z" reason: Error startedAt: "2024-09-19T08:11:20Z" hostIP: 172.18.0.3 hostIPs: - ip: 172.18.0.3 phase: Running ``` ``` k -n metallb-system get pods -l component=frr-k8s-webhook-server -o wide -w NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Pending 0 0s frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Pending 0 0s kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 ContainerCreating 0 0s kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Running 0 0s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Completed 0 1s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Running 1 (2s ago) 3s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Error 1 (4s ago) 5s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 CrashLoopBackOff 1 (7s ago) 10s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Running 2 (23s ago) 26s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Completed 2 (24s ago) 27s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 CrashLoopBackOff 2 (2s ago) 28s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 0/1 Running 3 (35s ago) 61s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 1/1 Running 3 (44s ago) 70s 10.244.2.11 kind-worker frr-k8s-webhook-server-6ffd7bc857-glnvr 1/1 Terminating 3 (76s ago) 102s 10.244.2.11 kind-worker ``` Signed-off-by: karampok --- test/e2e/functional/tests/e2e.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/test/e2e/functional/tests/e2e.go b/test/e2e/functional/tests/e2e.go index ad22e5883..1f8339e8a 100644 --- a/test/e2e/functional/tests/e2e.go +++ b/test/e2e/functional/tests/e2e.go @@ -280,7 +280,7 @@ var _ = Describe("metallb", func() { }, metallbutils.DeployTimeout, metallbutils.Interval).ShouldNot(HaveOccurred()) By("checking frr-k8s webhook deployment is in running state") - Eventually(func() error { + c := func() error { deploy, err := testclient.Client.Deployments(metallb.Namespace).Get(context.Background(), consts.FRRK8SWebhookDeploymentName, metav1.GetOptions{}) if err != nil { return err @@ -297,18 +297,21 @@ var _ = Describe("metallb", func() { } for _, pod := range pods.Items { - if pod.Status.Phase != corev1.PodRunning { + if !PodIsReady(&pod) { return fmt.Errorf("deployment %s pod %s is not running, expected status %s got %s", consts.MetalLBOperatorDeploymentName, pod.Name, corev1.PodRunning, pod.Status.Phase) } } return nil - }, metallbutils.DeployTimeout, metallbutils.Interval).ShouldNot(HaveOccurred()) + } + Eventually(c, metallbutils.DeployTimeout, metallbutils.Interval).ShouldNot(HaveOccurred()) + By("Consistently be running") + Consistently(c, 30*time.Second, metallbutils.Interval).ShouldNot(HaveOccurred()) }, Entry("Native Mode", metallbv1beta1.NativeMode), Entry("FRR Mode", metallbv1beta1.FRRMode), - Entry("FRR-K8s Mode", metallbv1beta1.FRRK8sMode), + FEntry("FRR-K8s Mode", metallbv1beta1.FRRK8sMode), ) }) @@ -799,3 +802,22 @@ var _ = Describe("metallb", func() { // Gomega transformation functions for v1.Container func envGetter(c v1.Container) []v1.EnvVar { return c.Env } func nameGetter(c v1.Container) string { return c.Name } + +func PodIsReady(p *corev1.Pod) bool { + return podConditionStatus(p, corev1.PodReady) == corev1.ConditionTrue && podConditionStatus(p, corev1.ContainersReady) == corev1.ConditionTrue +} + +// podConditionStatus returns the status of the condition for a given pod. +func podConditionStatus(p *corev1.Pod, condition corev1.PodConditionType) corev1.ConditionStatus { + if p == nil { + return corev1.ConditionUnknown + } + + for _, c := range p.Status.Conditions { + if c.Type == condition { + return c.Status + } + } + + return corev1.ConditionUnknown +}