Report when backoff limit is exceeded

Depends-On: openstack-k8s-operators/lib-common#504 Signed-off-by: Fabricio Aguiar <[email protected]>
openstack-k8s-operators · May 7, 2024 · 6db26ab · 6db26ab
1 parent d464430
commit 6db26ab
Show file tree

Hide file tree

Showing 8 changed files with 61 additions and 86 deletions.
diff --git a/api/bases/ansibleee.openstack.org_openstackansibleees.yaml b/api/bases/ansibleee.openstack.org_openstackansibleees.yaml
@@ -1575,6 +1575,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                format: int64
+                type: integer
               hash:
                 additionalProperties:
                   type: string

diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go
diff --git a/api/v1beta1/openstack_ansibleee_types.go b/api/v1beta1/openstack_ansibleee_types.go
@@ -135,6 +135,10 @@ type OpenStackAnsibleEEStatus struct {
 	// JobStatus status of the executed job (Pending/Running/Succeeded/Failed)
 	JobStatus string `json:"JobStatus,omitempty" optional:"true"`
 
+	// The number of pods which reached phase Failed.
+	// +optional
+	Failed int64 `json:"failed,omitempty"`
+
 	// ObservedGeneration - the most recent generation observed for this
 	// service. If the observed generation is less than the spec generation,
 	// then the controller has not processed the latest changes injected by
@@ -182,7 +186,7 @@ func init() {
 
 // IsReady - returns true if the OpenStackAnsibleEE is ready
 func (instance OpenStackAnsibleEE) IsReady() bool {
-	return instance.Status.Conditions.IsTrue(AnsibleExecutionJobReadyCondition)
+	return instance.Status.Conditions.IsTrue(condition.JobReadyCondition)
 }
 
 // SetupDefaults - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks)

diff --git a/config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml b/config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml
@@ -1575,6 +1575,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                format: int64
+                type: integer
               hash:
                 additionalProperties:
                   type: string

diff --git a/controllers/openstack_ansibleee_controller.go b/controllers/openstack_ansibleee_controller.go
@@ -134,7 +134,7 @@ func (r *OpenStackAnsibleEEReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 	cl := condition.CreateList(
 		condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage),
-		condition.UnknownCondition(ansibleeev1.AnsibleExecutionJobReadyCondition, condition.InitReason, ansibleeev1.AnsibleExecutionJobInitMessage),
+		condition.UnknownCondition(condition.JobReadyCondition, condition.InitReason, condition.JobReadyInitMessage),
 	)
 
 	instance.Status.Conditions.Init(&cl)
@@ -163,7 +163,7 @@ func (r *OpenStackAnsibleEEReconciler) Reconcile(ctx context.Context, req ctrl.R
 			instance.Status.Conditions.Set(condition.FalseCondition(
 				condition.NetworkAttachmentsReadyCondition,
 				condition.ErrorReason,
-				condition.SeverityWarning,
+				condition.SeverityError,
 				condition.NetworkAttachmentsReadyErrorMessage,
 				err.Error()))
 			return ctrl.Result{}, err
@@ -208,22 +208,29 @@ func (r *OpenStackAnsibleEEReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 	if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
-			ansibleeev1.AnsibleExecutionJobReadyCondition,
+			condition.JobReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
-			ansibleeev1.AnsibleExecutionJobWaitingMessage))
+			condition.JobReadyRunningMessage))
 		instance.Status.JobStatus = ansibleeev1.JobStatusRunning
 		return ctrlResult, nil
 	}
 
 	if err != nil {
+		errorReason := condition.ErrorReason
+		severity := condition.SeverityWarning
+		if ansibleeeJob.HasReachedLimit() {
+			errorReason = condition.JobReasonBackoffLimitExceeded
+			severity = condition.SeverityError
+		}
 		instance.Status.Conditions.Set(condition.FalseCondition(
-			ansibleeev1.AnsibleExecutionJobReadyCondition,
-			condition.ErrorReason,
-			condition.SeverityWarning,
-			ansibleeev1.AnsibleExecutionJobErrorMessage,
+			condition.JobReadyCondition,
+			errorReason,
+			severity,
+			condition.JobReadyErrorMessage,
 			err.Error()))
 		instance.Status.JobStatus = ansibleeev1.JobStatusFailed
+		instance.Status.Failed = ansibleeeJob.GetTotalFailedAttempts()
 		return ctrl.Result{}, err
 	}
 
@@ -232,7 +239,7 @@ func (r *OpenStackAnsibleEEReconciler) Reconcile(ctx context.Context, req ctrl.R
 		Log.Info(fmt.Sprintf("AnsibleEE CR '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[ansibleeeJobType]))
 	}
 
-	instance.Status.Conditions.MarkTrue(ansibleeev1.AnsibleExecutionJobReadyCondition, ansibleeev1.AnsibleExecutionJobReadyMessage)
+	instance.Status.Conditions.MarkTrue(condition.JobReadyCondition, condition.JobReadyMessage)
 	instance.Status.JobStatus = ansibleeev1.JobStatusSucceeded
 
 	// We reached the end of the Reconcile, update the Ready condition based on

diff --git a/docs/assemblies/openstack_ansibleee.adoc b/docs/assemblies/openstack_ansibleee.adoc
@@ -213,6 +213,11 @@ OpenStackAnsibleEEStatus defines the observed state of OpenStackAnsibleEE
 | string
 | false
 
+| failed
+| The number of pods which reached phase Failed.
+| int64
+| false
+
 | observedGeneration
 | ObservedGeneration - the most recent generation observed for this service. If the observed generation is less than the spec generation, then the controller has not processed the latest changes injected by the opentack-operator in the top-level CR (e.g. the ContainerImage)
 | int64

diff --git a/tests/functional/ansibleee_controller_test.go b/tests/functional/ansibleee_controller_test.go
@@ -27,7 +27,6 @@ import (
 
 	//revive:disable-next-line:dot-imports
 	. "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers"
-	"github.com/openstack-k8s-operators/openstack-ansibleee-operator/api/v1beta1"
 )
 
 var _ = Describe("Ansibleee controller", func() {
@@ -40,7 +39,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -61,7 +60,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectCondition(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionTrue,
 				)
 				th.ExpectCondition(
@@ -80,7 +79,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -102,7 +101,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.ErrorReason,
 					"AnsibleExecutionJob error occured Internal error occurred: Job Failed. Check job logs",
@@ -123,7 +122,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -140,7 +139,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectCondition(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionTrue,
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
@@ -163,7 +162,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -177,7 +176,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectCondition(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionTrue,
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
@@ -197,7 +196,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -218,7 +217,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectCondition(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionTrue,
 				)
 				th.ExpectCondition(
@@ -237,7 +236,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -259,7 +258,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.ErrorReason,
 					"AnsibleExecutionJob error occured Internal error occurred: Job Failed. Check job logs",
@@ -328,7 +327,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -349,7 +348,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectCondition(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionTrue,
 				)
 				th.ExpectCondition(
@@ -368,7 +367,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -390,7 +389,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.ErrorReason,
 					"AnsibleExecutionJob error occured Internal error occurred: Job Failed. Check job logs",
@@ -476,7 +475,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -497,7 +496,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectCondition(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionTrue,
 				)
 				th.ExpectCondition(
@@ -516,7 +515,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",
@@ -538,7 +537,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.ErrorReason,
 					"AnsibleExecutionJob error occured Internal error occurred: Job Failed. Check job logs",
@@ -569,7 +568,7 @@ var _ = Describe("Ansibleee controller", func() {
 				th.ExpectConditionWithDetails(
 					ansibleeeName,
 					ConditionGetterFunc(AnsibleeeConditionGetter),
-					v1beta1.AnsibleExecutionJobReadyCondition,
+					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.RequestedReason,
 					"AnsibleExecutionJob is running",

diff --git a/tests/kuttl/tests/run_failed_playbook/01-assert.yaml b/tests/kuttl/tests/run_failed_playbook/01-assert.yaml
@@ -23,19 +23,20 @@ spec:
   preserveJobs: true
 status:
   JobStatus: Failed
+  failed: 4
   conditions:
-  - message: 'AnsibleExecutionJob error occured Internal error occurred: Job Failed.
+  - message: 'Job error occured Internal error occurred: Job has reached the specified backoff limit.
       Check job logs'
-    reason: Error
-    severity: Warning
+    reason: BackoffLimitExceeded
+    severity: Error
     status: "False"
     type: Ready
-  - message: 'AnsibleExecutionJob error occured Internal error occurred: Job Failed.
+  - message: 'Job error occured Internal error occurred: Job has reached the specified backoff limit.
       Check job logs'
-    reason: Error
-    severity: Warning
+    reason: BackoffLimitExceeded
+    severity: Error
     status: "False"
-    type: AnsibleExecutionJobReady
+    type: JobReady
 ---
 apiVersion: v1
 kind: Pod