Report when backoff limit is exceeded

Depends-On: openstack-k8s-operators/lib-common#504 Signed-off-by: Fabricio Aguiar <[email protected]>
openstack-k8s-operators · May 23, 2024 · 3fce341 · 3fce341
1 parent c6e11d8
commit 3fce341
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 6 deletions.
diff --git a/api/bases/ansibleee.openstack.org_openstackansibleees.yaml b/api/bases/ansibleee.openstack.org_openstackansibleees.yaml
@@ -1575,6 +1575,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                format: int32
+                type: integer
               hash:
                 additionalProperties:
                   type: string

diff --git a/api/v1beta1/openstack_ansibleee_types.go b/api/v1beta1/openstack_ansibleee_types.go
@@ -135,6 +135,10 @@ type OpenStackAnsibleEEStatus struct {
 	// JobStatus status of the executed job (Pending/Running/Succeeded/Failed)
 	JobStatus string `json:"JobStatus,omitempty" optional:"true"`
 
+	// The number of pods which reached phase Failed.
+	// +optional
+	Failed int32 `json:"failed,omitempty"`
+
 	// ObservedGeneration - the most recent generation observed for this
 	// service. If the observed generation is less than the spec generation,
 	// then the controller has not processed the latest changes injected by

diff --git a/config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml b/config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml
@@ -1575,6 +1575,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                format: int32
+                type: integer
               hash:
                 additionalProperties:
                   type: string

diff --git a/controllers/openstack_ansibleee_controller.go b/controllers/openstack_ansibleee_controller.go
@@ -217,13 +217,21 @@ func (r *OpenStackAnsibleEEReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 
 	if err != nil {
+		var errorReason condition.Reason
+		errorReason = condition.ErrorReason
+		severity := condition.SeverityWarning
+		if ansibleeeJob.HasReachedLimit() {
+			errorReason = condition.JobReasonBackoffLimitExceeded
+			severity = condition.SeverityError
+		}
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.JobReadyCondition,
-			condition.ErrorReason,
-			condition.SeverityWarning,
+			errorReason,
+			severity,
 			condition.JobReadyErrorMessage,
 			err.Error()))
 		instance.Status.JobStatus = ansibleeev1.JobStatusFailed
+		instance.Status.Failed = ansibleeeJob.GetTotalFailedAttempts()
 		return ctrl.Result{}, err
 	}
 

diff --git a/docs/assemblies/openstack_ansibleee.adoc b/docs/assemblies/openstack_ansibleee.adoc
@@ -213,6 +213,11 @@ OpenStackAnsibleEEStatus defines the observed state of OpenStackAnsibleEE
 | string
 | false
 
+| failed
+| The number of pods which reached phase Failed.
+| int32
+| false
+
 | observedGeneration
 | ObservedGeneration - the most recent generation observed for this service. If the observed generation is less than the spec generation, then the controller has not processed the latest changes injected by the opentack-operator in the top-level CR (e.g. the ContainerImage)
 | int64

diff --git a/tests/kuttl/tests/run_failed_playbook/01-assert.yaml b/tests/kuttl/tests/run_failed_playbook/01-assert.yaml
@@ -23,17 +23,18 @@ spec:
   preserveJobs: true
 status:
   JobStatus: Failed
+  failed: 4
   conditions:
   - message: 'Job error occurred Internal error occurred: Job has reached the specified backoff limit.
       Check job logs'
-    reason: Error
-    severity: Warning
+    reason: BackoffLimitExceeded
+    severity: Error
     status: "False"
     type: Ready
   - message: 'Job error occurred Internal error occurred: Job has reached the specified backoff limit.
       Check job logs'
-    reason: Error
-    severity: Warning
+    reason: BackoffLimitExceeded
+    severity: Error
     status: "False"
     type: JobReady
 ---