Report when backoff limit is exceeded

Depends-On: openstack-k8s-operators/lib-common#504 Signed-off-by: Fabricio Aguiar <[email protected]>
openstack-k8s-operators · May 7, 2024 · 1dfa222 · 1dfa222
1 parent 3bb0a8b
commit 1dfa222
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 8 deletions.
diff --git a/api/bases/ansibleee.openstack.org_openstackansibleees.yaml b/api/bases/ansibleee.openstack.org_openstackansibleees.yaml
@@ -1575,6 +1575,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                format: int64
+                type: integer
               hash:
                 additionalProperties:
                   type: string

diff --git a/api/v1beta1/openstack_ansibleee_types.go b/api/v1beta1/openstack_ansibleee_types.go
@@ -135,6 +135,10 @@ type OpenStackAnsibleEEStatus struct {
 	// JobStatus status of the executed job (Pending/Running/Succeeded/Failed)
 	JobStatus string `json:"JobStatus,omitempty" optional:"true"`
 
+	// The number of pods which reached phase Failed.
+	// +optional
+	Failed int32 `json:"failed,omitempty"`
+
 	// ObservedGeneration - the most recent generation observed for this
 	// service. If the observed generation is less than the spec generation,
 	// then the controller has not processed the latest changes injected by

diff --git a/config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml b/config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml
@@ -1575,6 +1575,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                format: int64
+                type: integer
               hash:
                 additionalProperties:
                   type: string

diff --git a/controllers/openstack_ansibleee_controller.go b/controllers/openstack_ansibleee_controller.go
@@ -217,13 +217,20 @@ func (r *OpenStackAnsibleEEReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 
 	if err != nil {
+		errorReason := condition.ErrorReason
+		severity := condition.SeverityWarning
+		if ansibleeeJob.HasReachedLimit() {
+			errorReason = condition.JobReasonBackoffLimitExceeded
+			severity = condition.SeverityError
+		}
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.JobReadyCondition,
-			condition.ErrorReason,
-			condition.SeverityWarning,
+			errorReason,
+			severity,
 			condition.JobReadyErrorMessage,
 			err.Error()))
 		instance.Status.JobStatus = ansibleeev1.JobStatusFailed
+		instance.Status.Failed = ansibleeeJob.GetTotalFailedAttempts()
 		return ctrl.Result{}, err
 	}
 

diff --git a/docs/assemblies/openstack_ansibleee.adoc b/docs/assemblies/openstack_ansibleee.adoc
@@ -213,6 +213,11 @@ OpenStackAnsibleEEStatus defines the observed state of OpenStackAnsibleEE
 | string
 | false
 
+| failed
+| The number of pods which reached phase Failed.
+| int64
+| false
+
 | observedGeneration
 | ObservedGeneration - the most recent generation observed for this service. If the observed generation is less than the spec generation, then the controller has not processed the latest changes injected by the opentack-operator in the top-level CR (e.g. the ContainerImage)
 | int64

diff --git a/tests/kuttl/tests/run_failed_playbook/01-assert.yaml b/tests/kuttl/tests/run_failed_playbook/01-assert.yaml
@@ -23,17 +23,18 @@ spec:
   preserveJobs: true
 status:
   JobStatus: Failed
+  failed: 4
   conditions:
-  - message: 'Job error occured Internal error occurred: Job Failed.
+  - message: 'Job error occured Internal error occurred: Job has reached the specified backoff limit.
       Check job logs'
-    reason: Error
-    severity: Warning
+    reason: BackoffLimitExceeded
+    severity: Error
     status: "False"
     type: Ready
-  - message: 'Job error occured Internal error occurred: Job Failed.
+  - message: 'Job error occured Internal error occurred: Job has reached the specified backoff limit.
       Check job logs'
-    reason: Error
-    severity: Warning
+    reason: BackoffLimitExceeded
+    severity: Error
     status: "False"
     type: JobReady
 ---