Report when backoff limit is exceeded

Depends-On: openstack-k8s-operators/lib-common#504

Signed-off-by: Fabricio Aguiar <[email protected]>

api/bases/ansibleee.openstack.org_openstackansibleees.yaml

@@ -1575,6 +1575,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                format: int32
+                type: integer
               hash:
                 additionalProperties:
                   type: string

api/go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/go-playground/validator/v10 v10.19.0
 	github.com/onsi/ginkgo/v2 v2.18.0
 	github.com/onsi/gomega v1.33.1
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240429052447-09a614506ca6
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240523061957-78c308214ae3
 	github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240429052447-09a614506ca6
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.9

api/go.sum

@@ -90,8 +90,8 @@ github.com/onsi/ginkgo/v2 v2.18.0 h1:W9Y7IWXxPUpAit9ieMOLI7PJZGaW22DTKgiVAuhDTLc
 github.com/onsi/ginkgo/v2 v2.18.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
 github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
 github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240429052447-09a614506ca6 h1:WLsG3Ko+phW5xZJjncypLWGASoLqKrt05qN9Zxsad6g=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240429052447-09a614506ca6/go.mod h1:lYhFzul37AR/6gAhTAA1KKWbOlzB3F/7014lejn883c=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240523061957-78c308214ae3 h1:0vrMw/RB4y8skUl/4u1h3mRNZ1iUXeh4ZcUjSQqnU8o=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240523061957-78c308214ae3/go.mod h1:lYhFzul37AR/6gAhTAA1KKWbOlzB3F/7014lejn883c=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240429052447-09a614506ca6 h1:NNSOEpTZCa9RL5sZiF4ZOlB+agBrL7q7FB9pC58d4S8=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240429052447-09a614506ca6/go.mod h1:C/qUWW4lW3687riZxYd+YRCtOyHZKURu3Imv6S9OP7U=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

api/v1beta1/openstack_ansibleee_types.go

@@ -135,6 +135,10 @@ type OpenStackAnsibleEEStatus struct {
 	// JobStatus status of the executed job (Pending/Running/Succeeded/Failed)
 	JobStatus string `json:"JobStatus,omitempty" optional:"true"`
 
+	// The number of pods which reached phase Failed.
+	// +optional
+	Failed int32 `json:"failed,omitempty"`
+
 	// ObservedGeneration - the most recent generation observed for this
 	// service. If the observed generation is less than the spec generation,
 	// then the controller has not processed the latest changes injected by

config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml

@@ -1575,6 +1575,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                format: int32
+                type: integer
               hash:
                 additionalProperties:
                   type: string

controllers/openstack_ansibleee_controller.go

@@ -217,13 +217,21 @@ func (r *OpenStackAnsibleEEReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 
 	if err != nil {
+		var errorReason condition.Reason
+		errorReason = condition.ErrorReason
+		severity := condition.SeverityWarning
+		if ansibleeeJob.HasReachedLimit() {
+			errorReason = condition.JobReasonBackoffLimitExceeded
+			severity = condition.SeverityError
+		}
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.JobReadyCondition,
-			condition.ErrorReason,
-			condition.SeverityWarning,
+			errorReason,
+			severity,
 			condition.JobReadyErrorMessage,
 			err.Error()))
 		instance.Status.JobStatus = ansibleeev1.JobStatusFailed
+		instance.Status.Failed = ansibleeeJob.GetTotalFailedAttempts()
 		return ctrl.Result{}, err
 	}
 

docs/assemblies/openstack_ansibleee.adoc

@@ -213,6 +213,11 @@ OpenStackAnsibleEEStatus defines the observed state of OpenStackAnsibleEE
 | string
 | false
 
+| failed
+| The number of pods which reached phase Failed.
+| int32
+| false
+
 | observedGeneration
 | ObservedGeneration - the most recent generation observed for this service. If the observed generation is less than the spec generation, then the controller has not processed the latest changes injected by the opentack-operator in the top-level CR (e.g. the ContainerImage)
 | int64

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
 	github.com/onsi/ginkgo/v2 v2.18.0
 	github.com/onsi/gomega v1.33.1
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240429052447-09a614506ca6
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240523061957-78c308214ae3
 	github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240429052447-09a614506ca6
 	github.com/openstack-k8s-operators/openstack-ansibleee-operator/api v0.0.0-00010101000000-000000000000
 	gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -94,8 +94,8 @@ github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
 github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
 github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs=
 github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240429052447-09a614506ca6 h1:WLsG3Ko+phW5xZJjncypLWGASoLqKrt05qN9Zxsad6g=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240429052447-09a614506ca6/go.mod h1:lYhFzul37AR/6gAhTAA1KKWbOlzB3F/7014lejn883c=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240523061957-78c308214ae3 h1:0vrMw/RB4y8skUl/4u1h3mRNZ1iUXeh4ZcUjSQqnU8o=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240523061957-78c308214ae3/go.mod h1:lYhFzul37AR/6gAhTAA1KKWbOlzB3F/7014lejn883c=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240429052447-09a614506ca6 h1:NNSOEpTZCa9RL5sZiF4ZOlB+agBrL7q7FB9pC58d4S8=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240429052447-09a614506ca6/go.mod h1:C/qUWW4lW3687riZxYd+YRCtOyHZKURu3Imv6S9OP7U=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

tests/functional/ansibleee_controller_test.go

@@ -104,7 +104,7 @@ var _ = Describe("Ansibleee controller", func() {
 					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.ErrorReason,
-					"Job error occurred Internal error occurred: Job Failed. Check job logs",
+					"Job error occurred Internal error occurred: Job Attempt #1 Failed. Check job logs",
 				)
 				th.ExpectCondition(
 					ansibleeeName,
@@ -261,7 +261,7 @@ var _ = Describe("Ansibleee controller", func() {
 					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.ErrorReason,
-					"Job error occurred Internal error occurred: Job Failed. Check job logs",
+					"Job error occurred Internal error occurred: Job Attempt #1 Failed. Check job logs",
 				)
 				th.ExpectCondition(
 					ansibleeeName,
@@ -392,7 +392,7 @@ var _ = Describe("Ansibleee controller", func() {
 					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.ErrorReason,
-					"Job error occurred Internal error occurred: Job Failed. Check job logs",
+					"Job error occurred Internal error occurred: Job Attempt #1 Failed. Check job logs",
 				)
 				th.ExpectCondition(
 					ansibleeeName,
@@ -540,7 +540,7 @@ var _ = Describe("Ansibleee controller", func() {
 					condition.JobReadyCondition,
 					corev1.ConditionFalse,
 					condition.ErrorReason,
-					"Job error occurred Internal error occurred: Job Failed. Check job logs",
+					"Job error occurred Internal error occurred: Job Attempt #1 Failed. Check job logs",
 				)
 				th.ExpectCondition(
 					ansibleeeName,

tests/kuttl/tests/run_failed_playbook/01-assert.yaml

@@ -23,17 +23,18 @@ spec:
   preserveJobs: true
 status:
   JobStatus: Failed
+  failed: 4
   conditions:
-  - message: 'Job error occurred Internal error occurred: Job Failed.
+  - message: 'Job error occurred Internal error occurred: Job has reached the specified backoff limit.
       Check job logs'
-    reason: Error
-    severity: Warning
+    reason: BackoffLimitExceeded
+    severity: Error
     status: "False"
     type: Ready
-  - message: 'Job error occurred Internal error occurred: Job Failed.
+  - message: 'Job error occurred Internal error occurred: Job has reached the specified backoff limit.
       Check job logs'
-    reason: Error
-    severity: Warning
+    reason: BackoffLimitExceeded
+    severity: Error
     status: "False"
     type: JobReady
 ---