Add kuttl test for failed service

Signed-off-by: Fabricio Aguiar <[email protected]>
fao89 · May 31, 2024 · 16a86a7 · 16a86a7
1 parent 96c565b
commit 16a86a7
Show file tree

Hide file tree

Showing 6 changed files with 338 additions and 60 deletions.
diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go
@@ -99,11 +99,11 @@ const (
 	NodeSetDeploymentErrorMessage = "Deployment error occurred %s for NodeSet"
 
 	// NodeSetServiceDeploymentReadyMessage ready
-	NodeSetServiceDeploymentReadyMessage = "%s Deployment ready"
+	NodeSetServiceDeploymentReadyMessage = "Deployment ready for %s service"
 
 	// NodeSetServiceDeploymentReadyWaitingMessage not yet ready
-	NodeSetServiceDeploymentReadyWaitingMessage = "%s Deployment not yet ready"
+	NodeSetServiceDeploymentReadyWaitingMessage = "Deployment not yet ready for %s service"
 
 	// NodeSetServiceDeploymentErrorMessage error
-	NodeSetServiceDeploymentErrorMessage = "%s Deployment error occurred"
+	NodeSetServiceDeploymentErrorMessage = "Deployment error occurred in %s service"
 )
diff --git a/controllers/openstackdataplanedeployment_controller.go b/controllers/openstackdataplanedeployment_controller.go
@@ -214,14 +214,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 						condition.SeverityError,
 						dataplanev1.ServiceErrorMessage,
 						err.Error())
-					if len(instance.Spec.ServicesOverride) == 0 {
-						nsConditions.MarkFalse(
-							dataplanev1.NodeSetDeploymentReadyCondition,
-							condition.ErrorReason,
-							condition.SeverityError,
-							dataplanev1.ServiceErrorMessage,
-							err.Error())
-					}
+					nsConditions.MarkFalse(
+						dataplanev1.NodeSetDeploymentReadyCondition,
+						condition.ErrorReason,
+						condition.SeverityError,
+						dataplanev1.ServiceErrorMessage,
+						err.Error())
 					return ctrl.Result{}, err
 				}
 				if service.Spec.TLSCert != nil {
@@ -234,14 +232,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 							condition.SeverityError,
 							condition.TLSInputErrorMessage,
 							err.Error())
-						if len(instance.Spec.ServicesOverride) == 0 {
-							nsConditions.MarkFalse(
-								dataplanev1.NodeSetDeploymentReadyCondition,
-								condition.ErrorReason,
-								condition.SeverityError,
-								condition.TLSInputErrorMessage,
-								err.Error())
-						}
+						nsConditions.MarkFalse(
+							dataplanev1.NodeSetDeploymentReadyCondition,
+							condition.ErrorReason,
+							condition.SeverityError,
+							condition.TLSInputErrorMessage,
+							err.Error())
 						return ctrl.Result{}, err
 					} else if (*result != ctrl.Result{}) {
 						return *result, nil // requeue here
@@ -253,10 +249,11 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 
 	// All nodeSets successfully fetched.
 	// Mark InputReadyCondition=True
-	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.ReadyMessage)
+	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
 	shouldRequeue := false
 	haveError := false
 	deploymentErrMsg := ""
+	backoffLimitReached := false
 
 	globalInventorySecrets := map[string]string{}
 	globalSSHKeySecrets := map[string]string{}
@@ -347,12 +344,9 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 			} else {
 				deploymentErrMsg = fmt.Sprintf("%s & %s", deploymentErrMsg, errMsg)
 			}
-			nsConditions.MarkFalse(
-				dataplanev1.NodeSetDeploymentReadyCondition,
-				condition.ErrorReason,
-				condition.SeverityError,
-				condition.DeploymentReadyErrorMessage,
-				err.Error())
+			nsConditions.Set(nsConditions.Mirror(dataplanev1.NodeSetDeploymentReadyCondition))
+			errorReason := nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition).Reason
+			backoffLimitReached = errorReason == condition.JobReasonBackoffLimitExceeded
 		}
 
 		if deployResult != nil {
@@ -367,10 +361,17 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 	}
 
 	if haveError {
+		var reason condition.Reason
+		reason = condition.ErrorReason
+		severity := condition.SeverityWarning
+		if backoffLimitReached {
+			reason = condition.JobReasonBackoffLimitExceeded
+			severity = condition.SeverityError
+		}
 		instance.Status.Conditions.MarkFalse(
 			condition.DeploymentReadyCondition,
-			condition.ErrorReason,
-			condition.SeverityError,
+			reason,
+			severity,
 			condition.DeploymentReadyErrorMessage,
 			deploymentErrMsg)
 		return ctrl.Result{}, fmt.Errorf(deploymentErrMsg)

diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go
@@ -70,16 +70,26 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) {
 	copy(aeeSpecMounts, d.AeeSpec.ExtraMounts)
 	// Deploy the composable services
 	for _, service := range services {
+		deployName = service
+		readyCondition = condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(service)))
+		readyWaitingMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, deployName)
+		readyMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyMessage, deployName)
+		readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, deployName) + " error %s"
+
+		nsConditions := d.Status.NodeSetConditions[d.NodeSet.Name]
 		log.Info("Deploying service", "service", service)
 		foundService, err := GetService(d.Ctx, d.Helper, service)
 		if err != nil {
+			nsConditions.Set(condition.FalseCondition(
+				readyCondition,
+				condition.ErrorReason,
+				condition.SeverityError,
+				readyErrorMessage,
+				err.Error()))
+			d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions
 			return &ctrl.Result{}, err
 		}
-		deployName = foundService.Name
-		readyCondition = condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(service)))
-		readyWaitingMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, deployName)
-		readyMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyMessage, deployName)
-		readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, deployName)
+
 		containerImages := dataplaneutil.GetContainerImages(d.Version)
 		if containerImages.AnsibleeeImage != nil {
 			d.AeeSpec.OpenStackAnsibleEERunnerImage = *containerImages.AnsibleeeImage
@@ -94,6 +104,13 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) {
 		copy(d.AeeSpec.ExtraMounts, aeeSpecMounts)
 		d.AeeSpec, err = d.addServiceExtraMounts(foundService)
 		if err != nil {
+			nsConditions.Set(condition.FalseCondition(
+				readyCondition,
+				condition.ErrorReason,
+				condition.SeverityError,
+				readyErrorMessage,
+				err.Error()))
+			d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions
 			return &ctrl.Result{}, err
 		}
 
@@ -103,6 +120,13 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) {
 				d.AeeSpec, err = d.addCertMounts(services)
 			}
 			if err != nil {
+				nsConditions.Set(condition.FalseCondition(
+					readyCondition,
+					condition.ErrorReason,
+					condition.SeverityError,
+					readyErrorMessage,
+					err.Error()))
+				d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions
 				return &ctrl.Result{}, err
 			}
 		}
@@ -116,7 +140,7 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) {
 			foundService,
 		)
 
-		nsConditions := d.Status.NodeSetConditions[d.NodeSet.Name]
+		nsConditions = d.Status.NodeSetConditions[d.NodeSet.Name]
 		if err != nil || !nsConditions.IsTrue(readyCondition) {
 			log.Info(fmt.Sprintf("Condition %s not ready", readyCondition))
 			return &ctrl.Result{}, err

diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml
@@ -29,12 +29,16 @@ spec:
 status:
   observedGeneration: 1
   conditions:
-  - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found
+  - message: Deployment error occurred in this-service-does-not-exist service error
+      OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist"
+      not found
     reason: Error
     severity: Error
     status: "False"
     type: Ready
-  - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found
+  - message: Deployment error occurred in this-service-does-not-exist service error
+      OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist"
+      not found
     reason: Error
     severity: Error
     status: "False"
@@ -65,79 +69,87 @@ status:
       reason: Ready
       status: "True"
       type: NodeSetDeploymentReady
-    - message: bootstrap Deployment ready
+    - message: Deployment ready for bootstrap service
       reason: Ready
       status: "True"
       type: ServiceBootstrapDeploymentReady
-    - message: configure-network Deployment ready
+    - message: Deployment ready for configure-network service
       reason: Ready
       status: "True"
       type: ServiceConfigureNetworkDeploymentReady
-    - message: configure-os Deployment ready
+    - message: Deployment ready for configure-os service
       reason: Ready
       status: "True"
       type: ServiceConfigureOsDeploymentReady
-    - message: download-cache Deployment ready
+    - message: Deployment ready for download-cache service
       reason: Ready
       status: "True"
       type: ServiceDownloadCacheDeploymentReady
-    - message: install-certs Deployment ready
+    - message: Deployment ready for install-certs service
       reason: Ready
       status: "True"
       type: ServiceInstallCertsDeploymentReady
-    - message: install-os Deployment ready
+    - message: Deployment ready for install-os service
       reason: Ready
       status: "True"
       type: ServiceInstallOsDeploymentReady
-    - message: libvirt Deployment ready
+    - message: Deployment ready for libvirt service
       reason: Ready
       status: "True"
       type: ServiceLibvirtDeploymentReady
-    - message: neutron-dhcp Deployment ready
+    - message: Deployment ready for neutron-dhcp service
       reason: Ready
       status: "True"
       type: ServiceNeutronDhcpDeploymentReady
-    - message: neutron-metadata Deployment ready
+    - message: Deployment ready for neutron-metadata service
       reason: Ready
       status: "True"
       type: ServiceNeutronMetadataDeploymentReady
-    - message: neutron-ovn Deployment ready
+    - message: Deployment ready for neutron-ovn service
       reason: Ready
       status: "True"
       type: ServiceNeutronOvnDeploymentReady
-    - message: neutron-sriov Deployment ready
+    - message: Deployment ready for neutron-sriov service
       reason: Ready
       status: "True"
       type: ServiceNeutronSriovDeploymentReady
-    - message: nova Deployment ready
+    - message: Deployment ready for nova service
       reason: Ready
       status: "True"
       type: ServiceNovaDeploymentReady
-    - message: ovn Deployment ready
+    - message: Deployment ready for ovn service
       reason: Ready
       status: "True"
       type: ServiceOvnDeploymentReady
-    - message: run-os Deployment ready
+    - message: Deployment ready for run-os service
       reason: Ready
       status: "True"
       type: ServiceRunOsDeploymentReady
-    - message: validate-network Deployment ready
+    - message: Deployment ready for validate-network service
       reason: Ready
       status: "True"
       type: ServiceValidateNetworkDeploymentReady
     edpm-compute-no-nodes-non-existent-service:
-    - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org
-         "this-service-does-not-exist" not found
+    - message: Deployment error occurred in this-service-does-not-exist service error
+        OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist"
+        not found
       reason: Error
       severity: Error
       status: "False"
       type: NodeSetDeploymentReady
+    - message: Deployment error occurred in this-service-does-not-exist service error
+        OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist"
+        not found
+      reason: Error
+      severity: Error
+      status: "False"
+      type: ServiceThisServiceDoesNotExistDeploymentReady
     edpm-compute-no-nodes-ovrd:
     - message: Deployment completed
       reason: Ready
       status: "True"
       type: NodeSetDeploymentReady
-    - message: custom-svc Deployment ready
+    - message: Deployment ready for custom-svc service
       reason: Ready
       status: "True"
       type: ServiceCustomSvcDeploymentReady
@@ -146,7 +158,7 @@ status:
       reason: Ready
       status: "True"
       type: NodeSetDeploymentReady
-    - message: ovn Deployment ready
+    - message: Deployment ready for ovn service
       reason: Ready
       status: "True"
       type: ServiceOvnDeploymentReady
@@ -167,24 +179,32 @@ status:
   - message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: OpenStackDataPlaneService.dataplane.openstack.org
          "this-service-does-not-exist" not found'
     reason: Error
-    severity: Error
+    severity: Warning
     status: "False"
     type: Ready
   - message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: OpenStackDataPlaneService.dataplane.openstack.org
          "this-service-does-not-exist" not found'
     reason: Error
-    severity: Error
+    severity: Warning
     status: "False"
     type: DeploymentReady
-  - message: Setup complete
+  - message: Input data complete
     reason: Ready
     status: "True"
     type: InputReady
   nodeSetConditions:
     edpm-compute-no-nodes:
-    - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org
-        "this-service-does-not-exist" not found
+    - message: Deployment error occurred in this-service-does-not-exist service error
+        OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist"
+        not found
       reason: Error
       severity: Error
       status: "False"
       type: NodeSetDeploymentReady
+    - message: Deployment error occurred in this-service-does-not-exist service error
+        OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist"
+        not found
+      reason: Error
+      severity: Error
+      status: "False"
+      type: ServiceThisServiceDoesNotExistDeploymentReady