From 16a86a73d07c83a8f4e8b191525fb72ac207be26 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Fri, 31 May 2024 22:54:42 +0100 Subject: [PATCH] Add kuttl test for failed service Signed-off-by: Fabricio Aguiar --- api/v1beta1/conditions.go | 6 +- ...openstackdataplanedeployment_controller.go | 51 ++--- pkg/deployment/deployment.go | 36 +++- .../05-assert.yaml | 72 ++++--- .../dataplane-service-failure/00-assert.yaml | 195 ++++++++++++++++++ .../dataplane-service-failure/00-create.yaml | 38 ++++ 6 files changed, 338 insertions(+), 60 deletions(-) create mode 100644 tests/kuttl/tests/dataplane-service-failure/00-assert.yaml create mode 100644 tests/kuttl/tests/dataplane-service-failure/00-create.yaml diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index d54b9f4bd..33380b1e3 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -99,11 +99,11 @@ const ( NodeSetDeploymentErrorMessage = "Deployment error occurred %s for NodeSet" // NodeSetServiceDeploymentReadyMessage ready - NodeSetServiceDeploymentReadyMessage = "%s Deployment ready" + NodeSetServiceDeploymentReadyMessage = "Deployment ready for %s service" // NodeSetServiceDeploymentReadyWaitingMessage not yet ready - NodeSetServiceDeploymentReadyWaitingMessage = "%s Deployment not yet ready" + NodeSetServiceDeploymentReadyWaitingMessage = "Deployment not yet ready for %s service" // NodeSetServiceDeploymentErrorMessage error - NodeSetServiceDeploymentErrorMessage = "%s Deployment error occurred" + NodeSetServiceDeploymentErrorMessage = "Deployment error occurred in %s service" ) diff --git a/controllers/openstackdataplanedeployment_controller.go b/controllers/openstackdataplanedeployment_controller.go index aed4e2160..1bbb446e3 100644 --- a/controllers/openstackdataplanedeployment_controller.go +++ b/controllers/openstackdataplanedeployment_controller.go @@ -214,14 +214,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, condition.SeverityError, dataplanev1.ServiceErrorMessage, err.Error()) - if len(instance.Spec.ServicesOverride) == 0 { - nsConditions.MarkFalse( - dataplanev1.NodeSetDeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityError, - dataplanev1.ServiceErrorMessage, - err.Error()) - } + nsConditions.MarkFalse( + dataplanev1.NodeSetDeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityError, + dataplanev1.ServiceErrorMessage, + err.Error()) return ctrl.Result{}, err } if service.Spec.TLSCert != nil { @@ -234,14 +232,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, condition.SeverityError, condition.TLSInputErrorMessage, err.Error()) - if len(instance.Spec.ServicesOverride) == 0 { - nsConditions.MarkFalse( - dataplanev1.NodeSetDeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityError, - condition.TLSInputErrorMessage, - err.Error()) - } + nsConditions.MarkFalse( + dataplanev1.NodeSetDeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityError, + condition.TLSInputErrorMessage, + err.Error()) return ctrl.Result{}, err } else if (*result != ctrl.Result{}) { return *result, nil // requeue here @@ -253,10 +249,11 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, // All nodeSets successfully fetched. // Mark InputReadyCondition=True - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.ReadyMessage) + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) shouldRequeue := false haveError := false deploymentErrMsg := "" + backoffLimitReached := false globalInventorySecrets := map[string]string{} globalSSHKeySecrets := map[string]string{} @@ -347,12 +344,9 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, } else { deploymentErrMsg = fmt.Sprintf("%s & %s", deploymentErrMsg, errMsg) } - nsConditions.MarkFalse( - dataplanev1.NodeSetDeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityError, - condition.DeploymentReadyErrorMessage, - err.Error()) + nsConditions.Set(nsConditions.Mirror(dataplanev1.NodeSetDeploymentReadyCondition)) + errorReason := nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition).Reason + backoffLimitReached = errorReason == condition.JobReasonBackoffLimitExceeded } if deployResult != nil { @@ -367,10 +361,17 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, } if haveError { + var reason condition.Reason + reason = condition.ErrorReason + severity := condition.SeverityWarning + if backoffLimitReached { + reason = condition.JobReasonBackoffLimitExceeded + severity = condition.SeverityError + } instance.Status.Conditions.MarkFalse( condition.DeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityError, + reason, + severity, condition.DeploymentReadyErrorMessage, deploymentErrMsg) return ctrl.Result{}, fmt.Errorf(deploymentErrMsg) diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 2398b7292..6ab8e88c4 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -70,16 +70,26 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) { copy(aeeSpecMounts, d.AeeSpec.ExtraMounts) // Deploy the composable services for _, service := range services { + deployName = service + readyCondition = condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(service))) + readyWaitingMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, deployName) + readyMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyMessage, deployName) + readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, deployName) + " error %s" + + nsConditions := d.Status.NodeSetConditions[d.NodeSet.Name] log.Info("Deploying service", "service", service) foundService, err := GetService(d.Ctx, d.Helper, service) if err != nil { + nsConditions.Set(condition.FalseCondition( + readyCondition, + condition.ErrorReason, + condition.SeverityError, + readyErrorMessage, + err.Error())) + d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions return &ctrl.Result{}, err } - deployName = foundService.Name - readyCondition = condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(service))) - readyWaitingMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, deployName) - readyMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyMessage, deployName) - readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, deployName) + containerImages := dataplaneutil.GetContainerImages(d.Version) if containerImages.AnsibleeeImage != nil { d.AeeSpec.OpenStackAnsibleEERunnerImage = *containerImages.AnsibleeeImage @@ -94,6 +104,13 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) { copy(d.AeeSpec.ExtraMounts, aeeSpecMounts) d.AeeSpec, err = d.addServiceExtraMounts(foundService) if err != nil { + nsConditions.Set(condition.FalseCondition( + readyCondition, + condition.ErrorReason, + condition.SeverityError, + readyErrorMessage, + err.Error())) + d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions return &ctrl.Result{}, err } @@ -103,6 +120,13 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) { d.AeeSpec, err = d.addCertMounts(services) } if err != nil { + nsConditions.Set(condition.FalseCondition( + readyCondition, + condition.ErrorReason, + condition.SeverityError, + readyErrorMessage, + err.Error())) + d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions return &ctrl.Result{}, err } } @@ -116,7 +140,7 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) { foundService, ) - nsConditions := d.Status.NodeSetConditions[d.NodeSet.Name] + nsConditions = d.Status.NodeSetConditions[d.NodeSet.Name] if err != nil || !nsConditions.IsTrue(readyCondition) { log.Info(fmt.Sprintf("Condition %s not ready", readyCondition)) return &ctrl.Result{}, err diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml index 7b299c591..f0f625350 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml @@ -29,12 +29,16 @@ spec: status: observedGeneration: 1 conditions: - - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found + - message: Deployment error occurred in this-service-does-not-exist service error + OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" + not found reason: Error severity: Error status: "False" type: Ready - - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found + - message: Deployment error occurred in this-service-does-not-exist service error + OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" + not found reason: Error severity: Error status: "False" @@ -65,79 +69,87 @@ status: reason: Ready status: "True" type: NodeSetDeploymentReady - - message: bootstrap Deployment ready + - message: Deployment ready for bootstrap service reason: Ready status: "True" type: ServiceBootstrapDeploymentReady - - message: configure-network Deployment ready + - message: Deployment ready for configure-network service reason: Ready status: "True" type: ServiceConfigureNetworkDeploymentReady - - message: configure-os Deployment ready + - message: Deployment ready for configure-os service reason: Ready status: "True" type: ServiceConfigureOsDeploymentReady - - message: download-cache Deployment ready + - message: Deployment ready for download-cache service reason: Ready status: "True" type: ServiceDownloadCacheDeploymentReady - - message: install-certs Deployment ready + - message: Deployment ready for install-certs service reason: Ready status: "True" type: ServiceInstallCertsDeploymentReady - - message: install-os Deployment ready + - message: Deployment ready for install-os service reason: Ready status: "True" type: ServiceInstallOsDeploymentReady - - message: libvirt Deployment ready + - message: Deployment ready for libvirt service reason: Ready status: "True" type: ServiceLibvirtDeploymentReady - - message: neutron-dhcp Deployment ready + - message: Deployment ready for neutron-dhcp service reason: Ready status: "True" type: ServiceNeutronDhcpDeploymentReady - - message: neutron-metadata Deployment ready + - message: Deployment ready for neutron-metadata service reason: Ready status: "True" type: ServiceNeutronMetadataDeploymentReady - - message: neutron-ovn Deployment ready + - message: Deployment ready for neutron-ovn service reason: Ready status: "True" type: ServiceNeutronOvnDeploymentReady - - message: neutron-sriov Deployment ready + - message: Deployment ready for neutron-sriov service reason: Ready status: "True" type: ServiceNeutronSriovDeploymentReady - - message: nova Deployment ready + - message: Deployment ready for nova service reason: Ready status: "True" type: ServiceNovaDeploymentReady - - message: ovn Deployment ready + - message: Deployment ready for ovn service reason: Ready status: "True" type: ServiceOvnDeploymentReady - - message: run-os Deployment ready + - message: Deployment ready for run-os service reason: Ready status: "True" type: ServiceRunOsDeploymentReady - - message: validate-network Deployment ready + - message: Deployment ready for validate-network service reason: Ready status: "True" type: ServiceValidateNetworkDeploymentReady edpm-compute-no-nodes-non-existent-service: - - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org - "this-service-does-not-exist" not found + - message: Deployment error occurred in this-service-does-not-exist service error + OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" + not found reason: Error severity: Error status: "False" type: NodeSetDeploymentReady + - message: Deployment error occurred in this-service-does-not-exist service error + OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" + not found + reason: Error + severity: Error + status: "False" + type: ServiceThisServiceDoesNotExistDeploymentReady edpm-compute-no-nodes-ovrd: - message: Deployment completed reason: Ready status: "True" type: NodeSetDeploymentReady - - message: custom-svc Deployment ready + - message: Deployment ready for custom-svc service reason: Ready status: "True" type: ServiceCustomSvcDeploymentReady @@ -146,7 +158,7 @@ status: reason: Ready status: "True" type: NodeSetDeploymentReady - - message: ovn Deployment ready + - message: Deployment ready for ovn service reason: Ready status: "True" type: ServiceOvnDeploymentReady @@ -167,24 +179,32 @@ status: - message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found' reason: Error - severity: Error + severity: Warning status: "False" type: Ready - message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found' reason: Error - severity: Error + severity: Warning status: "False" type: DeploymentReady - - message: Setup complete + - message: Input data complete reason: Ready status: "True" type: InputReady nodeSetConditions: edpm-compute-no-nodes: - - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org - "this-service-does-not-exist" not found + - message: Deployment error occurred in this-service-does-not-exist service error + OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" + not found reason: Error severity: Error status: "False" type: NodeSetDeploymentReady + - message: Deployment error occurred in this-service-does-not-exist service error + OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" + not found + reason: Error + severity: Error + status: "False" + type: ServiceThisServiceDoesNotExistDeploymentReady diff --git a/tests/kuttl/tests/dataplane-service-failure/00-assert.yaml b/tests/kuttl/tests/dataplane-service-failure/00-assert.yaml new file mode 100644 index 000000000..cc4ea0319 --- /dev/null +++ b/tests/kuttl/tests/dataplane-service-failure/00-assert.yaml @@ -0,0 +1,195 @@ +--- +apiVersion: ansibleee.openstack.org/v1beta1 +kind: OpenStackAnsibleEE +metadata: + generation: 1 + labels: + openstackdataplanedeployment: edpm-compute-no-nodes + openstackdataplanenodeset: edpm-compute-no-nodes + openstackdataplaneservice: failed-service + name: failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + namespace: openstack + ownerReferences: + - apiVersion: dataplane.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: OpenStackDataPlaneDeployment + name: edpm-compute-no-nodes +spec: + backoffLimit: 3 + env: + - name: ANSIBLE_FORCE_COLOR + value: "True" + envConfigMapName: openstack-aee-default-env + extraMounts: + - mounts: + - mountPath: /runner/env/ssh_key + name: ssh-key + subPath: ssh_key + - mountPath: /runner/inventory/hosts + name: inventory + subPath: inventory + volumes: + - name: ssh-key + secret: + items: + - key: ssh-privatekey + path: ssh_key + secretName: dataplane-ansible-ssh-private-key-secret + - name: inventory + secret: + items: + - key: inventory + path: inventory + secretName: dataplanenodeset-edpm-compute-no-nodes + extraVars: + edpm_override_hosts: edpm-compute-no-nodes + edpm_service_type: failed-service + name: openstackansibleee + play: | + - hosts: localhost + gather_facts: no + name: kuttl play + tasks: + - name: Copy absent file + ansible.builtin.shell: | + set -euxo pipefail + cp absent failed_op + preserveJobs: true + restartPolicy: Never + serviceAccountName: edpm-compute-no-nodes + uid: 1001 +status: + JobStatus: Failed + conditions: + - message: 'Job error occurred Internal error occurred: Job has reached the specified + backoff limit. Check job logs' + reason: BackoffLimitExceeded + severity: Error + status: "False" + type: Ready + - message: 'Job error occurred Internal error occurred: Job has reached the specified + backoff limit. Check job logs' + reason: BackoffLimitExceeded + severity: Error + status: "False" + type: JobReady + observedGeneration: 1 +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneNodeSet +metadata: + generation: 1 + name: edpm-compute-no-nodes + namespace: openstack +spec: + nodeTemplate: + ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret + managementNetwork: ctlplane + nodes: {} + preProvisioned: true + services: + - failed-service + tlsEnabled: true +status: + conditions: + - message: 'Deployment error occurred in failed-service service error backoff limit + reached for execution.name failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + execution.namespace openstack execution.status.jobstatus: Failed' + reason: Error + severity: Error + status: "False" + type: Ready + - message: 'Deployment error occurred in failed-service service error backoff limit + reached for execution.name failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + execution.namespace openstack execution.status.jobstatus: Failed' + reason: Error + severity: Error + status: "False" + type: DeploymentReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: NodeSetDNSDataReady ready + reason: Ready + status: "True" + type: NodeSetDNSDataReady + - message: NodeSetIPReservationReady ready + reason: Ready + status: "True" + type: NodeSetIPReservationReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady + - message: Setup complete + reason: Ready + status: "True" + type: SetupReady + deploymentStatuses: + edpm-compute-no-nodes: + - message: 'Deployment error occurred in failed-service service error backoff + limit reached for execution.name failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + execution.namespace openstack execution.status.jobstatus: Failed' + reason: BackoffLimitExceeded + severity: Error + status: "False" + type: NodeSetDeploymentReady + - message: 'Deployment error occurred in failed-service service error backoff + limit reached for execution.name failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + execution.namespace openstack execution.status.jobstatus: Failed' + reason: BackoffLimitExceeded + severity: Error + status: "False" + type: ServiceFailedServiceDeploymentReady + observedGeneration: 1 +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + generation: 1 + name: edpm-compute-no-nodes + namespace: openstack +spec: + backoffLimit: 3 + deploymentRequeueTime: 15 + nodeSets: + - edpm-compute-no-nodes +status: + conditions: + - message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: backoff + limit reached for execution.name failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + execution.namespace openstack execution.status.jobstatus: Failed' + reason: BackoffLimitExceeded + severity: Error + status: "False" + type: Ready + - message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: backoff + limit reached for execution.name failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + execution.namespace openstack execution.status.jobstatus: Failed' + reason: BackoffLimitExceeded + severity: Error + status: "False" + type: DeploymentReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + nodeSetConditions: + edpm-compute-no-nodes: + - message: 'Deployment error occurred in failed-service service error backoff + limit reached for execution.name failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + execution.namespace openstack execution.status.jobstatus: Failed' + reason: BackoffLimitExceeded + severity: Error + status: "False" + type: NodeSetDeploymentReady + - message: 'Deployment error occurred in failed-service service error backoff + limit reached for execution.name failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + execution.namespace openstack execution.status.jobstatus: Failed' + reason: BackoffLimitExceeded + severity: Error + status: "False" + type: ServiceFailedServiceDeploymentReady + observedGeneration: 1 diff --git a/tests/kuttl/tests/dataplane-service-failure/00-create.yaml b/tests/kuttl/tests/dataplane-service-failure/00-create.yaml new file mode 100644 index 000000000..fded2483b --- /dev/null +++ b/tests/kuttl/tests/dataplane-service-failure/00-create.yaml @@ -0,0 +1,38 @@ +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneService +metadata: + name: failed-service +spec: + play: | + - hosts: localhost + gather_facts: no + name: kuttl play + tasks: + - name: Copy absent file + ansible.builtin.shell: | + set -euxo pipefail + cp absent failed_op +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneNodeSet +metadata: + name: edpm-compute-no-nodes +spec: + preProvisioned: true + env: + - name: ANSIBLE_FORCE_COLOR + value: "True" + nodes: {} + nodeTemplate: + ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret + services: + - failed-service +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-compute-no-nodes +spec: + backoffLimit: 3 + nodeSets: + - edpm-compute-no-nodes