From 428dfbbcf17721ebb195f89588d2927a946c2326 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Fri, 31 May 2024 12:22:31 +0100 Subject: [PATCH] Add kuttl test for failed service Signed-off-by: Fabricio Aguiar --- api/v1beta1/conditions.go | 6 +- ...openstackdataplanedeployment_controller.go | 51 ++--- docs/assemblies/interacting_with_ansible.adoc | 16 +- ...ng-data-plane-creation-and-deployment.adoc | 23 +-- pkg/deployment/deployment.go | 37 +++- .../01-assert.yaml | 64 +++--- .../02-assert.yaml | 8 +- .../02-assert.yaml | 8 +- .../01-assert.yaml | 60 +++--- .../02-assert.yaml | 4 +- .../04-assert.yaml | 4 +- .../05-assert.yaml | 53 +++-- .../06-assert.yaml | 8 +- .../dataplane-deploy-tls-test/02-assert.yaml | 8 +- .../dataplane-deploy-tls-test/03-assert.yaml | 12 +- .../dataplane-service-config/00-assert.yaml | 6 +- .../00-assert.yaml | 6 +- .../dataplane-service-failure/00-assert.yaml | 195 ++++++++++++++++++ .../dataplane-service-failure/00-create.yaml | 38 ++++ 19 files changed, 437 insertions(+), 170 deletions(-) create mode 100644 tests/kuttl/tests/dataplane-service-failure/00-assert.yaml create mode 100644 tests/kuttl/tests/dataplane-service-failure/00-create.yaml diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index d54b9f4bd..33380b1e3 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -99,11 +99,11 @@ const ( NodeSetDeploymentErrorMessage = "Deployment error occurred %s for NodeSet" // NodeSetServiceDeploymentReadyMessage ready - NodeSetServiceDeploymentReadyMessage = "%s Deployment ready" + NodeSetServiceDeploymentReadyMessage = "Deployment ready for %s service" // NodeSetServiceDeploymentReadyWaitingMessage not yet ready - NodeSetServiceDeploymentReadyWaitingMessage = "%s Deployment not yet ready" + NodeSetServiceDeploymentReadyWaitingMessage = "Deployment not yet ready for %s service" // NodeSetServiceDeploymentErrorMessage error - NodeSetServiceDeploymentErrorMessage = "%s Deployment error occurred" + NodeSetServiceDeploymentErrorMessage = "Deployment error occurred in %s service" ) diff --git a/controllers/openstackdataplanedeployment_controller.go b/controllers/openstackdataplanedeployment_controller.go index aed4e2160..1bbb446e3 100644 --- a/controllers/openstackdataplanedeployment_controller.go +++ b/controllers/openstackdataplanedeployment_controller.go @@ -214,14 +214,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, condition.SeverityError, dataplanev1.ServiceErrorMessage, err.Error()) - if len(instance.Spec.ServicesOverride) == 0 { - nsConditions.MarkFalse( - dataplanev1.NodeSetDeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityError, - dataplanev1.ServiceErrorMessage, - err.Error()) - } + nsConditions.MarkFalse( + dataplanev1.NodeSetDeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityError, + dataplanev1.ServiceErrorMessage, + err.Error()) return ctrl.Result{}, err } if service.Spec.TLSCert != nil { @@ -234,14 +232,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, condition.SeverityError, condition.TLSInputErrorMessage, err.Error()) - if len(instance.Spec.ServicesOverride) == 0 { - nsConditions.MarkFalse( - dataplanev1.NodeSetDeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityError, - condition.TLSInputErrorMessage, - err.Error()) - } + nsConditions.MarkFalse( + dataplanev1.NodeSetDeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityError, + condition.TLSInputErrorMessage, + err.Error()) return ctrl.Result{}, err } else if (*result != ctrl.Result{}) { return *result, nil // requeue here @@ -253,10 +249,11 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, // All nodeSets successfully fetched. // Mark InputReadyCondition=True - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.ReadyMessage) + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) shouldRequeue := false haveError := false deploymentErrMsg := "" + backoffLimitReached := false globalInventorySecrets := map[string]string{} globalSSHKeySecrets := map[string]string{} @@ -347,12 +344,9 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, } else { deploymentErrMsg = fmt.Sprintf("%s & %s", deploymentErrMsg, errMsg) } - nsConditions.MarkFalse( - dataplanev1.NodeSetDeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityError, - condition.DeploymentReadyErrorMessage, - err.Error()) + nsConditions.Set(nsConditions.Mirror(dataplanev1.NodeSetDeploymentReadyCondition)) + errorReason := nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition).Reason + backoffLimitReached = errorReason == condition.JobReasonBackoffLimitExceeded } if deployResult != nil { @@ -367,10 +361,17 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, } if haveError { + var reason condition.Reason + reason = condition.ErrorReason + severity := condition.SeverityWarning + if backoffLimitReached { + reason = condition.JobReasonBackoffLimitExceeded + severity = condition.SeverityError + } instance.Status.Conditions.MarkFalse( condition.DeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityError, + reason, + severity, condition.DeploymentReadyErrorMessage, deploymentErrMsg) return ctrl.Result{}, fmt.Errorf(deploymentErrMsg) diff --git a/docs/assemblies/interacting_with_ansible.adoc b/docs/assemblies/interacting_with_ansible.adoc index 5d72fdd7d..de2897f3a 100644 --- a/docs/assemblies/interacting_with_ansible.adoc +++ b/docs/assemblies/interacting_with_ansible.adoc @@ -22,14 +22,14 @@ retrieved from the API. Sample output when the default list of services: NAME NETWORKATTACHMENTS STATUS MESSAGE - configure-network-edpm-compute True AnsibleExecutionJob complete - configure-os-edpm-compute True AnsibleExecutionJob complete - install-os-edpm-compute True AnsibleExecutionJob complete - libvirt-edpm-compute True AnsibleExecutionJob complete - nova-edpm-compute True AnsibleExecutionJob complete - run-os-edpm-compute True AnsibleExecutionJob complete - telemetry-edpm-compute True AnsibleExecutionJob complete - validate-network-edpm-compute True AnsibleExecutionJob complete + configure-network-edpm-compute True Job completed + configure-os-edpm-compute True Job completed + install-os-edpm-compute True Job completed + libvirt-edpm-compute True Job completed + nova-edpm-compute True Job completed + run-os-edpm-compute True Job completed + telemetry-edpm-compute True Job completed + validate-network-edpm-compute True Job completed Querying for pods with the OpenStackAnsibleEE label diff --git a/docs/assemblies/proc_troubleshooting-data-plane-creation-and-deployment.adoc b/docs/assemblies/proc_troubleshooting-data-plane-creation-and-deployment.adoc index 3787d1e26..161efcc24 100644 --- a/docs/assemblies/proc_troubleshooting-data-plane-creation-and-deployment.adoc +++ b/docs/assemblies/proc_troubleshooting-data-plane-creation-and-deployment.adoc @@ -34,10 +34,10 @@ The following example output shows all services and their job condition for all $ oc get openstackansibleee NAME NETWORKATTACHMENTS STATUS MESSAGE -bootstrap-openstack-edpm ["ctlplane"] True AnsibleExecutionJob complete -download-cache-openstack-edpm ["ctlplane"] False AnsibleExecutionJob is running -repo-setup-openstack-edpm ["ctlplane"] True AnsibleExecutionJob complete -validate-network-another-osdpd ["ctlplane"] False AnsibleExecutionJob is running +bootstrap-openstack-edpm ["ctlplane"] True Job completed +download-cache-openstack-edpm ["ctlplane"] False Job in progress +repo-setup-openstack-edpm ["ctlplane"] True Job completed +validate-network-another-osdpd ["ctlplane"] False Job in progress ---- . Filter for the name and service for a specific deployment: @@ -54,20 +54,19 @@ The following example filters the list to only show services and their job condi $ oc get openstackansibleee -l openstackdataplanedeployment=openstack-edpm-ipam1 NAME NETWORKATTACHMENTS STATUS MESSAGE -bootstrap-openstack-edpm ["ctlplane"] True AnsibleExecutionJob complete -download-cache-openstack-edpm ["ctlplane"] False AnsibleExecutionJob is running -repo-setup-openstack-edpm ["ctlplane"] True AnsibleExecutionJob complete +bootstrap-openstack-edpm ["ctlplane"] True Job completed +download-cache-openstack-edpm ["ctlplane"] False Job in progress +repo-setup-openstack-edpm ["ctlplane"] True Job completed ---- .Job Condition Messages AnsibleEE jobs have an associated condition message that indicates the current state of the service job. This condition message is displayed in the `MESSAGE` field of the `oc get openstackansibleee` command output. Jobs return one of the following conditions when queried: -* `AnsibleExecutionJob not started`: The job has not started. -* `AnsibleExecutionJob not found`: The job could not be found. -* `AnsibleExecutionJob is running`: The job is currently running. -* `AnsibleExecutionJob complete`: The job execution is complete. -* `AnsibleExecutionJob error occured `: The job execution stopped unexpectedly. The `` is replaced with a specific error message. +* `Job not started`: The job has not started. +* `Job in progress`: The job is currently running. +* `Job completed`: The job execution is complete. +* `Job error occured `: The job execution stopped unexpectedly. The `` is replaced with a specific error message. To further investigate a service displaying a particular job condition message, use the command `oc logs job/` to display the logs associated with that service. For example, to display the logs for the `repo-setup-openstack-edpm` service, use the command `oc logs job/repo-setup-openstack-edpm`. diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index ebd56fd43..2a2d1e72d 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -70,16 +70,26 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) { copy(aeeSpecMounts, d.AeeSpec.ExtraMounts) // Deploy the composable services for _, service := range services { + deployName = service + readyCondition = condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(service))) + readyWaitingMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, deployName) + readyMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyMessage, deployName) + readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, deployName) + " error %s" + + nsConditions := d.Status.NodeSetConditions[d.NodeSet.Name] log.Info("Deploying service", "service", service) foundService, err := GetService(d.Ctx, d.Helper, service) if err != nil { + nsConditions.Set(condition.FalseCondition( + readyCondition, + condition.ErrorReason, + condition.SeverityError, + readyErrorMessage, + err.Error())) + d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions return &ctrl.Result{}, err } - deployName = foundService.Name - readyCondition = condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(service))) - readyWaitingMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, deployName) - readyMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyMessage, deployName) - readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, deployName) + containerImages := dataplaneutil.GetContainerImages(d.Version) if containerImages.AnsibleeeImage != nil { d.AeeSpec.OpenStackAnsibleEERunnerImage = *containerImages.AnsibleeeImage @@ -94,6 +104,13 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) { copy(d.AeeSpec.ExtraMounts, aeeSpecMounts) d.AeeSpec, err = d.addServiceExtraMounts(foundService) if err != nil { + nsConditions.Set(condition.FalseCondition( + readyCondition, + condition.ErrorReason, + condition.SeverityError, + readyErrorMessage, + err.Error())) + d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions return &ctrl.Result{}, err } @@ -103,6 +120,13 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) { d.AeeSpec, err = d.addCertMounts(services) } if err != nil { + nsConditions.Set(condition.FalseCondition( + readyCondition, + condition.ErrorReason, + condition.SeverityError, + readyErrorMessage, + err.Error())) + d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions return &ctrl.Result{}, err } } @@ -116,7 +140,6 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) { foundService, ) - nsConditions := d.Status.NodeSetConditions[d.NodeSet.Name] if err != nil || !nsConditions.IsTrue(readyCondition) { log.Info(fmt.Sprintf("Condition %s not ready", readyCondition)) return &ctrl.Result{}, err @@ -206,7 +229,7 @@ func (d *Deployer) ConditionalDeploy( if ansibleEE.Status.JobStatus == ansibleeev1.JobStatusFailed { errorMsg := fmt.Sprintf("execution.name %s execution.namespace %s execution.status.jobstatus: %s", ansibleEE.Name, ansibleEE.Namespace, ansibleEE.Status.JobStatus) - ansibleCondition := ansibleEE.Status.Conditions.Get(condition.JobReadyCondition) + ansibleCondition := ansibleEE.Status.Conditions.Get(condition.ReadyCondition) if ansibleCondition.Reason == condition.JobReasonBackoffLimitExceeded { errorMsg = fmt.Sprintf("backoff limit reached for execution.name %s execution.namespace %s execution.status.jobstatus: %s", ansibleEE.Name, ansibleEE.Namespace, ansibleEE.Status.JobStatus) } diff --git a/tests/kuttl/tests/dataplane-deploy-global-service-test/01-assert.yaml b/tests/kuttl/tests/dataplane-deploy-global-service-test/01-assert.yaml index 5672d30da..9931440e2 100644 --- a/tests/kuttl/tests/dataplane-deploy-global-service-test/01-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-global-service-test/01-assert.yaml @@ -130,10 +130,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -180,10 +180,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -230,10 +230,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 @@ -281,10 +281,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -331,10 +331,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -381,10 +381,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -432,10 +432,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -482,10 +482,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -532,10 +532,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -594,10 +594,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -685,10 +685,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -747,10 +747,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -809,10 +809,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -871,10 +871,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -934,10 +934,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -1020,7 +1020,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-deploy-global-service-test/02-assert.yaml b/tests/kuttl/tests/dataplane-deploy-global-service-test/02-assert.yaml index 42e9ae1dc..c70c8a66b 100644 --- a/tests/kuttl/tests/dataplane-deploy-global-service-test/02-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-global-service-test/02-assert.yaml @@ -110,10 +110,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -160,7 +160,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-deploy-multiple-secrets/02-assert.yaml b/tests/kuttl/tests/dataplane-deploy-multiple-secrets/02-assert.yaml index 2df0c2a2d..92f4636c8 100644 --- a/tests/kuttl/tests/dataplane-deploy-multiple-secrets/02-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-multiple-secrets/02-assert.yaml @@ -164,10 +164,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -211,7 +211,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-assert.yaml index a1096ef70..3529bf843 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-assert.yaml @@ -103,10 +103,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -155,10 +155,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 @@ -206,10 +206,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -256,10 +256,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -306,10 +306,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -357,10 +357,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -407,10 +407,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -457,10 +457,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -519,10 +519,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -610,10 +610,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -672,10 +672,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -734,10 +734,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -796,10 +796,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -859,10 +859,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -945,7 +945,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/02-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/02-assert.yaml index ec07c64fa..0a24745ce 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/02-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/02-assert.yaml @@ -56,7 +56,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/04-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/04-assert.yaml index 021793207..504c745fb 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/04-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/04-assert.yaml @@ -115,7 +115,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml index 7b299c591..be6871687 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml @@ -29,12 +29,16 @@ spec: status: observedGeneration: 1 conditions: - - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found + - message: Deployment error occurred in this-service-does-not-exist service error + OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" + not found reason: Error severity: Error status: "False" type: Ready - - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found + - message: Deployment error occurred in this-service-does-not-exist service error + OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" + not found reason: Error severity: Error status: "False" @@ -65,68 +69,68 @@ status: reason: Ready status: "True" type: NodeSetDeploymentReady - - message: bootstrap Deployment ready + - message: Deployment ready for bootstrap service reason: Ready status: "True" type: ServiceBootstrapDeploymentReady - - message: configure-network Deployment ready + - message: Deployment ready for configure-network service reason: Ready status: "True" type: ServiceConfigureNetworkDeploymentReady - - message: configure-os Deployment ready + - message: Deployment ready for configure-os service reason: Ready status: "True" type: ServiceConfigureOsDeploymentReady - - message: download-cache Deployment ready + - message: Deployment ready for download-cache service reason: Ready status: "True" type: ServiceDownloadCacheDeploymentReady - - message: install-certs Deployment ready + - message: Deployment ready for install-certs service reason: Ready status: "True" type: ServiceInstallCertsDeploymentReady - - message: install-os Deployment ready + - message: Deployment ready for install-os service reason: Ready status: "True" type: ServiceInstallOsDeploymentReady - - message: libvirt Deployment ready + - message: Deployment ready for libvirt service reason: Ready status: "True" type: ServiceLibvirtDeploymentReady - - message: neutron-dhcp Deployment ready + - message: Deployment ready for neutron-dhcp service reason: Ready status: "True" type: ServiceNeutronDhcpDeploymentReady - - message: neutron-metadata Deployment ready + - message: Deployment ready for neutron-metadata service reason: Ready status: "True" type: ServiceNeutronMetadataDeploymentReady - - message: neutron-ovn Deployment ready + - message: Deployment ready for neutron-ovn service reason: Ready status: "True" type: ServiceNeutronOvnDeploymentReady - - message: neutron-sriov Deployment ready + - message: Deployment ready for neutron-sriov service reason: Ready status: "True" type: ServiceNeutronSriovDeploymentReady - - message: nova Deployment ready + - message: Deployment ready for nova service reason: Ready status: "True" type: ServiceNovaDeploymentReady - - message: ovn Deployment ready + - message: Deployment ready for ovn service reason: Ready status: "True" type: ServiceOvnDeploymentReady - - message: run-os Deployment ready + - message: Deployment ready for run-os service reason: Ready status: "True" type: ServiceRunOsDeploymentReady - - message: validate-network Deployment ready + - message: Deployment ready for validate-network service reason: Ready status: "True" type: ServiceValidateNetworkDeploymentReady edpm-compute-no-nodes-non-existent-service: - - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org + - message: Service error occurred OpenStackDataPlaneService.dataplane.openstack.org "this-service-does-not-exist" not found reason: Error severity: Error @@ -137,7 +141,7 @@ status: reason: Ready status: "True" type: NodeSetDeploymentReady - - message: custom-svc Deployment ready + - message: Deployment ready for custom-svc service reason: Ready status: "True" type: ServiceCustomSvcDeploymentReady @@ -146,7 +150,7 @@ status: reason: Ready status: "True" type: NodeSetDeploymentReady - - message: ovn Deployment ready + - message: Deployment ready for ovn service reason: Ready status: "True" type: ServiceOvnDeploymentReady @@ -176,7 +180,7 @@ status: severity: Error status: "False" type: DeploymentReady - - message: Setup complete + - message: Input data complete reason: Ready status: "True" type: InputReady @@ -188,3 +192,10 @@ status: severity: Error status: "False" type: NodeSetDeploymentReady + - message: Deployment error occurred OpenStackDataPlaneService.dataplane.openstack.org + "this-service-does-not-exist" not found + reason: Error + severity: Error + status: "False" + type: ServiceThisServiceDoesNotExistDeploymentReady + diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/06-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/06-assert.yaml index 6cd10c1d7..3451af5f2 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/06-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/06-assert.yaml @@ -108,10 +108,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -158,7 +158,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-deploy-tls-test/02-assert.yaml b/tests/kuttl/tests/dataplane-deploy-tls-test/02-assert.yaml index 246e561f7..89680d9d0 100644 --- a/tests/kuttl/tests/dataplane-deploy-tls-test/02-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-tls-test/02-assert.yaml @@ -129,10 +129,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -176,7 +176,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-deploy-tls-test/03-assert.yaml b/tests/kuttl/tests/dataplane-deploy-tls-test/03-assert.yaml index de738bee2..e367eec99 100644 --- a/tests/kuttl/tests/dataplane-deploy-tls-test/03-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-tls-test/03-assert.yaml @@ -209,10 +209,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -256,10 +256,10 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady --- apiVersion: ansibleee.openstack.org/v1beta1 kind: OpenStackAnsibleEE @@ -303,7 +303,7 @@ status: reason: Ready status: "True" type: Ready - - message: AnsibleExecutionJob complete + - message: Job completed reason: Ready status: "True" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-service-config/00-assert.yaml b/tests/kuttl/tests/dataplane-service-config/00-assert.yaml index f0892e60c..9ea626712 100644 --- a/tests/kuttl/tests/dataplane-service-config/00-assert.yaml +++ b/tests/kuttl/tests/dataplane-service-config/00-assert.yaml @@ -107,13 +107,13 @@ spec: status: JobStatus: Running conditions: - - message: AnsibleExecutionJob is running + - message: Job in progress reason: Requested severity: Info status: "False" type: Ready - - message: AnsibleExecutionJob is running + - message: Job in progress reason: Requested severity: Info status: "False" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-service-custom-image/00-assert.yaml b/tests/kuttl/tests/dataplane-service-custom-image/00-assert.yaml index e2ff03aea..e06165732 100644 --- a/tests/kuttl/tests/dataplane-service-custom-image/00-assert.yaml +++ b/tests/kuttl/tests/dataplane-service-custom-image/00-assert.yaml @@ -88,13 +88,13 @@ spec: status: JobStatus: Running conditions: - - message: AnsibleExecutionJob is running + - message: Job in progress reason: Requested severity: Info status: "False" type: Ready - - message: AnsibleExecutionJob is running + - message: Job in progress reason: Requested severity: Info status: "False" - type: AnsibleExecutionJobReady + type: JobReady diff --git a/tests/kuttl/tests/dataplane-service-failure/00-assert.yaml b/tests/kuttl/tests/dataplane-service-failure/00-assert.yaml new file mode 100644 index 000000000..ff2e30c39 --- /dev/null +++ b/tests/kuttl/tests/dataplane-service-failure/00-assert.yaml @@ -0,0 +1,195 @@ +--- +apiVersion: ansibleee.openstack.org/v1beta1 +kind: OpenStackAnsibleEE +metadata: + generation: 1 + labels: + openstackdataplanedeployment: edpm-compute-no-nodes + openstackdataplanenodeset: edpm-compute-no-nodes + openstackdataplaneservice: failed-service + name: failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes + namespace: openstack + ownerReferences: + - apiVersion: dataplane.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: OpenStackDataPlaneDeployment + name: edpm-compute-no-nodes +spec: + backoffLimit: 3 + env: + - name: ANSIBLE_FORCE_COLOR + value: "True" + envConfigMapName: openstack-aee-default-env + extraMounts: + - mounts: + - mountPath: /runner/env/ssh_key + name: ssh-key + subPath: ssh_key + - mountPath: /runner/inventory/hosts + name: inventory + subPath: inventory + volumes: + - name: ssh-key + secret: + items: + - key: ssh-privatekey + path: ssh_key + secretName: dataplane-ansible-ssh-private-key-secret + - name: inventory + secret: + items: + - key: inventory + path: inventory + secretName: dataplanenodeset-edpm-compute-no-nodes + extraVars: + edpm_override_hosts: edpm-compute-no-nodes + edpm_service_type: failed-service + name: openstackansibleee + play: | + - hosts: localhost + gather_facts: no + name: kuttl play + tasks: + - name: Copy absent file + ansible.builtin.shell: | + set -euxo pipefail + cp absent failed_op + preserveJobs: true + restartPolicy: Never + serviceAccountName: edpm-compute-no-nodes + uid: 1001 +status: + JobStatus: Failed + conditions: + - message: 'Job error occurred Internal error occurred: Job has + reached the specified backoff limit. Check job logs' + reason: Error + severity: Warning + status: "False" + type: Ready + - message: 'Job error occurred Internal error occurred: Job has + reached the specified backoff limit. Check job logs' + reason: Error + severity: Warning + status: "False" + type: JobReady + observedGeneration: 1 +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneNodeSet +metadata: + generation: 1 + name: edpm-compute-no-nodes + namespace: openstack +spec: + nodeTemplate: + ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret + managementNetwork: ctlplane + nodes: {} + preProvisioned: true + services: + - failed-service + tlsEnabled: true +status: + conditions: + - message: 'Deployment error occurred in failed-service service error execution.name + failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace + openstack execution.status.jobstatus: Failed' + reason: Error + severity: Error + status: "False" + type: Ready + - message: 'Deployment error occurred in failed-service service error execution.name + failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace + openstack execution.status.jobstatus: Failed' + reason: Error + severity: Error + status: "False" + type: DeploymentReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: NodeSetDNSDataReady ready + reason: Ready + status: "True" + type: NodeSetDNSDataReady + - message: NodeSetIPReservationReady ready + reason: Ready + status: "True" + type: NodeSetIPReservationReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady + - message: Setup complete + reason: Ready + status: "True" + type: SetupReady + deploymentStatuses: + edpm-compute-no-nodes: + - message: 'Deployment error occurred in failed-service service error execution.name + failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace + openstack execution.status.jobstatus: Failed' + reason: Error + severity: Warning + status: "False" + type: NodeSetDeploymentReady + - message: 'Deployment error occurred in failed-service service error execution.name + failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace + openstack execution.status.jobstatus: Failed' + reason: Error + severity: Warning + status: "False" + type: ServiceFailedServiceDeploymentReady + observedGeneration: 1 +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + generation: 1 + name: edpm-compute-no-nodes + namespace: openstack +spec: + backoffLimit: 3 + deploymentRequeueTime: 15 + nodeSets: + - edpm-compute-no-nodes +status: + conditions: + - message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: execution.name + failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace + openstack execution.status.jobstatus: Failed' + reason: Error + severity: Warning + status: "False" + type: Ready + - message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: execution.name + failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace + openstack execution.status.jobstatus: Failed' + reason: Error + severity: Warning + status: "False" + type: DeploymentReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + nodeSetConditions: + edpm-compute-no-nodes: + - message: 'Deployment error occurred in failed-service service error execution.name + failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace + openstack execution.status.jobstatus: Failed' + reason: Error + severity: Warning + status: "False" + type: NodeSetDeploymentReady + - message: 'Deployment error occurred in failed-service service error execution.name + failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace + openstack execution.status.jobstatus: Failed' + reason: Error + severity: Warning + status: "False" + type: ServiceFailedServiceDeploymentReady + observedGeneration: 1 diff --git a/tests/kuttl/tests/dataplane-service-failure/00-create.yaml b/tests/kuttl/tests/dataplane-service-failure/00-create.yaml new file mode 100644 index 000000000..fded2483b --- /dev/null +++ b/tests/kuttl/tests/dataplane-service-failure/00-create.yaml @@ -0,0 +1,38 @@ +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneService +metadata: + name: failed-service +spec: + play: | + - hosts: localhost + gather_facts: no + name: kuttl play + tasks: + - name: Copy absent file + ansible.builtin.shell: | + set -euxo pipefail + cp absent failed_op +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneNodeSet +metadata: + name: edpm-compute-no-nodes +spec: + preProvisioned: true + env: + - name: ANSIBLE_FORCE_COLOR + value: "True" + nodes: {} + nodeTemplate: + ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret + services: + - failed-service +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-compute-no-nodes +spec: + backoffLimit: 3 + nodeSets: + - edpm-compute-no-nodes