Skip to content

Commit

Permalink
Add kuttl test for failed service
Browse files Browse the repository at this point in the history
Signed-off-by: Fabricio Aguiar <[email protected]>
  • Loading branch information
fao89 committed May 29, 2024
1 parent bf4ecbc commit 4c7b2a6
Show file tree
Hide file tree
Showing 6 changed files with 268 additions and 32 deletions.
6 changes: 3 additions & 3 deletions api/v1beta1/conditions.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,11 @@ const (
NodeSetDeploymentErrorMessage = "Deployment error occurred %s for NodeSet"

// NodeSetServiceDeploymentReadyMessage ready
NodeSetServiceDeploymentReadyMessage = "%s Deployment ready"
NodeSetServiceDeploymentReadyMessage = "Deployment ready for %s service"

// NodeSetServiceDeploymentReadyWaitingMessage not yet ready
NodeSetServiceDeploymentReadyWaitingMessage = "%s Deployment not yet ready"
NodeSetServiceDeploymentReadyWaitingMessage = "Deployment not yet ready for %s service"

// NodeSetServiceDeploymentErrorMessage error
NodeSetServiceDeploymentErrorMessage = "%s Deployment error occurred"
NodeSetServiceDeploymentErrorMessage = "Deployment error occurred in %s service"
)
23 changes: 14 additions & 9 deletions controllers/openstackdataplanedeployment_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -253,10 +253,11 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,

// All nodeSets successfully fetched.
// Mark InputReadyCondition=True
instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.ReadyMessage)
instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
shouldRequeue := false
haveError := false
deploymentErrMsg := ""
backoffLimitReached := false

globalInventorySecrets := map[string]string{}
globalSSHKeySecrets := map[string]string{}
Expand Down Expand Up @@ -347,12 +348,9 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
} else {
deploymentErrMsg = fmt.Sprintf("%s & %s", deploymentErrMsg, errMsg)
}
nsConditions.MarkFalse(
dataplanev1.NodeSetDeploymentReadyCondition,
condition.ErrorReason,
condition.SeverityError,
condition.DeploymentReadyErrorMessage,
err.Error())
nsConditions.Set(nsConditions.Mirror(dataplanev1.NodeSetDeploymentReadyCondition))
errorReason := nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition).Reason
backoffLimitReached = errorReason == condition.JobReasonBackoffLimitExceeded
}

if deployResult != nil {
Expand All @@ -367,10 +365,17 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
}

if haveError {
var reason condition.Reason
reason = condition.ErrorReason
severity := condition.SeverityWarning
if backoffLimitReached {
reason = condition.JobReasonBackoffLimitExceeded
severity = condition.SeverityError
}
instance.Status.Conditions.MarkFalse(
condition.DeploymentReadyCondition,
condition.ErrorReason,
condition.SeverityError,
reason,
severity,
condition.DeploymentReadyErrorMessage,
deploymentErrMsg)
return ctrl.Result{}, fmt.Errorf(deploymentErrMsg)
Expand Down
4 changes: 2 additions & 2 deletions pkg/deployment/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func (d *Deployer) Deploy(services []string) (*ctrl.Result, error) {
readyCondition = condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(service)))
readyWaitingMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, deployName)
readyMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyMessage, deployName)
readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, deployName)
readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, deployName) + " error %s"
containerImages := dataplaneutil.GetContainerImages(d.Version)
if containerImages.AnsibleeeImage != nil {
d.AeeSpec.OpenStackAnsibleEERunnerImage = *containerImages.AnsibleeeImage
Expand Down Expand Up @@ -206,7 +206,7 @@ func (d *Deployer) ConditionalDeploy(

if ansibleEE.Status.JobStatus == ansibleeev1.JobStatusFailed {
errorMsg := fmt.Sprintf("execution.name %s execution.namespace %s execution.status.jobstatus: %s", ansibleEE.Name, ansibleEE.Namespace, ansibleEE.Status.JobStatus)
ansibleCondition := ansibleEE.Status.Conditions.Get(condition.JobReadyCondition)
ansibleCondition := ansibleEE.Status.Conditions.Get(condition.ReadyCondition)
if ansibleCondition.Reason == condition.JobReasonBackoffLimitExceeded {
errorMsg = fmt.Sprintf("backoff limit reached for execution.name %s execution.namespace %s execution.status.jobstatus: %s", ansibleEE.Name, ansibleEE.Namespace, ansibleEE.Status.JobStatus)
}
Expand Down
36 changes: 18 additions & 18 deletions tests/kuttl/tests/dataplane-deploy-no-nodes-test/05-assert.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,63 +65,63 @@ status:
reason: Ready
status: "True"
type: NodeSetDeploymentReady
- message: bootstrap Deployment ready
- message: Deployment ready for bootstrap service
reason: Ready
status: "True"
type: ServiceBootstrapDeploymentReady
- message: configure-network Deployment ready
- message: Deployment ready for configure-network service
reason: Ready
status: "True"
type: ServiceConfigureNetworkDeploymentReady
- message: configure-os Deployment ready
- message: Deployment ready for configure-os service
reason: Ready
status: "True"
type: ServiceConfigureOsDeploymentReady
- message: download-cache Deployment ready
- message: Deployment ready for download-cache service
reason: Ready
status: "True"
type: ServiceDownloadCacheDeploymentReady
- message: install-certs Deployment ready
- message: Deployment ready for install-certs service
reason: Ready
status: "True"
type: ServiceInstallCertsDeploymentReady
- message: install-os Deployment ready
- message: Deployment ready for install-os service
reason: Ready
status: "True"
type: ServiceInstallOsDeploymentReady
- message: libvirt Deployment ready
- message: Deployment ready for libvirt service
reason: Ready
status: "True"
type: ServiceLibvirtDeploymentReady
- message: neutron-dhcp Deployment ready
- message: Deployment ready for neutron-dhcp service
reason: Ready
status: "True"
type: ServiceNeutronDhcpDeploymentReady
- message: neutron-metadata Deployment ready
- message: Deployment ready for neutron-metadata service
reason: Ready
status: "True"
type: ServiceNeutronMetadataDeploymentReady
- message: neutron-ovn Deployment ready
- message: Deployment ready for neutron-ovn service
reason: Ready
status: "True"
type: ServiceNeutronOvnDeploymentReady
- message: neutron-sriov Deployment ready
- message: Deployment ready for neutron-sriov service
reason: Ready
status: "True"
type: ServiceNeutronSriovDeploymentReady
- message: nova Deployment ready
- message: Deployment ready for nova service
reason: Ready
status: "True"
type: ServiceNovaDeploymentReady
- message: ovn Deployment ready
- message: Deployment ready for ovn service
reason: Ready
status: "True"
type: ServiceOvnDeploymentReady
- message: run-os Deployment ready
- message: Deployment ready for run-os service
reason: Ready
status: "True"
type: ServiceRunOsDeploymentReady
- message: validate-network Deployment ready
- message: Deployment ready for validate-network service
reason: Ready
status: "True"
type: ServiceValidateNetworkDeploymentReady
Expand All @@ -137,7 +137,7 @@ status:
reason: Ready
status: "True"
type: NodeSetDeploymentReady
- message: custom-svc Deployment ready
- message: Deployment ready for custom-svc service
reason: Ready
status: "True"
type: ServiceCustomSvcDeploymentReady
Expand All @@ -146,7 +146,7 @@ status:
reason: Ready
status: "True"
type: NodeSetDeploymentReady
- message: ovn Deployment ready
- message: Deployment ready for ovn service
reason: Ready
status: "True"
type: ServiceOvnDeploymentReady
Expand Down Expand Up @@ -176,7 +176,7 @@ status:
severity: Error
status: "False"
type: DeploymentReady
- message: Setup complete
- message: Input data complete
reason: Ready
status: "True"
type: InputReady
Expand Down
193 changes: 193 additions & 0 deletions tests/kuttl/tests/dataplane-service-failure/00-assert.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
---
apiVersion: ansibleee.openstack.org/v1beta1
kind: OpenStackAnsibleEE
metadata:
generation: 1
labels:
openstackdataplanedeployment: edpm-compute-no-nodes
openstackdataplanenodeset: edpm-compute-no-nodes
openstackdataplaneservice: failed-service
name: failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes
namespace: openstack
ownerReferences:
- apiVersion: dataplane.openstack.org/v1beta1
blockOwnerDeletion: true
controller: true
kind: OpenStackDataPlaneDeployment
name: edpm-compute-no-nodes
spec:
backoffLimit: 3
env:
envConfigMapName: openstack-aee-default-env
extraMounts:
- mounts:
- mountPath: /runner/env/ssh_key
name: ssh-key
subPath: ssh_key
- mountPath: /runner/inventory/hosts
name: inventory
subPath: inventory
volumes:
- name: ssh-key
secret:
items:
- key: ssh-privatekey
path: ssh_key
secretName: dataplane-ansible-ssh-private-key-secret
- name: inventory
secret:
items:
- key: inventory
path: inventory
secretName: dataplanenodeset-edpm-compute-no-nodes
extraVars:
edpm_override_hosts: edpm-compute-no-nodes
edpm_service_type: failed-service
name: openstackansibleee
play: |
- hosts: localhost
gather_facts: no
name: kuttl play
tasks:
- name: Copy absent file
ansible.builtin.shell: |
set -euxo pipefail
cp absent failed_op
preserveJobs: true
restartPolicy: Never
serviceAccountName: edpm-compute-no-nodes
uid: 1001
status:
JobStatus: Failed
conditions:
- message: 'AnsibleExecutionJob error occurred Internal error occurred: Job has
reached the specified backoff limit. Check job logs'
reason: Error
severity: Warning
status: "False"
type: Ready
- message: 'AnsibleExecutionJob error occurred Internal error occurred: Job has
reached the specified backoff limit. Check job logs'
reason: Error
severity: Warning
status: "False"
type: AnsibleExecutionJobReady
observedGeneration: 1
---
apiVersion: dataplane.openstack.org/v1beta1
kind: OpenStackDataPlaneNodeSet
metadata:
generation: 1
name: edpm-compute-no-nodes
namespace: openstack
spec:
nodeTemplate:
ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret
managementNetwork: ctlplane
nodes: {}
preProvisioned: true
services:
- failed-service
tlsEnabled: true
status:
conditions:
- message: 'Deployment error occurred in failed-service service error execution.name
failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace
openstack execution.status.jobstatus: Failed'
reason: Error
severity: Error
status: "False"
type: Ready
- message: 'Deployment error occurred in failed-service service error execution.name
failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace
openstack execution.status.jobstatus: Failed'
reason: Error
severity: Error
status: "False"
type: DeploymentReady
- message: Input data complete
reason: Ready
status: "True"
type: InputReady
- message: NodeSetDNSDataReady ready
reason: Ready
status: "True"
type: NodeSetDNSDataReady
- message: NodeSetIPReservationReady ready
reason: Ready
status: "True"
type: NodeSetIPReservationReady
- message: ServiceAccount created
reason: Ready
status: "True"
type: ServiceAccountReady
- message: Setup complete
reason: Ready
status: "True"
type: SetupReady
deploymentStatuses:
edpm-compute-no-nodes:
- message: 'Deployment error occurred in failed-service service error execution.name
failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace
openstack execution.status.jobstatus: Failed'
reason: Error
severity: Warning
status: "False"
type: NodeSetDeploymentReady
- message: 'Deployment error occurred in failed-service service error execution.name
failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace
openstack execution.status.jobstatus: Failed'
reason: Error
severity: Warning
status: "False"
type: ServiceFailedServiceDeploymentReady
observedGeneration: 1
---
apiVersion: dataplane.openstack.org/v1beta1
kind: OpenStackDataPlaneDeployment
metadata:
generation: 1
name: edpm-compute-no-nodes
namespace: openstack
spec:
backoffLimit: 3
deploymentRequeueTime: 15
nodeSets:
- edpm-compute-no-nodes
status:
conditions:
- message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: execution.name
failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace
openstack execution.status.jobstatus: Failed'
reason: Error
severity: Warning
status: "False"
type: Ready
- message: 'Deployment error occurred nodeSet: edpm-compute-no-nodes error: execution.name
failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace
openstack execution.status.jobstatus: Failed'
reason: Error
severity: Warning
status: "False"
type: DeploymentReady
- message: Input data complete
reason: Ready
status: "True"
type: InputReady
nodeSetConditions:
edpm-compute-no-nodes:
- message: 'Deployment error occurred in failed-service service error execution.name
failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace
openstack execution.status.jobstatus: Failed'
reason: Error
severity: Warning
status: "False"
type: NodeSetDeploymentReady
- message: 'Deployment error occurred in failed-service service error execution.name
failed-service-edpm-compute-no-nodes-edpm-compute-no-nodes execution.namespace
openstack execution.status.jobstatus: Failed'
reason: Error
severity: Warning
status: "False"
type: ServiceFailedServiceDeploymentReady
observedGeneration: 1
Loading

0 comments on commit 4c7b2a6

Please sign in to comment.