From c4f27505992364d88df1efb8547244366b7b267c Mon Sep 17 00:00:00 2001 From: Alexander Wels Date: Tue, 9 Jul 2024 10:21:55 -0500 Subject: [PATCH] Add storage live migration support to direct DirectVolumeMigration Add prometheus progress reporting to VM Live Migration Validate the plan if the live migration checkbox is set, ensure that the source and destination cluster have a valid kubevirt version and configuration. Allow canceling of migration. This includes aborting live migrations and cleaning up rsync pods if they are still running. Added a finalizer to the DirectVolumeMigration which is only removed once everything is cleaned up properly. Properly handle rollback of direct volume migrations that include VMs. For both running and stopped VMs as well as live migration of running VMs, or quiesce and starting them after migration. Signed-off-by: Alexander Wels --- .github/workflows/pr-make.yml | 2 +- Makefile | 2 +- ...n.openshift.io_directvolumemigrations.yaml | 655 +++++- .../crds/migration.openshift.io_migplans.yaml | 5 + go.mod | 12 +- go.sum | 14 +- .../v1alpha1/directvolumemigration_types.go | 112 +- .../migration/v1alpha1/migcluster_types.go | 7 + pkg/apis/migration/v1alpha1/migplan_types.go | 9 + .../v1alpha1/zz_generated.deepcopy.go | 111 + pkg/compat/fake/client.go | 4 + .../directvolumemigration_controller.go | 187 +- .../directvolumemigration_controller_test.go | 208 ++ .../directvolumemigration/pvcs_test.go | 12 +- pkg/controller/directvolumemigration/rsync.go | 680 ++++--- .../directvolumemigration/rsync_test.go | 1087 +++++++++- pkg/controller/directvolumemigration/task.go | 134 +- .../directvolumemigration/validation.go | 21 +- pkg/controller/directvolumemigration/vm.go | 575 ++++++ .../directvolumemigration/vm_test.go | 1795 +++++++++++++++++ pkg/controller/migmigration/description.go | 10 +- pkg/controller/migmigration/dvm.go | 93 +- pkg/controller/migmigration/dvm_test.go | 133 ++ .../migmigration_controller_test.go | 3 +- pkg/controller/migmigration/quiesce.go | 88 +- pkg/controller/migmigration/quiesce_test.go | 194 +- pkg/controller/migmigration/rollback.go | 73 +- pkg/controller/migmigration/rollback_test.go | 166 ++ pkg/controller/migmigration/storage.go | 124 +- pkg/controller/migmigration/storage_test.go | 435 ++++ pkg/controller/migmigration/task.go | 360 ++-- pkg/controller/migmigration/task_test.go | 191 +- pkg/controller/migplan/validation.go | 152 +- pkg/controller/migplan/validation_test.go | 411 +++- 34 files changed, 7325 insertions(+), 740 deletions(-) create mode 100644 pkg/controller/directvolumemigration/vm.go create mode 100644 pkg/controller/directvolumemigration/vm_test.go create mode 100644 pkg/controller/migmigration/rollback_test.go create mode 100644 pkg/controller/migmigration/storage_test.go diff --git a/.github/workflows/pr-make.yml b/.github/workflows/pr-make.yml index fa4d2d7310..c81015c6fa 100644 --- a/.github/workflows/pr-make.yml +++ b/.github/workflows/pr-make.yml @@ -8,7 +8,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.21 + go-version: 1.22 - name: Setup dependencies run: | sudo apt-get update && sudo apt-get install -y libgpgme-dev libdevmapper-dev btrfs-progs libbtrfs-dev diff --git a/Makefile b/Makefile index b050d46467..eacadad425 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ ifeq (, $(shell which controller-gen)) CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ cd $$CONTROLLER_GEN_TMP_DIR ;\ go mod init tmp ;\ - go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.0 ;\ + go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0 ;\ rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ } CONTROLLER_GEN=$(GOBIN)/controller-gen diff --git a/config/crds/migration.openshift.io_directvolumemigrations.yaml b/config/crds/migration.openshift.io_directvolumemigrations.yaml index 57efcb6f43..1c302ef797 100644 --- a/config/crds/migration.openshift.io_directvolumemigrations.yaml +++ b/config/crds/migration.openshift.io_directvolumemigrations.yaml @@ -107,6 +107,13 @@ spec: type: string type: object x-kubernetes-map-type: atomic + liveMigrate: + description: Specifies if any volumes associated with a VM should + be live storage migrated instead of offline migrated + type: boolean + migrationType: + description: Specifies if this is the final DVM in the migration plan + type: string persistentVolumeClaims: description: Holds all the PVCs that are to be migrated with direct volume migration @@ -272,6 +279,86 @@ spec: items: type: string type: array + failedLiveMigration: + items: + properties: + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + message: + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + totalElapsedTime: + type: string + vmName: + type: string + vmNamespace: + type: string + type: object + type: array failedPods: items: properties: @@ -383,37 +470,14 @@ spec: type: string observedDigest: type: string - pendingPods: + pendingLiveMigration: items: properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string lastObservedProgressPercent: type: string lastObservedTransferRate: type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + message: type: string pvcRef: description: "ObjectReference contains enough information to @@ -478,36 +542,73 @@ spec: type: string type: object x-kubernetes-map-type: atomic - resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string totalElapsedTime: type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + vmName: + type: string + vmNamespace: type: string type: object - x-kubernetes-map-type: atomic type: array - phase: - type: string - phaseDescription: - type: string - rsyncOperations: + pendingPods: items: - description: RsyncOperation defines observed state of an Rsync Operation properties: - currentAttempt: - description: CurrentAttempt current ongoing attempt of an Rsync - operation - type: integer - failed: - description: Failed whether operation as a whole failed - type: boolean - pvcReference: - description: PVCReference pvc to which this Rsync operation - corresponds to + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." properties: apiVersion: description: API version of the referent. @@ -544,12 +645,19 @@ spec: type: string type: object x-kubernetes-map-type: atomic - succeeded: - description: Succeeded whether operation as a whole succeded - type: boolean + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + totalElapsedTime: + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string type: object + x-kubernetes-map-type: atomic type: array - runningPods: + pendingSinceTimeLimitPods: items: properties: apiVersion: @@ -656,10 +764,447 @@ spec: type: object x-kubernetes-map-type: atomic type: array - startTimestamp: - format: date-time + phase: type: string - successfulPods: + phaseDescription: + type: string + rsyncOperations: + items: + description: RsyncOperation defines observed state of an Rsync Operation + properties: + currentAttempt: + description: CurrentAttempt current ongoing attempt of an Rsync + operation + type: integer + failed: + description: Failed whether operation as a whole failed + type: boolean + pvcReference: + description: PVCReference pvc to which this Rsync operation + corresponds to + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + succeeded: + description: Succeeded whether operation as a whole succeded + type: boolean + type: object + type: array + runningLiveMigration: + items: + properties: + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + message: + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + totalElapsedTime: + type: string + vmName: + type: string + vmNamespace: + type: string + type: object + type: array + runningPods: + items: + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + totalElapsedTime: + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + skippedVolumes: + items: + type: string + type: array + startTimestamp: + format: date-time + type: string + successfulLiveMigration: + items: + properties: + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + message: + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + totalElapsedTime: + type: string + vmName: + type: string + vmNamespace: + type: string + type: object + type: array + successfulPods: + items: + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + totalElapsedTime: + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + unknownPods: items: properties: apiVersion: diff --git a/config/crds/migration.openshift.io_migplans.yaml b/config/crds/migration.openshift.io_migplans.yaml index 05194a2345..46ab17692b 100644 --- a/config/crds/migration.openshift.io_migplans.yaml +++ b/config/crds/migration.openshift.io_migplans.yaml @@ -275,6 +275,11 @@ spec: type: object type: object x-kubernetes-map-type: atomic + liveMigrate: + description: LiveMigrate optional flag to enable live migration of + VMs during direct volume migration Only running VMs when the plan + is executed will be live migrated + type: boolean migStorageRef: description: "ObjectReference contains enough information to let you inspect or modify the referred object. --- New uses of this type diff --git a/go.mod b/go.mod index 888e709845..8978e02990 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/konveyor/mig-controller -go 1.21 +go 1.22.0 + +toolchain go1.22.5 require ( cloud.google.com/go/storage v1.30.1 @@ -17,7 +19,7 @@ require ( github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.3.0 github.com/konveyor/controller v0.12.0 - github.com/konveyor/crane-lib v0.1.2 + github.com/konveyor/crane-lib v0.1.3 github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d github.com/mattn/go-sqlite3 v1.14.22 github.com/onsi/ginkgo v1.16.4 @@ -27,6 +29,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/common v0.53.0 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/vmware-tanzu/velero v1.7.1 go.uber.org/zap v1.27.0 @@ -36,7 +39,7 @@ require ( k8s.io/apimachinery v0.30.0 k8s.io/client-go v1.5.2 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 - kubevirt.io/api v1.2.0 + kubevirt.io/api v1.3.0 kubevirt.io/containerized-data-importer-api v1.59.0 sigs.k8s.io/controller-runtime v0.18.1 ) @@ -156,7 +159,6 @@ require ( github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/proglottis/gpgme v0.1.3 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.14.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect @@ -208,7 +210,7 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.4 // indirect + k8s.io/apiextensions-apiserver v0.30.0 // indirect k8s.io/component-base v0.30.0 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect diff --git a/go.sum b/go.sum index 37fe20f5a1..871ca4cca7 100644 --- a/go.sum +++ b/go.sum @@ -1542,6 +1542,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -1585,8 +1586,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konveyor/controller v0.12.0 h1:TYEGrb6TxegMqMH1ZPy/TMyOeecH+JHKQNqKvSUGWkM= github.com/konveyor/controller v0.12.0/go.mod h1:kGFv+5QxjuRo1wUO+bO/HGpULkdkR1pb1tEsVMPAz3s= -github.com/konveyor/crane-lib v0.1.2 h1:HSQd2u7UJB0vV8c75Gxd5KqDm9gevpT9bsB02P9MoZ4= -github.com/konveyor/crane-lib v0.1.2/go.mod h1:oSqLMhvUa3kNC/IaKXdGV/Tfs2bdwoZpYrbiV4KpVdY= +github.com/konveyor/crane-lib v0.1.3 h1:dlVe8uGfLhu5cs9GFFikXmTL1ysukHopv5dba54iBcM= +github.com/konveyor/crane-lib v0.1.3/go.mod h1:oSqLMhvUa3kNC/IaKXdGV/Tfs2bdwoZpYrbiV4KpVdY= github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d h1:tETgPq+JxXhVhnrcLc7rcW9BURax36VUsix8DAU0wuY= github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d/go.mod h1:Yk0xQ4N5rwE1+NWLSFQUQLgNT1/8p4Uor60LlgQfymg= github.com/konveyor/velero v0.10.2-0.20220124204642-f91d69bb9a5e h1:cjeGzY/zgPJCSLpC6Yurpn1li9ERrMtrF0nixFz2fmc= @@ -1719,6 +1720,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= @@ -3151,8 +3153,8 @@ k8s.io/api v0.29.4/go.mod h1:DetSv0t4FBTcEpfA84NJV3g9a7+rSzlUHk5ADAYHUv0= k8s.io/apiextensions-apiserver v0.17.1/go.mod h1:DRIFH5x3jalE4rE7JP0MQKby9zdYk9lUJQuMmp+M/L0= k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= -k8s.io/apiextensions-apiserver v0.29.4 h1:M7hbuHU/ckbibR7yPbe6DyNWgTFKNmZDbdZKD8q1Smk= -k8s.io/apiextensions-apiserver v0.29.4/go.mod h1:TTDC9fB+0kHY2rogf5hgBR03KBKCwED+GHUsXGpR7SM= +k8s.io/apiextensions-apiserver v0.30.0 h1:jcZFKMqnICJfRxTgnC4E+Hpcq8UEhT8B2lhBcQ+6uAs= +k8s.io/apiextensions-apiserver v0.30.0/go.mod h1:N9ogQFGcrbWqAY9p2mUAL5mGxsLqwgtUce127VtRX5Y= k8s.io/apimachinery v0.29.4 h1:RaFdJiDmuKs/8cm1M6Dh1Kvyh59YQFDcFuFTSmXes6Q= k8s.io/apimachinery v0.29.4/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= k8s.io/apiserver v0.17.1/go.mod h1:BQEUObJv8H6ZYO7DeKI5vb50tjk6paRJ4ZhSyJsiSco= @@ -3217,8 +3219,8 @@ k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.2.0 h1:1f8XQLPl4BuHPsc6SHTPnYSYeDxucKCQGa8CdrGJSRc= -kubevirt.io/api v1.2.0/go.mod h1:SbeR9ma4EwnaOZEUkh/lNz0kzYm5LPpEDE30vKXC5Zg= +kubevirt.io/api v1.3.0 h1:9sGElMmnRU50pGED+MPPD2OwQl4S5lvjCUjm+t0mI90= +kubevirt.io/api v1.3.0/go.mod h1:e6LkElYZZm8NcP2gKlFVHZS9pgNhIARHIjSBSfeiP1s= kubevirt.io/containerized-data-importer-api v1.59.0 h1:GdDt9BlR0qHejpMaPfASbsG8JWDmBf1s7xZBj5W9qn0= kubevirt.io/containerized-data-importer-api v1.59.0/go.mod h1:4yOGtCE7HvgKp7wftZZ3TBvDJ0x9d6N6KaRjRYcUFpE= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= diff --git a/pkg/apis/migration/v1alpha1/directvolumemigration_types.go b/pkg/apis/migration/v1alpha1/directvolumemigration_types.go index bfd8c2348f..4bf7115a61 100644 --- a/pkg/apis/migration/v1alpha1/directvolumemigration_types.go +++ b/pkg/apis/migration/v1alpha1/directvolumemigration_types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( "fmt" + "slices" kapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -65,22 +66,43 @@ type DirectVolumeMigrationSpec struct { // Specifies if progress reporting CRs needs to be deleted or not DeleteProgressReportingCRs bool `json:"deleteProgressReportingCRs,omitempty"` + + // Specifies if any volumes associated with a VM should be live storage migrated instead of offline migrated + LiveMigrate *bool `json:"liveMigrate,omitempty"` + + // Specifies if this is the final DVM in the migration plan + MigrationType *DirectVolumeMigrationType `json:"migrationType,omitempty"` } +type DirectVolumeMigrationType string + +const ( + MigrationTypeStage DirectVolumeMigrationType = "Stage" + MigrationTypeFinal DirectVolumeMigrationType = "CutOver" + MigrationTypeRollback DirectVolumeMigrationType = "Rollback" +) + // DirectVolumeMigrationStatus defines the observed state of DirectVolumeMigration type DirectVolumeMigrationStatus struct { - Conditions `json:","` - ObservedDigest string `json:"observedDigest"` - StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` - PhaseDescription string `json:"phaseDescription"` - Phase string `json:"phase,omitempty"` - Itinerary string `json:"itinerary,omitempty"` - Errors []string `json:"errors,omitempty"` - SuccessfulPods []*PodProgress `json:"successfulPods,omitempty"` - FailedPods []*PodProgress `json:"failedPods,omitempty"` - RunningPods []*PodProgress `json:"runningPods,omitempty"` - PendingPods []*PodProgress `json:"pendingPods,omitempty"` - RsyncOperations []*RsyncOperation `json:"rsyncOperations,omitempty"` + Conditions `json:","` + ObservedDigest string `json:"observedDigest"` + StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` + PhaseDescription string `json:"phaseDescription"` + Phase string `json:"phase,omitempty"` + Itinerary string `json:"itinerary,omitempty"` + Errors []string `json:"errors,omitempty"` + SuccessfulPods []*PodProgress `json:"successfulPods,omitempty"` + FailedPods []*PodProgress `json:"failedPods,omitempty"` + RunningPods []*PodProgress `json:"runningPods,omitempty"` + PendingPods []*PodProgress `json:"pendingPods,omitempty"` + UnknownPods []*PodProgress `json:"unknownPods,omitempty"` + PendingSinceTimeLimitPods []*PodProgress `json:"pendingSinceTimeLimitPods,omitempty"` + SuccessfulLiveMigrations []*LiveMigrationProgress `json:"successfulLiveMigration,omitempty"` + RunningLiveMigrations []*LiveMigrationProgress `json:"runningLiveMigration,omitempty"` + PendingLiveMigrations []*LiveMigrationProgress `json:"pendingLiveMigration,omitempty"` + FailedLiveMigrations []*LiveMigrationProgress `json:"failedLiveMigration,omitempty"` + RsyncOperations []*RsyncOperation `json:"rsyncOperations,omitempty"` + SkippedVolumes []string `json:"skippedVolumes,omitempty"` } // GetRsyncOperationStatusForPVC returns RsyncOperation from status for matching PVC, creates new one if doesn't exist already @@ -117,6 +139,42 @@ func (ds *DirectVolumeMigrationStatus) AddRsyncOperation(podStatus *RsyncOperati ds.RsyncOperations = append(ds.RsyncOperations, podStatus) } +func (dvm *DirectVolumeMigration) IsCompleted() bool { + return len(dvm.Status.SuccessfulPods)+ + len(dvm.Status.FailedPods)+ + len(dvm.Status.SkippedVolumes)+ + len(dvm.Status.SuccessfulLiveMigrations)+ + len(dvm.Status.FailedLiveMigrations) == len(dvm.Spec.PersistentVolumeClaims) +} + +func (dvm *DirectVolumeMigration) IsCutover() bool { + return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeFinal +} + +func (dvm *DirectVolumeMigration) IsRollback() bool { + return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeRollback +} + +func (dvm *DirectVolumeMigration) IsStage() bool { + return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeStage +} + +func (dvm *DirectVolumeMigration) SkipVolume(volumeName, namespace string) { + dvm.Status.SkippedVolumes = append(dvm.Status.SkippedVolumes, fmt.Sprintf("%s/%s", namespace, volumeName)) +} + +func (dvm *DirectVolumeMigration) IsLiveMigrate() bool { + return dvm.Spec.LiveMigrate != nil && *dvm.Spec.LiveMigrate +} + +func (dvm *DirectVolumeMigration) AllReportingCompleted() bool { + isCompleted := dvm.IsCompleted() + isAnyPending := len(dvm.Status.PendingPods) > 0 || len(dvm.Status.PendingLiveMigrations) > 0 + isAnyRunning := len(dvm.Status.RunningPods) > 0 || len(dvm.Status.RunningLiveMigrations) > 0 + isAnyUnknown := len(dvm.Status.UnknownPods) > 0 + return !isAnyRunning && !isAnyPending && !isAnyUnknown && isCompleted +} + // TODO: Explore how to reliably get stunnel+rsync logs/status reported back to // DirectVolumeMigrationStatus @@ -151,6 +209,16 @@ type PodProgress struct { TotalElapsedTime *metav1.Duration `json:"totalElapsedTime,omitempty"` } +type LiveMigrationProgress struct { + VMName string `json:"vmName,omitempty"` + VMNamespace string `json:"vmNamespace,omitempty"` + PVCReference *kapi.ObjectReference `json:"pvcRef,omitempty"` + LastObservedProgressPercent string `json:"lastObservedProgressPercent,omitempty"` + LastObservedTransferRate string `json:"lastObservedTransferRate,omitempty"` + TotalElapsedTime *metav1.Duration `json:"totalElapsedTime,omitempty"` + Message string `json:"message,omitempty"` +} + // RsyncOperation defines observed state of an Rsync Operation type RsyncOperation struct { // PVCReference pvc to which this Rsync operation corresponds to @@ -205,6 +273,26 @@ func (r *DirectVolumeMigration) GetMigrationForDVM(client k8sclient.Client) (*Mi return GetMigrationForDVM(client, r.OwnerReferences) } +func (r *DirectVolumeMigration) GetSourceNamespaces() []string { + namespaces := []string{} + for _, pvc := range r.Spec.PersistentVolumeClaims { + if pvc.Namespace != "" && !slices.Contains(namespaces, pvc.Namespace) { + namespaces = append(namespaces, pvc.Namespace) + } + } + return namespaces +} + +func (r *DirectVolumeMigration) GetDestinationNamespaces() []string { + namespaces := []string{} + for _, pvc := range r.Spec.PersistentVolumeClaims { + if pvc.TargetNamespace != "" && !slices.Contains(namespaces, pvc.TargetNamespace) { + namespaces = append(namespaces, pvc.TargetNamespace) + } + } + return namespaces +} + // Add (de-duplicated) errors. func (r *DirectVolumeMigration) AddErrors(errors []string) { m := map[string]bool{} diff --git a/pkg/apis/migration/v1alpha1/migcluster_types.go b/pkg/apis/migration/v1alpha1/migcluster_types.go index 31cbb6966d..d4492f2c1a 100644 --- a/pkg/apis/migration/v1alpha1/migcluster_types.go +++ b/pkg/apis/migration/v1alpha1/migcluster_types.go @@ -804,6 +804,13 @@ var accessModeList = []provisionerAccessModes{ kapi.PersistentVolumeBlock: {kapi.ReadWriteOnce, kapi.ReadOnlyMany, kapi.ReadWriteMany}, }, }, + provisionerAccessModes{ + Provisioner: "csi.kubevirt.io", + AccessModes: map[kapi.PersistentVolumeMode][]kapi.PersistentVolumeAccessMode{ + kapi.PersistentVolumeFilesystem: {kapi.ReadWriteOnce}, + kapi.PersistentVolumeBlock: {kapi.ReadWriteOnce}, + }, + }, } // Get the list of k8s StorageClasses from the cluster. diff --git a/pkg/apis/migration/v1alpha1/migplan_types.go b/pkg/apis/migration/v1alpha1/migplan_types.go index 4c4f18616e..cc0b46148f 100644 --- a/pkg/apis/migration/v1alpha1/migplan_types.go +++ b/pkg/apis/migration/v1alpha1/migplan_types.go @@ -115,6 +115,11 @@ type MigPlanSpec struct { // LabelSelector optional label selector on the included resources in Velero Backup // +kubebuilder:validation:Optional LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` + + // LiveMigrate optional flag to enable live migration of VMs during direct volume migration + // Only running VMs when the plan is executed will be live migrated + // +kubebuilder:validation:Optional + LiveMigrate *bool `json:"liveMigrate,omitempty"` } // MigPlanStatus defines the observed state of MigPlan @@ -218,6 +223,10 @@ type PlanResources struct { DestMigCluster *MigCluster } +func (r *MigPlan) LiveMigrationChecked() bool { + return r.Spec.LiveMigrate != nil && *r.Spec.LiveMigrate +} + // GetRefResources gets referenced resources from a MigPlan. func (r *MigPlan) GetRefResources(client k8sclient.Client) (*PlanResources, error) { isIntraCluster, err := r.IsIntraCluster(client) diff --git a/pkg/apis/migration/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/migration/v1alpha1/zz_generated.deepcopy.go index 164fd13781..2f571abf1c 100644 --- a/pkg/apis/migration/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/migration/v1alpha1/zz_generated.deepcopy.go @@ -570,6 +570,16 @@ func (in *DirectVolumeMigrationSpec) DeepCopyInto(out *DirectVolumeMigrationSpec (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.LiveMigrate != nil { + in, out := &in.LiveMigrate, &out.LiveMigrate + *out = new(bool) + **out = **in + } + if in.MigrationType != nil { + in, out := &in.MigrationType, &out.MigrationType + *out = new(DirectVolumeMigrationType) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DirectVolumeMigrationSpec. @@ -639,6 +649,72 @@ func (in *DirectVolumeMigrationStatus) DeepCopyInto(out *DirectVolumeMigrationSt } } } + if in.UnknownPods != nil { + in, out := &in.UnknownPods, &out.UnknownPods + *out = make([]*PodProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(PodProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.PendingSinceTimeLimitPods != nil { + in, out := &in.PendingSinceTimeLimitPods, &out.PendingSinceTimeLimitPods + *out = make([]*PodProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(PodProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.SuccessfulLiveMigrations != nil { + in, out := &in.SuccessfulLiveMigrations, &out.SuccessfulLiveMigrations + *out = make([]*LiveMigrationProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LiveMigrationProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.RunningLiveMigrations != nil { + in, out := &in.RunningLiveMigrations, &out.RunningLiveMigrations + *out = make([]*LiveMigrationProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LiveMigrationProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.PendingLiveMigrations != nil { + in, out := &in.PendingLiveMigrations, &out.PendingLiveMigrations + *out = make([]*LiveMigrationProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LiveMigrationProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.FailedLiveMigrations != nil { + in, out := &in.FailedLiveMigrations, &out.FailedLiveMigrations + *out = make([]*LiveMigrationProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LiveMigrationProgress) + (*in).DeepCopyInto(*out) + } + } + } if in.RsyncOperations != nil { in, out := &in.RsyncOperations, &out.RsyncOperations *out = make([]*RsyncOperation, len(*in)) @@ -650,6 +726,11 @@ func (in *DirectVolumeMigrationStatus) DeepCopyInto(out *DirectVolumeMigrationSt } } } + if in.SkippedVolumes != nil { + in, out := &in.SkippedVolumes, &out.SkippedVolumes + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DirectVolumeMigrationStatus. @@ -749,6 +830,31 @@ func (in *IncompatibleNamespace) DeepCopy() *IncompatibleNamespace { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LiveMigrationProgress) DeepCopyInto(out *LiveMigrationProgress) { + *out = *in + if in.PVCReference != nil { + in, out := &in.PVCReference, &out.PVCReference + *out = new(v1.ObjectReference) + **out = **in + } + if in.TotalElapsedTime != nil { + in, out := &in.TotalElapsedTime, &out.TotalElapsedTime + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LiveMigrationProgress. +func (in *LiveMigrationProgress) DeepCopy() *LiveMigrationProgress { + if in == nil { + return nil + } + out := new(LiveMigrationProgress) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MigAnalytic) DeepCopyInto(out *MigAnalytic) { *out = *in @@ -1416,6 +1522,11 @@ func (in *MigPlanSpec) DeepCopyInto(out *MigPlanSpec) { *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } + if in.LiveMigrate != nil { + in, out := &in.LiveMigrate, &out.LiveMigrate + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MigPlanSpec. diff --git a/pkg/compat/fake/client.go b/pkg/compat/fake/client.go index 05afc3e031..8307501e8c 100644 --- a/pkg/compat/fake/client.go +++ b/pkg/compat/fake/client.go @@ -15,6 +15,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -106,5 +107,8 @@ func getSchemeForFakeClient() (*runtime.Scheme, error) { if err := virtv1.AddToScheme(scheme.Scheme); err != nil { return nil, err } + if err := cdiv1.AddToScheme(scheme.Scheme); err != nil { + return nil, err + } return scheme.Scheme, nil } diff --git a/pkg/controller/directvolumemigration/directvolumemigration_controller.go b/pkg/controller/directvolumemigration/directvolumemigration_controller.go index f9e0bd91bd..7499b47043 100644 --- a/pkg/controller/directvolumemigration/directvolumemigration_controller.go +++ b/pkg/controller/directvolumemigration/directvolumemigration_controller.go @@ -22,10 +22,15 @@ import ( "github.com/konveyor/controller/pkg/logging" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/compat" migref "github.com/konveyor/mig-controller/pkg/reference" "github.com/opentracing/opentracing-go" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -34,6 +39,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) +const ( + dvmFinalizer = "migration.openshift.io/directvolumemigrationfinalizer" +) + var ( sink = logging.WithName("directvolume") log = sink.Real @@ -47,7 +56,7 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileDirectVolumeMigration{Client: mgr.GetClient(), scheme: mgr.GetScheme()} + return &ReconcileDirectVolumeMigration{Config: mgr.GetConfig(), Client: mgr.GetClient(), scheme: mgr.GetScheme()} } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -86,6 +95,7 @@ var _ reconcile.Reconciler = &ReconcileDirectVolumeMigration{} // ReconcileDirectVolumeMigration reconciles a DirectVolumeMigration object type ReconcileDirectVolumeMigration struct { + *rest.Config client.Client scheme *runtime.Scheme tracer opentracing.Tracer @@ -104,6 +114,8 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request // Set values tracer := logging.WithName("directvolume", "dvm", request.Name) log := tracer.Real + // Default to PollReQ, can be overridden by r.migrate phase-specific ReQ interval + requeueAfter := time.Duration(PollReQ) // Fetch the DirectVolumeMigration instance direct := &migapi.DirectVolumeMigration{} @@ -120,6 +132,9 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request // Set MigMigration name key on logger migration, err := direct.GetMigrationForDVM(r) + if err != nil { + return reconcile.Result{}, err + } if migration != nil { log = log.WithValues("migMigration", migration.Name) } @@ -131,8 +146,32 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request defer reconcileSpan.Finish() } - // Check if completed - if direct.Status.Phase == Completed { + if direct.DeletionTimestamp != nil { + sourceDeleted, err := r.cleanupSourceResourcesInNamespace(direct) + if err != nil { + return reconcile.Result{}, err + } + targetDeleted, err := r.cleanupTargetResourcesInNamespaces(direct) + if err != nil { + return reconcile.Result{}, err + } + if sourceDeleted && targetDeleted { + log.V(5).Info("DirectVolumeMigration resources deleted. removing finalizer") + // Remove finalizer + RemoveFinalizer(direct, dvmFinalizer) + } else { + // Requeue + log.V(5).Info("Requeing waiting for cleanup", "after", requeueAfter) + return reconcile.Result{RequeueAfter: requeueAfter}, nil + } + } else { + // Add finalizer + AddFinalizer(direct, dvmFinalizer) + } + + // Check if completed and return if not deleted, otherwise need to update to + // remove the finalizer. + if direct.Status.Phase == Completed && direct.DeletionTimestamp == nil { return reconcile.Result{Requeue: false}, nil } @@ -146,10 +185,7 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request return reconcile.Result{Requeue: true}, nil } - // Default to PollReQ, can be overridden by r.migrate phase-specific ReQ interval - requeueAfter := time.Duration(PollReQ) - - if !direct.Status.HasBlockerCondition() { + if !direct.Status.HasBlockerCondition() && direct.DeletionTimestamp == nil { requeueAfter, err = r.migrate(ctx, direct) if err != nil { tracer.Trace(err) @@ -182,3 +218,140 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request // Done return reconcile.Result{Requeue: false}, nil } + +// AddFinalizer adds a finalizer to a resource +func AddFinalizer(obj metav1.Object, name string) { + if HasFinalizer(obj, name) { + return + } + + obj.SetFinalizers(append(obj.GetFinalizers(), name)) +} + +// RemoveFinalizer removes a finalizer from a resource +func RemoveFinalizer(obj metav1.Object, name string) { + if !HasFinalizer(obj, name) { + return + } + + var finalizers []string + for _, f := range obj.GetFinalizers() { + if f != name { + finalizers = append(finalizers, f) + } + } + + obj.SetFinalizers(finalizers) +} + +// HasFinalizer returns true if a resource has a specific finalizer +func HasFinalizer(object metav1.Object, value string) bool { + for _, f := range object.GetFinalizers() { + if f == value { + return true + } + } + return false +} + +func (r *ReconcileDirectVolumeMigration) cleanupSourceResourcesInNamespace(direct *migapi.DirectVolumeMigration) (bool, error) { + sourceCluster, err := direct.GetSourceCluster(r) + if err != nil { + return false, err + } + + // Cleanup source resources + client, err := sourceCluster.GetClient(r) + if err != nil { + return false, err + } + + liveMigrationCanceled, err := r.cancelLiveMigrations(client, direct) + if err != nil { + return false, err + } + + completed, err := r.cleanupResourcesInNamespaces(client, direct.GetUID(), direct.GetSourceNamespaces()) + return completed && liveMigrationCanceled, err +} + +func (r *ReconcileDirectVolumeMigration) cancelLiveMigrations(client compat.Client, direct *migapi.DirectVolumeMigration) (bool, error) { + if direct.IsCutover() && direct.IsLiveMigrate() { + // Cutover and live migration is enabled, attempt to cancel migrations in progress. + namespaces := direct.GetSourceNamespaces() + allLiveMigrationCompleted := true + for _, ns := range namespaces { + volumeVmMap, err := getRunningVmVolumeMap(client, ns) + if err != nil { + return false, err + } + vmVolumeMap := make(map[string]*vmVolumes) + for i := range direct.Spec.PersistentVolumeClaims { + sourceName := direct.Spec.PersistentVolumeClaims[i].Name + targetName := direct.Spec.PersistentVolumeClaims[i].TargetName + vmName, found := volumeVmMap[sourceName] + if !found { + continue + } + volumes := vmVolumeMap[vmName] + if volumes == nil { + volumes = &vmVolumes{ + sourceVolumes: []string{sourceName}, + targetVolumes: []string{targetName}, + } + } else { + volumes.sourceVolumes = append(volumes.sourceVolumes, sourceName) + volumes.targetVolumes = append(volumes.targetVolumes, targetName) + } + vmVolumeMap[vmName] = volumes + } + vmNames := make([]string, 0) + for vmName, volumes := range vmVolumeMap { + if err := cancelLiveMigration(client, vmName, ns, volumes, log); err != nil { + return false, err + } + vmNames = append(vmNames, vmName) + } + migrated, err := liveMigrationsCompleted(client, ns, vmNames) + if err != nil { + return false, err + } + allLiveMigrationCompleted = allLiveMigrationCompleted && migrated + } + return allLiveMigrationCompleted, nil + } + return true, nil +} + +func (r *ReconcileDirectVolumeMigration) cleanupTargetResourcesInNamespaces(direct *migapi.DirectVolumeMigration) (bool, error) { + destinationCluster, err := direct.GetDestinationCluster(r) + if err != nil { + return false, err + } + + // Cleanup source resources + client, err := destinationCluster.GetClient(r) + if err != nil { + return false, err + } + return r.cleanupResourcesInNamespaces(client, direct.GetUID(), direct.GetDestinationNamespaces()) +} + +func (r *ReconcileDirectVolumeMigration) cleanupResourcesInNamespaces(client compat.Client, uid types.UID, namespaces []string) (bool, error) { + selector := labels.SelectorFromSet(map[string]string{ + "directvolumemigration": string(uid), + }) + + sourceDeleted := true + for _, ns := range namespaces { + if err := findAndDeleteNsResources(client, ns, selector, log); err != nil { + return false, err + } + err, deleted := areRsyncNsResourcesDeleted(client, ns, selector, log) + if err != nil { + return false, err + } + sourceDeleted = sourceDeleted && deleted + } + return sourceDeleted, nil +} diff --git a/pkg/controller/directvolumemigration/directvolumemigration_controller_test.go b/pkg/controller/directvolumemigration/directvolumemigration_controller_test.go index 39f9b6f6af..912e30b916 100644 --- a/pkg/controller/directvolumemigration/directvolumemigration_controller_test.go +++ b/pkg/controller/directvolumemigration/directvolumemigration_controller_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" migrationv1alpha1 "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" "github.com/onsi/gomega" "golang.org/x/net/context" @@ -27,6 +28,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -82,3 +85,208 @@ func TestReconcile(t *testing.T) { defer c.Delete(context.TODO(), instance) g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) } + +func TestCleanupTargetResourcesInNamespaces(t *testing.T) { + g := gomega.NewGomegaWithT(t) + reconciler := &ReconcileDirectVolumeMigration{} + instance := &migrationv1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: migrationv1alpha1.OpenshiftMigrationNamespace, + UID: "1234", + }, + } + client := getFakeClientWithObjs( + createDVMPod("namespace1", string(instance.GetUID())), + createDVMSecret("namespace1", string(instance.GetUID())), + createDVMSecret("namespace2", string(instance.GetUID())), + ) + + _, err := reconciler.cleanupResourcesInNamespaces(client, instance.GetUID(), []string{"namespace1", "namespace2"}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // ensure the pod is gone + pod := &kapi.Pod{} + err = client.Get(context.TODO(), types.NamespacedName{Name: "dvm-pod", Namespace: "namespace1"}, pod) + g.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue()) + + // ensure the secrets are gone + secret := &kapi.Secret{} + err = client.Get(context.TODO(), types.NamespacedName{Name: "dvm-secret", Namespace: "namespace1"}, secret) + g.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue()) + secret = &kapi.Secret{} + err = client.Get(context.TODO(), types.NamespacedName{Name: "dvm-secret", Namespace: "namespace2"}, secret) + g.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue()) +} + +func TestCancelLiveMigrationsStage(t *testing.T) { + g := gomega.NewGomegaWithT(t) + reconciler := &ReconcileDirectVolumeMigration{} + instance := &migrationv1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: migrationv1alpha1.OpenshiftMigrationNamespace, + UID: "1234", + }, + Spec: migrationv1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeStage), + }, + } + client := getFakeClientWithObjs() + allCompleted, err := reconciler.cancelLiveMigrations(client, instance) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(allCompleted).To(gomega.BeTrue()) +} + +func TestCancelLiveMigrationsCutoverAllCompleted(t *testing.T) { + g := gomega.NewGomegaWithT(t) + reconciler := &ReconcileDirectVolumeMigration{} + instance := &migrationv1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: migrationv1alpha1.OpenshiftMigrationNamespace, + UID: "1234", + }, + Spec: migrationv1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeFinal), + LiveMigrate: ptr.To[bool](true), + PersistentVolumeClaims: []migapi.PVCToMigrate{ + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc1", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc1", + }, + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc2", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc2", + }, + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc3", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc3", + }, + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc4", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc4", + }, + }, + }, + } + client := getFakeClientWithObjs( + createVirtualMachineWithVolumes("vm1", testNamespace, []virtv1.Volume{ + { + Name: "pvc1", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "pvc1", + }, + }, + }, + }), + createVirtlauncherPod("vm1", testNamespace, []string{"pvc1"}), + createVirtualMachineWithVolumes("vm2", testNamespace, []virtv1.Volume{ + { + Name: "pvc2", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "pvc2", + }, + }, + }, + { + Name: "pvc3", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "pvc3", + }, + }, + }, + }), + createVirtlauncherPod("vm2", testNamespace, []string{"pvc2", "pvc3"}), + createVirtualMachine("vm3", testNamespace), + ) + allCompleted, err := reconciler.cancelLiveMigrations(client, instance) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(allCompleted).To(gomega.BeTrue()) +} + +func TestCancelLiveMigrationsCutoverNotCompleted(t *testing.T) { + g := gomega.NewGomegaWithT(t) + reconciler := &ReconcileDirectVolumeMigration{} + instance := &migrationv1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: migrationv1alpha1.OpenshiftMigrationNamespace, + UID: "1234", + }, + Spec: migrationv1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeFinal), + LiveMigrate: ptr.To[bool](true), + PersistentVolumeClaims: []migapi.PVCToMigrate{ + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc1", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc1", + }, + }, + }, + } + client := getFakeClientWithObjs( + createVirtualMachineWithVolumes("vm1", testNamespace, []virtv1.Volume{ + { + Name: "pvc1", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "pvc1", + }, + }, + }, + }), + createVirtlauncherPod("vm1", testNamespace, []string{"pvc1"}), + createInProgressVirtualMachineMigration("vmim", testNamespace, "vm1"), + ) + allCompleted, err := reconciler.cancelLiveMigrations(client, instance) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(allCompleted).To(gomega.BeFalse()) +} + +func createDVMPod(namespace, uid string) *kapi.Pod { + return &kapi.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dvm-pod", + Namespace: namespace, + Labels: map[string]string{ + "directvolumemigration": uid, + }, + }, + } +} + +func createDVMSecret(namespace, uid string) *kapi.Secret { + return &kapi.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dvm-secret", + Namespace: namespace, + Labels: map[string]string{ + "directvolumemigration": uid, + }, + }, + } +} diff --git a/pkg/controller/directvolumemigration/pvcs_test.go b/pkg/controller/directvolumemigration/pvcs_test.go index 05735b63c3..e001720bf6 100644 --- a/pkg/controller/directvolumemigration/pvcs_test.go +++ b/pkg/controller/directvolumemigration/pvcs_test.go @@ -25,7 +25,7 @@ const ( func Test_CreateDestinationPVCsNewPVCMigrationOwner(t *testing.T) { RegisterTestingT(t) migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) client := getFakeCompatClient(&migapi.MigCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster", @@ -68,7 +68,7 @@ func Test_CreateDestinationPVCsNewPVCMigrationOwner(t *testing.T) { func Test_CreateDestinationPVCsNewPVCDVMOwnerOtherNS(t *testing.T) { RegisterTestingT(t) migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) client := getFakeCompatClient(&migapi.MigCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster", @@ -110,7 +110,7 @@ func Test_CreateDestinationPVCsExpandPVSize(t *testing.T) { settings.Settings.DvmOpts.EnablePVResizing = true defer func() { settings.Settings.DvmOpts.EnablePVResizing = false }() migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) migPlan.Spec.PersistentVolumes.List = append(migPlan.Spec.PersistentVolumes.List, migapi.PV{ PVC: migapi.PVC{ Namespace: testNamespace, @@ -154,7 +154,7 @@ func Test_CreateDestinationPVCsExpandProposedPVSize(t *testing.T) { settings.Settings.DvmOpts.EnablePVResizing = true defer func() { settings.Settings.DvmOpts.EnablePVResizing = false }() migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) migPlan.Spec.PersistentVolumes.List = append(migPlan.Spec.PersistentVolumes.List, migapi.PV{ PVC: migapi.PVC{ Namespace: testNamespace, @@ -197,7 +197,7 @@ func Test_CreateDestinationPVCsExpandProposedPVSize(t *testing.T) { func Test_CreateDestinationPVCsExistingTargetPVC(t *testing.T) { RegisterTestingT(t) migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) client := getFakeCompatClient(&migapi.MigCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster", @@ -246,7 +246,7 @@ func createMigPlan() *migapi.MigPlan { } } -func createSourcePvc(name, namespace string) *kapi.PersistentVolumeClaim { +func createPvc(name, namespace string) *kapi.PersistentVolumeClaim { return &kapi.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, diff --git a/pkg/controller/directvolumemigration/rsync.go b/pkg/controller/directvolumemigration/rsync.go index 66762aa886..59cc967647 100644 --- a/pkg/controller/directvolumemigration/rsync.go +++ b/pkg/controller/directvolumemigration/rsync.go @@ -10,10 +10,12 @@ import ( "path" "reflect" "regexp" + "slices" "strconv" "strings" "time" + "github.com/go-logr/logr" liberr "github.com/konveyor/controller/pkg/error" "github.com/konveyor/crane-lib/state_transfer/endpoint" routeendpoint "github.com/konveyor/crane-lib/state_transfer/endpoint/route" @@ -36,6 +38,8 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/set" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -63,11 +67,6 @@ const ( // ensureRsyncEndpoints ensures that new Endpoints are created for Rsync and Blockrsync Transfers func (t *Task) ensureRsyncEndpoints() error { - destClient, err := t.getDestinationClient() - if err != nil { - return liberr.Wrap(err) - } - dvmLabels := t.buildDVMLabels() dvmLabels["purpose"] = DirectVolumeMigrationRsync blockdvmLabels := t.buildDVMLabels() @@ -76,7 +75,8 @@ func (t *Task) ensureRsyncEndpoints() error { hostnames := []string{} if t.EndpointType == migapi.NodePort { - hostnames, err = getWorkerNodeHostnames(destClient) + var err error + hostnames, err = getWorkerNodeHostnames(t.destinationClient) if err != nil { return liberr.Wrap(err) } @@ -120,7 +120,7 @@ func (t *Task) ensureRsyncEndpoints() error { if err != nil { t.Log.Info("failed to get cluster_subdomain, attempting to get cluster's ingress domain", "error", err) ingressConfig := &configv1.Ingress{} - err = destClient.Get(context.TODO(), types.NamespacedName{Name: "cluster"}, ingressConfig) + err = t.destinationClient.Get(context.TODO(), types.NamespacedName{Name: "cluster"}, ingressConfig) if err != nil { t.Log.Error(err, "failed to retrieve cluster's ingress domain, extremely long namespace names will cause route creation failure") } else { @@ -148,12 +148,10 @@ func (t *Task) ensureRsyncEndpoints() error { ) } - err = endpoint.Create(destClient) - if err != nil { + if err := endpoint.Create(t.destinationClient); err != nil { return liberr.Wrap(err) } - err = blockEndpoint.Create(destClient) - if err != nil { + if err := blockEndpoint.Create(t.destinationClient); err != nil { return liberr.Wrap(err) } } @@ -250,6 +248,9 @@ func (t *Task) getRsyncClientMutations(srcClient compat.Client, destClient compa if err != nil { return nil, liberr.Wrap(err) } + if migration == nil { + return transferOptions, nil + } containerMutation.SecurityContext, err = t.getSecurityContext(srcClient, namespace, migration) if err != nil { @@ -281,6 +282,9 @@ func (t *Task) getRsyncTransferServerMutations(client compat.Client, namespace s if err != nil { return nil, liberr.Wrap(err) } + if migration == nil { + return transferOptions, nil + } containerMutation.SecurityContext, err = t.getSecurityContext(client, namespace, migration) if err != nil { @@ -402,22 +406,12 @@ func (t *Task) getSecurityContext(client compat.Client, namespace string, migrat // ensureRsyncTransferServer ensures that server component of the Transfer is created func (t *Task) ensureRsyncTransferServer() error { - destClient, err := t.getDestinationClient() - if err != nil { - return liberr.Wrap(err) - } - - srcClient, err := t.getSourceClient() - if err != nil { - return liberr.Wrap(err) - } - nsMap, err := t.getNamespacedPVCPairs() if err != nil { return liberr.Wrap(err) } - err = t.buildDestinationLimitRangeMap(nsMap, destClient) + err = t.buildDestinationLimitRangeMap(nsMap, t.destinationClient) if err != nil { return liberr.Wrap(err) } @@ -427,16 +421,16 @@ func (t *Task) ensureRsyncTransferServer() error { return liberr.Wrap(err) } - if err := t.ensureFilesystemRsyncTransferServer(srcClient, destClient, nsMap, transportOptions); err != nil { + if err := t.ensureFilesystemRsyncTransferServer(nsMap, transportOptions); err != nil { return err } - if err := t.ensureBlockRsyncTransferServer(srcClient, destClient, nsMap, transportOptions); err != nil { + if err := t.ensureBlockRsyncTransferServer(nsMap, transportOptions); err != nil { return err } return nil } -func (t *Task) ensureFilesystemRsyncTransferServer(srcClient, destClient compat.Client, nsMap map[string][]transfer.PVCPair, transportOptions *transport.Options) error { +func (t *Task) ensureFilesystemRsyncTransferServer(nsMap map[string][]transfer.PVCPair, transportOptions *transport.Options) error { for bothNs, pvcPairs := range nsMap { srcNs := getSourceNs(bothNs) destNs := getDestNs(bothNs) @@ -444,12 +438,12 @@ func (t *Task) ensureFilesystemRsyncTransferServer(srcClient, destClient compat. types.NamespacedName{Name: DirectVolumeMigrationRsyncTransfer, Namespace: srcNs}, types.NamespacedName{Name: DirectVolumeMigrationRsyncTransfer, Namespace: destNs}, ) - endpoints, err := t.getEndpoints(destClient, destNs) + endpoints, err := t.getEndpoints(t.destinationClient, destNs) if err != nil { return liberr.Wrap(err) } stunnelTransport, err := stunneltransport.GetTransportFromKubeObjects( - srcClient, destClient, "fs", nnPair, endpoints[0], transportOptions) + t.sourceClient, t.destinationClient, "fs", nnPair, endpoints[0], transportOptions) if err != nil { return liberr.Wrap(err) } @@ -464,21 +458,21 @@ func (t *Task) ensureFilesystemRsyncTransferServer(srcClient, destClient compat. if err != nil { return liberr.Wrap(err) } - mutations, err := t.getRsyncTransferServerMutations(destClient, destNs) + mutations, err := t.getRsyncTransferServerMutations(t.destinationClient, destNs) if err != nil { return liberr.Wrap(err) } rsyncOptions = append(rsyncOptions, mutations...) rsyncOptions = append(rsyncOptions, rsynctransfer.WithDestinationPodLabels(labels)) transfer, err := rsynctransfer.NewTransfer( - stunnelTransport, endpoints[0], srcClient, destClient, filesystemPvcList, t.Log, rsyncOptions...) + stunnelTransport, endpoints[0], t.sourceClient, t.destinationClient, filesystemPvcList, t.Log, rsyncOptions...) if err != nil { return liberr.Wrap(err) } if transfer == nil { return fmt.Errorf("transfer %s/%s not found", nnPair.Source().Namespace, nnPair.Source().Name) } - err = transfer.CreateServer(destClient) + err = transfer.CreateServer(t.destinationClient) if err != nil { return liberr.Wrap(err) } @@ -487,7 +481,7 @@ func (t *Task) ensureFilesystemRsyncTransferServer(srcClient, destClient compat. return nil } -func (t *Task) ensureBlockRsyncTransferServer(srcClient, destClient compat.Client, nsMap map[string][]transfer.PVCPair, transportOptions *transport.Options) error { +func (t *Task) ensureBlockRsyncTransferServer(nsMap map[string][]transfer.PVCPair, transportOptions *transport.Options) error { for bothNs, pvcPairs := range nsMap { srcNs := getSourceNs(bothNs) destNs := getDestNs(bothNs) @@ -495,19 +489,26 @@ func (t *Task) ensureBlockRsyncTransferServer(srcClient, destClient compat.Clien types.NamespacedName{Name: DirectVolumeMigrationRsyncTransfer, Namespace: srcNs}, types.NamespacedName{Name: DirectVolumeMigrationRsyncTransfer, Namespace: destNs}, ) - endpoints, err := t.getEndpoints(destClient, destNs) + endpoints, err := t.getEndpoints(t.destinationClient, destNs) if err != nil { return liberr.Wrap(err) } stunnelTransports, err := stunneltransport.GetTransportFromKubeObjects( - srcClient, destClient, "block", nnPair, endpoints[1], transportOptions) + t.sourceClient, t.destinationClient, "block", nnPair, endpoints[1], transportOptions) if err != nil { return liberr.Wrap(err) } + blockOrVMPvcList, err := transfer.NewBlockOrVMDiskPVCPairList(pvcPairs...) if err != nil { return liberr.Wrap(err) } + if t.PlanResources.MigPlan.LiveMigrationChecked() { + blockOrVMPvcList, err = t.filterRunningVMs(blockOrVMPvcList) + if err != nil { + return liberr.Wrap(err) + } + } if len(blockOrVMPvcList) > 0 { labels := t.buildDVMLabels() labels["app"] = DirectVolumeMigrationRsyncTransferBlock @@ -521,14 +522,14 @@ func (t *Task) ensureBlockRsyncTransferServer(srcClient, destClient compat.Clien } transfer, err := blockrsynctransfer.NewTransfer( - stunnelTransports, endpoints[1], srcClient, destClient, blockOrVMPvcList, t.Log, transferOptions) + stunnelTransports, endpoints[1], t.sourceClient, t.destinationClient, blockOrVMPvcList, t.Log, transferOptions) if err != nil { return liberr.Wrap(err) } if transfer == nil { return fmt.Errorf("transfer %s/%s not found", nnPair.Source().Namespace, nnPair.Source().Name) } - err = transfer.CreateServer(destClient) + err = transfer.CreateServer(t.destinationClient) if err != nil { return liberr.Wrap(err) } @@ -537,9 +538,29 @@ func (t *Task) ensureBlockRsyncTransferServer(srcClient, destClient compat.Clien return nil } +// Return only PVCPairs that are NOT associated with a running VM +func (t *Task) filterRunningVMs(unfilteredPVCPairs []transfer.PVCPair) ([]transfer.PVCPair, error) { + runningVMPairs := []transfer.PVCPair{} + ns := set.New[string]() + for _, pvcPair := range unfilteredPVCPairs { + ns.Insert(pvcPair.Source().Claim().Namespace) + } + nsVolumes, err := t.getRunningVMVolumes(ns.SortedList()) + if err != nil { + return nil, err + } + + for _, pvcPair := range unfilteredPVCPairs { + if !slices.Contains(nsVolumes, fmt.Sprintf("%s/%s", pvcPair.Source().Claim().Namespace, pvcPair.Source().Claim().Name)) { + runningVMPairs = append(runningVMPairs, pvcPair) + } + } + return runningVMPairs, nil +} + func (t *Task) createRsyncTransferClients(srcClient compat.Client, - destClient compat.Client, nsMap map[string][]transfer.PVCPair) (*rsyncClientOperationStatusList, error) { - statusList := &rsyncClientOperationStatusList{} + destClient compat.Client, nsMap map[string][]transfer.PVCPair) (*migrationOperationStatusList, error) { + statusList := &migrationOperationStatusList{} pvcNodeMap, err := t.getPVCNodeNameMap(srcClient) if err != nil { @@ -624,7 +645,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, ) if lastObservedOperationStatus.IsComplete() { statusList.Add( - rsyncClientOperationStatus{ + migrationOperationStatus{ failed: lastObservedOperationStatus.Failed, succeeded: lastObservedOperationStatus.Succeeded, operation: lastObservedOperationStatus, @@ -634,7 +655,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, } newOperation := lastObservedOperationStatus - currentStatus := rsyncClientOperationStatus{ + currentStatus := migrationOperationStatus{ operation: newOperation, } pod, err := t.getLatestPodForOperation(srcClient, *lastObservedOperationStatus) @@ -654,11 +675,20 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, } blockOrVMPvcList, err := transfer.NewBlockOrVMDiskPVCPairList(pvc) if err != nil { - t.Log.Error(err, "failed creating PVC pair", "pvc", newOperation) + t.Log.Error(err, "failed creating block PVC pair", "pvc", newOperation) currentStatus.AddError(err) statusList.Add(currentStatus) continue } + if t.PlanResources.MigPlan.LiveMigrationChecked() { + blockOrVMPvcList, err = t.filterRunningVMs(blockOrVMPvcList) + if err != nil { + t.Log.Error(err, "failed filtering block PVC pairs", "pvc", newOperation) + currentStatus.AddError(err) + statusList.Add(currentStatus) + continue + } + } // Force schedule Rsync Pod on the application node nodeName := pvcNodeMap[fmt.Sprintf("%s/%s", srcNs, pvc.Source().Claim().Name)] @@ -699,7 +729,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, if pod != nil { newOperation.CurrentAttempt, _ = strconv.Atoi(pod.Labels[RsyncAttemptLabel]) updateOperationStatus(¤tStatus, pod) - if currentStatus.failed && currentStatus.operation.CurrentAttempt < GetRsyncPodBackOffLimit(*t.Owner) { + if currentStatus.failed && currentStatus.operation.CurrentAttempt < GetRsyncPodBackOffLimit(t.Owner) { // since we have not yet attempted all retries, // reset the failed status and set the pending status currentStatus.failed = false @@ -726,6 +756,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, SourcePodMeta: transfer.ResourceMetadata{ Labels: labels, }, + NodeName: nodeName, } transfer, err := blockrsynctransfer.NewTransfer( blockStunnelTransport, endpoints[1], srcClient, destClient, blockOrVMPvcList, t.Log, &transferOptions) @@ -793,6 +824,8 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, transferOptions.SourcePodMeta = transfer.ResourceMetadata{ Labels: labels, } + transferOptions.NodeName = nodeName + transfer, err := blockrsynctransfer.NewTransfer( blockStunnelTransport, endpoints[1], srcClient, destClient, blockOrVMPvcList, t.Log, transferOptions) if err != nil { @@ -809,7 +842,6 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, } } statusList.Add(currentStatus) - t.Log.Info("adding status of pvc", "pvc", currentStatus.operation, "errors", currentStatus.errors) } } return statusList, nil @@ -818,7 +850,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, func (t *Task) createClientPodForTransfer( name, namespace string, tr transfer.Transfer, - currentStatus *rsyncClientOperationStatus) *rsyncClientOperationStatus { + currentStatus *migrationOperationStatus) *migrationOperationStatus { if tr == nil { currentStatus.AddError( fmt.Errorf("transfer %s/%s not found", namespace, name)) @@ -833,12 +865,6 @@ func (t *Task) createClientPodForTransfer( } func (t *Task) areRsyncTransferPodsRunning() (arePodsRunning bool, nonRunningPods []*corev1.Pod, e error) { - // Get client for destination - destClient, err := t.getDestinationClient() - if err != nil { - return false, nil, err - } - pvcMap := t.getPVCNamespaceMap() dvmLabels := t.buildDVMLabels() dvmLabels["purpose"] = DirectVolumeMigrationRsync @@ -847,7 +873,7 @@ func (t *Task) areRsyncTransferPodsRunning() (arePodsRunning bool, nonRunningPod for bothNs, _ := range pvcMap { ns := getDestNs(bothNs) pods := corev1.PodList{} - err = destClient.List( + err := t.destinationClient.List( context.TODO(), &pods, &k8sclient.ListOptions{ @@ -861,7 +887,7 @@ func (t *Task) areRsyncTransferPodsRunning() (arePodsRunning bool, nonRunningPod if pod.Status.Phase != corev1.PodRunning { // Log abnormal events for Rsync transfer Pod if any are found migevent.LogAbnormalEventsForResource( - destClient, t.Log, + t.destinationClient, t.Log, "Found abnormal event for Rsync transfer Pod on destination cluster", types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, pod.UID, "Pod") @@ -960,16 +986,6 @@ func (p pvcWithSecurityContextInfo) Get(srcClaimName string, srcClaimNamespace s // With namespace mapping, the destination cluster namespace may be different than that in the source cluster. // This function maps PVCs to the appropriate src:dest namespace pairs. func (t *Task) getNamespacedPVCPairs() (map[string][]transfer.PVCPair, error) { - srcClient, err := t.getSourceClient() - if err != nil { - return nil, err - } - - destClient, err := t.getDestinationClient() - if err != nil { - return nil, err - } - nsMap := map[string][]transfer.PVCPair{} for _, pvc := range t.Owner.Spec.PersistentVolumeClaims { srcNs := pvc.Namespace @@ -978,12 +994,12 @@ func (t *Task) getNamespacedPVCPairs() (map[string][]transfer.PVCPair, error) { destNs = pvc.TargetNamespace } srcPvc := corev1.PersistentVolumeClaim{} - err := srcClient.Get(context.TODO(), types.NamespacedName{Name: pvc.Name, Namespace: srcNs}, &srcPvc) + err := t.sourceClient.Get(context.TODO(), types.NamespacedName{Name: pvc.Name, Namespace: srcNs}, &srcPvc) if err != nil { return nil, err } destPvc := corev1.PersistentVolumeClaim{} - err = destClient.Get(context.TODO(), types.NamespacedName{Name: pvc.TargetName, Namespace: destNs}, &destPvc) + err = t.destinationClient.Get(context.TODO(), types.NamespacedName{Name: pvc.TargetName, Namespace: destNs}, &destPvc) if err != nil { return nil, err } @@ -1062,10 +1078,6 @@ func getDestNs(bothNs string) string { func (t *Task) areRsyncRoutesAdmitted() (bool, []string, error) { messages := []string{} // Get client for destination - destClient, err := t.getDestinationClient() - if err != nil { - return false, messages, err - } nsMap := t.getPVCNamespaceMap() for bothNs := range nsMap { namespace := getDestNs(bothNs) @@ -1075,13 +1087,12 @@ func (t *Task) areRsyncRoutesAdmitted() (bool, []string, error) { route := routev1.Route{} key := types.NamespacedName{Name: DirectVolumeMigrationRsyncTransferRoute, Namespace: namespace} - err = destClient.Get(context.TODO(), key, &route) - if err != nil { + if err := t.destinationClient.Get(context.TODO(), key, &route); err != nil { return false, messages, err } // Logs abnormal events related to route if any are found migevent.LogAbnormalEventsForResource( - destClient, t.Log, + t.destinationClient, t.Log, "Found abnormal event for Rsync Route on destination cluster", types.NamespacedName{Namespace: route.Namespace, Name: route.Name}, route.UID, "Route") @@ -1110,8 +1121,7 @@ func (t *Task) areRsyncRoutesAdmitted() (bool, []string, error) { messages = append(messages, message) } default: - _, err = t.getEndpoints(destClient, namespace) - if err != nil { + if _, err := t.getEndpoints(t.destinationClient, namespace); err != nil { t.Log.Info("rsync transfer service is not healthy", "namespace", namespace) messages = append(messages, fmt.Sprintf("rsync transfer service is not healthy in namespace %s", namespace)) } @@ -1318,7 +1328,17 @@ func (t *Task) createPVProgressCR() error { labels := t.Owner.GetCorrelationLabels() for bothNs, vols := range pvcMap { ns := getSourceNs(bothNs) + volumeNames, err := t.getRunningVMVolumes([]string{ns}) + if err != nil { + return liberr.Wrap(err) + } for _, vol := range vols { + matchString := fmt.Sprintf("%s/%s", ns, vol.Name) + if t.PlanResources.MigPlan.LiveMigrationChecked() && + slices.Contains(volumeNames, matchString) { + t.Log.V(3).Info("Skipping Rsync Progress CR creation for running VM", "volume", matchString) + continue + } dvmp := migapi.DirectVolumeMigrationProgress{ ObjectMeta: metav1.ObjectMeta{ Name: getMD5Hash(t.Owner.Name + vol.Name + ns), @@ -1365,79 +1385,189 @@ func getMD5Hash(s string) string { // hasAllProgressReportingCompleted reads DVMP CR and status of Rsync Operations present for all PVCs and generates progress information in CR status // returns True when progress reporting for all Rsync Pods is complete func (t *Task) hasAllProgressReportingCompleted() (bool, error) { - t.Owner.Status.RunningPods = []*migapi.PodProgress{} - t.Owner.Status.FailedPods = []*migapi.PodProgress{} - t.Owner.Status.SuccessfulPods = []*migapi.PodProgress{} - t.Owner.Status.PendingPods = []*migapi.PodProgress{} - unknownPods := []*migapi.PodProgress{} - var pendingSinceTimeLimitPods []string + // Keep current progress in case looking up progress fails, this way we don't wipe out + // the progress until next time the update succeeds. + currentProgress := t.getCurrentLiveMigrationProgress() + t.resetProgressCounters() pvcMap := t.getPVCNamespaceMap() for bothNs, vols := range pvcMap { ns := getSourceNs(bothNs) + if err := t.populateVMMappings(ns); err != nil { + return false, err + } for _, vol := range vols { - operation := t.Owner.Status.GetRsyncOperationStatusForPVC(&corev1.ObjectReference{ - Namespace: ns, - Name: vol.Name, - }) - dvmp := migapi.DirectVolumeMigrationProgress{} - err := t.Client.Get(context.TODO(), types.NamespacedName{ - Name: getMD5Hash(t.Owner.Name + vol.Name + ns), - Namespace: migapi.OpenshiftMigrationNamespace, - }, &dvmp) - if err != nil { - return false, err + matchString := fmt.Sprintf("%s/%s", ns, vol.Name) + if t.PlanResources.MigPlan.LiveMigrationChecked() && + slices.Contains(t.VirtualMachineMappings.runningVMVolumeNames, matchString) { + // Only count skipped during staging. During cutover we need to live migrate + // PVCs for running VMs. For reporting purposes we won't count skipped PVCs + // here since they will get reported with the live migration status. + if !(t.Owner.IsCutover() || t.Owner.IsRollback()) { + t.Owner.SkipVolume(vol.Name, ns) + } else { + if err := t.updateVolumeLiveMigrationProgressStatus(vol.Name, ns, currentProgress); err != nil { + return false, err + } + } + } else { + // On rollback we are only interested in live migration volumes, skip the rest + if t.Owner.IsRollback() { + t.Owner.SkipVolume(vol.Name, ns) + } + if err := t.updateRsyncProgressStatus(vol.Name, ns); err != nil { + return false, err + } } - podProgress := &migapi.PodProgress{ - ObjectReference: &corev1.ObjectReference{ - Namespace: ns, - Name: dvmp.Status.PodName, - }, - PVCReference: &corev1.ObjectReference{ - Namespace: ns, - Name: vol.Name, - }, - LastObservedProgressPercent: dvmp.Status.TotalProgressPercentage, - LastObservedTransferRate: dvmp.Status.LastObservedTransferRate, - TotalElapsedTime: dvmp.Status.RsyncElapsedTime, + } + } + + return t.Owner.AllReportingCompleted(), nil +} + +func (t *Task) updateVolumeLiveMigrationProgressStatus(volumeName, namespace string, currentProgress map[string]*migapi.LiveMigrationProgress) error { + matchString := fmt.Sprintf("%s/%s", namespace, volumeName) + + liveMigrationProgress := &migapi.LiveMigrationProgress{ + VMName: "", + VMNamespace: namespace, + PVCReference: &corev1.ObjectReference{ + Namespace: namespace, + Name: volumeName, + }, + TotalElapsedTime: nil, + LastObservedProgressPercent: "", + } + // Look up VirtualMachineInstanceMigration CR to get the status of the migration + if vmim, exists := t.VirtualMachineMappings.volumeVMIMMap[matchString]; exists { + liveMigrationProgress.VMName = vmim.Spec.VMIName + elapsedTime := getVMIMElapsedTime(vmim) + liveMigrationProgress.TotalElapsedTime = &elapsedTime + if vmim.Status.MigrationState.FailureReason != "" { + liveMigrationProgress.Message = vmim.Status.MigrationState.FailureReason + } + switch vmim.Status.Phase { + case virtv1.MigrationSucceeded: + liveMigrationProgress.LastObservedProgressPercent = "100%" + t.Owner.Status.SuccessfulLiveMigrations = append(t.Owner.Status.SuccessfulLiveMigrations, liveMigrationProgress) + case virtv1.MigrationFailed: + + t.Owner.Status.FailedLiveMigrations = append(t.Owner.Status.FailedLiveMigrations, liveMigrationProgress) + case virtv1.MigrationRunning: + progressPercent, err := t.getLastObservedProgressPercent(vmim.Spec.VMIName, namespace, currentProgress) + if err != nil { + return err } - switch { - case dvmp.Status.PodPhase == corev1.PodRunning: - t.Owner.Status.RunningPods = append(t.Owner.Status.RunningPods, podProgress) - case operation.Failed: - t.Owner.Status.FailedPods = append(t.Owner.Status.FailedPods, podProgress) - case dvmp.Status.PodPhase == corev1.PodSucceeded: - t.Owner.Status.SuccessfulPods = append(t.Owner.Status.SuccessfulPods, podProgress) - case dvmp.Status.PodPhase == corev1.PodPending: - t.Owner.Status.PendingPods = append(t.Owner.Status.PendingPods, podProgress) - if dvmp.Status.CreationTimestamp != nil { - if time.Now().UTC().Sub(dvmp.Status.CreationTimestamp.Time.UTC()) > PendingPodWarningTimeLimit { - pendingSinceTimeLimitPods = append(pendingSinceTimeLimitPods, fmt.Sprintf("%s/%s", podProgress.Namespace, podProgress.Name)) + liveMigrationProgress.LastObservedProgressPercent = progressPercent + t.Owner.Status.RunningLiveMigrations = append(t.Owner.Status.RunningLiveMigrations, liveMigrationProgress) + case virtv1.MigrationPending: + t.Owner.Status.PendingLiveMigrations = append(t.Owner.Status.PendingLiveMigrations, liveMigrationProgress) + } + } else { + // VMIM doesn't exist, check if the VMI is in error. + vmName := t.VirtualMachineMappings.volumeVMNameMap[volumeName] + message, err := virtualMachineMigrationStatus(t.sourceClient, vmName, namespace, t.Log) + if err != nil { + return err + } + liveMigrationProgress.VMName = vmName + if message != "" { + vmMatchString := fmt.Sprintf("%s/%s", namespace, vmName) + liveMigrationProgress.Message = message + if currentProgress[vmMatchString] != nil && currentProgress[vmMatchString].TotalElapsedTime != nil { + liveMigrationProgress.TotalElapsedTime = currentProgress[vmMatchString].TotalElapsedTime + } else { + if t.Owner.Status.StartTimestamp != nil { + dvmStart := *t.Owner.Status.StartTimestamp + liveMigrationProgress.TotalElapsedTime = &metav1.Duration{ + Duration: time.Since(dvmStart.Time), } } - case dvmp.Status.PodPhase == "": - unknownPods = append(unknownPods, podProgress) - case !operation.Failed: - t.Owner.Status.RunningPods = append(t.Owner.Status.RunningPods, podProgress) } + t.Owner.Status.FailedLiveMigrations = append(t.Owner.Status.FailedLiveMigrations, liveMigrationProgress) + } else { + t.Owner.Status.PendingLiveMigrations = append(t.Owner.Status.PendingLiveMigrations, liveMigrationProgress) } } + return nil +} - isCompleted := len(t.Owner.Status.SuccessfulPods)+len(t.Owner.Status.FailedPods) == len(t.Owner.Spec.PersistentVolumeClaims) - isAnyPending := len(t.Owner.Status.PendingPods) > 0 - isAnyRunning := len(t.Owner.Status.RunningPods) > 0 - isAnyUnknown := len(unknownPods) > 0 - if len(pendingSinceTimeLimitPods) > 0 { - pendingMessage := fmt.Sprintf("Rsync Client Pods [%s] are stuck in Pending state for more than 10 mins", strings.Join(pendingSinceTimeLimitPods[:], ", ")) - t.Log.Info(pendingMessage) - t.Owner.Status.SetCondition(migapi.Condition{ - Type: RsyncClientPodsPending, - Status: migapi.True, - Reason: "PodStuckInContainerCreating", - Category: migapi.Warn, - Message: pendingMessage, - }) +func (t *Task) updateRsyncProgressStatus(volumeName, namespace string) error { + operation := t.Owner.Status.GetRsyncOperationStatusForPVC(&corev1.ObjectReference{ + Namespace: namespace, + Name: volumeName, + }) + dvmp := migapi.DirectVolumeMigrationProgress{} + err := t.Client.Get(context.TODO(), types.NamespacedName{ + Name: getMD5Hash(t.Owner.Name + volumeName + namespace), + Namespace: migapi.OpenshiftMigrationNamespace, + }, &dvmp) + if err != nil && !k8serror.IsNotFound(err) { + return err + } else if k8serror.IsNotFound(err) { + return nil + } + podProgress := &migapi.PodProgress{ + ObjectReference: &corev1.ObjectReference{ + Namespace: namespace, + Name: dvmp.Status.PodName, + }, + PVCReference: &corev1.ObjectReference{ + Namespace: namespace, + Name: volumeName, + }, + LastObservedProgressPercent: dvmp.Status.TotalProgressPercentage, + LastObservedTransferRate: dvmp.Status.LastObservedTransferRate, + TotalElapsedTime: dvmp.Status.RsyncElapsedTime, + } + switch { + case dvmp.Status.PodPhase == corev1.PodRunning: + t.Owner.Status.RunningPods = append(t.Owner.Status.RunningPods, podProgress) + case operation.Failed: + t.Owner.Status.FailedPods = append(t.Owner.Status.FailedPods, podProgress) + case dvmp.Status.PodPhase == corev1.PodSucceeded: + t.Owner.Status.SuccessfulPods = append(t.Owner.Status.SuccessfulPods, podProgress) + case dvmp.Status.PodPhase == corev1.PodPending: + t.Owner.Status.PendingPods = append(t.Owner.Status.PendingPods, podProgress) + if dvmp.Status.CreationTimestamp != nil { + if time.Now().UTC().Sub(dvmp.Status.CreationTimestamp.Time.UTC()) > PendingPodWarningTimeLimit { + t.Owner.Status.PendingSinceTimeLimitPods = append(t.Owner.Status.PendingSinceTimeLimitPods, podProgress) + } + } + case dvmp.Status.PodPhase == "": + t.Owner.Status.UnknownPods = append(t.Owner.Status.UnknownPods, podProgress) + case !operation.Failed: + t.Owner.Status.RunningPods = append(t.Owner.Status.RunningPods, podProgress) + } + return nil +} + +func (t *Task) resetProgressCounters() { + t.Owner.Status.RunningPods = []*migapi.PodProgress{} + t.Owner.Status.FailedPods = []*migapi.PodProgress{} + t.Owner.Status.SuccessfulPods = []*migapi.PodProgress{} + t.Owner.Status.PendingPods = []*migapi.PodProgress{} + t.Owner.Status.UnknownPods = []*migapi.PodProgress{} + t.Owner.Status.RunningLiveMigrations = []*migapi.LiveMigrationProgress{} + t.Owner.Status.FailedLiveMigrations = []*migapi.LiveMigrationProgress{} + t.Owner.Status.SuccessfulLiveMigrations = []*migapi.LiveMigrationProgress{} + t.Owner.Status.PendingLiveMigrations = []*migapi.LiveMigrationProgress{} +} + +func (t *Task) getCurrentLiveMigrationProgress() map[string]*migapi.LiveMigrationProgress { + currentProgress := make(map[string]*migapi.LiveMigrationProgress) + for _, progress := range t.Owner.Status.RunningLiveMigrations { + currentProgress[fmt.Sprintf("%s/%s", progress.VMNamespace, progress.VMName)] = progress + } + for _, progress := range t.Owner.Status.FailedLiveMigrations { + currentProgress[fmt.Sprintf("%s/%s", progress.VMNamespace, progress.VMName)] = progress } - return !isAnyRunning && !isAnyPending && !isAnyUnknown && isCompleted, nil + for _, progress := range t.Owner.Status.SuccessfulLiveMigrations { + currentProgress[fmt.Sprintf("%s/%s", progress.VMNamespace, progress.VMName)] = progress + } + for _, progress := range t.Owner.Status.PendingLiveMigrations { + currentProgress[fmt.Sprintf("%s/%s", progress.VMNamespace, progress.VMName)] = progress + } + return currentProgress } func (t *Task) hasAllRsyncClientPodsTimedOut() (bool, error) { @@ -1489,29 +1619,17 @@ func (t *Task) isAllRsyncClientPodsNoRouteToHost() (bool, error) { // Delete rsync resources func (t *Task) deleteRsyncResources() error { - // Get client for source + destination - srcClient, err := t.getSourceClient() - if err != nil { - return err - } - destClient, err := t.getDestinationClient() - if err != nil { - return err - } - t.Log.Info("Checking for stale Rsync resources on source MigCluster", "migCluster", path.Join(t.Owner.Spec.SrcMigClusterRef.Namespace, t.Owner.Spec.SrcMigClusterRef.Name)) t.Log.Info("Checking for stale Rsync resources on destination MigCluster", "migCluster", path.Join(t.Owner.Spec.DestMigClusterRef.Namespace, t.Owner.Spec.DestMigClusterRef.Name)) - err = t.findAndDeleteResources(srcClient, destClient, t.getPVCNamespaceMap()) - if err != nil { + if err := t.findAndDeleteResources(t.sourceClient, t.destinationClient, t.getPVCNamespaceMap()); err != nil { return err } - err = t.deleteRsyncPassword() - if err != nil { + if err := t.deleteRsyncPassword(); err != nil { return err } @@ -1521,68 +1639,49 @@ func (t *Task) deleteRsyncResources() error { t.Log.Info("Checking for stale DVMP resources on host MigCluster", "migCluster", "host") - err = t.deleteProgressReportingCRs(t.Client) - if err != nil { + if err := t.deleteProgressReportingCRs(t.Client); err != nil { return err } return nil } -func (t *Task) waitForRsyncResourcesDeleted() (error, bool) { - srcClient, err := t.getSourceClient() - if err != nil { - return err, false - } - destClient, err := t.getDestinationClient() - if err != nil { - return err, false - } +func (t *Task) waitForRsyncResourcesDeleted() (bool, error) { t.Log.Info("Checking if Rsync resource deletion has completed on source and destination MigClusters") - err, deleted := t.areRsyncResourcesDeleted(srcClient, destClient, t.getPVCNamespaceMap()) - if err != nil { - return err, false - } - if !deleted { - return nil, false - } - return nil, true -} - -func (t *Task) areRsyncResourcesDeleted(srcClient, destClient compat.Client, pvcMap map[string][]pvcMapElement) (error, bool) { + pvcMap := t.getPVCNamespaceMap() selector := labels.SelectorFromSet(map[string]string{ "app": DirectVolumeMigrationRsyncTransfer, }) - for bothNs, _ := range pvcMap { + for bothNs := range pvcMap { srcNs := getSourceNs(bothNs) destNs := getDestNs(bothNs) t.Log.Info("Searching source namespace for leftover Rsync Pods, ConfigMaps, "+ "Services, Secrets, Routes with label.", "searchNamespace", srcNs, "labelSelector", selector) - err, areDeleted := t.areRsyncNsResourcesDeleted(srcClient, srcNs, selector) + err, areDeleted := areRsyncNsResourcesDeleted(t.sourceClient, srcNs, selector, t.Log) if err != nil { - return err, false + return false, err } if !areDeleted { - return nil, false + return false, nil } t.Log.Info("Searching destination namespace for leftover Rsync Pods, ConfigMaps, "+ "Services, Secrets, Routes with label.", "searchNamespace", destNs, "labelSelector", selector) - err, areDeleted = t.areRsyncNsResourcesDeleted(destClient, destNs, selector) + err, areDeleted = areRsyncNsResourcesDeleted(t.destinationClient, destNs, selector, t.Log) if err != nil { - return err, false + return false, err } if !areDeleted { - return nil, false + return false, nil } } - return nil, true + return true, nil } -func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selector labels.Selector) (error, bool) { +func areRsyncNsResourcesDeleted(client compat.Client, ns string, selector labels.Selector, log logr.Logger) (error, bool) { podList := corev1.PodList{} cmList := corev1.ConfigMapList{} svcList := corev1.ServiceList{} @@ -1601,7 +1700,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(podList.Items) > 0 { - t.Log.Info("Found stale Rsync Pod.", + log.Info("Found stale Rsync Pod.", "pod", path.Join(podList.Items[0].Namespace, podList.Items[0].Name), "podPhase", podList.Items[0].Status.Phase) return nil, false @@ -1618,7 +1717,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(secretList.Items) > 0 { - t.Log.Info("Found stale Rsync Secret.", + log.Info("Found stale Rsync Secret.", "secret", path.Join(secretList.Items[0].Namespace, secretList.Items[0].Name)) return nil, false } @@ -1634,7 +1733,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(cmList.Items) > 0 { - t.Log.Info("Found stale Rsync ConfigMap.", + log.Info("Found stale Rsync ConfigMap.", "configMap", path.Join(cmList.Items[0].Namespace, cmList.Items[0].Name)) return nil, false } @@ -1650,7 +1749,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(svcList.Items) > 0 { - t.Log.Info("Found stale Rsync Service.", + log.Info("Found stale Rsync Service.", "service", path.Join(svcList.Items[0].Namespace, svcList.Items[0].Name)) return nil, false } @@ -1667,7 +1766,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(routeList.Items) > 0 { - t.Log.Info("Found stale Rsync Route.", + log.Info("Found stale Rsync Route.", "route", path.Join(routeList.Items[0].Namespace, routeList.Items[0].Name)) return nil, false } @@ -1685,11 +1784,11 @@ func (t *Task) findAndDeleteResources(srcClient, destClient compat.Client, pvcMa for bothNs := range pvcMap { srcNs := getSourceNs(bothNs) destNs := getDestNs(bothNs) - err := t.findAndDeleteNsResources(srcClient, srcNs, selector) + err := findAndDeleteNsResources(srcClient, srcNs, selector, t.Log) if err != nil { return err } - err = t.findAndDeleteNsResources(destClient, destNs, selector) + err = findAndDeleteNsResources(destClient, destNs, selector, t.Log) if err != nil { return err } @@ -1697,7 +1796,7 @@ func (t *Task) findAndDeleteResources(srcClient, destClient compat.Client, pvcMa return nil } -func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selector labels.Selector) error { +func findAndDeleteNsResources(client compat.Client, ns string, selector labels.Selector, log logr.Logger) error { podList := corev1.PodList{} cmList := corev1.ConfigMapList{} svcList := corev1.ServiceList{} @@ -1765,7 +1864,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete pods for _, pod := range podList.Items { - t.Log.Info("Deleting stale DVM Pod", + log.Info("Deleting stale DVM Pod", "pod", path.Join(pod.Namespace, pod.Name)) err = client.Delete(context.TODO(), &pod, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1775,7 +1874,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete secrets for _, secret := range secretList.Items { - t.Log.Info("Deleting stale DVM Secret", + log.Info("Deleting stale DVM Secret", "secret", path.Join(secret.Namespace, secret.Name)) err = client.Delete(context.TODO(), &secret, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1785,7 +1884,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete routes for _, route := range routeList.Items { - t.Log.Info("Deleting stale DVM Route", + log.Info("Deleting stale DVM Route", "route", path.Join(route.Namespace, route.Name)) err = client.Delete(context.TODO(), &route, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1795,7 +1894,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete svcs for _, svc := range svcList.Items { - t.Log.Info("Deleting stale DVM Service", + log.Info("Deleting stale DVM Service", "service", path.Join(svc.Namespace, svc.Name)) err = client.Delete(context.TODO(), &svc, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1805,7 +1904,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete configmaps for _, cm := range cmList.Items { - t.Log.Info("Deleting stale DVM ConfigMap", + log.Info("Deleting stale DVM ConfigMap", "configMap", path.Join(cm.Namespace, cm.Name)) err = client.Delete(context.TODO(), &cm, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1855,7 +1954,7 @@ func isPrivilegedLabelPresent(client compat.Client, namespace string) (bool, err return false, nil } -func GetRsyncPodBackOffLimit(dvm migapi.DirectVolumeMigration) int { +func GetRsyncPodBackOffLimit(dvm *migapi.DirectVolumeMigration) int { overriddenBackOffLimit := settings.Settings.DvmOpts.RsyncOpts.BackOffLimit // when both the spec and the overridden backoff limits are not set, use default if dvm.Spec.BackOffLimit == 0 && overriddenBackOffLimit == 0 { @@ -1873,41 +1972,160 @@ func GetRsyncPodBackOffLimit(dvm migapi.DirectVolumeMigration) int { // returns whether or not all operations are completed, whether any of the operation is failed, and a list of failure reasons func (t *Task) runRsyncOperations() (bool, bool, []string, error) { var failureReasons []string - destClient, err := t.getDestinationClient() - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) - } - srcClient, err := t.getSourceClient() - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) - } pvcMap, err := t.getNamespacedPVCPairs() if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + return false, false, failureReasons, err } - err = t.buildSourceLimitRangeMap(pvcMap, srcClient) + err = t.buildSourceLimitRangeMap(pvcMap, t.sourceClient) if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + return false, false, failureReasons, err } - status, err := t.createRsyncTransferClients(srcClient, destClient, pvcMap) - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + var ( + status *migrationOperationStatusList + progressCompleted, + rsyncOperationsCompleted, + anyRsyncFailed bool + ) + + if !t.Owner.IsRollback() { + t.Log.V(3).Info("Creating Rsync Transfer Clients") + status, err = t.createRsyncTransferClients(t.sourceClient, t.destinationClient, pvcMap) + if err != nil { + return false, false, failureReasons, err + } + t.podPendingSinceTimeLimit() } // report progress of pods - progressCompleted, err := t.hasAllProgressReportingCompleted() - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + progressCompleted, err = t.hasAllProgressReportingCompleted() + if err != nil { + return false, false, failureReasons, err + } + migrationOperationsCompleted := true + anyMigrationFailed := false + if t.Owner.IsCutover() || t.Owner.IsRollback() { + liveMigrationPVCMap := pvcMap + if t.Owner.IsRollback() { + // Swap source and destination PVCs for rollback + liveMigrationPVCMap = swapSourceDestination(pvcMap) + } + var migrationFailureReasons []string + if t.Owner.IsLiveMigrate() { + t.Log.V(3).Info("Starting live migrations") + // Doing a cutover or rollback, start any live migrations if needed. + failureReasons, err = t.startLiveMigrations(liveMigrationPVCMap) + if err != nil { + return false, len(failureReasons) > 0, failureReasons, err + } + migrationOperationsCompleted, anyMigrationFailed, migrationFailureReasons, err = t.processMigrationOperationStatus(pvcMap, t.sourceClient) + if err != nil { + return false, len(migrationFailureReasons) > 0, migrationFailureReasons, err + } + failureReasons = append(failureReasons, migrationFailureReasons...) + } } - operationsCompleted, anyFailed, failureReasons, err := t.processRsyncOperationStatus(status, []error{}) - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + + if !t.Owner.IsRollback() { + var rsyncFailureReasons []string + rsyncOperationsCompleted, anyRsyncFailed, rsyncFailureReasons, err = t.processRsyncOperationStatus(status, []error{}) + if err != nil { + return false, len(failureReasons) > 0, failureReasons, err + } + failureReasons = append(failureReasons, rsyncFailureReasons...) + } else { + rsyncOperationsCompleted = true + } + t.Log.V(3).Info("Migration Operations Completed", "MigrationOperationsCompleted", migrationOperationsCompleted, "RsyncOperationsCompleted", rsyncOperationsCompleted, "ProgressCompleted", progressCompleted) + return migrationOperationsCompleted && rsyncOperationsCompleted && progressCompleted, anyRsyncFailed || anyMigrationFailed, failureReasons, nil +} + +func swapSourceDestination(pvcMap map[string][]transfer.PVCPair) map[string][]transfer.PVCPair { + swappedMap := make(map[string][]transfer.PVCPair) + for bothNs, volumes := range pvcMap { + swappedVolumes := make([]transfer.PVCPair, 0) + for _, volume := range volumes { + swappedVolumes = append(swappedVolumes, transfer.NewPVCPair(volume.Destination().Claim(), volume.Source().Claim())) + } + ns := getSourceNs(bothNs) + destNs := getDestNs(bothNs) + swappedMap[fmt.Sprintf("%s:%s", destNs, ns)] = swappedVolumes } - return operationsCompleted && progressCompleted, anyFailed, failureReasons, nil + return swappedMap +} + +func (t *Task) podPendingSinceTimeLimit() { + if len(t.Owner.Status.PendingSinceTimeLimitPods) > 0 { + pendingPods := make([]string, 0) + for _, pod := range t.Owner.Status.PendingSinceTimeLimitPods { + pendingPods = append(pendingPods, fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) + } + + pendingMessage := fmt.Sprintf("Rsync Client Pods [%s] are stuck in Pending state for more than 10 mins", strings.Join(pendingPods[:], ", ")) + t.Owner.Status.SetCondition(migapi.Condition{ + Type: RsyncClientPodsPending, + Status: migapi.True, + Reason: "PodStuckInContainerCreating", + Category: migapi.Warn, + Message: pendingMessage, + }) + } +} + +// Count the number of VMs, and the number of migrations in error/completed. If they match total isComplete needs to be true +// returns: +// isComplete: whether all migrations are completed, false if no pvc pairs are found +// anyFailed: whether any of the migrations failed +// failureReasons: list of failure reasons +// error: error if any +func (t *Task) processMigrationOperationStatus(nsMap map[string][]transfer.PVCPair, sourceClient k8sclient.Client) (bool, bool, []string, error) { + isComplete, anyFailed, failureReasons := false, false, make([]string, 0) + vmVolumeMap := make(map[string]vmVolumes) + + for k, v := range nsMap { + namespace, err := getNamespace(k) + if err != nil { + failureReasons = append(failureReasons, err.Error()) + return isComplete, anyFailed, failureReasons, err + } + volumeVmMap, err := getRunningVmVolumeMap(sourceClient, namespace) + if err != nil { + failureReasons = append(failureReasons, err.Error()) + return isComplete, anyFailed, failureReasons, err + } + for _, pvcPair := range v { + if vmName, found := volumeVmMap[pvcPair.Source().Claim().Name]; found { + vmVolumeMap[vmName] = vmVolumes{ + sourceVolumes: append(vmVolumeMap[vmName].sourceVolumes, pvcPair.Source().Claim().Name), + targetVolumes: append(vmVolumeMap[vmName].targetVolumes, pvcPair.Destination().Claim().Name), + } + } + } + isComplete = true + completeCount := 0 + failedCount := 0 + for vmName := range vmVolumeMap { + message, err := virtualMachineMigrationStatus(sourceClient, vmName, namespace, t.Log) + if err != nil { + failureReasons = append(failureReasons, message) + return isComplete, anyFailed, failureReasons, err + } + if message == "" { + // Completed + completeCount++ + } else { + // Failed + failedCount++ + anyFailed = true + failureReasons = append(failureReasons, message) + } + } + isComplete = true + } + return isComplete, anyFailed, failureReasons, nil } // processRsyncOperationStatus processes status of Rsync operations by reading the status list // returns whether all operations are completed and whether any of the operation is failed -func (t *Task) processRsyncOperationStatus(status *rsyncClientOperationStatusList, garbageCollectionErrors []error) (bool, bool, []string, error) { +func (t *Task) processRsyncOperationStatus(status *migrationOperationStatusList, garbageCollectionErrors []error) (bool, bool, []string, error) { isComplete, anyFailed, failureReasons := false, false, make([]string, 0) if status.AllCompleted() { isComplete = true @@ -1916,7 +2134,7 @@ func (t *Task) processRsyncOperationStatus(status *rsyncClientOperationStatusLis if status.Failed() > 0 { anyFailed = true // attempt to categorize failures in any of the special failure categories we defined - failureReasons, err := t.reportAdvancedErrorHeuristics() + failureReasons, err := t.reportAdvancedRsyncErrorHeuristics() if err != nil { return isComplete, anyFailed, failureReasons, liberr.Wrap(err) } @@ -1964,7 +2182,7 @@ func (t *Task) processRsyncOperationStatus(status *rsyncClientOperationStatusLis // for all errored pods, attempts to determine whether the errors fall into any // of the special categories we can identify and reports them as conditions // returns reasons and error for reconcile decisions -func (t *Task) reportAdvancedErrorHeuristics() ([]string, error) { +func (t *Task) reportAdvancedRsyncErrorHeuristics() ([]string, error) { reasons := make([]string, 0) // check if the pods are failing due to a network misconfiguration causing Stunnel to timeout isStunnelTimeout, err := t.hasAllRsyncClientPodsTimedOut() @@ -2007,8 +2225,8 @@ func (t *Task) reportAdvancedErrorHeuristics() ([]string, error) { return reasons, nil } -// rsyncClientOperationStatus defines status of one Rsync operation -type rsyncClientOperationStatus struct { +// migrationOperationStatus defines status of one Rsync operation +type migrationOperationStatus struct { operation *migapi.RsyncOperation // When set,.means that all attempts have been exhausted resulting in a failure failed bool @@ -2024,33 +2242,33 @@ type rsyncClientOperationStatus struct { // HasErrors Checks whether there were errors in processing this operation // presence of errors indicates that the status information may not be accurate, demands a retry -func (e *rsyncClientOperationStatus) HasErrors() bool { +func (e *migrationOperationStatus) HasErrors() bool { return len(e.errors) > 0 } -func (e *rsyncClientOperationStatus) AddError(err error) { +func (e *migrationOperationStatus) AddError(err error) { if e.errors == nil { e.errors = make([]error, 0) } e.errors = append(e.errors, err) } -// rsyncClientOperationStatusList managed list of all ongoing Rsync operations -type rsyncClientOperationStatusList struct { +// migrationOperationStatusList managed list of all ongoing Rsync operations +type migrationOperationStatusList struct { // ops list of operations - ops []rsyncClientOperationStatus + ops []migrationOperationStatus } -func (r *rsyncClientOperationStatusList) Add(s rsyncClientOperationStatus) { +func (r *migrationOperationStatusList) Add(s migrationOperationStatus) { if r.ops == nil { - r.ops = make([]rsyncClientOperationStatus, 0) + r.ops = make([]migrationOperationStatus, 0) } r.ops = append(r.ops, s) } // AllCompleted checks whether all of the Rsync attempts are in a terminal state // If true, reconcile can move to next phase -func (r *rsyncClientOperationStatusList) AllCompleted() bool { +func (r *migrationOperationStatusList) AllCompleted() bool { for _, attempt := range r.ops { if attempt.pending || attempt.running || attempt.HasErrors() { return false @@ -2060,7 +2278,7 @@ func (r *rsyncClientOperationStatusList) AllCompleted() bool { } // AnyErrored checks whether any of the operation is resulting in an error -func (r *rsyncClientOperationStatusList) AnyErrored() bool { +func (r *migrationOperationStatusList) AnyErrored() bool { for _, attempt := range r.ops { if attempt.HasErrors() { return true @@ -2070,7 +2288,7 @@ func (r *rsyncClientOperationStatusList) AnyErrored() bool { } // Failed returns number of failed operations -func (r *rsyncClientOperationStatusList) Failed() int { +func (r *migrationOperationStatusList) Failed() int { i := 0 for _, attempt := range r.ops { if attempt.failed { @@ -2081,7 +2299,7 @@ func (r *rsyncClientOperationStatusList) Failed() int { } // Succeeded returns number of failed operations -func (r *rsyncClientOperationStatusList) Succeeded() int { +func (r *migrationOperationStatusList) Succeeded() int { i := 0 for _, attempt := range r.ops { if attempt.succeeded { @@ -2092,7 +2310,7 @@ func (r *rsyncClientOperationStatusList) Succeeded() int { } // Pending returns number of pending operations -func (r *rsyncClientOperationStatusList) Pending() int { +func (r *migrationOperationStatusList) Pending() int { i := 0 for _, attempt := range r.ops { if attempt.pending { @@ -2103,7 +2321,7 @@ func (r *rsyncClientOperationStatusList) Pending() int { } // Running returns number of running operations -func (r *rsyncClientOperationStatusList) Running() int { +func (r *migrationOperationStatusList) Running() int { i := 0 for _, attempt := range r.ops { if attempt.running { @@ -2161,7 +2379,7 @@ func (t *Task) getLatestPodForOperation(client compat.Client, operation migapi.R } // updateOperationStatus given a Rsync Pod and operation status, updates operation status with pod status -func updateOperationStatus(status *rsyncClientOperationStatus, pod *corev1.Pod) { +func updateOperationStatus(status *migrationOperationStatus, pod *corev1.Pod) { switch pod.Status.Phase { case corev1.PodFailed: status.failed = true diff --git a/pkg/controller/directvolumemigration/rsync_test.go b/pkg/controller/directvolumemigration/rsync_test.go index e7face64e2..a1085cb580 100644 --- a/pkg/controller/directvolumemigration/rsync_test.go +++ b/pkg/controller/directvolumemigration/rsync_test.go @@ -18,16 +18,24 @@ import ( fakecompat "github.com/konveyor/mig-controller/pkg/compat/fake" configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + testVolume = "test-volume" + testDVM = "test-dvm" +) + func getTestRsyncPodForPVC(podName string, pvcName string, ns string, attemptNo string, timestamp time.Time) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -668,7 +676,7 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { Owner *migapi.DirectVolumeMigration } type args struct { - status rsyncClientOperationStatusList + status migrationOperationStatusList garbageCollectionErrors []error } tests := []struct { @@ -688,7 +696,7 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { Client: getFakeCompatClient(), Owner: &migapi.DirectVolumeMigration{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dvm", Namespace: "openshift-migration", + Name: testDVM, Namespace: "openshift-migration", }, Status: migapi.DirectVolumeMigrationStatus{ Conditions: migapi.Conditions{ @@ -700,8 +708,8 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { }, }, args: args{ - status: rsyncClientOperationStatusList{ - ops: []rsyncClientOperationStatus{ + status: migrationOperationStatusList{ + ops: []migrationOperationStatus{ {errors: []error{fmt.Errorf("failed creating pod")}}, }, }, @@ -718,7 +726,7 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { Client: getFakeCompatClient(), Owner: &migapi.DirectVolumeMigration{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dvm", Namespace: "openshift-migration", + Name: testDVM, Namespace: "openshift-migration", }, Status: migapi.DirectVolumeMigrationStatus{ Conditions: migapi.Conditions{ @@ -730,8 +738,8 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { }, }, args: args{ - status: rsyncClientOperationStatusList{ - ops: []rsyncClientOperationStatus{ + status: migrationOperationStatusList{ + ops: []migrationOperationStatus{ {errors: []error{fmt.Errorf("failed creating pod")}}, }, }, @@ -770,6 +778,926 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { } } +func TestTask_processMigrationOperationStatus(t *testing.T) { + tests := []struct { + name string + nsMap map[string][]transfer.PVCPair + expectCompleted bool + expectFailed bool + expectedFailureReasons []string + client compat.Client + wantErr bool + }{ + { + name: "empty namespace pair", + nsMap: map[string][]transfer.PVCPair{}, + expectedFailureReasons: []string{}, + client: getFakeCompatClient(), + }, + { + name: "invalid namespace pair", + nsMap: map[string][]transfer.PVCPair{ + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient(), + expectedFailureReasons: []string{"invalid namespace pair: test-namespace"}, + wantErr: true, + }, + { + name: "no running VMs", + nsMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient(), + expectedFailureReasons: []string{}, + expectCompleted: true, + }, + { + name: "running VMs, no matching volumes", + nsMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient(createVirtualMachine("vm", testNamespace), createVirtlauncherPod("vm", testNamespace, []string{"dv"})), + expectedFailureReasons: []string{}, + expectCompleted: true, + }, + { + name: "running VMs, no matching volumes", + nsMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient( + createVirtualMachine("vm", testNamespace), + createVirtlauncherPod("vm", testNamespace, []string{"pvc1"}), + createVirtualMachineInstance("vm", testNamespace, virtv1.Running), + ), + expectedFailureReasons: []string{}, + expectCompleted: true, + }, + { + name: "failed migration, no matching volumes", + nsMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient( + createVirtualMachine("vm", testNamespace), + createVirtlauncherPod("vm", testNamespace, []string{"pvc1"}), + createVirtualMachineInstance("vm", testNamespace, virtv1.Failed), + createCanceledVirtualMachineMigration("vmim", testNamespace, "vm", virtv1.MigrationAbortSucceeded), + ), + expectCompleted: true, + expectFailed: true, + expectedFailureReasons: []string{"Migration canceled"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.sourceClient = tt.client + isComplete, anyFailed, failureReasons, err := task.processMigrationOperationStatus(tt.nsMap, task.sourceClient) + if err != nil && !tt.wantErr { + t.Errorf("Unexpected() error = %v", err) + t.FailNow() + } else if err == nil && tt.wantErr { + t.Errorf("Expected error, got nil") + t.FailNow() + } + if isComplete != tt.expectCompleted { + t.Errorf("Expected completed to be %t, got %t", tt.expectCompleted, isComplete) + t.FailNow() + } + if anyFailed != tt.expectFailed { + t.Errorf("Expected failed to be %t, got %t", tt.expectFailed, anyFailed) + t.FailNow() + } + if !reflect.DeepEqual(failureReasons, tt.expectedFailureReasons) { + t.Errorf("Unexpected() got = %v, want %v", failureReasons, tt.expectedFailureReasons) + t.FailNow() + } + }) + } +} + +func TestTask_podPendingSinceTimeLimit(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + expectedCondition *migapi.Condition + }{ + { + name: "No pending pods", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + PendingSinceTimeLimitPods: []*migapi.PodProgress{}, + }, + }, + expectedCondition: nil, + }, + { + name: "Pending pods", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + PendingSinceTimeLimitPods: []*migapi.PodProgress{ + { + ObjectReference: &corev1.ObjectReference{ + Name: "test-pod", + Namespace: testNamespace, + }, + }, + }, + }, + }, + expectedCondition: &migapi.Condition{ + Type: RsyncClientPodsPending, + Status: migapi.True, + Reason: "PodStuckInContainerCreating", + Category: migapi.Warn, + Message: "", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{ + Owner: tt.dvm, + } + task.podPendingSinceTimeLimit() + if tt.expectedCondition != nil { + if !tt.dvm.Status.HasCondition(tt.expectedCondition.Type) { + t.Errorf("Condition %s not found", tt.expectedCondition.Type) + } + } else { + if tt.dvm.Status.HasCondition(RsyncClientPodsPending) { + t.Errorf("Condition %s found", RsyncClientPodsPending) + } + } + }) + } +} + +func TestTask_swapSourceDestination(t *testing.T) { + tests := []struct { + name string + pvcMap map[string][]transfer.PVCPair + expectedMap map[string][]transfer.PVCPair + }{ + { + name: "empty map", + pvcMap: map[string][]transfer.PVCPair{}, + expectedMap: map[string][]transfer.PVCPair{}, + }, + { + name: "one namespace, one pair", + pvcMap: map[string][]transfer.PVCPair{ + "foo:bar": {transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace))}, + }, + expectedMap: map[string][]transfer.PVCPair{ + "bar:foo": {transfer.NewPVCPair(createPvc("pvc2", testNamespace), createPvc("pvc1", testNamespace))}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := swapSourceDestination(tt.pvcMap) + if !reflect.DeepEqual(out, tt.expectedMap) { + t.Errorf("swapSourceDestination() = %v, want %v", out, tt.expectedMap) + } + }) + } +} + +func TestTask_runRsyncOperations(t *testing.T) { + tests := []struct { + name string + client compat.Client + dvm *migapi.DirectVolumeMigration + expectComplete bool + expectFailed bool + expectedReasons []string + }{ + { + name: "no PVCs, stage migration type", + client: getFakeCompatClient(), + dvm: &migapi.DirectVolumeMigration{ + Spec: migapi.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []migapi.PVCToMigrate{}, + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeStage), + }, + }, + expectComplete: true, + }, + { + name: "no PVCs, cutover migration type, no live migration", + client: getFakeCompatClient(), + dvm: &migapi.DirectVolumeMigration{ + Spec: migapi.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []migapi.PVCToMigrate{}, + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeFinal), + }, + }, + expectComplete: true, + }, + { + name: "no PVCs, cutover migration type, live migration", + client: getFakeCompatClient(), + dvm: &migapi.DirectVolumeMigration{ + Spec: migapi.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []migapi.PVCToMigrate{}, + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeFinal), + LiveMigrate: ptr.To[bool](true), + }, + }, + }, + { + name: "no PVCs, rollback migration type, live migration", + client: getFakeCompatClient(), + dvm: &migapi.DirectVolumeMigration{ + Spec: migapi.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []migapi.PVCToMigrate{}, + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeRollback), + LiveMigrate: ptr.To[bool](true), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.sourceClient = tt.client + task.destinationClient = tt.client + task.Client = tt.client + task.Owner = tt.dvm + completed, failed, reasons, err := task.runRsyncOperations() + if err != nil { + t.Errorf("runRsyncOperations() error = %v", err) + t.FailNow() + } + if completed != tt.expectComplete { + t.Errorf("Expected completed to be %t, got %t", tt.expectComplete, completed) + t.FailNow() + } + if failed != tt.expectFailed { + t.Errorf("Expected failed to be %t, got %t", tt.expectFailed, failed) + t.FailNow() + } + if len(reasons) != len(tt.expectedReasons) { + t.Errorf("%v is not the same length as %v", reasons, tt.expectedReasons) + t.FailNow() + } + for i, s := range reasons { + if s != tt.expectedReasons[i] { + t.Errorf("%s is not equal to %s", s, tt.expectedReasons[i]) + t.FailNow() + } + } + }) + } +} + +func TestTask_getCurrentLiveMigrationProgress(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + expectedProgress map[string]*migapi.LiveMigrationProgress + }{ + { + name: "no progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{}, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{}, + }, + { + name: "live migration progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + RunningLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + { + VMName: "test-vm-2", + VMNamespace: testNamespace, + }, + }, + }, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/test-vm", testNamespace): { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + fmt.Sprintf("%s/test-vm-2", testNamespace): { + VMName: "test-vm-2", + VMNamespace: testNamespace, + }, + }, + }, + { + name: "failed live migration progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + FailedLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/test-vm", testNamespace): { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + { + name: "successful live migration progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + SuccessfulLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/test-vm", testNamespace): { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + { + name: "pending live migration progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + PendingLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/test-vm", testNamespace): { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.Owner = tt.dvm + progress := task.getCurrentLiveMigrationProgress() + if len(progress) != len(tt.expectedProgress) { + t.Errorf("getCurrentLiveMigrationProgress() = %v, want %v", progress, tt.expectedProgress) + t.FailNow() + } + for k, v := range progress { + if !reflect.DeepEqual(v, tt.expectedProgress[k]) { + t.Errorf("getCurrentLiveMigrationProgress() = %v, want %v", progress, tt.expectedProgress) + t.FailNow() + } + } + }) + } +} + +func TestTask_updateRsyncProgressStatus(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + volumeName string + client compat.Client + expectedSuccessfulPods []*migapi.PodProgress + expectedFailedPods []*migapi.PodProgress + expectedRunningPods []*migapi.PodProgress + expectedPendingPods []*migapi.PodProgress + expectedUnknownPods []*migapi.PodProgress + expectedPendingSince []*migapi.PodProgress + }{ + { + name: "no progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(), + }, + { + name: "running pod progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodRunning)), + expectedRunningPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "failed pod progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{ + RsyncOperations: []*migapi.RsyncOperation{ + { + Failed: true, + PVCReference: &corev1.ObjectReference{ + Namespace: testNamespace, + Name: testVolume, + }, + }, + }, + }, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodFailed)), + expectedFailedPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "failed pod, operation hasn't failed progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodFailed)), + expectedRunningPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "pending pod, progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodPending)), + expectedPendingPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "pending pod older than 10 minutes, progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createOldDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodPending)), + expectedPendingPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + expectedPendingSince: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "unknown pod, progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, "")), + expectedUnknownPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "successful pod, progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodSucceeded)), + expectedSuccessfulPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.Client = tt.client + task.Owner = tt.dvm + err := task.updateRsyncProgressStatus(tt.volumeName, testNamespace) + if err != nil { + t.Errorf("Unexpected error: %v", err) + t.FailNow() + } + if len(tt.expectedSuccessfulPods) != len(tt.dvm.Status.SuccessfulPods) { + t.Errorf("Expected %d successful pods, got %d", len(tt.expectedSuccessfulPods), len(tt.dvm.Status.SuccessfulPods)) + t.FailNow() + } + if len(tt.expectedFailedPods) != len(tt.dvm.Status.FailedPods) { + t.Errorf("Expected %d failed pods, got %d", len(tt.expectedFailedPods), len(tt.dvm.Status.FailedPods)) + t.FailNow() + } + if len(tt.expectedRunningPods) != len(tt.dvm.Status.RunningPods) { + t.Errorf("Expected %d running pods, got %d", len(tt.expectedRunningPods), len(tt.dvm.Status.RunningPods)) + t.FailNow() + } + if len(tt.expectedPendingPods) != len(tt.dvm.Status.PendingPods) { + t.Errorf("Expected %d pending pods, got %d", len(tt.expectedPendingPods), len(tt.dvm.Status.PendingPods)) + t.FailNow() + } + if len(tt.expectedUnknownPods) != len(tt.dvm.Status.UnknownPods) { + t.Errorf("Expected %d unknown pods, got %d", len(tt.expectedUnknownPods), len(tt.dvm.Status.UnknownPods)) + t.FailNow() + } + if len(tt.expectedPendingSince) != len(tt.dvm.Status.PendingSinceTimeLimitPods) { + t.Errorf("Expected %d pending since pods, got %d", len(tt.expectedPendingSince), len(tt.dvm.Status.PendingSinceTimeLimitPods)) + t.FailNow() + } + }) + } +} + +func TestTask_updateVolumeLiveMigrationProgressStatus(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + volumeName string + client compat.Client + virtualMachineMappings VirtualMachineMappings + existingProgress map[string]*migapi.LiveMigrationProgress + expectedSuccessfulLiveMigrations []*migapi.LiveMigrationProgress + expectedFailedLiveMigrations []*migapi.LiveMigrationProgress + expectedRunningLiveMigrations []*migapi.LiveMigrationProgress + expectedPendingLiveMigrations []*migapi.LiveMigrationProgress + promQuery func(context.Context, string, time.Time, ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) + }{ + { + name: "no progress", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedFailedLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + }, + { + name: "failed vmim with failure reason", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedFailedLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMIMMap: map[string]*virtv1.VirtualMachineInstanceMigration{ + fmt.Sprintf("%s/%s", testNamespace, testVolume): { + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + FailureReason: "test failure", + }, + Phase: virtv1.MigrationFailed, + }, + }, + }, + }, + }, + { + name: "successful vmim without failure reason", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedSuccessfulLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMIMMap: map[string]*virtv1.VirtualMachineInstanceMigration{ + fmt.Sprintf("%s/%s", testNamespace, testVolume): { + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{}, + Phase: virtv1.MigrationSucceeded, + }, + }, + }, + }, + }, + { + name: "running vmim without failure reason", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedRunningLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMIMMap: map[string]*virtv1.VirtualMachineInstanceMigration{ + fmt.Sprintf("%s/%s", testNamespace, testVolume): { + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{}, + Phase: virtv1.MigrationRunning, + }, + }, + }, + }, + promQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> 59.3 @", + }, nil, nil + }, + }, + { + name: "pending vmim without failure reason", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedPendingLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMIMMap: map[string]*virtv1.VirtualMachineInstanceMigration{ + fmt.Sprintf("%s/%s", testNamespace, testVolume): { + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{}, + Phase: virtv1.MigrationPending, + }, + }, + }, + }, + }, + { + name: "no vmim, but VMI exists", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(createVirtualMachineInstance("test-vm", testNamespace, virtv1.Running)), + existingProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/%s", testNamespace, "test-vm"): createExpectedLiveMigrationProgress(nil), + }, + expectedPendingLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMNameMap: map[string]string{ + testVolume: "test-vm", + }, + }, + }, + { + name: "no vmim, but VMI exists, unable to live migrate", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(createVirtualMachineInstanceWithConditions("test-vm", testNamespace, []virtv1.VirtualMachineInstanceCondition{ + { + Type: virtv1.VirtualMachineInstanceVolumesChange, + Status: corev1.ConditionTrue, + }, + { + Type: virtv1.VirtualMachineInstanceIsMigratable, + Status: corev1.ConditionFalse, + Message: "Unable to live migrate because of the test reason", + }, + })), + existingProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/%s", testNamespace, "test-vm"): createExpectedLiveMigrationProgress(&metav1.Duration{ + Duration: time.Second * 10, + }), + }, + expectedFailedLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMNameMap: map[string]string{ + testVolume: "test-vm", + }, + }, + }, + { + name: "no vmim, but VMI exists, unable to live migrate", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + StartTimestamp: &metav1.Time{Time: time.Now().Add(-time.Minute * 11)}, + }, + }, + volumeName: testVolume, + client: getFakeCompatClient(createVirtualMachineInstanceWithConditions("test-vm", testNamespace, []virtv1.VirtualMachineInstanceCondition{ + { + Type: virtv1.VirtualMachineInstanceVolumesChange, + Status: corev1.ConditionTrue, + }, + { + Type: virtv1.VirtualMachineInstanceIsMigratable, + Status: corev1.ConditionFalse, + Message: "Unable to live migrate because of the test reason", + }, + })), + existingProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/%s", testNamespace, "test-vm"): createExpectedLiveMigrationProgress(nil), + }, + expectedFailedLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMNameMap: map[string]string{ + testVolume: "test-vm", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.sourceClient = tt.client + task.Client = tt.client + task.Owner = tt.dvm + task.PrometheusAPI = prometheusv1.NewAPI(nil) + task.PromQuery = tt.promQuery + task.VirtualMachineMappings = tt.virtualMachineMappings + err := task.updateVolumeLiveMigrationProgressStatus(tt.volumeName, testNamespace, tt.existingProgress) + if err != nil { + t.Errorf("Unexpected error: %v", err) + t.FailNow() + } + if len(tt.expectedSuccessfulLiveMigrations) != len(tt.dvm.Status.SuccessfulLiveMigrations) { + t.Errorf("Expected %d successful live migrations, got %d", len(tt.expectedSuccessfulLiveMigrations), len(tt.dvm.Status.SuccessfulLiveMigrations)) + t.FailNow() + } + if len(tt.expectedFailedLiveMigrations) != len(tt.dvm.Status.FailedLiveMigrations) { + t.Errorf("Expected %d failed live migrations, got %d", len(tt.expectedFailedLiveMigrations), len(tt.dvm.Status.FailedLiveMigrations)) + t.FailNow() + } + if len(tt.expectedRunningLiveMigrations) != len(tt.dvm.Status.RunningLiveMigrations) { + t.Errorf("Expected %d running live migrations, got %d", len(tt.expectedRunningLiveMigrations), len(tt.dvm.Status.RunningPods)) + t.FailNow() + } + if len(tt.expectedPendingLiveMigrations) != len(tt.dvm.Status.PendingLiveMigrations) { + t.Errorf("Expected %d pending live migrations, got %d", len(tt.expectedPendingLiveMigrations), len(tt.dvm.Status.PendingLiveMigrations)) + t.FailNow() + } + }) + } +} + +func TestTask_filterRunningVMs(t *testing.T) { + tests := []struct { + name string + client compat.Client + input []transfer.PVCPair + expected []transfer.PVCPair + }{ + { + name: "no input", + client: getFakeCompatClient(), + input: []transfer.PVCPair{}, + expected: []transfer.PVCPair{}, + }, + { + name: "single PVCPair with running VM", + client: getFakeCompatClient(createVirtualMachineWithVolumes("test-vm", testNamespace, []virtv1.Volume{ + { + Name: "source", + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "source", + }, + }, + }, + }, + }), + createVirtlauncherPod("test-vm", testNamespace, []string{"source"}), + ), + input: []transfer.PVCPair{ + transfer.NewPVCPair(createPvc("source", testNamespace), createPvc("target", testNamespace)), + }, + expected: []transfer.PVCPair{}, + }, + { + name: "two PVCPairs one running VM, one without", + client: getFakeCompatClient(createVirtualMachineWithVolumes("test-vm", testNamespace, []virtv1.Volume{ + { + Name: "source", + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "source", + }, + }, + }, + }, + }), + createVirtlauncherPod("test-vm", testNamespace, []string{"source"}), + ), + input: []transfer.PVCPair{ + transfer.NewPVCPair(createPvc("source", testNamespace), createPvc("target", testNamespace)), + transfer.NewPVCPair(createPvc("source-remain", testNamespace), createPvc("target-remain", testNamespace)), + }, + expected: []transfer.PVCPair{ + transfer.NewPVCPair(createPvc("source-remain", testNamespace), createPvc("target-remain", testNamespace)), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.sourceClient = tt.client + out, err := task.filterRunningVMs(tt.input) + if err != nil { + t.Errorf("Unexpected error: %v", err) + t.FailNow() + } + if len(out) != len(tt.expected) { + t.Errorf("Expected %d, got %d", len(tt.expected), len(out)) + t.FailNow() + } + for i, p := range out { + if p.Source().Claim().Name != tt.expected[i].Source().Claim().Name { + t.Errorf("Expected %s, got %s", tt.expected[i].Source().Claim().Name, p.Source().Claim().Name) + t.FailNow() + } + if p.Destination().Claim().Name != tt.expected[i].Destination().Claim().Name { + t.Errorf("Expected %s, got %s", tt.expected[i].Destination().Claim().Name, p.Destination().Claim().Name) + t.FailNow() + } + } + }) + } +} + +func createExpectedLiveMigrationProgress(elapsedTime *metav1.Duration) *migapi.LiveMigrationProgress { + return &migapi.LiveMigrationProgress{ + VMName: "test-vm", + VMNamespace: testNamespace, + TotalElapsedTime: elapsedTime, + } +} + +func createExpectedPodProgress() *migapi.PodProgress { + return &migapi.PodProgress{ + ObjectReference: &corev1.ObjectReference{ + Namespace: testNamespace, + Name: "test-pod", + }, + PVCReference: &corev1.ObjectReference{ + Namespace: testNamespace, + Name: testVolume, + }, + LastObservedProgressPercent: "23%", + LastObservedTransferRate: "10MiB/s", + TotalElapsedTime: &metav1.Duration{Duration: time.Second * 10}, + } +} + +func createDirectVolumeMigrationProgress(dvmName, volumeName, namespace string, podPhase corev1.PodPhase) *migapi.DirectVolumeMigrationProgress { + return &migapi.DirectVolumeMigrationProgress{ + ObjectMeta: metav1.ObjectMeta{ + Name: getMD5Hash(dvmName + volumeName + namespace), + Namespace: migapi.OpenshiftMigrationNamespace, + }, + Spec: migapi.DirectVolumeMigrationProgressSpec{}, + Status: migapi.DirectVolumeMigrationProgressStatus{ + RsyncPodStatus: migapi.RsyncPodStatus{ + PodPhase: podPhase, + PodName: "test-pod", + LastObservedTransferRate: "10MiB/s", + }, + TotalProgressPercentage: "23", + RsyncElapsedTime: &metav1.Duration{ + Duration: time.Second * 10, + }, + }, + } +} + +func createOldDirectVolumeMigrationProgress(dvmName, volumeName, namespace string, podPhase corev1.PodPhase) *migapi.DirectVolumeMigrationProgress { + dvmp := createDirectVolumeMigrationProgress(dvmName, volumeName, namespace, podPhase) + dvmp.Status.CreationTimestamp = &metav1.Time{Time: time.Now().Add(-time.Minute * 11)} + return dvmp +} + func getPVCPair(name string, namespace string, volumeMode corev1.PersistentVolumeMode) transfer.PVCPair { pvcPair := transfer.NewPVCPair( &corev1.PersistentVolumeClaim{ @@ -916,31 +1844,31 @@ func TestTask_createRsyncTransferClients(t *testing.T) { fields fields wantPods []*corev1.Pod wantErr bool - wantReturn rsyncClientOperationStatusList + wantReturn migrationOperationStatusList wantCRStatus []*migapi.RsyncOperation }{ { name: "when there are 0 existing Rsync Pods in source and no PVCs to migrate, status list should be empty", fields: fields{ - SrcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - DestClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + SrcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + DestClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, wantPods: []*corev1.Pod{}, wantErr: false, - wantReturn: rsyncClientOperationStatusList{}, + wantReturn: migrationOperationStatusList{}, wantCRStatus: []*migapi.RsyncOperation{}, }, { name: "when there are 0 existing Rsync Pods in source and 1 new fs PVC is provided as input, 1 Rsync Pod must be created in source namespace", fields: fields{ - SrcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - DestClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + SrcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + DestClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, PVCPairMap: map[string][]transfer.PVCPair{ @@ -953,16 +1881,16 @@ func TestTask_createRsyncTransferClients(t *testing.T) { getTestRsyncPodForPVC("pod-0", "pvc-0", "test-ns", "1", metav1.Now().Time), }, wantErr: false, - wantReturn: rsyncClientOperationStatusList{}, + wantReturn: migrationOperationStatusList{}, wantCRStatus: []*migapi.RsyncOperation{}, }, { name: "when there are 0 existing Rsync Pods in source and 1 new block PVC is provided as input, 1 Rsync Pod must be created in source namespace", fields: fields{ - SrcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - DestClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + SrcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + DestClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, PVCPairMap: map[string][]transfer.PVCPair{ @@ -975,16 +1903,16 @@ func TestTask_createRsyncTransferClients(t *testing.T) { getTestRsyncPodForPVC("pod-0", "pvc-0", "test-ns", "1", metav1.Now().Time), }, wantErr: false, - wantReturn: rsyncClientOperationStatusList{}, + wantReturn: migrationOperationStatusList{}, wantCRStatus: []*migapi.RsyncOperation{}, }, { name: "when there are 0 existing Rsync Pods in source and 2 new fs PVCs and one block PVC are provided as input, 3 Rsync Pod must be created in source namespace", fields: fields{ - SrcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - DestClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + SrcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + DestClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, PVCPairMap: map[string][]transfer.PVCPair{ @@ -1001,7 +1929,7 @@ func TestTask_createRsyncTransferClients(t *testing.T) { getTestRsyncPodForPVC("pod-0", "pvc-0", "ns-02", "1", metav1.Now().Time), }, wantErr: false, - wantReturn: rsyncClientOperationStatusList{}, + wantReturn: migrationOperationStatusList{}, wantCRStatus: []*migapi.RsyncOperation{}, }, } @@ -1011,6 +1939,13 @@ func TestTask_createRsyncTransferClients(t *testing.T) { Log: log.WithName("test-logger"), Client: tt.fields.DestClient, Owner: tt.fields.Owner, + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + LiveMigrate: ptr.To[bool](false), + }, + }, + }, } got, err := tr.createRsyncTransferClients(tt.fields.SrcClient, tt.fields.DestClient, tt.fields.PVCPairMap) if (err != nil) != tt.wantErr { @@ -1085,7 +2020,7 @@ func Test_getSecurityContext(t *testing.T) { fields: fields{ client: getFakeCompatClientWithVersion(1, 24), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, Migration: &migapi.MigMigration{ @@ -1118,7 +2053,7 @@ func Test_getSecurityContext(t *testing.T) { fields: fields{ client: getFakeCompatClientWithVersion(1, 24), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, Migration: &migapi.MigMigration{ @@ -1149,7 +2084,7 @@ func Test_getSecurityContext(t *testing.T) { fields: fields{ client: getFakeCompatClientWithVersion(1, 24), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, Migration: &migapi.MigMigration{ @@ -1180,7 +2115,7 @@ func Test_getSecurityContext(t *testing.T) { fields: fields{ client: getFakeCompatClientWithVersion(1, 24), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, Migration: &migapi.MigMigration{ @@ -1238,8 +2173,10 @@ func Test_ensureRsyncEndpoints(t *testing.T) { checkRoute bool }{ { - name: "error getting destination client", - fields: fields{}, + name: "error getting destination client", + fields: fields{ + owner: &migapi.DirectVolumeMigration{}, + }, wantErr: true, }, { @@ -1249,7 +2186,7 @@ func Test_ensureRsyncEndpoints(t *testing.T) { destClient: getFakeCompatClient(createNode("worker1"), createNode("worker2")), endpointType: migapi.NodePort, owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{ BackOffLimit: 2, PersistentVolumeClaims: []migapi.PVCToMigrate{ @@ -1269,7 +2206,16 @@ func Test_ensureRsyncEndpoints(t *testing.T) { destClient: getFakeCompatClient(createMigCluster("test-cluster"), createClusterIngress()), endpointType: migapi.Route, owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{ + Name: testDVM, + Namespace: migapi.OpenshiftMigrationNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "migmigration", + }, + }, + UID: "test-uid", + }, Spec: migapi.DirectVolumeMigrationSpec{ BackOffLimit: 2, PersistentVolumeClaims: []migapi.PVCToMigrate{ @@ -1292,7 +2238,16 @@ func Test_ensureRsyncEndpoints(t *testing.T) { destClient: getFakeCompatClient(createMigCluster("test-cluster")), endpointType: migapi.Route, owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{ + Name: testDVM, + Namespace: migapi.OpenshiftMigrationNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "migmigration", + }, + }, + UID: "test-uid", + }, Spec: migapi.DirectVolumeMigrationSpec{ BackOffLimit: 2, PersistentVolumeClaims: []migapi.PVCToMigrate{ @@ -1315,7 +2270,16 @@ func Test_ensureRsyncEndpoints(t *testing.T) { destClient: getFakeCompatClientWithSubdomain(createMigCluster("test-cluster")), endpointType: migapi.Route, owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{ + Name: testDVM, + Namespace: migapi.OpenshiftMigrationNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "migmigration", + }, + }, + UID: "test-uid", + }, Spec: migapi.DirectVolumeMigrationSpec{ BackOffLimit: 2, PersistentVolumeClaims: []migapi.PVCToMigrate{ @@ -1363,7 +2327,7 @@ func Test_ensureRsyncEndpoints(t *testing.T) { } } -func verifyServiceHealthy(name, namespace, appLabel string, c client.Client, t *testing.T) { +func verifyServiceHealthy(name, namespace, appLabel string, c k8sclient.Client, t *testing.T) { service := &corev1.Service{} if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, service); err != nil { t.Fatalf("ensureRsyncEndpoints() failed to get rsync service in namespace %s", namespace) @@ -1377,7 +2341,7 @@ func verifyServiceHealthy(name, namespace, appLabel string, c client.Client, t * } } -func verifyRouteHealthy(name, namespace, appLabel string, c client.Client, t *testing.T) { +func verifyRouteHealthy(name, namespace, appLabel string, c k8sclient.Client, t *testing.T) { route := &routev1.Route{} if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, route); err != nil { if !k8serrors.IsNotFound(err) { @@ -1417,15 +2381,15 @@ func Test_ensureFilesystemRsyncTransferServer(t *testing.T) { { name: "get single pvc filesystem server pod", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { getPVCPair("pvc", "test-ns", corev1.PersistentVolumeFilesystem), }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1434,15 +2398,15 @@ func Test_ensureFilesystemRsyncTransferServer(t *testing.T) { { name: "no filesystem pod available, no server pods", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { getPVCPair("pvc", "test-ns", corev1.PersistentVolumeBlock), }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1450,15 +2414,15 @@ func Test_ensureFilesystemRsyncTransferServer(t *testing.T) { { name: "error with invalid PVCPair", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { invalidPVCPair{}, }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1470,10 +2434,11 @@ func Test_ensureFilesystemRsyncTransferServer(t *testing.T) { tr := &Task{ Log: log.WithName("test-logger"), destinationClient: tt.fields.destClient, + sourceClient: tt.fields.srcClient, Owner: tt.fields.Owner, Client: tt.fields.destClient, } - if err := tr.ensureFilesystemRsyncTransferServer(tt.fields.srcClient, tt.fields.destClient, tt.fields.PVCPairMap, tt.fields.transportOptions); err != nil && !tt.wantErr { + if err := tr.ensureFilesystemRsyncTransferServer(tt.fields.PVCPairMap, tt.fields.transportOptions); err != nil && !tt.wantErr { t.Fatalf("ensureFilesystemRsyncTransferServer() error = %v, wantErr %v", err, tt.wantErr) } // Verify the server pod is created in the destination namespace @@ -1512,15 +2477,15 @@ func Test_ensureBlockRsyncTransferServer(t *testing.T) { { name: "get single pvc block server pod", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { getPVCPair("pvc", "test-ns", corev1.PersistentVolumeBlock), }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1529,15 +2494,15 @@ func Test_ensureBlockRsyncTransferServer(t *testing.T) { { name: "no block pod available, no server pods", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { getPVCPair("pvc", "test-ns", corev1.PersistentVolumeFilesystem), }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1545,15 +2510,15 @@ func Test_ensureBlockRsyncTransferServer(t *testing.T) { { name: "error with invalid PVCPair", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { invalidPVCPair{}, }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1565,10 +2530,18 @@ func Test_ensureBlockRsyncTransferServer(t *testing.T) { tr := &Task{ Log: log.WithName("test-logger"), destinationClient: tt.fields.destClient, + sourceClient: tt.fields.srcClient, Owner: tt.fields.Owner, Client: tt.fields.destClient, + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + LiveMigrate: ptr.To[bool](false), + }, + }, + }, } - if err := tr.ensureBlockRsyncTransferServer(tt.fields.srcClient, tt.fields.destClient, tt.fields.PVCPairMap, tt.fields.transportOptions); err != nil && !tt.wantErr { + if err := tr.ensureBlockRsyncTransferServer(tt.fields.PVCPairMap, tt.fields.transportOptions); err != nil && !tt.wantErr { t.Fatalf("ensureBlockRsyncTransferServer() error = %v, wantErr %v", err, tt.wantErr) } // Verify the server pod is created in the destination namespace diff --git a/pkg/controller/directvolumemigration/task.go b/pkg/controller/directvolumemigration/task.go index c4a75ea0a6..c7b8422b41 100644 --- a/pkg/controller/directvolumemigration/task.go +++ b/pkg/controller/directvolumemigration/task.go @@ -12,7 +12,10 @@ import ( migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" "github.com/konveyor/mig-controller/pkg/compat" "github.com/opentracing/opentracing-go" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" corev1 "k8s.io/api/core/v1" + virtv1 "kubevirt.io/api/core/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -23,27 +26,28 @@ var NoReQ = time.Duration(0) // Phases const ( - Created = "" - Started = "Started" - Prepare = "Prepare" - CleanStaleRsyncResources = "CleanStaleRsyncResources" - CreateDestinationNamespaces = "CreateDestinationNamespaces" - DestinationNamespacesCreated = "DestinationNamespacesCreated" - CreateDestinationPVCs = "CreateDestinationPVCs" - DestinationPVCsCreated = "DestinationPVCsCreated" - CreateStunnelConfig = "CreateStunnelConfig" - CreateRsyncConfig = "CreateRsyncConfig" - CreateRsyncRoute = "CreateRsyncRoute" - EnsureRsyncRouteAdmitted = "EnsureRsyncRouteAdmitted" - CreateRsyncTransferPods = "CreateRsyncTransferPods" - WaitForRsyncTransferPodsRunning = "WaitForRsyncTransferPodsRunning" - CreatePVProgressCRs = "CreatePVProgressCRs" - RunRsyncOperations = "RunRsyncOperations" - DeleteRsyncResources = "DeleteRsyncResources" - WaitForRsyncResourcesTerminated = "WaitForRsyncResourcesTerminated" - WaitForStaleRsyncResourcesTerminated = "WaitForStaleRsyncResourcesTerminated" - Completed = "Completed" - MigrationFailed = "MigrationFailed" + Created = "" + Started = "Started" + Prepare = "Prepare" + CleanStaleRsyncResources = "CleanStaleRsyncResources" + CreateDestinationNamespaces = "CreateDestinationNamespaces" + DestinationNamespacesCreated = "DestinationNamespacesCreated" + CreateDestinationPVCs = "CreateDestinationPVCs" + DestinationPVCsCreated = "DestinationPVCsCreated" + CreateStunnelConfig = "CreateStunnelConfig" + CreateRsyncConfig = "CreateRsyncConfig" + CreateRsyncRoute = "CreateRsyncRoute" + EnsureRsyncRouteAdmitted = "EnsureRsyncRouteAdmitted" + CreateRsyncTransferPods = "CreateRsyncTransferPods" + WaitForRsyncTransferPodsRunning = "WaitForRsyncTransferPodsRunning" + CreatePVProgressCRs = "CreatePVProgressCRs" + RunRsyncOperations = "RunRsyncOperations" + DeleteRsyncResources = "DeleteRsyncResources" + WaitForRsyncResourcesTerminated = "WaitForRsyncResourcesTerminated" + WaitForStaleRsyncResourcesTerminated = "WaitForStaleRsyncResourcesTerminated" + Completed = "Completed" + MigrationFailed = "MigrationFailed" + DeleteStaleVirtualMachineInstanceMigrations = "DeleteStaleVirtualMachineInstanceMigrations" ) // labels @@ -67,6 +71,13 @@ const ( MigratedByDirectVolumeMigration = "migration.openshift.io/migrated-by-directvolumemigration" // (dvm UID) ) +// Itinerary names +const ( + VolumeMigrationItinerary = "VolumeMigration" + VolumeMigrationFailedItinerary = "VolumeMigrationFailed" + VolumeMigrationRollbackItinerary = "VolumeMigrationRollback" +) + // Flags // TODO: are there any phases to skip? /*const ( @@ -104,7 +115,7 @@ func (r Itinerary) progressReport(phase string) (string, int, int) { } var VolumeMigration = Itinerary{ - Name: "VolumeMigration", + Name: VolumeMigrationItinerary, Steps: []Step{ {phase: Created}, {phase: Started}, @@ -115,6 +126,7 @@ var VolumeMigration = Itinerary{ {phase: DestinationNamespacesCreated}, {phase: CreateDestinationPVCs}, {phase: DestinationPVCsCreated}, + {phase: DeleteStaleVirtualMachineInstanceMigrations}, {phase: CreateRsyncRoute}, {phase: EnsureRsyncRouteAdmitted}, {phase: CreateRsyncConfig}, @@ -130,13 +142,27 @@ var VolumeMigration = Itinerary{ } var FailedItinerary = Itinerary{ - Name: "VolumeMigrationFailed", + Name: VolumeMigrationFailedItinerary, Steps: []Step{ {phase: MigrationFailed}, {phase: Completed}, }, } +var RollbackItinerary = Itinerary{ + Name: VolumeMigrationRollbackItinerary, + Steps: []Step{ + {phase: Created}, + {phase: Started}, + {phase: Prepare}, + {phase: CleanStaleRsyncResources}, + {phase: DeleteStaleVirtualMachineInstanceMigrations}, + {phase: WaitForStaleRsyncResourcesTerminated}, + {phase: RunRsyncOperations}, + {phase: Completed}, + }, +} + // A task that provides the complete migration workflow. // Log - A controller's logger. // Client - A controller's (local) client. @@ -149,6 +175,7 @@ var FailedItinerary = Itinerary{ type Task struct { Log logr.Logger Client k8sclient.Client + PrometheusAPI prometheusv1.API sourceClient compat.Client destinationClient compat.Client Owner *migapi.DirectVolumeMigration @@ -164,9 +191,10 @@ type Task struct { SparseFileMap sparseFilePVCMap SourceLimitRangeMapping limitRangeMap DestinationLimitRangeMapping limitRangeMap - - Tracer opentracing.Tracer - ReconcileSpan opentracing.Span + VirtualMachineMappings VirtualMachineMappings + PromQuery func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) + Tracer opentracing.Tracer + ReconcileSpan opentracing.Span } type limitRangeMap map[string]corev1.LimitRange @@ -176,17 +204,51 @@ type sshKeys struct { PrivateKey *rsa.PrivateKey } +type VirtualMachineMappings struct { + volumeVMIMMap map[string]*virtv1.VirtualMachineInstanceMigration + volumeVMNameMap map[string]string + runningVMVolumeNames []string +} + func (t *Task) init() error { t.RsyncRoutes = make(map[string]string) t.Requeue = FastReQ if t.failed() { t.Itinerary = FailedItinerary + } else if t.Owner.IsRollback() { + t.Itinerary = RollbackItinerary } else { t.Itinerary = VolumeMigration } if t.Itinerary.Name != t.Owner.Status.Itinerary { t.Phase = t.Itinerary.Steps[0].phase } + // Initialize the source and destination clients + _, err := t.getSourceClient() + if err != nil { + return err + } + _, err = t.getDestinationClient() + return err +} + +func (t *Task) populateVMMappings(namespace string) error { + t.VirtualMachineMappings = VirtualMachineMappings{} + volumeVMIMMap, err := t.getVolumeVMIMInNamespaces([]string{namespace}) + if err != nil { + return err + } + volumeVMMap, err := getRunningVmVolumeMap(t.sourceClient, namespace) + if err != nil { + return err + } + volumeNames, err := t.getRunningVMVolumes([]string{namespace}) + if err != nil { + return err + } + t.VirtualMachineMappings.volumeVMIMMap = volumeVMIMMap + t.VirtualMachineMappings.volumeVMNameMap = volumeVMMap + t.VirtualMachineMappings.runningVMVolumeNames = volumeNames return nil } @@ -209,11 +271,7 @@ func (t *Task) Run(ctx context.Context) error { // Run the current phase. switch t.Phase { - case Created, Started: - if err = t.next(); err != nil { - return liberr.Wrap(err) - } - case Prepare: + case Created, Started, Prepare: if err = t.next(); err != nil { return liberr.Wrap(err) } @@ -377,7 +435,7 @@ func (t *Task) Run(ctx context.Context) error { msg := fmt.Sprintf("Rsync Transfer Pod(s) on destination cluster have not started Running within 3 minutes. "+ "Run these command(s) to check Pod warning events: [%s]", - fmt.Sprintf("%s", strings.Join(nonRunningPodStrings, ", "))) + strings.Join(nonRunningPodStrings, ", ")) t.Log.Info(msg) t.Owner.Status.SetCondition( @@ -425,8 +483,15 @@ func (t *Task) Run(ctx context.Context) error { if err = t.next(); err != nil { return liberr.Wrap(err) } + case DeleteStaleVirtualMachineInstanceMigrations: + if err := t.deleteStaleVirtualMachineInstanceMigrations(); err != nil { + return liberr.Wrap(err) + } + if err = t.next(); err != nil { + return liberr.Wrap(err) + } case WaitForStaleRsyncResourcesTerminated, WaitForRsyncResourcesTerminated: - err, deleted := t.waitForRsyncResourcesDeleted() + deleted, err := t.waitForRsyncResourcesDeleted() if err != nil { return liberr.Wrap(err) } @@ -436,6 +501,7 @@ func (t *Task) Run(ctx context.Context) error { return liberr.Wrap(err) } } + t.Log.Info("Stale Rsync resources are still terminating. Waiting.") t.Requeue = PollReQ case Completed: @@ -528,6 +594,7 @@ func (t *Task) getSourceClient() (compat.Client, error) { if err != nil { return nil, err } + t.sourceClient = client return client, nil } @@ -547,6 +614,7 @@ func (t *Task) getDestinationClient() (compat.Client, error) { if err != nil { return nil, err } + t.destinationClient = client return client, nil } diff --git a/pkg/controller/directvolumemigration/validation.go b/pkg/controller/directvolumemigration/validation.go index 0636cc7daf..2bd1f16a9e 100644 --- a/pkg/controller/directvolumemigration/validation.go +++ b/pkg/controller/directvolumemigration/validation.go @@ -28,6 +28,7 @@ const ( Running = "Running" Failed = "Failed" RsyncClientPodsPending = "RsyncClientPodsPending" + LiveMigrationsPending = "LiveMigrationsPending" Succeeded = "Succeeded" SourceToDestinationNetworkError = "SourceToDestinationNetworkError" FailedCreatingRsyncPods = "FailedCreatingRsyncPods" @@ -43,6 +44,7 @@ const ( NotReady = "NotReady" RsyncTimeout = "RsyncTimedOut" RsyncNoRouteToHost = "RsyncNoRouteToHost" + NotSupported = "NotSupported" ) // Messages @@ -75,28 +77,25 @@ const ( ) // Validate the direct resource -func (r ReconcileDirectVolumeMigration) validate(ctx context.Context, direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validate(ctx context.Context, direct *migapi.DirectVolumeMigration) error { if opentracing.SpanFromContext(ctx) != nil { var span opentracing.Span span, ctx = opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validate") defer span.Finish() } - err := r.validateSrcCluster(ctx, direct) - if err != nil { + if err := r.validateSrcCluster(ctx, direct); err != nil { return liberr.Wrap(err) } - err = r.validateDestCluster(ctx, direct) - if err != nil { + if err := r.validateDestCluster(ctx, direct); err != nil { return liberr.Wrap(err) } - err = r.validatePVCs(ctx, direct) - if err != nil { + if err := r.validatePVCs(ctx, direct); err != nil { return liberr.Wrap(err) } return nil } -func (r ReconcileDirectVolumeMigration) validateSrcCluster(ctx context.Context, direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validateSrcCluster(ctx context.Context, direct *migapi.DirectVolumeMigration) error { if opentracing.SpanFromContext(ctx) != nil { span, _ := opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validateSrcCluster") defer span.Finish() @@ -146,7 +145,7 @@ func (r ReconcileDirectVolumeMigration) validateSrcCluster(ctx context.Context, return nil } -func (r ReconcileDirectVolumeMigration) validateDestCluster(ctx context.Context, direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validateDestCluster(ctx context.Context, direct *migapi.DirectVolumeMigration) error { if opentracing.SpanFromContext(ctx) != nil { span, _ := opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validateDestCluster") defer span.Finish() @@ -198,11 +197,11 @@ func (r ReconcileDirectVolumeMigration) validateDestCluster(ctx context.Context, // TODO: Validate that storage class mappings have valid storage class selections // Leaving as TODO because this is technically already validated from the // migplan, so not necessary from directvolumemigration controller to be fair -func (r ReconcileDirectVolumeMigration) validateStorageClassMappings(direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validateStorageClassMappings(direct *migapi.DirectVolumeMigration) error { return nil } -func (r ReconcileDirectVolumeMigration) validatePVCs(ctx context.Context, direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validatePVCs(ctx context.Context, direct *migapi.DirectVolumeMigration) error { if opentracing.SpanFromContext(ctx) != nil { span, _ := opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validatePVCs") defer span.Finish() diff --git a/pkg/controller/directvolumemigration/vm.go b/pkg/controller/directvolumemigration/vm.go new file mode 100644 index 0000000000..9aa42656dc --- /dev/null +++ b/pkg/controller/directvolumemigration/vm.go @@ -0,0 +1,575 @@ +package directvolumemigration + +import ( + "context" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "slices" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/konveyor/crane-lib/state_transfer/transfer" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + routev1 "github.com/openshift/api/route/v1" + prometheusapi "github.com/prometheus/client_golang/api" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + prometheusURLKey = "PROMETHEUS_URL" + prometheusRoute = "prometheus-k8s" + progressQuery = "kubevirt_vmi_migration_data_processed_bytes{name=\"%s\"} / (kubevirt_vmi_migration_data_processed_bytes{name=\"%s\"} + kubevirt_vmi_migration_data_remaining_bytes{name=\"%s\"}) * 100" +) + +var ErrVolumesDoNotMatch = errors.New("volumes do not match") + +type vmVolumes struct { + sourceVolumes []string + targetVolumes []string +} + +func (t *Task) startLiveMigrations(nsMap map[string][]transfer.PVCPair) ([]string, error) { + reasons := []string{} + vmVolumeMap := make(map[string]vmVolumes) + sourceClient := t.sourceClient + if t.Owner.IsRollback() { + sourceClient = t.destinationClient + } + for k, v := range nsMap { + namespace, err := getNamespace(k) + if err != nil { + reasons = append(reasons, err.Error()) + return reasons, err + } + volumeVmMap, err := getRunningVmVolumeMap(sourceClient, namespace) + if err != nil { + reasons = append(reasons, err.Error()) + return reasons, err + } + for _, pvcPair := range v { + if vmName, found := volumeVmMap[pvcPair.Source().Claim().Name]; found { + vmVolumeMap[vmName] = vmVolumes{ + sourceVolumes: append(vmVolumeMap[vmName].sourceVolumes, pvcPair.Source().Claim().Name), + targetVolumes: append(vmVolumeMap[vmName].targetVolumes, pvcPair.Destination().Claim().Name), + } + } + } + for vmName, volumes := range vmVolumeMap { + if err := t.storageLiveMigrateVM(vmName, namespace, &volumes); err != nil { + switch err { + case ErrVolumesDoNotMatch: + reasons = append(reasons, fmt.Sprintf("source and target volumes do not match for VM %s", vmName)) + continue + default: + reasons = append(reasons, err.Error()) + return reasons, err + } + } + } + } + return reasons, nil +} + +func getNamespace(colonDelimitedString string) (string, error) { + namespacePair := strings.Split(colonDelimitedString, ":") + if len(namespacePair) != 2 { + return "", fmt.Errorf("invalid namespace pair: %s", colonDelimitedString) + } + if namespacePair[0] != namespacePair[1] && namespacePair[0] != "" { + return "", fmt.Errorf("source and target namespaces must match: %s", colonDelimitedString) + } + return namespacePair[0], nil +} + +// Return a list of namespace/volume combinations that are currently in use by running VMs +func (t *Task) getRunningVMVolumes(namespaces []string) ([]string, error) { + runningVMVolumes := []string{} + + for _, ns := range namespaces { + volumesVmMap, err := getRunningVmVolumeMap(t.sourceClient, ns) + if err != nil { + return nil, err + } + for volume := range volumesVmMap { + runningVMVolumes = append(runningVMVolumes, fmt.Sprintf("%s/%s", ns, volume)) + } + } + return runningVMVolumes, nil +} + +func getRunningVmVolumeMap(client k8sclient.Client, namespace string) (map[string]string, error) { + volumesVmMap := make(map[string]string) + vmMap, err := getVMNamesInNamespace(client, namespace) + if err != nil { + return nil, err + } + for vmName := range vmMap { + podList := corev1.PodList{} + client.List(context.TODO(), &podList, k8sclient.InNamespace(namespace)) + for _, pod := range podList.Items { + for _, owner := range pod.OwnerReferences { + if owner.Name == vmName && owner.Kind == "VirtualMachineInstance" { + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + volumesVmMap[volume.PersistentVolumeClaim.ClaimName] = vmName + } + } + } + } + } + } + return volumesVmMap, nil +} + +func getVMNamesInNamespace(client k8sclient.Client, namespace string) (map[string]interface{}, error) { + vms := make(map[string]interface{}) + vmList := virtv1.VirtualMachineList{} + err := client.List(context.TODO(), &vmList, k8sclient.InNamespace(namespace)) + if err != nil { + if meta.IsNoMatchError(err) { + return nil, nil + } + return nil, err + } + for _, vm := range vmList.Items { + vms[vm.Name] = nil + } + return vms, nil +} + +func (t *Task) storageLiveMigrateVM(vmName, namespace string, volumes *vmVolumes) error { + sourceClient := t.sourceClient + if t.Owner.IsRollback() { + sourceClient = t.destinationClient + } + vm := &virtv1.VirtualMachine{} + if err := sourceClient.Get(context.TODO(), k8sclient.ObjectKey{Namespace: namespace, Name: vmName}, vm); err != nil { + return err + } + // Check if the source volumes match before attempting to migrate. + if !t.compareVolumes(vm, volumes.sourceVolumes) { + // Check if the target volumes match, if so, the migration is already in progress or complete. + if t.compareVolumes(vm, volumes.targetVolumes) { + t.Log.V(5).Info("Volumes already updated for VM", "vm", vmName) + return nil + } else { + return ErrVolumesDoNotMatch + } + } + + // Volumes match, create patch to update the VM with the target volumes + return updateVM(sourceClient, vm, volumes.sourceVolumes, volumes.targetVolumes, t.Log) +} + +func (t *Task) compareVolumes(vm *virtv1.VirtualMachine, volumes []string) bool { + foundCount := 0 + if vm.Spec.Template == nil { + return true + } + for _, vmVolume := range vm.Spec.Template.Spec.Volumes { + if vmVolume.PersistentVolumeClaim == nil && vmVolume.DataVolume == nil { + // Skip all non PVC or DataVolume volumes + continue + } + if vmVolume.PersistentVolumeClaim != nil { + if slices.Contains(volumes, vmVolume.PersistentVolumeClaim.ClaimName) { + foundCount++ + } + } else if vmVolume.DataVolume != nil { + if slices.Contains(volumes, vmVolume.DataVolume.Name) { + foundCount++ + } + } + } + return foundCount == len(volumes) +} + +func findVirtualMachineInstanceMigration(client k8sclient.Client, vmName, namespace string) (*virtv1.VirtualMachineInstanceMigration, error) { + vmimList := &virtv1.VirtualMachineInstanceMigrationList{} + if err := client.List(context.TODO(), vmimList, k8sclient.InNamespace(namespace)); err != nil { + return nil, err + } + for _, vmim := range vmimList.Items { + if vmim.Spec.VMIName == vmName { + return &vmim, nil + } + } + return nil, nil +} + +func virtualMachineMigrationStatus(client k8sclient.Client, vmName, namespace string, log logr.Logger) (string, error) { + log.Info("Checking migration status for VM", "vm", vmName) + vmim, err := findVirtualMachineInstanceMigration(client, vmName, namespace) + if err != nil { + return err.Error(), err + } + if vmim != nil { + log.V(5).Info("Found VMIM", "vmim", vmim.Name, "namespace", vmim.Namespace) + if vmim.Status.MigrationState != nil { + if vmim.Status.MigrationState.Failed { + return vmim.Status.MigrationState.FailureReason, nil + } else if vmim.Status.MigrationState.AbortStatus == virtv1.MigrationAbortSucceeded { + return "Migration canceled", nil + } else if vmim.Status.MigrationState.AbortStatus == virtv1.MigrationAbortFailed { + return "Migration canceled failed", nil + } else if vmim.Status.MigrationState.AbortStatus == virtv1.MigrationAbortInProgress { + return "Migration cancel in progress", nil + } + if vmim.Status.MigrationState.Completed { + return "", nil + } + } + } + + vmi := &virtv1.VirtualMachineInstance{} + if err := client.Get(context.TODO(), k8sclient.ObjectKey{Namespace: namespace, Name: vmName}, vmi); err != nil { + if k8serrors.IsNotFound(err) { + return fmt.Sprintf("VMI %s not found in namespace %s", vmName, namespace), nil + } else { + return err.Error(), err + } + } + volumeChange := false + liveMigrateable := false + liveMigrateableMessage := "" + for _, condition := range vmi.Status.Conditions { + if condition.Type == virtv1.VirtualMachineInstanceVolumesChange { + volumeChange = condition.Status == corev1.ConditionTrue + } + if condition.Type == virtv1.VirtualMachineInstanceIsMigratable { + liveMigrateable = condition.Status == corev1.ConditionTrue + liveMigrateableMessage = condition.Message + } + } + if volumeChange && !liveMigrateable { + // Unable to storage live migrate because something else is preventing migration + return liveMigrateableMessage, nil + } + return "", nil +} + +func cancelLiveMigration(client k8sclient.Client, vmName, namespace string, volumes *vmVolumes, log logr.Logger) error { + vm := &virtv1.VirtualMachine{} + if err := client.Get(context.TODO(), k8sclient.ObjectKey{Namespace: namespace, Name: vmName}, vm); err != nil { + return err + } + + log.V(3).Info("Canceling live migration", "vm", vmName) + if err := updateVM(client, vm, volumes.targetVolumes, volumes.sourceVolumes, log); err != nil { + return err + } + return nil +} + +func liveMigrationsCompleted(client k8sclient.Client, namespace string, vmNames []string) (bool, error) { + vmim := &virtv1.VirtualMachineInstanceMigrationList{} + if err := client.List(context.TODO(), vmim, k8sclient.InNamespace(namespace)); err != nil { + return false, err + } + completed := true + for _, migration := range vmim.Items { + if slices.Contains(vmNames, migration.Spec.VMIName) { + if migration.Status.Phase != virtv1.MigrationSucceeded { + completed = false + break + } + } + } + return completed, nil +} + +func updateVM(client k8sclient.Client, vm *virtv1.VirtualMachine, sourceVolumes, targetVolumes []string, log logr.Logger) error { + if vm == nil || vm.Name == "" { + return nil + } + vmCopy := vm.DeepCopy() + // Ensure the VM migration strategy is set properly. + log.V(5).Info("Setting volume migration strategy to migration", "vm", vmCopy.Name) + vmCopy.Spec.UpdateVolumesStrategy = ptr.To[virtv1.UpdateVolumesStrategy](virtv1.UpdateVolumesStrategyMigration) + + for i := 0; i < len(sourceVolumes); i++ { + // Check if we need to update DataVolumeTemplates. + for j, dvTemplate := range vmCopy.Spec.DataVolumeTemplates { + if dvTemplate.Name == sourceVolumes[i] { + log.V(5).Info("Updating DataVolumeTemplate", "source", sourceVolumes[i], "target", targetVolumes[i]) + vmCopy.Spec.DataVolumeTemplates[j].Name = targetVolumes[i] + } + } + for j, volume := range vm.Spec.Template.Spec.Volumes { + if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == sourceVolumes[i] { + log.V(5).Info("Updating PersistentVolumeClaim", "source", sourceVolumes[i], "target", targetVolumes[i]) + vmCopy.Spec.Template.Spec.Volumes[j].PersistentVolumeClaim.ClaimName = targetVolumes[i] + } + if volume.DataVolume != nil && volume.DataVolume.Name == sourceVolumes[i] { + log.V(5).Info("Updating DataVolume", "source", sourceVolumes[i], "target", targetVolumes[i]) + if err := CreateNewDataVolume(client, sourceVolumes[i], targetVolumes[i], vmCopy.Namespace, log); err != nil { + return err + } + vmCopy.Spec.Template.Spec.Volumes[j].DataVolume.Name = targetVolumes[i] + } + } + } + if !reflect.DeepEqual(vm, vmCopy) { + log.V(5).Info("Calling VM update", "vm", vm.Name) + if err := client.Update(context.TODO(), vmCopy); err != nil { + return err + } + } else { + log.V(5).Info("No changes to VM", "vm", vm.Name) + } + return nil +} + +func CreateNewDataVolume(client k8sclient.Client, sourceDvName, targetDvName, ns string, log logr.Logger) error { + log.V(3).Info("Create new adopting datavolume from source datavolume", "namespace", ns, "source name", sourceDvName, "target name", targetDvName) + originalDv := &cdiv1.DataVolume{} + if err := client.Get(context.TODO(), k8sclient.ObjectKey{Namespace: ns, Name: sourceDvName}, originalDv); err != nil { + log.Error(err, "Failed to get source datavolume", "namespace", ns, "name", sourceDvName) + return err + } + + // Create adopting datavolume. + adoptingDV := originalDv.DeepCopy() + adoptingDV.Name = targetDvName + if adoptingDV.Annotations == nil { + adoptingDV.Annotations = make(map[string]string) + } + adoptingDV.Annotations["cdi.kubevirt.io/allowClaimAdoption"] = "true" + adoptingDV.ResourceVersion = "" + adoptingDV.ManagedFields = nil + adoptingDV.UID = "" + adoptingDV.Spec.Source = &cdiv1.DataVolumeSource{ + Blank: &cdiv1.DataVolumeBlankImage{}, + } + err := client.Create(context.TODO(), adoptingDV) + if err != nil && !k8serrors.IsAlreadyExists(err) { + log.Error(err, "Failed to create adopting datavolume", "namespace", ns, "name", targetDvName) + return err + } + return nil +} + +// Gets all the VirtualMachineInstanceMigration objects by volume in the passed in namespace. +func (t *Task) getVolumeVMIMInNamespaces(namespaces []string) (map[string]*virtv1.VirtualMachineInstanceMigration, error) { + vmimMap := make(map[string]*virtv1.VirtualMachineInstanceMigration) + for _, namespace := range namespaces { + volumeVMMap, err := getRunningVmVolumeMap(t.sourceClient, namespace) + if err != nil { + return nil, err + } + for volumeName, vmName := range volumeVMMap { + vmimList := &virtv1.VirtualMachineInstanceMigrationList{} + if err := t.sourceClient.List(context.TODO(), vmimList, k8sclient.InNamespace(namespace)); err != nil { + return nil, err + } + for _, vmim := range vmimList.Items { + if vmim.Spec.VMIName == vmName { + vmimMap[fmt.Sprintf("%s/%s", namespace, volumeName)] = &vmim + } + } + } + } + return vmimMap, nil +} + +func getVMIMElapsedTime(vmim *virtv1.VirtualMachineInstanceMigration) metav1.Duration { + if vmim == nil || vmim.Status.MigrationState == nil { + return metav1.Duration{ + Duration: 0, + } + } + if vmim.Status.MigrationState.StartTimestamp == nil { + for _, timestamps := range vmim.Status.PhaseTransitionTimestamps { + if timestamps.Phase == virtv1.MigrationRunning { + vmim.Status.MigrationState.StartTimestamp = ×tamps.PhaseTransitionTimestamp + } + } + } + if vmim.Status.MigrationState.StartTimestamp == nil { + return metav1.Duration{ + Duration: 0, + } + } + if vmim.Status.MigrationState.EndTimestamp != nil { + return metav1.Duration{ + Duration: vmim.Status.MigrationState.EndTimestamp.Sub(vmim.Status.MigrationState.StartTimestamp.Time), + } + } + return metav1.Duration{ + Duration: time.Since(vmim.Status.MigrationState.StartTimestamp.Time), + } +} + +func (t *Task) getLastObservedProgressPercent(vmName, namespace string, currentProgress map[string]*migapi.LiveMigrationProgress) (string, error) { + if err := t.buildPrometheusAPI(); err != nil { + return "", err + } + + result, warnings, err := t.PromQuery(context.TODO(), fmt.Sprintf(progressQuery, vmName, vmName, vmName), time.Now()) + if err != nil { + t.Log.Error(err, "Failed to query prometheus, returning previous progress") + if progress, found := currentProgress[fmt.Sprintf("%s/%s", namespace, vmName)]; found { + return progress.LastObservedProgressPercent, nil + } + return "", nil + } + if len(warnings) > 0 { + t.Log.Info("Warnings", "warnings", warnings) + } + t.Log.V(5).Info("Prometheus query result", "type", result.Type(), "value", result.String()) + progress := parseProgress(result.String()) + if progress != "" { + return progress + "%", nil + } + return "", nil +} + +func (t *Task) buildPrometheusAPI() error { + if t.PrometheusAPI != nil { + return nil + } + restConfig, err := t.PlanResources.SrcMigCluster.BuildRestConfig(t.Client) + if err != nil { + return err + } + + url, err := t.buildSourcePrometheusEndPointURL() + if err != nil { + return err + } + + // Prometheus URL not found, return blank progress + if url == "" { + return nil + } + httpClient, err := rest.HTTPClientFor(restConfig) + if err != nil { + return err + } + client, err := prometheusapi.NewClient(prometheusapi.Config{ + Address: url, + Client: httpClient, + }) + if err != nil { + return err + } + t.PrometheusAPI = prometheusv1.NewAPI(client) + t.PromQuery = t.PrometheusAPI.Query + return nil +} + +func parseProgress(progress string) string { + regExp := regexp.MustCompile(`\=\> (\d{1,3})\.\d* @`) + + if regExp.MatchString(progress) { + return regExp.FindStringSubmatch(progress)[1] + } + return "" +} + +// Find the URL that contains the prometheus metrics on the source cluster. +func (t *Task) buildSourcePrometheusEndPointURL() (string, error) { + urlString, err := t.getPrometheusURLFromConfig() + if err != nil { + return "", err + } + if urlString == "" { + // URL not found in config map, attempt to get the open shift prometheus route. + routes := &routev1.RouteList{} + req, err := labels.NewRequirement("app.kubernetes.io/part-of", selection.Equals, []string{"openshift-monitoring"}) + if err != nil { + return "", err + } + if err := t.sourceClient.List(context.TODO(), routes, k8sclient.InNamespace("openshift-monitoring"), &k8sclient.ListOptions{ + LabelSelector: labels.NewSelector().Add(*req), + }); err != nil { + return "", err + } + for _, r := range routes.Items { + if r.Spec.To.Name == prometheusRoute { + urlString = r.Spec.Host + break + } + } + } + if urlString == "" { + // Don't return error just return empty and skip the progress report + return "", nil + } + parsedUrl, err := url.Parse(urlString) + if err != nil { + return "", err + } + parsedUrl.Scheme = "https" + urlString = parsedUrl.String() + t.Log.V(3).Info("Prometheus route URL", "url", urlString) + return urlString, nil +} + +// The key in the config map should be in format _PROMETHEUS_URL +// For instance if the cluster name is "cluster1" the key should be "cluster1_PROMETHEUS_URL" +func (t *Task) getPrometheusURLFromConfig() (string, error) { + migControllerConfigMap := &corev1.ConfigMap{} + if err := t.sourceClient.Get(context.TODO(), k8sclient.ObjectKey{Namespace: migapi.OpenshiftMigrationNamespace, Name: "migration-controller"}, migControllerConfigMap); err != nil { + return "", err + } + key := fmt.Sprintf("%s_%s", t.PlanResources.SrcMigCluster.Name, prometheusURLKey) + if prometheusURL, found := migControllerConfigMap.Data[key]; found { + return prometheusURL, nil + } + return "", nil +} + +func (t *Task) deleteStaleVirtualMachineInstanceMigrations() error { + pvcMap, err := t.getNamespacedPVCPairs() + if err != nil { + return err + } + + for namespacePair := range pvcMap { + namespace, err := getNamespace(namespacePair) + if err != nil { + return err + } + vmMap, err := getVMNamesInNamespace(t.sourceClient, namespace) + if err != nil { + return err + } + + vmimList := &virtv1.VirtualMachineInstanceMigrationList{} + if err := t.sourceClient.List(context.TODO(), vmimList, k8sclient.InNamespace(namespace)); err != nil { + if meta.IsNoMatchError(err) { + return nil + } + return err + } + for _, vmim := range vmimList.Items { + if _, ok := vmMap[vmim.Spec.VMIName]; ok { + if vmim.Status.Phase == virtv1.MigrationSucceeded && + vmim.Status.MigrationState != nil && + vmim.Status.MigrationState.EndTimestamp.Before(&t.Owner.CreationTimestamp) { + // Old VMIM that has completed and succeeded before the migration was created, delete the VMIM + if err := t.sourceClient.Delete(context.TODO(), &vmim); err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + } + } + } + return nil +} diff --git a/pkg/controller/directvolumemigration/vm_test.go b/pkg/controller/directvolumemigration/vm_test.go new file mode 100644 index 0000000000..d4f7c1cadb --- /dev/null +++ b/pkg/controller/directvolumemigration/vm_test.go @@ -0,0 +1,1795 @@ +package directvolumemigration + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" + + transfer "github.com/konveyor/crane-lib/state_transfer/transfer" + "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/compat" + fakecompat "github.com/konveyor/mig-controller/pkg/compat/fake" + routev1 "github.com/openshift/api/route/v1" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + sourcePVC = "source-pvc" + sourceNs = "source-ns" + targetPVC = "target-pvc" + targetNs = "target-ns" + targetDv = "target-dv" +) + +func TestTask_startLiveMigrations(t *testing.T) { + tests := []struct { + name string + task *Task + client compat.Client + pairMap map[string][]transfer.PVCPair + expectedReasons []string + wantErr bool + }{ + { + name: "same namespace, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace)), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + }, + { + name: "different namespace, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace)), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace2: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace2)), + }, + }, + wantErr: true, + expectedReasons: []string{"source and target namespaces must match: test-namespace:test-namespace2"}, + }, + { + name: "running VM, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace), createVirtlauncherPod("vm", testNamespace, []string{"dv"})), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + }, + { + name: "running VM, matching volume", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + }), createVirtlauncherPod("vm", testNamespace, []string{"dv"}), + createDataVolume("dv", testNamespace)), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("dv", testNamespace), createPvc("dv2", testNamespace)), + }, + }, + }, + { + name: "running VM, no matching volume", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-nomatch", + }, + }, + }, + }), createVirtlauncherPod("vm", testNamespace, []string{"dv"}), + createDataVolume("dv", testNamespace)), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("dv", testNamespace), createPvc("dv2", testNamespace)), + }, + }, + wantErr: false, + expectedReasons: []string{"source and target volumes do not match for VM vm"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.task.Owner.Spec.MigrationType == nil || tt.task.Owner.Spec.MigrationType == ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback) { + tt.task.sourceClient = tt.client + } else { + tt.task.destinationClient = tt.client + } + reasons, err := tt.task.startLiveMigrations(tt.pairMap) + if err != nil && !tt.wantErr { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } else if err == nil && tt.wantErr { + t.Errorf("expected error, got nil") + t.FailNow() + } + if len(reasons) != len(tt.expectedReasons) { + t.Errorf("expected %v, got %v", tt.expectedReasons, reasons) + t.FailNow() + } + for i, r := range reasons { + if r != tt.expectedReasons[i] { + t.Errorf("expected %v, got %v", tt.expectedReasons, reasons) + t.FailNow() + } + } + }) + } +} + +func TestGetNamespace(t *testing.T) { + _, err := getNamespace("nocolon") + if err == nil { + t.Errorf("expected error, got nil") + t.FailNow() + } +} + +func TestTaskGetRunningVMVolumes(t *testing.T) { + tests := []struct { + name string + task *Task + client compat.Client + expectedVolumes []string + }{ + { + name: "no VMs", + task: &Task{}, + client: getFakeClientWithObjs(), + expectedVolumes: []string{}, + }, + { + name: "no running vms", + task: &Task{}, + client: getFakeClientWithObjs(createVirtualMachine("vm", "ns1"), createVirtualMachine("vm2", "ns2")), + expectedVolumes: []string{}, + }, + { + name: "running all vms", + task: &Task{}, + client: getFakeClientWithObjs( + createVirtualMachine("vm", "ns1"), + createVirtualMachine("vm2", "ns2"), + createVirtlauncherPod("vm", "ns1", []string{"dv"}), + createVirtlauncherPod("vm2", "ns2", []string{"dv2"})), + expectedVolumes: []string{"ns1/dv", "ns2/dv2"}, + }, + { + name: "running single vms", + task: &Task{}, + client: getFakeClientWithObjs( + createVirtualMachine("vm", "ns1"), + createVirtualMachine("vm2", "ns2"), + createVirtlauncherPod("vm2", "ns2", []string{"dv2"})), + expectedVolumes: []string{"ns2/dv2"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + volumes, err := tt.task.getRunningVMVolumes([]string{"ns1", "ns2"}) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if len(volumes) != len(tt.expectedVolumes) { + t.Errorf("expected %v, got %v", tt.expectedVolumes, volumes) + t.FailNow() + } + for i, v := range volumes { + if v != tt.expectedVolumes[i] { + t.Errorf("expected %v, got %v", tt.expectedVolumes, volumes) + t.FailNow() + } + } + }) + } +} + +func TestTaskStorageLiveMigrateVM(t *testing.T) { + tests := []struct { + name string + task *Task + client compat.Client + volumes *vmVolumes + wantErr bool + }{ + { + name: "no vm, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(), + volumes: &vmVolumes{}, + wantErr: true, + }, + { + name: "vm, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace)), + volumes: &vmVolumes{}, + wantErr: false, + }, + { + name: "vm, matching volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + }), createDataVolume("dv", testNamespace)), + volumes: &vmVolumes{ + sourceVolumes: []string{"dv"}, + targetVolumes: []string{targetDv}, + }, + wantErr: false, + }, + { + name: "vm, matching volumes, already modified", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: targetDv, + }, + }, + }, + }), createDataVolume("dv", testNamespace)), + volumes: &vmVolumes{ + sourceVolumes: []string{"dv"}, + targetVolumes: []string{targetDv}, + }, + wantErr: false, + }, + { + name: "vm, non matching volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "unmatched", + }, + }, + }, + }), createDataVolume("dv", testNamespace)), + volumes: &vmVolumes{ + sourceVolumes: []string{"dv"}, + targetVolumes: []string{targetDv}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.task.Owner.Spec.MigrationType == nil || tt.task.Owner.Spec.MigrationType == ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback) { + tt.task.sourceClient = tt.client + } else { + tt.task.destinationClient = tt.client + } + err := tt.task.storageLiveMigrateVM("vm", testNamespace, tt.volumes) + if err != nil && !tt.wantErr { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } else if err == nil && tt.wantErr { + t.Errorf("expected error, got nil") + t.FailNow() + } + }) + } +} + +func TestVirtualMachineMigrationStatus(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + expectedStatus string + }{ + { + name: "In progress VMIM", + client: getFakeClientWithObjs(createInProgressVirtualMachineMigration("vmim", testNamespace, "vm")), + expectedStatus: fmt.Sprintf("VMI %s not found in namespace %s", "vm", testNamespace), + }, + { + name: "No VMIM or VMI", + client: getFakeClientWithObjs(), + expectedStatus: fmt.Sprintf("VMI %s not found in namespace %s", "vm", testNamespace), + }, + { + name: "Failed VMIM with message", + client: getFakeClientWithObjs(createFailedVirtualMachineMigration("vmim", testNamespace, "vm", "failed")), + expectedStatus: "failed", + }, + { + name: "Canceled VMIM", + client: getFakeClientWithObjs(createCanceledVirtualMachineMigration("vmim", testNamespace, "vm", virtv1.MigrationAbortSucceeded)), + expectedStatus: "Migration canceled", + }, + { + name: "Canceled VMIM inprogress", + client: getFakeClientWithObjs(createCanceledVirtualMachineMigration("vmim", testNamespace, "vm", virtv1.MigrationAbortInProgress)), + expectedStatus: "Migration cancel in progress", + }, + { + name: "Canceled VMIM failed", + client: getFakeClientWithObjs(createCanceledVirtualMachineMigration("vmim", testNamespace, "vm", virtv1.MigrationAbortFailed)), + expectedStatus: "Migration canceled failed", + }, + { + name: "Completed VMIM", + client: getFakeClientWithObjs(createCompletedVirtualMachineMigration("vmim", testNamespace, "vm")), + expectedStatus: "", + }, + { + name: "VMI without conditions", + client: getFakeClientWithObjs(createVirtualMachineInstance("vm", testNamespace, virtv1.Running)), + expectedStatus: "", + }, + { + name: "VMI with update strategy conditions, and live migrate possible", + client: getFakeClientWithObjs(createVirtualMachineInstanceWithConditions("vm", testNamespace, []virtv1.VirtualMachineInstanceCondition{ + { + Type: virtv1.VirtualMachineInstanceVolumesChange, + Status: corev1.ConditionTrue, + }, + { + Type: virtv1.VirtualMachineInstanceIsMigratable, + Status: corev1.ConditionTrue, + }, + })), + expectedStatus: "", + }, + { + name: "VMI with update strategy conditions, and live migrate not possible", + client: getFakeClientWithObjs(createVirtualMachineInstanceWithConditions("vm", testNamespace, []virtv1.VirtualMachineInstanceCondition{ + { + Type: virtv1.VirtualMachineInstanceVolumesChange, + Status: corev1.ConditionTrue, + }, + { + Type: virtv1.VirtualMachineInstanceIsMigratable, + Status: corev1.ConditionFalse, + Message: "Migration not possible", + }, + })), + expectedStatus: "Migration not possible", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status, err := virtualMachineMigrationStatus(tt.client, "vm", testNamespace, log.WithName(tt.name)) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if status != tt.expectedStatus { + t.Errorf("expected %s, got %s", tt.expectedStatus, status) + t.FailNow() + } + }) + } +} + +func TestCancelLiveMigration(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + vmVolumes *vmVolumes + expectErr bool + }{ + { + name: "no changed volumes", + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{})), + vmVolumes: &vmVolumes{}, + }, + { + name: "no virtual machine", + client: getFakeClientWithObjs(), + vmVolumes: &vmVolumes{}, + expectErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := cancelLiveMigration(tt.client, "vm", testNamespace, tt.vmVolumes, log.WithName(tt.name)) + if err != nil && !tt.expectErr { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + }) + } +} + +func TestLiveMigrationsCompleted(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + vmNames []string + expectComplete bool + }{ + { + name: "no VMIMs", + client: getFakeClientWithObjs(), + vmNames: []string{"vm1", "vm2"}, + expectComplete: true, + }, + { + name: "all VMIMs completed, but no matching VMs", + client: getFakeClientWithObjs(createCompletedVirtualMachineMigration("vmim1", testNamespace, "vm")), + vmNames: []string{"vm1", "vm2"}, + expectComplete: true, + }, + { + name: "all VMIMs completed, and one matching VM", + client: getFakeClientWithObjs( + createCompletedVirtualMachineMigration("vmim1", testNamespace, "vm1"), + createCompletedVirtualMachineMigration("vmim2", testNamespace, "vm2")), + vmNames: []string{"vm1"}, + expectComplete: true, + }, + { + name: "not all VMIMs completed, and matching VM", + client: getFakeClientWithObjs( + createCompletedVirtualMachineMigration("vmim1", testNamespace, "vm1"), + createInProgressVirtualMachineMigration("vmim2", testNamespace, "vm2")), + vmNames: []string{"vm1"}, + expectComplete: true, + }, + { + name: "not all VMIMs completed, and matching VM", + client: getFakeClientWithObjs( + createCompletedVirtualMachineMigration("vmim1", testNamespace, "vm1"), + createInProgressVirtualMachineMigration("vmim2", testNamespace, "vm2")), + vmNames: []string{"vm1", "vm2"}, + expectComplete: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + complete, err := liveMigrationsCompleted(tt.client, testNamespace, tt.vmNames) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if complete != tt.expectComplete { + t.Errorf("expected %t, got %t", tt.expectComplete, complete) + t.FailNow() + } + }) + } +} + +func TestUpdateVM(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + vmName string + sourceVolumes []string + targetVolumes []string + expectedVM *virtv1.VirtualMachine + expectedUpdate bool + }{ + { + name: "name nil vm", + client: getFakeClientWithObjs(), + vmName: "", + }, + { + name: "vm without volumes", + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace)), + vmName: "vm", + expectedVM: createVirtualMachine("vm", testNamespace), + expectedUpdate: true, + }, + { + name: "already migrated vm, no update", + client: getFakeClientWithObjs(createVirtualMachineWithUpdateStrategy("vm", testNamespace, []virtv1.Volume{})), + vmName: "vm", + expectedVM: createVirtualMachineWithUpdateStrategy("vm", testNamespace, []virtv1.Volume{}), + expectedUpdate: false, + }, + { + name: "update volumes in VM, no datavolume template", + client: getFakeClientWithObjs( + createDataVolume("volume-source", testNamespace), + createDataVolume("volume-source2", testNamespace), + createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-source", + }, + }, + }, + { + Name: "dv2", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-source2", + }, + }, + }, + })), + vmName: "vm", + expectedVM: createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-target", + }, + }, + }, + { + Name: "dv2", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-target2", + }, + }, + }, + }), + expectedUpdate: true, + sourceVolumes: []string{"volume-source", "volume-source2"}, + targetVolumes: []string{"volume-target", "volume-target2"}, + }, + { + name: "update volume in VM, with datavolume template", + client: getFakeClientWithObjs( + createDataVolume("volume-source", testNamespace), + createVirtualMachineWithTemplateAndVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-source", + }, + }, + }, + })), + vmName: "vm", + expectedVM: createVirtualMachineWithTemplateAndVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-target", + }, + }, + }, + }), + expectedUpdate: true, + sourceVolumes: []string{"volume-source"}, + targetVolumes: []string{"volume-target"}, + }, + { + name: "update persisten volumes in VM, no datavolume template", + client: getFakeClientWithObjs( + createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "pvc", + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "volume-source", + }, + }, + }, + }, + })), + vmName: "vm", + expectedVM: createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "pvc", + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "volume-target", + }, + }, + }, + }, + }), + expectedUpdate: true, + sourceVolumes: []string{"volume-source"}, + targetVolumes: []string{"volume-target"}, + }} + for _, tt := range tests { + sourceVM := &virtv1.VirtualMachine{} + err := tt.client.Get(context.Background(), k8sclient.ObjectKey{Name: "vm", Namespace: testNamespace}, sourceVM) + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + t.Run(tt.name, func(t *testing.T) { + err := updateVM(tt.client, sourceVM, tt.sourceVolumes, tt.targetVolumes, log.WithName(tt.name)) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if tt.expectedVM != nil { + vm := &virtv1.VirtualMachine{} + err = tt.client.Get(context.Background(), k8sclient.ObjectKey{Name: "vm", Namespace: testNamespace}, vm) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if vm.Spec.Template != nil && len(tt.expectedVM.Spec.Template.Spec.Volumes) != len(vm.Spec.Template.Spec.Volumes) { + t.Errorf("expected volumes to be equal") + t.FailNow() + } else if vm.Spec.Template != nil { + for i, v := range vm.Spec.Template.Spec.Volumes { + if v.VolumeSource.DataVolume != nil { + if v.VolumeSource.DataVolume.Name != tt.expectedVM.Spec.Template.Spec.Volumes[i].VolumeSource.DataVolume.Name { + t.Errorf("expected volumes to be equal") + t.FailNow() + } + } + if v.VolumeSource.PersistentVolumeClaim != nil { + if v.VolumeSource.PersistentVolumeClaim.ClaimName != tt.expectedVM.Spec.Template.Spec.Volumes[i].VolumeSource.PersistentVolumeClaim.ClaimName { + t.Errorf("expected volumes to be equal") + t.FailNow() + } + } + } + for i, tp := range vm.Spec.DataVolumeTemplates { + if tp.Name != tt.expectedVM.Spec.DataVolumeTemplates[i].Name { + t.Errorf("expected data volume templates to be equal") + t.FailNow() + } + } + } + if vm.Spec.UpdateVolumesStrategy == nil || *vm.Spec.UpdateVolumesStrategy != virtv1.UpdateVolumesStrategyMigration { + t.Errorf("expected update volumes strategy to be migration") + t.FailNow() + } + if tt.expectedUpdate { + newVersion, _ := strconv.Atoi(vm.GetResourceVersion()) + oldVersion, _ := strconv.Atoi(sourceVM.GetResourceVersion()) + if newVersion <= oldVersion { + t.Errorf("expected resource version to be updated, originalVersion: %s, updatedVersion: %s", sourceVM.GetResourceVersion(), vm.GetResourceVersion()) + t.FailNow() + } + } else { + if vm.GetResourceVersion() != sourceVM.GetResourceVersion() { + t.Errorf("expected resource version to be the same, originalVersion: %s, updatedVersion: %s", sourceVM.GetResourceVersion(), vm.GetResourceVersion()) + t.FailNow() + } + } + } + }) + } +} + +func TestCreateNewDataVolume(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + sourceDv *cdiv1.DataVolume + expectedDv *cdiv1.DataVolume + expectedNewDv bool + }{ + { + name: "create new data volume", + client: getFakeClientWithObjs(createDataVolume("source-dv", testNamespace)), + sourceDv: &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-dv", + }, + }, + expectedNewDv: true, + expectedDv: createDataVolume("source-dv", testNamespace), + }, + { + name: "don't update existing new data volume", + client: getFakeClientWithObjs(createDataVolume("source-dv", testNamespace), createDataVolume(targetDv, testNamespace)), + sourceDv: &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-dv", + }, + }, + expectedNewDv: false, + expectedDv: createDataVolume("source-dv", testNamespace), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := CreateNewDataVolume(tt.client, tt.sourceDv.Name, targetDv, testNamespace, log.WithName(tt.name)) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + dv := &cdiv1.DataVolume{} + err = tt.client.Get(context.Background(), k8sclient.ObjectKey{Name: targetDv, Namespace: testNamespace}, dv) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if tt.expectedNewDv { + if dv.GetAnnotations()["cdi.kubevirt.io/allowClaimAdoption"] != "true" { + t.Errorf("expected allowClaimAdoption annotation to be true") + t.FailNow() + } + if dv.Spec.Source == nil { + t.Errorf("expected source to be set") + t.FailNow() + } + if dv.Spec.Source.Blank == nil { + t.Errorf("expected source blank to be set") + t.FailNow() + } + } else { + if _, ok := dv.GetAnnotations()["cdi.kubevirt.io/allowClaimAdoption"]; ok { + t.Errorf("expected allowClaimAdoption annotation to not be set") + t.FailNow() + } + } + }) + } + +} + +func TestTask_getVolumeVMIMInNamespaces(t *testing.T) { + tests := []struct { + name string + sourceNamespace string + task *Task + client compat.Client + expectedVMIM map[string]*virtv1.VirtualMachineInstanceMigration + }{ + { + name: "empty volume name, due to no VMs", + task: &Task{}, + sourceNamespace: sourceNs, + client: getFakeClientWithObjs(), + }, + { + name: "empty volume name, due to no running VMs", + task: &Task{}, + sourceNamespace: sourceNs, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", sourceNs, []virtv1.Volume{ + { + Name: "data", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + })), + }, + { + name: "empty volume name, due to no migrations", + task: &Task{}, + sourceNamespace: sourceNs, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", sourceNs, []virtv1.Volume{ + { + Name: "data", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + }), createVirtlauncherPod("vm", sourceNs, []string{"dv"})), + }, + { + name: "running VMIM", + task: &Task{}, + sourceNamespace: sourceNs, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", sourceNs, []virtv1.Volume{ + { + Name: "data", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + }), createVirtlauncherPod("vm", sourceNs, []string{"dv"}), + createCompletedVirtualMachineMigration("vmim", sourceNs, "vm")), + expectedVMIM: map[string]*virtv1.VirtualMachineInstanceMigration{ + "source-ns/dv": createCompletedVirtualMachineMigration("vmim", sourceNs, "vm"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + out, err := tt.task.getVolumeVMIMInNamespaces([]string{tt.sourceNamespace}) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + for k, v := range out { + if tt.expectedVMIM[k] == nil { + t.Errorf("unexpected VMIM %s", k) + t.FailNow() + } + if tt.expectedVMIM[k] == nil { + t.Errorf("got unexpected VMIM %s", k) + t.FailNow() + } + if v.Name != tt.expectedVMIM[k].Name { + t.Errorf("expected %s, got %s", tt.expectedVMIM[k].Name, v.Name) + t.FailNow() + } + } + }) + } +} + +func TestGetVMIMElapsedTime(t *testing.T) { + now := metav1.Now() + tests := []struct { + name string + vmim *virtv1.VirtualMachineInstanceMigration + expectedValue metav1.Duration + }{ + { + name: "nil VMIM", + vmim: nil, + expectedValue: metav1.Duration{ + Duration: 0, + }, + }, + { + name: "nil VMIM.Status.MigrationState", + vmim: &virtv1.VirtualMachineInstanceMigration{ + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: nil, + }, + }, + expectedValue: metav1.Duration{ + Duration: 0, + }, + }, + { + name: "VMIM.Status.MigrationState.StartTimestamp nil, and no replacement from PhaseTransition", + vmim: &virtv1.VirtualMachineInstanceMigration{ + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + StartTimestamp: nil, + }, + }, + }, + expectedValue: metav1.Duration{ + Duration: 0, + }, + }, + { + name: "VMIM.Status.MigrationState.StartTimestamp nil, and replacement from PhaseTransition", + vmim: &virtv1.VirtualMachineInstanceMigration{ + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + StartTimestamp: nil, + }, + PhaseTransitionTimestamps: []virtv1.VirtualMachineInstanceMigrationPhaseTransitionTimestamp{ + { + Phase: virtv1.MigrationRunning, + PhaseTransitionTimestamp: metav1.Time{ + Time: now.Add(-1 * time.Hour), + }, + }, + }, + }, + }, + expectedValue: metav1.Duration{ + Duration: time.Hour, + }, + }, + { + name: "VMIM.Status.MigrationState.StartTimestamp set, and end timestamp set", + vmim: &virtv1.VirtualMachineInstanceMigration{ + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + StartTimestamp: &metav1.Time{ + Time: now.Add(-1 * time.Hour), + }, + EndTimestamp: &now, + }, + }, + }, + expectedValue: metav1.Duration{ + Duration: time.Hour, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := getVMIMElapsedTime(tt.vmim) + // Round to nearest to avoid issues with time.Duration precision + // we could still get really unlucky and just be on the edge of a minute + // but it is unlikely + if out.Round(time.Minute) != tt.expectedValue.Round(time.Minute) { + t.Errorf("expected %s, got %s", tt.expectedValue, out) + } + }) + } +} + +func TestTaskGetLastObservedProgressPercent(t *testing.T) { + tests := []struct { + name string + vmName string + sourceNamespace string + currentProgress map[string]*migapi.LiveMigrationProgress + task *Task + client compat.Client + expectedPercent string + }{ + { + name: "valid result from query", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> 59.3 @", + }, nil, nil + }, + }, + expectedPercent: "59%", + }, + { + name: "invalid result from query", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> x59.3 @", + }, nil, nil + }, + }, + expectedPercent: "", + }, + { + name: "error result from query", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> x59.3 @", + }, nil, fmt.Errorf("error") + }, + }, + currentProgress: map[string]*migapi.LiveMigrationProgress{ + "source-ns/vm": &migapi.LiveMigrationProgress{ + LastObservedProgressPercent: "43%", + }, + }, + vmName: "vm", + sourceNamespace: sourceNs, + expectedPercent: "43%", + }, + { + name: "error result from query, no existing progress", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> x59.3 @", + }, nil, fmt.Errorf("error") + }, + }, + vmName: "vm", + sourceNamespace: sourceNs, + expectedPercent: "", + }, + { + name: "warning result from query, no existing progress", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> 21.7 @", + }, prometheusv1.Warnings{"warning"}, nil + }, + }, + vmName: "vm", + sourceNamespace: sourceNs, + expectedPercent: "21%", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + percent, err := tt.task.getLastObservedProgressPercent(tt.vmName, tt.sourceNamespace, tt.currentProgress) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if percent != tt.expectedPercent { + t.Errorf("expected %s, got %s", tt.expectedPercent, percent) + } + }) + } +} + +func TestTaskBuildPrometheusAPI(t *testing.T) { + tests := []struct { + name string + task *Task + client compat.Client + apiNil bool + }{ + { + name: "API already built", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{}), + }, + apiNil: false, + }, + { + name: "API not built, should build", + client: getFakeClientWithObjs( + createControllerConfigMap( + "migration-controller", + migapi.OpenshiftMigrationNamespace, + map[string]string{ + "source-cluster_PROMETHEUS_URL": "http://prometheus", + }, + ), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + Spec: migapi.MigClusterSpec{ + IsHostCluster: true, + }, + }, + }, + }, + apiNil: false, + }, + { + name: "API not built, should build", + client: getFakeClientWithObjs( + createControllerConfigMap( + "migration-controller", + migapi.OpenshiftMigrationNamespace, + map[string]string{ + "source-cluster_PROMETHEUS_URL": "", + }, + ), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + Spec: migapi.MigClusterSpec{ + IsHostCluster: true, + }, + }, + }, + }, + apiNil: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + err := tt.task.buildPrometheusAPI() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if tt.apiNil && tt.task.PrometheusAPI != nil { + t.Errorf("expected API to be nil") + } + }) + } +} + +func TestParseProgress(t *testing.T) { + tests := []struct { + name string + intput string + expectedValue string + }{ + { + name: "Empty string", + intput: "", + expectedValue: "", + }, + { + name: "Valid progress", + intput: "=> 59.3 @", + expectedValue: "59", + }, + { + name: "Invalid progress", + intput: "=> x59.3 @", + expectedValue: "", + }, + { + name: "Invalid progress over 100", + intput: "=> 1159.3 @", + expectedValue: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := parseProgress(tt.intput) + if out != tt.expectedValue { + t.Errorf("expected %s, got %s", tt.expectedValue, out) + } + }) + } +} + +func TestTaskBuildSourcePrometheusEndPointURL(t *testing.T) { + tests := []struct { + name string + client compat.Client + task *Task + expectedError bool + expectedErrorMessage string + expectedValue string + }{ + { + name: "No prometheus config map, should return not found error", + client: getFakeClientWithObjs(), + task: &Task{}, + expectedError: true, + expectedErrorMessage: "not found", + }, + { + name: "Prometheus config map exists, but no prometheus url, should return empty url", + client: getFakeClientWithObjs( + createControllerConfigMap("migration-controller", migapi.OpenshiftMigrationNamespace, nil), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "", + }, + { + name: "Prometheus config map exists, with prometheus url, should return correct url", + client: getFakeClientWithObjs( + createControllerConfigMap( + "migration-controller", + migapi.OpenshiftMigrationNamespace, + map[string]string{ + "source-cluster_PROMETHEUS_URL": "http://prometheus", + }, + ), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "https://prometheus", + }, + { + name: "Prometheus config map exists, but no prometheus url, but route, should return route url", + client: getFakeClientWithObjs( + createControllerConfigMap("migration-controller", migapi.OpenshiftMigrationNamespace, nil), + createRoute("prometheus-route", "openshift-monitoring", "https://route.prometheus"), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "https://route.prometheus", + }, + { + name: "Prometheus config map exists, but no prometheus url, but route, should return route url", + client: getFakeClientWithObjs( + createControllerConfigMap("migration-controller", migapi.OpenshiftMigrationNamespace, nil), + createRoute("prometheus-route", "openshift-monitoring", "http://route.prometheus"), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "https://route.prometheus", + }, + { + name: "Prometheus config map exists, with invalid prometheus url, should return blank", + client: getFakeClientWithObjs( + createControllerConfigMap("migration-controller", migapi.OpenshiftMigrationNamespace, + map[string]string{ + "source-cluster_PROMETHEUS_URL": "%#$invalid", + }, + ), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + value, err := tt.task.buildSourcePrometheusEndPointURL() + if tt.expectedError { + if err == nil { + t.Errorf("expected error but got nil") + t.FailNow() + } + if !strings.Contains(err.Error(), tt.expectedErrorMessage) { + t.Errorf("expected error message to contain %s, got %s", tt.expectedErrorMessage, err.Error()) + } + } + if value != tt.expectedValue { + t.Errorf("expected %s, got %s", tt.expectedValue, value) + } + }) + } +} + +func TestTaskDeleteStaleVirtualMachineInstanceMigrations(t *testing.T) { + tests := []struct { + name string + client compat.Client + task *Task + expectedError bool + expectedMsg string + expectedVMIMs []*virtv1.VirtualMachineInstanceMigration + }{ + { + name: "No pvcs in either namespace, but persistent volume claims in DVM, should error on missing PVCs", + client: getFakeClientWithObjs(), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: targetNs, + }, + }, + }, + }, + }, + expectedError: true, + expectedMsg: "persistentvolumeclaims \"source-pvc\" not found", + }, + { + name: "PVCs in different namespaces, and persistent volume claims in DVM, should error on mismatched namespaces", + client: getFakeClientWithObjs(createPvc(sourcePVC, sourceNs), createPvc(targetPVC, targetNs)), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: targetNs, + }, + }, + }, + }, + }, + expectedError: true, + expectedMsg: "source and target namespaces must match", + }, + { + name: "PVCs in same namespace, and persistent volume claims in DVM, but no VMs or VMIMs, should not return error", + client: getFakeClientWithObjs(createPvc(sourcePVC, sourceNs), createPvc(targetPVC, sourceNs)), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: sourceNs, + }, + }, + }, + }, + }, + }, + { + name: "PVCs in same namespace, and persistent volume claims in DVM, VMs and VMIMs created before the DVM, should not return error, should delete VMIM", + client: getFakeClientWithObjs( + createPvc(sourcePVC, sourceNs), + createPvc(targetPVC, sourceNs), + createVirtualMachine("vm", sourceNs), + createCompletedVirtualMachineMigration("vmim", sourceNs, "vm"), + ), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: metav1.Now(), + }, + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: sourceNs, + }, + }, + }, + }, + }, + expectedVMIMs: []*virtv1.VirtualMachineInstanceMigration{}, + }, + { + name: "PVCs in same namespace, and persistent volume claims in DVM, VMs and VMIMs created after the DVM, should not return error, should NOT delete VMIM", + client: getFakeClientWithObjs( + createPvc(sourcePVC, sourceNs), + createPvc(targetPVC, sourceNs), + createVirtualMachine("vm", sourceNs), + createCompletedVirtualMachineMigration("vmim", sourceNs, "vm"), + ), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: metav1.Time{ + Time: metav1.Now().Add(-2 * time.Hour), + }, + }, + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: sourceNs, + }, + }, + }, + }, + }, + expectedVMIMs: []*virtv1.VirtualMachineInstanceMigration{ + createCompletedVirtualMachineMigration("vmim", sourceNs, "vm"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + tt.task.destinationClient = tt.client + err := tt.task.deleteStaleVirtualMachineInstanceMigrations() + if tt.expectedError { + if err == nil { + t.Errorf("expected error but got nil") + t.FailNow() + } else if !strings.Contains(err.Error(), tt.expectedMsg) { + t.Errorf("expected error message to contain %s, got %s", tt.expectedMsg, err.Error()) + } + } + remainingVMIMs := &virtv1.VirtualMachineInstanceMigrationList{} + err = tt.client.List(context.Background(), remainingVMIMs, k8sclient.InNamespace(sourceNs), &k8sclient.ListOptions{}) + if err != nil { + t.Errorf("error listing VMIMs: %v", err) + t.FailNow() + } + if len(remainingVMIMs.Items) != len(tt.expectedVMIMs) { + t.Errorf("expected %d VMIMs, got %d", len(tt.expectedVMIMs), len(remainingVMIMs.Items)) + t.FailNow() + } + for _, remainingVMIM := range remainingVMIMs.Items { + found := false + for _, expectedVMIM := range tt.expectedVMIMs { + if remainingVMIM.Name == expectedVMIM.Name { + found = true + break + } + } + if !found { + t.Errorf("unexpected VMIM %s", remainingVMIM.Name) + t.FailNow() + } + } + }) + } +} + +func getFakeClientWithObjs(obj ...k8sclient.Object) compat.Client { + client, _ := fakecompat.NewFakeClient(obj...) + return client +} + +func createVirtualMachine(name, namespace string) *virtv1.VirtualMachine { + return &virtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } +} + +func createCompletedVirtualMachineMigration(name, namespace, vmName string) *virtv1.VirtualMachineInstanceMigration { + return &virtv1.VirtualMachineInstanceMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineInstanceMigrationSpec{ + VMIName: vmName, + }, + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + Phase: virtv1.MigrationSucceeded, + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + EndTimestamp: &metav1.Time{ + Time: metav1.Now().Add(-1 * time.Hour), + }, + Completed: true, + }, + }, + } +} + +func createFailedVirtualMachineMigration(name, namespace, vmName, failedMessage string) *virtv1.VirtualMachineInstanceMigration { + return &virtv1.VirtualMachineInstanceMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineInstanceMigrationSpec{ + VMIName: vmName, + }, + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + Phase: virtv1.MigrationFailed, + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + EndTimestamp: &metav1.Time{ + Time: metav1.Now().Add(-1 * time.Hour), + }, + FailureReason: failedMessage, + Failed: true, + }, + }, + } +} + +func createCanceledVirtualMachineMigration(name, namespace, vmName string, reason virtv1.MigrationAbortStatus) *virtv1.VirtualMachineInstanceMigration { + return &virtv1.VirtualMachineInstanceMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineInstanceMigrationSpec{ + VMIName: vmName, + }, + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + Phase: virtv1.MigrationFailed, + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + EndTimestamp: &metav1.Time{ + Time: metav1.Now().Add(-1 * time.Hour), + }, + AbortStatus: reason, + }, + }, + } +} + +func createInProgressVirtualMachineMigration(name, namespace, vmName string) *virtv1.VirtualMachineInstanceMigration { + return &virtv1.VirtualMachineInstanceMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineInstanceMigrationSpec{ + VMIName: vmName, + }, + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + Phase: virtv1.MigrationRunning, + }, + } +} + +func createVirtualMachineWithVolumes(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachine(name, namespace) + vm.Spec = virtv1.VirtualMachineSpec{ + Template: &virtv1.VirtualMachineInstanceTemplateSpec{ + Spec: virtv1.VirtualMachineInstanceSpec{ + Volumes: volumes, + }, + }, + } + return vm +} + +func createVirtualMachineWithUpdateStrategy(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachineWithVolumes(name, namespace, volumes) + vm.Spec.UpdateVolumesStrategy = ptr.To[virtv1.UpdateVolumesStrategy](virtv1.UpdateVolumesStrategyMigration) + return vm +} + +func createVirtualMachineWithTemplateAndVolumes(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachineWithVolumes(name, namespace, volumes) + for _, volume := range volumes { + vm.Spec.DataVolumeTemplates = append(vm.Spec.DataVolumeTemplates, virtv1.DataVolumeTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: volume.DataVolume.Name, + }, + }) + } + return vm +} + +func createControllerConfigMap(name, namespace string, data map[string]string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: data, + } +} + +func createRoute(name, namespace, url string) *routev1.Route { + return &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/part-of": "openshift-monitoring", + }, + }, + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: prometheusRoute, + }, + Host: url, + }, + } +} + +type mockPrometheusClient struct { + fakeUrl string + responseBody string + expectedQuery string +} + +func (m *mockPrometheusClient) URL(ep string, args map[string]string) *url.URL { + url, err := url.Parse(m.fakeUrl) + if err != nil { + panic(err) + } + return url +} + +func (m *mockPrometheusClient) Do(ctx context.Context, req *http.Request) (*http.Response, []byte, error) { + if req.Body != nil { + defer req.Body.Close() + } + b, err := io.ReadAll(req.Body) + queryBody := string(b) + if !strings.Contains(queryBody, m.expectedQuery) { + return nil, nil, fmt.Errorf("expected query %s, got %s", m.expectedQuery, queryBody) + } + if err != nil { + return nil, nil, err + } + + out := []byte(m.responseBody) + + t := &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Body: io.NopCloser(bytes.NewBuffer(out)), + ContentLength: int64(len(out)), + Request: req, + Header: make(http.Header, 0), + } + return t, out, nil +} + +func createVirtlauncherPod(vmName, namespace string, dataVolumes []string) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("virt-launcher-%s", vmName), + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: vmName, + Kind: "VirtualMachineInstance", + }, + }, + }, + Spec: corev1.PodSpec{}, + } + for _, dv := range dataVolumes { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: dv, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: dv, + }, + }, + }) + } + return pod +} + +func createDataVolume(name, namespace string) *cdiv1.DataVolume { + return &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } +} + +func createVirtualMachineInstance(name, namespace string, phase virtv1.VirtualMachineInstancePhase) *virtv1.VirtualMachineInstance { + return &virtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Status: virtv1.VirtualMachineInstanceStatus{ + Phase: phase, + }, + } +} + +func createVirtualMachineInstanceWithConditions(name, namespace string, conditions []virtv1.VirtualMachineInstanceCondition) *virtv1.VirtualMachineInstance { + vm := createVirtualMachineInstance(name, namespace, virtv1.Running) + vm.Status.Conditions = append(vm.Status.Conditions, conditions...) + return vm +} diff --git a/pkg/controller/migmigration/description.go b/pkg/controller/migmigration/description.go index c2e67db6ff..15fc9a1652 100644 --- a/pkg/controller/migmigration/description.go +++ b/pkg/controller/migmigration/description.go @@ -52,8 +52,10 @@ var PhaseDescriptions = map[string]string{ WaitForResticReady: "Waiting for Restic Pods to restart, ensuring latest PVC mounts are available for PVC backups.", RestartVelero: "Restarting Velero Pods, ensuring work queue is empty.", WaitForVeleroReady: "Waiting for Velero Pods to restart, ensuring work queue is empty.", - QuiesceApplications: "Quiescing (Scaling to 0 replicas): Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines.", - EnsureQuiesced: "Waiting for Quiesce (Scaling to 0 replicas) to finish for Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines.", + QuiesceSourceApplications: "Quiescing (Scaling to 0 replicas): Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines in source namespace.", + QuiesceDestinationApplications: "Quiescing (Scaling to 0 replicas): Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines in target namespace.", + EnsureSrcQuiesced: "Waiting for Quiesce (Scaling to 0 replicas) to finish for Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines in source namespace.", + EnsureDestQuiesced: "Waiting for Quiesce (Scaling to 0 replicas) to finish for Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines in target namespace.", UnQuiesceSrcApplications: "UnQuiescing (Scaling to N replicas) source cluster Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines.", UnQuiesceDestApplications: "UnQuiescing (Scaling to N replicas) target cluster Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines.", EnsureStageBackup: "Creating a stage backup.", @@ -70,7 +72,9 @@ var PhaseDescriptions = map[string]string{ Verification: "Verifying health of migrated Pods.", Rollback: "Starting rollback", CreateDirectImageMigration: "Creating Direct Image Migration", - CreateDirectVolumeMigration: "Creating Direct Volume Migration", + CreateDirectVolumeMigrationStage: "Creating Direct Volume Migration for staging", + CreateDirectVolumeMigrationFinal: "Creating Direct Volume Migration for cutover", + CreateDirectVolumeMigrationRollback: "Creating Direct Volume Migration for rollback", WaitForDirectImageMigrationToComplete: "Waiting for Direct Image Migration to complete.", WaitForDirectVolumeMigrationToComplete: "Waiting for Direct Volume Migration to complete.", EnsureStagePodsDeleted: "Deleting any leftover stage Pods.", diff --git a/pkg/controller/migmigration/dvm.go b/pkg/controller/migmigration/dvm.go index 3ccd70a756..7cbc1aca00 100644 --- a/pkg/controller/migmigration/dvm.go +++ b/pkg/controller/migmigration/dvm.go @@ -17,7 +17,7 @@ import ( k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func (t *Task) createDirectVolumeMigration() error { +func (t *Task) createDirectVolumeMigration(migType *migapi.DirectVolumeMigrationType) error { existingDvm, err := t.getDirectVolumeMigration() if err != nil { return err @@ -31,11 +31,13 @@ func (t *Task) createDirectVolumeMigration() error { if dvm == nil { return errors.New("failed to build directvolumeclaim list") } + if migType != nil { + dvm.Spec.MigrationType = migType + } t.Log.Info("Creating DirectVolumeMigration on host cluster", "directVolumeMigration", path.Join(dvm.Namespace, dvm.Name)) err = t.Client.Create(context.TODO(), dvm) return err - } func (t *Task) buildDirectVolumeMigration() *migapi.DirectVolumeMigration { @@ -46,6 +48,10 @@ func (t *Task) buildDirectVolumeMigration() *migapi.DirectVolumeMigration { if pvcList == nil { return nil } + migrationType := migapi.MigrationTypeStage + if t.Owner.Spec.QuiescePods { + migrationType = migapi.MigrationTypeFinal + } dvm := &migapi.DirectVolumeMigration{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -57,6 +63,8 @@ func (t *Task) buildDirectVolumeMigration() *migapi.DirectVolumeMigration { DestMigClusterRef: t.PlanResources.DestMigCluster.GetObjectReference(), PersistentVolumeClaims: pvcList, CreateDestinationNamespaces: true, + LiveMigrate: t.PlanResources.MigPlan.Spec.LiveMigrate, + MigrationType: &migrationType, }, } migapi.SetOwnerReference(t.Owner, t.Owner, dvm) @@ -86,29 +94,15 @@ func (t *Task) getDirectVolumeMigration() (*migapi.DirectVolumeMigration, error) // Check if the DVM has completed. // Returns if it has completed, why it failed, and it's progress results func (t *Task) hasDirectVolumeMigrationCompleted(dvm *migapi.DirectVolumeMigration) (completed bool, failureReasons, progress []string) { - totalVolumes := len(dvm.Spec.PersistentVolumeClaims) - successfulPods := 0 - failedPods := 0 - runningPods := 0 - if dvm.Status.SuccessfulPods != nil { - successfulPods = len(dvm.Status.SuccessfulPods) - } - if dvm.Status.FailedPods != nil { - failedPods = len(dvm.Status.FailedPods) - } - if dvm.Status.RunningPods != nil { - runningPods = len(dvm.Status.RunningPods) - } - volumeProgress := fmt.Sprintf("%v total volumes; %v successful; %v running; %v failed", - totalVolumes, - successfulPods, - runningPods, - failedPods) + len(dvm.Spec.PersistentVolumeClaims), + len(dvm.Status.SuccessfulPods)+len(dvm.Status.SuccessfulLiveMigrations), + len(dvm.Status.RunningPods)+len(dvm.Status.FailedLiveMigrations), + len(dvm.Status.FailedPods)+len(dvm.Status.RunningLiveMigrations)) switch { - //case dvm.Status.Phase != "" && dvm.Status.Phase != dvmc.Completed: - // // TODO: Update this to check on the associated dvmp resources and build up a progress indicator back to - case dvm.Status.Phase == dvmc.Completed && dvm.Status.Itinerary == "VolumeMigration" && dvm.Status.HasCondition(dvmc.Succeeded): + // case dvm.Status.Phase != "" && dvm.Status.Phase != dvmc.Completed: + // TODO: Update this to check on the associated dvmp resources and build up a progress indicator back to + case dvm.Status.Phase == dvmc.Completed && (dvm.Status.Itinerary == dvmc.VolumeMigrationItinerary || dvm.Status.Itinerary == dvmc.VolumeMigrationRollbackItinerary) && dvm.Status.HasCondition(dvmc.Succeeded): // completed successfully completed = true case (dvm.Status.Phase == dvmc.MigrationFailed || dvm.Status.Phase == dvmc.Completed) && dvm.Status.HasCondition(dvmc.Failed): @@ -117,14 +111,14 @@ func (t *Task) hasDirectVolumeMigrationCompleted(dvm *migapi.DirectVolumeMigrati default: progress = append(progress, volumeProgress) } - progress = append(progress, t.getDVMPodProgress(*dvm)...) + progress = append(progress, t.getDVMProgress(dvm)...) // sort the progress report so we dont have flapping for the same progress info sort.Strings(progress) return completed, failureReasons, progress } -func (t *Task) getWarningForDVM(dvm *migapi.DirectVolumeMigration) (*migapi.Condition, error) { +func (t *Task) getWarningForDVM(dvm *migapi.DirectVolumeMigration) *migapi.Condition { conditions := dvm.Status.Conditions.FindConditionByCategory(dvmc.Warn) if len(conditions) > 0 { return &migapi.Condition{ @@ -133,10 +127,10 @@ func (t *Task) getWarningForDVM(dvm *migapi.DirectVolumeMigration) (*migapi.Cond Reason: migapi.NotReady, Category: migapi.Warn, Message: joinConditionMessages(conditions), - }, nil + } } - return nil, nil + return nil } func joinConditionMessages(conditions []*migapi.Condition) string { @@ -160,7 +154,12 @@ func (t *Task) setDirectVolumeMigrationFailureWarning(dvm *migapi.DirectVolumeMi }) } -func (t *Task) getDVMPodProgress(dvm migapi.DirectVolumeMigration) []string { +func (t *Task) getDVMProgress(dvm *migapi.DirectVolumeMigration) []string { + progress := getDVMPodProgress(dvm) + return append(progress, getDVMLiveMigrationProgress(dvm)...) +} + +func getDVMPodProgress(dvm *migapi.DirectVolumeMigration) []string { progress := []string{} podProgressIterator := map[string][]*migapi.PodProgress{ "Pending": dvm.Status.PendingPods, @@ -202,6 +201,44 @@ func (t *Task) getDVMPodProgress(dvm migapi.DirectVolumeMigration) []string { return progress } +// Live migration progress +func getDVMLiveMigrationProgress(dvm *migapi.DirectVolumeMigration) []string { + progress := []string{} + if dvm == nil { + return progress + } + liveMigrationProgressIterator := map[string][]*migapi.LiveMigrationProgress{ + "Running": dvm.Status.RunningLiveMigrations, + "Completed": dvm.Status.SuccessfulLiveMigrations, + "Failed": dvm.Status.FailedLiveMigrations, + "Pending": dvm.Status.PendingLiveMigrations, + } + for state, liveMigrations := range liveMigrationProgressIterator { + for _, liveMigration := range liveMigrations { + p := fmt.Sprintf( + "[%s] Live Migration %s: %s", + liveMigration.PVCReference.Name, + path.Join(liveMigration.VMNamespace, liveMigration.VMName), + state) + if liveMigration.Message != "" { + p += fmt.Sprintf(" %s", liveMigration.Message) + } + if liveMigration.LastObservedProgressPercent != "" { + p += fmt.Sprintf(" %s", liveMigration.LastObservedProgressPercent) + } + if liveMigration.LastObservedTransferRate != "" { + p += fmt.Sprintf(" (Transfer rate %s)", liveMigration.LastObservedTransferRate) + } + if liveMigration.TotalElapsedTime != nil { + p += fmt.Sprintf(" (%s)", liveMigration.TotalElapsedTime.Duration.Round(time.Second)) + } + progress = append(progress, p) + } + + } + return progress +} + func (t *Task) getDirectVolumeClaimList() []migapi.PVCToMigrate { nsMapping := t.PlanResources.MigPlan.GetNamespaceMapping() var pvcList []migapi.PVCToMigrate diff --git a/pkg/controller/migmigration/dvm_test.go b/pkg/controller/migmigration/dvm_test.go index 15f1c5a199..0489c8e6ca 100644 --- a/pkg/controller/migmigration/dvm_test.go +++ b/pkg/controller/migmigration/dvm_test.go @@ -2,11 +2,13 @@ package migmigration import ( "reflect" + "slices" "testing" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" dvmc "github.com/konveyor/mig-controller/pkg/controller/directvolumemigration" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestTask_hasDirectVolumeMigrationCompleted(t1 *testing.T) { @@ -160,3 +162,134 @@ func TestTask_hasDirectVolumeMigrationCompleted(t1 *testing.T) { }) } } + +func TestTask_getDVMLiveMigrationProgress(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + expectedProgress []string + }{ + { + name: "no dvm", + }, + { + name: "dvm with running live migrations, no message, no progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + RunningLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Running"}, + }, + { + name: "dvm with failed live migrations, message, no progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + FailedLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + Message: "Failed because of test", + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Failed Failed because of test"}, + }, + { + name: "dvm with completed live migrations, message, progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + SuccessfulLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + Message: "Successfully completed", + LastObservedProgressPercent: "100%", + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Completed Successfully completed 100%"}, + }, + { + name: "dvm with pending live migrations, message, blank progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + PendingLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + Message: "Pending", + LastObservedProgressPercent: "", + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Pending Pending"}, + }, + { + name: "dvm with running live migrations, message, progress, transferrate, and elapsed time", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + RunningLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + Message: "Running", + LastObservedProgressPercent: "50%", + LastObservedTransferRate: "10MB/s", + TotalElapsedTime: &metav1.Duration{ + Duration: 1000, + }, + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Running Running 50% (Transfer rate 10MB/s) (0s)"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := getDVMLiveMigrationProgress(tt.dvm) + if len(res) != len(tt.expectedProgress) { + t.Errorf("getDVMLiveMigrationProgress() = %v, want %v", res, tt.expectedProgress) + } + for _, p := range tt.expectedProgress { + if !slices.Contains(res, p) { + t.Errorf("getDVMLiveMigrationProgress() = %v, want %v", res, tt.expectedProgress) + } + } + }) + } +} diff --git a/pkg/controller/migmigration/migmigration_controller_test.go b/pkg/controller/migmigration/migmigration_controller_test.go index 71bc1b6183..53f21d37fa 100644 --- a/pkg/controller/migmigration/migmigration_controller_test.go +++ b/pkg/controller/migmigration/migmigration_controller_test.go @@ -17,7 +17,6 @@ limitations under the License. package migmigration import ( - "reflect" "testing" "time" @@ -103,5 +102,5 @@ func Test_Itineraries(t *testing.T) { } } - g.Expect(reflect.DeepEqual(stage.Phases, common.Phases)).To(gomega.BeTrue()) + g.Expect(stage.Phases).To(gomega.ContainElements(common.Phases)) } diff --git a/pkg/controller/migmigration/quiesce.go b/pkg/controller/migmigration/quiesce.go index 5a36e934ca..c8579109ac 100644 --- a/pkg/controller/migmigration/quiesce.go +++ b/pkg/controller/migmigration/quiesce.go @@ -29,8 +29,7 @@ const ( vmSubresourceURLFmt = "/apis/subresources.kubevirt.io/%s/namespaces/%s/virtualmachines/%s/%s" ) -// Quiesce applications on source cluster -func (t *Task) quiesceApplications() error { +func (t *Task) quiesceSourceApplications() error { client, err := t.getSourceClient() if err != nil { return liberr.Wrap(err) @@ -39,7 +38,24 @@ func (t *Task) quiesceApplications() error { if err != nil { return liberr.Wrap(err) } - err = t.quiesceCronJobs(client) + return t.quiesceApplications(client, restConfig) +} + +func (t *Task) quiesceDestinationApplications() error { + client, err := t.getDestinationClient() + if err != nil { + return liberr.Wrap(err) + } + restConfig, err := t.getDestinationRestConfig() + if err != nil { + return liberr.Wrap(err) + } + return t.quiesceApplications(client, restConfig) +} + +// Quiesce applications on source cluster +func (t *Task) quiesceApplications(client compat.Client, restConfig *rest.Config) error { + err := t.quiesceCronJobs(client) if err != nil { return liberr.Wrap(err) } @@ -67,15 +83,16 @@ func (t *Task) quiesceApplications() error { if err != nil { return liberr.Wrap(err) } - restClient, err := t.createRestClient(restConfig) - if err != nil { - return liberr.Wrap(err) - } - err = t.quiesceVirtualMachines(client, restClient) - if err != nil { - return liberr.Wrap(err) + if !t.PlanResources.MigPlan.LiveMigrationChecked() { + restClient, err := t.createRestClient(restConfig) + if err != nil { + return liberr.Wrap(err) + } + err = t.quiesceVirtualMachines(client, restClient) + if err != nil { + return liberr.Wrap(err) + } } - return nil } @@ -88,7 +105,7 @@ func (t *Task) unQuiesceSrcApplications() error { if err != nil { return liberr.Wrap(err) } - t.Log.Info("Unquiescing applications on source cluster.") + t.Log.V(3).Info("Unquiescing applications on source cluster.") err = t.unQuiesceApplications(srcClient, restConfig, t.sourceNamespaces()) if err != nil { return liberr.Wrap(err) @@ -105,7 +122,7 @@ func (t *Task) unQuiesceDestApplications() error { if err != nil { return liberr.Wrap(err) } - t.Log.Info("Unquiescing applications on destination cluster.") + t.Log.V(3).Info("Unquiescing applications on destination cluster.") err = t.unQuiesceApplications(destClient, restConfig, t.destinationNamespaces()) if err != nil { return liberr.Wrap(err) @@ -143,15 +160,16 @@ func (t *Task) unQuiesceApplications(client compat.Client, restConfig *rest.Conf if err != nil { return liberr.Wrap(err) } - restClient, err := t.createRestClient(restConfig) - if err != nil { - return liberr.Wrap(err) - } - err = t.unQuiesceVirtualMachines(client, restClient, namespaces) - if err != nil { - return liberr.Wrap(err) + if !t.PlanResources.MigPlan.LiveMigrationChecked() { + restClient, err := t.createRestClient(restConfig) + if err != nil { + return liberr.Wrap(err) + } + err = t.unQuiesceVirtualMachines(client, restClient, namespaces) + if err != nil { + return liberr.Wrap(err) + } } - return nil } @@ -971,27 +989,41 @@ func (t *Task) startVM(vm *virtv1.VirtualMachine, client k8sclient.Client, restC return nil } +func (t *Task) ensureSourceQuiescedPodsTerminated() (bool, error) { + client, err := t.getSourceClient() + if err != nil { + return false, liberr.Wrap(err) + } + return t.ensureQuiescedPodsTerminated(client, t.sourceNamespaces()) +} + +func (t *Task) ensureDestinationQuiescedPodsTerminated() (bool, error) { + client, err := t.getDestinationClient() + if err != nil { + return false, liberr.Wrap(err) + } + return t.ensureQuiescedPodsTerminated(client, t.destinationNamespaces()) +} + // Ensure scaled down pods have terminated. // Returns: `true` when all pods terminated. -func (t *Task) ensureQuiescedPodsTerminated() (bool, error) { +func (t *Task) ensureQuiescedPodsTerminated(client compat.Client, namespaces []string) (bool, error) { kinds := map[string]bool{ "ReplicationController": true, "StatefulSet": true, "ReplicaSet": true, "DaemonSet": true, "Job": true, - "VirtualMachine": true, + } + if !t.PlanResources.MigPlan.LiveMigrationChecked() { + kinds["VirtualMachineInstance"] = true } skippedPhases := map[v1.PodPhase]bool{ v1.PodSucceeded: true, v1.PodFailed: true, v1.PodUnknown: true, } - client, err := t.getSourceClient() - if err != nil { - return false, liberr.Wrap(err) - } - for _, ns := range t.sourceNamespaces() { + for _, ns := range namespaces { list := v1.PodList{} options := k8sclient.InNamespace(ns) err := client.List( diff --git a/pkg/controller/migmigration/quiesce_test.go b/pkg/controller/migmigration/quiesce_test.go index 6c9e15931f..654789d33d 100644 --- a/pkg/controller/migmigration/quiesce_test.go +++ b/pkg/controller/migmigration/quiesce_test.go @@ -362,6 +362,162 @@ func TestUnQuiesceVirtualMachine(t *testing.T) { } } +func TestEnsureDestinationQuiescedPodsTerminated(t *testing.T) { + tests := []struct { + name string + client compat.Client + task *Task + allterminated bool + }{ + { + name: "no pods", + client: getFakeClientWithObjs(), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + { + name: "no pods with owner ref", + client: getFakeClientWithObjs(createPodWithOwner("pod", "tgt-namespace", "", "", "")), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + { + name: "pods with deployment owner ref", + client: getFakeClientWithObjs(createPodWithOwner("pod", "tgt-namespace", "v1", "ReplicaSet", "deployment")), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: false, + }, + { + name: "skipped pods with vm owner ref", + client: getFakeClientWithObjs(createPodWithOwnerAndPhase("pod", "tgt-namespace", "v1", "VirtualMachineInstance", "virt-launcher", corev1.PodSucceeded)), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.destinationClient = tt.client + allTerminated, err := tt.task.ensureDestinationQuiescedPodsTerminated() + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if allTerminated != tt.allterminated { + t.Errorf("ensureDestinationQuiescedPodsTerminated() allTerminated = %v, want %v", allTerminated, tt.allterminated) + } + }) + } +} + +func TestEnsureSourceQuiescedPodsTerminated(t *testing.T) { + tests := []struct { + name string + client compat.Client + task *Task + allterminated bool + }{ + { + name: "no pods", + client: getFakeClientWithObjs(), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + { + name: "no pods with owner ref", + client: getFakeClientWithObjs(createPodWithOwner("pod", "src-namespace", "", "", "")), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + { + name: "pods with deployment owner ref", + client: getFakeClientWithObjs(createPodWithOwner("pod", "src-namespace", "v1", "ReplicationController", "controller")), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: false, + }, + { + name: "skipped pods with vm owner ref", + client: getFakeClientWithObjs(createPodWithOwnerAndPhase("pod", "src-namespace", "v1", "VirtualMachineInstance", "virt-launcher", corev1.PodFailed)), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + allTerminated, err := tt.task.ensureSourceQuiescedPodsTerminated() + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if allTerminated != tt.allterminated { + t.Errorf("ensureDestinationQuiescedPodsTerminated() allTerminated = %v, want %v", allTerminated, tt.allterminated) + } + }) + } +} + func getFakeClientWithObjs(obj ...k8sclient.Object) compat.Client { client, _ := fakecompat.NewFakeClient(obj...) return client @@ -401,17 +557,37 @@ func createVMWithAnnotation(name, namespace string, ann map[string]string) *virt } func createVirtlauncherPod(vmName, namespace string) *corev1.Pod { - return &corev1.Pod{ + pod := createPodWithOwner(vmName+"-virt-launcher", namespace, "kubevirt.io/v1", "VirtualMachineInstance", vmName) + pod.Labels = map[string]string{ + "kubevirt.io": "virt-launcher", + } + return pod +} + +func createPodWithOwner(name, namespace, apiversion, kind, ownerName string) *corev1.Pod { + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: vmName + "-virt-launcher", + Name: name, Namespace: namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "kubevirt.io/v1", - Kind: "VirtualMachineInstance", - Name: vmName, - }, - }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, }, } + if apiversion != "" && kind != "" && ownerName != "" { + pod.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: apiversion, + Kind: kind, + Name: ownerName, + }, + } + } + return pod +} + +func createPodWithOwnerAndPhase(name, namespace, apiversion, kind, ownerName string, phase corev1.PodPhase) *corev1.Pod { + pod := createPodWithOwner(name, namespace, apiversion, kind, ownerName) + pod.Status.Phase = phase + return pod } diff --git a/pkg/controller/migmigration/rollback.go b/pkg/controller/migmigration/rollback.go index b82b41c0b8..e439c569ae 100644 --- a/pkg/controller/migmigration/rollback.go +++ b/pkg/controller/migmigration/rollback.go @@ -18,18 +18,19 @@ import ( // Delete namespace and cluster-scoped resources on dest cluster func (t *Task) deleteMigrated() error { // Delete 'deployer' and 'hooks' Pods that DeploymentConfig leaves behind upon DC deletion. - err := t.deleteDeploymentConfigLeftoverPods() - if err != nil { + if err := t.deleteDeploymentConfigLeftoverPods(); err != nil { return liberr.Wrap(err) } - err = t.deleteMigratedNamespaceScopedResources() - if err != nil { + if err := t.deleteMigratedNamespaceScopedResources(); err != nil { return liberr.Wrap(err) } - err = t.deleteMovedNfsPVs() - if err != nil { + if err := t.deleteMovedNfsPVs(); err != nil { + return liberr.Wrap(err) + } + + if err := t.deleteLiveMigrationCompletedPods(); err != nil { return liberr.Wrap(err) } @@ -227,6 +228,66 @@ func (t *Task) deleteMovedNfsPVs() error { return nil } +// Completed virt-launcher pods remain after migration is complete. These pods reference +// the migrated PVCs and prevent the PVCs from being deleted. This function deletes +// the completed virt-launcher pods. +func (t *Task) deleteLiveMigrationCompletedPods() error { + if t.PlanResources.MigPlan.LiveMigrationChecked() { + // Possible live migrations were performed, check for completed virt-launcher pods. + destClient, err := t.getDestinationClient() + if err != nil { + return liberr.Wrap(err) + } + namespaceVolumeNamesMap := t.getDestinationVolumeNames() + for _, namespace := range t.destinationNamespaces() { + pods := corev1.PodList{} + destClient.List(context.TODO(), &pods, k8sclient.InNamespace(namespace), k8sclient.MatchingLabels(map[string]string{ + "kubevirt.io": "virt-launcher", + })) + for _, pod := range pods.Items { + if !t.isLiveMigrationCompletedPod(&pod, namespaceVolumeNamesMap[namespace]) { + continue + } + t.Log.V(3).Info("Deleting virt-launcher pod that has completed live migration", "namespace", pod.Namespace, "name", pod.Name) + err := destClient.Delete(context.TODO(), &pod) + if err != nil && !k8serror.IsNotFound(err) { + return err + } + } + } + } + return nil +} + +func (t *Task) getDestinationVolumeNames() map[string][]string { + namespaceVolumeNamesMap := make(map[string][]string) + pvcs := t.getDirectVolumeClaimList() + for _, pvc := range pvcs { + namespaceVolumeNamesMap[pvc.Namespace] = append(namespaceVolumeNamesMap[pvc.Namespace], pvc.TargetName) + } + return namespaceVolumeNamesMap +} + +func (t *Task) isLiveMigrationCompletedPod(pod *corev1.Pod, volumeNames []string) bool { + if len(pod.Spec.Volumes) == 0 { + return false + } + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + found := false + for _, volumeName := range volumeNames { + if volumeName == volume.PersistentVolumeClaim.ClaimName { + found = true + } + } + if !found { + return false + } + } + } + return pod.Status.Phase == corev1.PodSucceeded +} + func (t *Task) ensureMigratedResourcesDeleted() (bool, error) { t.Log.Info("Scanning all GVKs in all migrated namespaces to ensure " + "resources have finished deleting.") diff --git a/pkg/controller/migmigration/rollback_test.go b/pkg/controller/migmigration/rollback_test.go new file mode 100644 index 0000000000..940bbeae6e --- /dev/null +++ b/pkg/controller/migmigration/rollback_test.go @@ -0,0 +1,166 @@ +// FILEPATH: /home/awels/go/src/github.com/awels/mig-controller/pkg/controller/migmigration/rollback_test.go + +package migmigration + +import ( + "context" + "testing" + + "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestTask_DeleteLiveMigrationCompletedPods(t *testing.T) { + tests := []struct { + name string + objects []client.Object + liveMigrate bool + expectedPods []*corev1.Pod + deletedPods []*corev1.Pod + }{ + { + name: "live migrate is not checked", + liveMigrate: false, + }, + { + name: "live migrate is checked, no running pods", + liveMigrate: true, + }, + { + name: "live migrate is checked, running and completed pods, should delete completed pods", + liveMigrate: true, + objects: []client.Object{ + createVirtlauncherPodWithStatus("pod1", "ns1", corev1.PodRunning), + createVirtlauncherPodWithStatus("pod1", "ns2", corev1.PodSucceeded), + createVirtlauncherPod("pod2", "ns1"), + }, + expectedPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "pod1-virt-launcher", + }, + }, + }, + deletedPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns2", + Name: "pod1-virt-launcher", + }, + }, + }, + }, + { + name: "live migrate is checked, running and completed pods, but non matching volumes, should delete completed pods", + liveMigrate: true, + objects: []client.Object{ + createVirtlauncherPodWithStatus("pod1", "ns1", corev1.PodRunning), + createVirtlauncherPodWithExtraVolume("pod1", "ns2", corev1.PodSucceeded), + createVirtlauncherPod("pod2", "ns1"), + }, + expectedPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "pod1-virt-launcher", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := getFakeClientWithObjs(tt.objects...) + task := &Task{ + PlanResources: &v1alpha1.PlanResources{ + MigPlan: &v1alpha1.MigPlan{ + Spec: v1alpha1.MigPlanSpec{ + Namespaces: []string{"ns1:ns1", "ns2"}, + LiveMigrate: ptr.To[bool](tt.liveMigrate), + PersistentVolumes: v1alpha1.PersistentVolumes{ + List: []v1alpha1.PV{ + { + PVC: v1alpha1.PVC{ + Namespace: "ns1", + Name: "pvc1", + }, + Selection: v1alpha1.Selection{ + Action: v1alpha1.PvCopyAction, + CopyMethod: v1alpha1.PvBlockCopyMethod, + StorageClass: "sc2", + AccessMode: "ReadWriteOnce", + }, + StorageClass: "sc1", + }, + { + PVC: v1alpha1.PVC{ + Namespace: "ns2", + Name: "pvc1", + }, + Selection: v1alpha1.Selection{ + Action: v1alpha1.PvCopyAction, + CopyMethod: v1alpha1.PvFilesystemCopyMethod, + StorageClass: "sc2", + AccessMode: "ReadWriteOnce", + }, + StorageClass: "sc1", + }, + }, + }, + }, + }, + }, + destinationClient: c, + } + err := task.deleteLiveMigrationCompletedPods() + if err != nil { + t.Errorf("Task.deleteLiveMigrationCompletedPods() error = %v", err) + } + for _, pod := range tt.expectedPods { + res := &corev1.Pod{} + err := c.Get(context.TODO(), client.ObjectKeyFromObject(pod), res) + if err != nil { + t.Errorf("Task.deleteLiveMigrationCompletedPods() pod not found, while it should remain: %s/%s, %v", pod.Namespace, pod.Name, err) + } + } + for _, pod := range tt.deletedPods { + res := &corev1.Pod{} + err := c.Get(context.TODO(), client.ObjectKeyFromObject(pod), res) + if err == nil { + t.Errorf("Task.deleteLiveMigrationCompletedPods() pod %s/%s found, while it should be deleted", pod.Namespace, pod.Name) + } + } + }) + } +} + +func createVirtlauncherPodWithStatus(name, namespace string, phase corev1.PodPhase) *corev1.Pod { + pod := createVirtlauncherPod(name, namespace) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc1", + }, + }, + }) + pod.Status.Phase = phase + return pod +} + +func createVirtlauncherPodWithExtraVolume(name, namespace string, phase corev1.PodPhase) *corev1.Pod { + pod := createVirtlauncherPodWithStatus(name, namespace, phase) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "extra-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc2", + }, + }, + }) + return pod +} diff --git a/pkg/controller/migmigration/storage.go b/pkg/controller/migmigration/storage.go index bb7427d5a3..0413381ccc 100644 --- a/pkg/controller/migmigration/storage.go +++ b/pkg/controller/migmigration/storage.go @@ -11,12 +11,14 @@ import ( "github.com/go-logr/logr" liberr "github.com/konveyor/controller/pkg/error" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + dvmc "github.com/konveyor/mig-controller/pkg/controller/directvolumemigration" ocappsv1 "github.com/openshift/api/apps/v1" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" batchv1beta "k8s.io/api/batch/v1beta1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" k8smeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/rest" "k8s.io/utils/ptr" @@ -127,7 +129,11 @@ func (t *Task) swapPVCReferences() (reasons []string, err error) { reasons = append(reasons, fmt.Sprintf("Failed updating PVC references on StatefulSets [%s]", strings.Join(failedStatefulSetNames, ","))) } - failedVirtualMachineSwaps := t.swapVirtualMachinePVCRefs(client, restConfig, mapping) + restClient, err := t.createRestClient(restConfig) + if err != nil { + t.Log.Error(err, "failed creating rest client") + } + failedVirtualMachineSwaps := t.swapVirtualMachinePVCRefs(client, restClient, mapping) if len(failedVirtualMachineSwaps) > 0 { reasons = append(reasons, fmt.Sprintf("Failed updating PVC references on VirtualMachines [%s]", strings.Join(failedVirtualMachineSwaps, ","))) @@ -636,11 +642,10 @@ func (t *Task) swapCronJobsPVCRefs(client k8sclient.Client, mapping pvcNameMappi return } -func (t *Task) swapVirtualMachinePVCRefs(client k8sclient.Client, restConfig *rest.Config, mapping pvcNameMapping) (failedVirtualMachines []string) { +func (t *Task) swapVirtualMachinePVCRefs(client k8sclient.Client, restClient rest.Interface, mapping pvcNameMapping) (failedVirtualMachines []string) { for _, ns := range t.destinationNamespaces() { list := &virtv1.VirtualMachineList{} - options := k8sclient.InNamespace(ns) - if err := client.List(context.TODO(), list, options); err != nil { + if err := client.List(context.TODO(), list, k8sclient.InNamespace(ns)); err != nil { if k8smeta.IsNoMatchError(err) { continue } @@ -648,59 +653,77 @@ func (t *Task) swapVirtualMachinePVCRefs(client k8sclient.Client, restConfig *re continue } for _, vm := range list.Items { - for i, volume := range vm.Spec.Template.Spec.Volumes { - if volume.PersistentVolumeClaim != nil { - if isFailed := updatePVCRef(&vm.Spec.Template.Spec.Volumes[i].PersistentVolumeClaim.PersistentVolumeClaimVolumeSource, vm.Namespace, mapping); isFailed { - failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("%s/%s", vm.Namespace, vm.Name)) + retryCount := 1 + retry := true + for retry && retryCount <= 3 { + message, err := t.swapVirtualMachinePVCRef(client, restClient, &vm, mapping) + if err != nil && !k8serrors.IsConflict(err) { + failedVirtualMachines = append(failedVirtualMachines, message) + return + } else if k8serrors.IsConflict(err) { + t.Log.Info("Conflict updating VM, retrying after reloading VM resource") + // Conflict, reload VM and try again + if err := client.Get(context.TODO(), k8sclient.ObjectKey{Namespace: ns, Name: vm.Name}, &vm); err != nil { + failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("failed reloading %s/%s", ns, vm.Name)) } - } - if volume.DataVolume != nil { - isFailed, err := updateDataVolumeRef(client, vm.Spec.Template.Spec.Volumes[i].DataVolume, vm.Namespace, mapping, t.Log) - if err != nil || isFailed { - failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("%s/%s", vm.Namespace, vm.Name)) - } else { - // Update datavolume template if it exists. - for i, dvt := range vm.Spec.DataVolumeTemplates { - if destinationDVName, exists := mapping.Get(ns, dvt.Name); exists { - vm.Spec.DataVolumeTemplates[i].Name = destinationDVName - } - } + retryCount++ + } else { + retry = false + if message != "" { + failedVirtualMachines = append(failedVirtualMachines, message) } } } - for _, dvt := range vm.Spec.DataVolumeTemplates { - t.Log.Info("DataVolumeTemplate", "dv", dvt) + } + } + return +} + +func (t *Task) swapVirtualMachinePVCRef(client k8sclient.Client, restClient rest.Interface, vm *virtv1.VirtualMachine, mapping pvcNameMapping) (string, error) { + if vm.Spec.Template == nil { + return "", nil + } + for i, volume := range vm.Spec.Template.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + if isFailed := updatePVCRef(&vm.Spec.Template.Spec.Volumes[i].PersistentVolumeClaim.PersistentVolumeClaimVolumeSource, vm.Namespace, mapping); isFailed { + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), nil } - for _, volume := range vm.Spec.Template.Spec.Volumes { - if volume.DataVolume != nil { - t.Log.Info("datavolume", "dv", volume.DataVolume) + } + if volume.DataVolume != nil { + isFailed, err := updateDataVolumeRef(client, vm.Spec.Template.Spec.Volumes[i].DataVolume, vm.Namespace, mapping, t.Log) + if err != nil || isFailed { + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err + } + // Update datavolume template if it exists. + for i, dvt := range vm.Spec.DataVolumeTemplates { + if destinationDVName, exists := mapping.Get(vm.Namespace, dvt.Name); exists { + vm.Spec.DataVolumeTemplates[i].Name = destinationDVName } } - if shouldStartVM(&vm) { - if !isVMActive(&vm, client) { - restClient, err := t.createRestClient(restConfig) - if err != nil { - t.Log.Error(err, "failed creating rest client", "namespace", vm.Namespace, "name", vm.Name) - } - if err := t.startVM(&vm, client, restClient); err != nil { - t.Log.Error(err, "failed starting VM", "namespace", vm.Namespace, "name", vm.Name) - failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("%s/%s", vm.Namespace, vm.Name)) - } - } - } else { - if err := client.Update(context.Background(), &vm); err != nil { - t.Log.Error(err, "failed updating VM", "namespace", vm.Namespace, "name", vm.Name) - failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("%s/%s", vm.Namespace, vm.Name)) - } + } + } + if !isVMActive(vm, client) { + if err := client.Update(context.Background(), vm); err != nil { + t.Log.Error(err, "failed updating VM", "namespace", vm.Namespace, "name", vm.Name) + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err + } + if err := client.Get(context.Background(), k8sclient.ObjectKey{Namespace: vm.Namespace, Name: vm.Name}, vm); err != nil { + t.Log.Error(err, "failed getting VM", "namespace", vm.Namespace, "name", vm.Name) + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err + } + if shouldStartVM(vm) { + if err := t.startVM(vm, client, restClient); err != nil { + t.Log.Error(err, "failed starting VM", "namespace", vm.Namespace, "name", vm.Name) + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err } } } - return + return "", nil } // updatePVCRef given a PVCSource, namespace and a mapping of pvc names, swaps the claim // present in the pvc source with the mapped pvc name found in the mapping -// returns whether the swap was successful or not +// returns whether the swap was successful or not, true is failure, false is success func updatePVCRef(pvcSource *v1.PersistentVolumeClaimVolumeSource, ns string, mapping pvcNameMapping) bool { if pvcSource != nil { originalName := pvcSource.ClaimName @@ -729,26 +752,13 @@ func updateDataVolumeRef(client k8sclient.Client, dv *virtv1.DataVolumeSource, n } if destinationDVName, exists := mapping.Get(ns, originalName); exists { - log.Info("Found DataVolume mapping", "namespace", ns, "name", originalName, "destination", destinationDVName) dv.Name = destinationDVName - // Create adopting datavolume. - adoptingDV := originalDv.DeepCopy() - adoptingDV.Name = destinationDVName - if adoptingDV.Annotations == nil { - adoptingDV.Annotations = make(map[string]string) - } - adoptingDV.Annotations["cdi.kubevirt.io/allowClaimAdoption"] = "true" - adoptingDV.ResourceVersion = "" - adoptingDV.ManagedFields = nil - adoptingDV.UID = "" - - err := client.Create(context.Background(), adoptingDV) + err := dvmc.CreateNewDataVolume(client, originalDv.Name, destinationDVName, ns, log) if err != nil && !errors.IsAlreadyExists(err) { log.Error(err, "failed creating DataVolume", "namespace", ns, "name", destinationDVName) return true, err } } else { - log.Info("DataVolume reference already updated", "namespace", ns, "name", originalName) // attempt to figure out whether the current DV reference // already points to the new migrated PVC. This is needed to // guarantee idempotency of the operation diff --git a/pkg/controller/migmigration/storage_test.go b/pkg/controller/migmigration/storage_test.go new file mode 100644 index 0000000000..57afaef6e7 --- /dev/null +++ b/pkg/controller/migmigration/storage_test.go @@ -0,0 +1,435 @@ +package migmigration + +import ( + "context" + "slices" + "testing" + + "github.com/go-logr/logr" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestTask_updateDataVolumeRef(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + dvSource *virtv1.DataVolumeSource + ns string + mapping pvcNameMapping + log logr.Logger + expectedFailure bool + wantErr bool + newDV *cdiv1.DataVolume + }{ + { + name: "no dv source", + expectedFailure: false, + wantErr: false, + }, + { + name: "dv source set to unknown dv", + dvSource: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + expectedFailure: true, + wantErr: true, + }, + { + name: "dv source set to known dv, but missing in mapping", + objects: []runtime.Object{ + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + dvSource: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + ns: "ns-0", + expectedFailure: true, + wantErr: false, + }, + { + name: "dv source set to known dv, but mapping as value only", + objects: []runtime.Object{ + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + dvSource: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + mapping: pvcNameMapping{"ns-0/src-0": "dv-0"}, + ns: "ns-0", + expectedFailure: false, + wantErr: false, + }, + { + name: "dv source set to known dv, but mapping as key", + objects: []runtime.Object{ + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + dvSource: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + mapping: pvcNameMapping{"ns-0/dv-0": "tgt-0"}, + ns: "ns-0", + expectedFailure: false, + wantErr: false, + newDV: &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tgt-0", + Namespace: "ns-0", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := scheme.Scheme + err := cdiv1.AddToScheme(s) + if err != nil { + panic(err) + } + c := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(tt.objects...).Build() + res, err := updateDataVolumeRef(c, tt.dvSource, tt.ns, tt.mapping, tt.log) + if (err != nil) != tt.wantErr { + t.Errorf("updateDataVolumeRef() error = %v, wantErr %v", err, tt.wantErr) + t.FailNow() + } + if res != tt.expectedFailure { + t.Errorf("updateDataVolumeRef() expected failure = %v, want %v", res, tt.expectedFailure) + t.FailNow() + } + if tt.newDV != nil { + err := c.Get(context.TODO(), client.ObjectKeyFromObject(tt.newDV), tt.newDV) + if err != nil { + t.Errorf("updateDataVolumeRef() failed to create new DV: %v", err) + t.FailNow() + } + } else { + dvs := &cdiv1.DataVolumeList{} + err := c.List(context.TODO(), dvs, client.InNamespace(tt.ns)) + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("Error reading datavolumes: %v", err) + t.FailNow() + } else if err == nil && len(dvs.Items) > 0 { + for _, dv := range dvs.Items { + if dv.Name != tt.dvSource.Name { + t.Errorf("updateDataVolumeRef() created new DV when it shouldn't have, %v", dvs.Items) + t.FailNow() + } + } + } + } + }) + } +} + +func TestTask_swapVirtualMachinePVCRefs(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + restConfig rest.Interface + pvcMapping pvcNameMapping + expectedFailures []string + expectedNewName string + shouldStartVM bool + }{ + { + name: "no VMs, should return no failed VMs", + restConfig: getFakeRestClient(), + }, + { + name: "VMs without volumes, should return no failed VMs", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + &virtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vm-0", + Namespace: "ns-0", + }, + }, + }, + }, + { + name: "VMs with DVs, no mapping, should return all VMs", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachine("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + expectedFailures: []string{"ns-0/vm-0"}, + }, + { + name: "VMs with PVCs, no mapping, should return all VMs", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachine("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-0", + }, + }, + }, + }, + }), + }, + expectedFailures: []string{"ns-0/vm-0"}, + }, + { + name: "VMs with DVs, mapping to new name, should return no failures", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachine("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + pvcMapping: pvcNameMapping{"ns-0/dv-0": "dv-1"}, + expectedNewName: "dv-1", + }, + { + name: "VMs with DVTemplatess, mapping to new name, should return no failures", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachineWithDVTemplate("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + pvcMapping: pvcNameMapping{"ns-0/dv-0": "dv-1"}, + expectedNewName: "dv-1", + }, + { + name: "VMs with DVs, mapping to new name, but running VM, should return no failures, and no updates", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachine("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + createVirtlauncherPod("vm-0", "ns-0"), + }, + pvcMapping: pvcNameMapping{"ns-0/dv-0": "dv-1"}, + }, + { + name: "VMs with DVs, mapping to new name, should return no failures", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachineWithAnnotation("vm-0", "ns-0", migapi.StartVMAnnotation, "true", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + pvcMapping: pvcNameMapping{"ns-0/dv-0": "dv-1"}, + expectedNewName: "dv-1", + shouldStartVM: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"ns-0:ns-0", "ns-1:ns-1"}, + }, + }, + }, + } + s := scheme.Scheme + err := cdiv1.AddToScheme(s) + if err != nil { + panic(err) + } + err = virtv1.AddToScheme(s) + if err != nil { + panic(err) + } + c := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(tt.objects...).Build() + failedVMs := task.swapVirtualMachinePVCRefs(c, tt.restConfig, tt.pvcMapping) + if len(failedVMs) != len(tt.expectedFailures) { + t.Errorf("swapVirtualMachinePVCRefs() failed to swap PVC refs for VMs: %v, expected failures: %v", failedVMs, tt.expectedFailures) + t.FailNow() + } + for _, failedVM := range failedVMs { + if !slices.Contains(tt.expectedFailures, failedVM) { + t.Errorf("unexpected failed VM: %s, expected failures: %v", failedVM, tt.expectedFailures) + t.FailNow() + } + } + vm := &virtv1.VirtualMachine{} + if err := c.Get(context.TODO(), client.ObjectKey{Namespace: "ns-0", Name: "vm-0"}, vm); err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("failed to get VM: %v", err) + t.FailNow() + } + if vm.Spec.Template != nil { + found := false + for _, volume := range vm.Spec.Template.Spec.Volumes { + if volume.VolumeSource.DataVolume != nil && volume.VolumeSource.DataVolume.Name == tt.expectedNewName { + found = true + } + if volume.VolumeSource.PersistentVolumeClaim != nil && volume.VolumeSource.PersistentVolumeClaim.ClaimName == tt.expectedNewName { + found = true + } + } + if !found && tt.expectedNewName != "" { + t.Errorf("Didn't find new volume name %s", tt.expectedNewName) + t.FailNow() + } else if tt.expectedNewName == "" && vm.ObjectMeta.ResourceVersion == "1000" { + t.Errorf("VM updated when it shouldn't have") + t.FailNow() + } + // Check DVTemplates + if len(vm.Spec.DataVolumeTemplates) > 0 { + found = false + for _, dvTemplate := range vm.Spec.DataVolumeTemplates { + if dvTemplate.Name == tt.expectedNewName { + found = true + } + } + if !found && tt.expectedNewName != "" { + t.Errorf("Didn't find new volume name %s in DVTemplate", tt.expectedNewName) + t.FailNow() + } else if found && tt.expectedNewName == "" { + t.Errorf("Found new volume name %s in DVTemplate when it shouldn't have", tt.expectedNewName) + t.FailNow() + } + } + if tt.shouldStartVM { + if _, ok := vm.GetAnnotations()[migapi.StartVMAnnotation]; ok { + t.Errorf("VM should have started") + t.FailNow() + } + } + } + }) + } +} + +func createVirtualMachine(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + return &virtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineSpec{ + Template: &virtv1.VirtualMachineInstanceTemplateSpec{ + Spec: virtv1.VirtualMachineInstanceSpec{ + Volumes: volumes, + }, + }, + }, + } +} + +func createVirtualMachineWithDVTemplate(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachine(name, namespace, volumes) + for _, volume := range volumes { + if volume.VolumeSource.DataVolume != nil { + vm.Spec.DataVolumeTemplates = append(vm.Spec.DataVolumeTemplates, virtv1.DataVolumeTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: volume.VolumeSource.DataVolume.Name, + Namespace: namespace, + }, + Spec: cdiv1.DataVolumeSpec{ + Storage: &cdiv1.StorageSpec{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }) + } + } + return vm +} + +func createVirtualMachineWithAnnotation(name, namespace, key, value string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachine(name, namespace, volumes) + vm.Annotations = map[string]string{key: value} + return vm +} diff --git a/pkg/controller/migmigration/task.go b/pkg/controller/migmigration/task.go index 67d60062ee..88f5b5cada 100644 --- a/pkg/controller/migmigration/task.go +++ b/pkg/controller/migmigration/task.go @@ -27,78 +27,83 @@ var NoReQ = time.Duration(0) // Phases const ( - Created = "" - Started = "Started" - CleanStaleAnnotations = "CleanStaleAnnotations" - CleanStaleVeleroCRs = "CleanStaleVeleroCRs" - CleanStaleResticCRs = "CleanStaleResticCRs" - CleanStaleStagePods = "CleanStaleStagePods" - WaitForStaleStagePodsTerminated = "WaitForStaleStagePodsTerminated" - StartRefresh = "StartRefresh" - WaitForRefresh = "WaitForRefresh" - CreateRegistries = "CreateRegistries" - CreateDirectImageMigration = "CreateDirectImageMigration" - WaitForDirectImageMigrationToComplete = "WaitForDirectImageMigrationToComplete" - EnsureCloudSecretPropagated = "EnsureCloudSecretPropagated" - PreBackupHooks = "PreBackupHooks" - PostBackupHooks = "PostBackupHooks" - PreRestoreHooks = "PreRestoreHooks" - PostRestoreHooks = "PostRestoreHooks" - PreBackupHooksFailed = "PreBackupHooksFailed" - PostBackupHooksFailed = "PostBackupHooksFailed" - PreRestoreHooksFailed = "PreRestoreHooksFailed" - PostRestoreHooksFailed = "PostRestoreHooksFailed" - EnsureInitialBackup = "EnsureInitialBackup" - InitialBackupCreated = "InitialBackupCreated" - InitialBackupFailed = "InitialBackupFailed" - AnnotateResources = "AnnotateResources" - EnsureStagePodsFromRunning = "EnsureStagePodsFromRunning" - EnsureStagePodsFromTemplates = "EnsureStagePodsFromTemplates" - EnsureStagePodsFromOrphanedPVCs = "EnsureStagePodsFromOrphanedPVCs" - StagePodsCreated = "StagePodsCreated" - StagePodsFailed = "StagePodsFailed" - SourceStagePodsFailed = "SourceStagePodsFailed" - RestartVelero = "RestartVelero" - WaitForVeleroReady = "WaitForVeleroReady" - RestartRestic = "RestartRestic" - WaitForResticReady = "WaitForResticReady" - QuiesceApplications = "QuiesceApplications" - EnsureQuiesced = "EnsureQuiesced" - UnQuiesceSrcApplications = "UnQuiesceSrcApplications" - UnQuiesceDestApplications = "UnQuiesceDestApplications" - SwapPVCReferences = "SwapPVCReferences" - WaitForRegistriesReady = "WaitForRegistriesReady" - EnsureStageBackup = "EnsureStageBackup" - StageBackupCreated = "StageBackupCreated" - StageBackupFailed = "StageBackupFailed" - EnsureInitialBackupReplicated = "EnsureInitialBackupReplicated" - EnsureStageBackupReplicated = "EnsureStageBackupReplicated" - EnsureStageRestore = "EnsureStageRestore" - StageRestoreCreated = "StageRestoreCreated" - StageRestoreFailed = "StageRestoreFailed" - CreateDirectVolumeMigration = "CreateDirectVolumeMigration" - WaitForDirectVolumeMigrationToComplete = "WaitForDirectVolumeMigrationToComplete" - DirectVolumeMigrationFailed = "DirectVolumeMigrationFailed" - EnsureFinalRestore = "EnsureFinalRestore" - FinalRestoreCreated = "FinalRestoreCreated" - FinalRestoreFailed = "FinalRestoreFailed" - Verification = "Verification" - EnsureStagePodsDeleted = "EnsureStagePodsDeleted" - EnsureStagePodsTerminated = "EnsureStagePodsTerminated" - EnsureAnnotationsDeleted = "EnsureAnnotationsDeleted" - EnsureMigratedDeleted = "EnsureMigratedDeleted" - DeleteRegistries = "DeleteRegistries" - DeleteMigrated = "DeleteMigrated" - DeleteBackups = "DeleteBackups" - DeleteRestores = "DeleteRestores" - DeleteHookJobs = "DeleteHookJobs" - DeleteDirectVolumeMigrationResources = "DeleteDirectVolumeMigrationResources" - DeleteDirectImageMigrationResources = "DeleteDirectImageMigrationResources" - MigrationFailed = "MigrationFailed" - Canceling = "Canceling" - Canceled = "Canceled" - Rollback = "Rollback" - Completed = "Completed" + Created = "" + Started = "Started" + CleanStaleAnnotations = "CleanStaleAnnotations" + CleanStaleVeleroCRs = "CleanStaleVeleroCRs" + CleanStaleResticCRs = "CleanStaleResticCRs" + CleanStaleStagePods = "CleanStaleStagePods" + WaitForStaleStagePodsTerminated = "WaitForStaleStagePodsTerminated" + StartRefresh = "StartRefresh" + WaitForRefresh = "WaitForRefresh" + CreateRegistries = "CreateRegistries" + CreateDirectImageMigration = "CreateDirectImageMigration" + WaitForDirectImageMigrationToComplete = "WaitForDirectImageMigrationToComplete" + EnsureCloudSecretPropagated = "EnsureCloudSecretPropagated" + PreBackupHooks = "PreBackupHooks" + PostBackupHooks = "PostBackupHooks" + PreRestoreHooks = "PreRestoreHooks" + PostRestoreHooks = "PostRestoreHooks" + PreBackupHooksFailed = "PreBackupHooksFailed" + PostBackupHooksFailed = "PostBackupHooksFailed" + PreRestoreHooksFailed = "PreRestoreHooksFailed" + PostRestoreHooksFailed = "PostRestoreHooksFailed" + EnsureInitialBackup = "EnsureInitialBackup" + InitialBackupCreated = "InitialBackupCreated" + InitialBackupFailed = "InitialBackupFailed" + AnnotateResources = "AnnotateResources" + EnsureStagePodsFromRunning = "EnsureStagePodsFromRunning" + EnsureStagePodsFromTemplates = "EnsureStagePodsFromTemplates" + EnsureStagePodsFromOrphanedPVCs = "EnsureStagePodsFromOrphanedPVCs" + StagePodsCreated = "StagePodsCreated" + StagePodsFailed = "StagePodsFailed" + SourceStagePodsFailed = "SourceStagePodsFailed" + RestartVelero = "RestartVelero" + WaitForVeleroReady = "WaitForVeleroReady" + RestartRestic = "RestartRestic" + WaitForResticReady = "WaitForResticReady" + QuiesceSourceApplications = "QuiesceSourceApplications" + QuiesceDestinationApplications = "QuiesceDestinationApplications" + EnsureSrcQuiesced = "EnsureSrcQuiesced" + EnsureDestQuiesced = "EnsureDestQuiesced" + UnQuiesceSrcApplications = "UnQuiesceSrcApplications" + UnQuiesceDestApplications = "UnQuiesceDestApplications" + SwapPVCReferences = "SwapPVCReferences" + WaitForRegistriesReady = "WaitForRegistriesReady" + EnsureStageBackup = "EnsureStageBackup" + StageBackupCreated = "StageBackupCreated" + StageBackupFailed = "StageBackupFailed" + EnsureInitialBackupReplicated = "EnsureInitialBackupReplicated" + EnsureStageBackupReplicated = "EnsureStageBackupReplicated" + EnsureStageRestore = "EnsureStageRestore" + StageRestoreCreated = "StageRestoreCreated" + StageRestoreFailed = "StageRestoreFailed" + CreateDirectVolumeMigrationStage = "CreateDirectVolumeMigrationStage" + CreateDirectVolumeMigrationFinal = "CreateDirectVolumeMigrationFinal" + CreateDirectVolumeMigrationRollback = "CreateDirectVolumeMigrationRollback" + WaitForDirectVolumeMigrationToComplete = "WaitForDirectVolumeMigrationToComplete" + WaitForDirectVolumeMigrationRollbackToComplete = "WaitForDirectVolumeMigrationToRollbackComplete" + DirectVolumeMigrationFailed = "DirectVolumeMigrationFailed" + EnsureFinalRestore = "EnsureFinalRestore" + FinalRestoreCreated = "FinalRestoreCreated" + FinalRestoreFailed = "FinalRestoreFailed" + Verification = "Verification" + EnsureStagePodsDeleted = "EnsureStagePodsDeleted" + EnsureStagePodsTerminated = "EnsureStagePodsTerminated" + EnsureAnnotationsDeleted = "EnsureAnnotationsDeleted" + EnsureMigratedDeleted = "EnsureMigratedDeleted" + DeleteRegistries = "DeleteRegistries" + DeleteMigrated = "DeleteMigrated" + DeleteBackups = "DeleteBackups" + DeleteRestores = "DeleteRestores" + DeleteHookJobs = "DeleteHookJobs" + DeleteDirectVolumeMigrationResources = "DeleteDirectVolumeMigrationResources" + DeleteDirectImageMigrationResources = "DeleteDirectImageMigrationResources" + MigrationFailed = "MigrationFailed" + Canceling = "Canceling" + Canceled = "Canceled" + Rollback = "Rollback" + Completed = "Completed" ) // Flags @@ -124,18 +129,28 @@ const ( // Migration steps const ( - StepPrepare = "Prepare" - StepDirectImage = "DirectImage" - StepDirectVolume = "DirectVolume" - StepBackup = "Backup" - StepStageBackup = "StageBackup" - StepStageRestore = "StageRestore" - StepRestore = "Restore" - StepCleanup = "Cleanup" - StepCleanupVelero = "CleanupVelero" - StepCleanupHelpers = "CleanupHelpers" - StepCleanupMigrated = "CleanupMigrated" - StepCleanupUnquiesce = "CleanupUnquiesce" + StepPrepare = "Prepare" + StepDirectImage = "DirectImage" + StepDirectVolume = "DirectVolume" + StepBackup = "Backup" + StepStageBackup = "StageBackup" + StepStageRestore = "StageRestore" + StepRestore = "Restore" + StepCleanup = "Cleanup" + StepCleanupVelero = "CleanupVelero" + StepCleanupHelpers = "CleanupHelpers" + StepCleanupMigrated = "CleanupMigrated" + StepCleanupUnquiesce = "CleanupUnquiesce" + StepRollbackLiveMigration = "RollbackLiveMigration" +) + +// Itinerary names +const ( + StageItineraryName = "Stage" + FinalItineraryName = "Final" + CancelItineraryName = "Cancel" + FailedItineraryName = "Failed" + RollbackItineraryName = "Rollback" ) // Itinerary defines itinerary @@ -145,7 +160,7 @@ type Itinerary struct { } var StageItinerary = Itinerary{ - Name: "Stage", + Name: StageItineraryName, Phases: []Phase{ {Name: Created, Step: StepPrepare}, {Name: Started, Step: StepPrepare}, @@ -159,9 +174,9 @@ var StageItinerary = Itinerary{ {Name: WaitForStaleStagePodsTerminated, Step: StepPrepare}, {Name: CreateRegistries, Step: StepPrepare, all: IndirectImage | EnableImage | HasISs}, {Name: CreateDirectImageMigration, Step: StepStageBackup, all: DirectImage | EnableImage}, - {Name: QuiesceApplications, Step: StepStageBackup, all: Quiesce}, - {Name: EnsureQuiesced, Step: StepStageBackup, all: Quiesce}, - {Name: CreateDirectVolumeMigration, Step: StepStageBackup, all: DirectVolume | EnableVolume}, + {Name: QuiesceSourceApplications, Step: StepStageBackup, all: Quiesce}, + {Name: EnsureSrcQuiesced, Step: StepStageBackup, all: Quiesce}, + {Name: CreateDirectVolumeMigrationStage, Step: StepStageBackup, all: DirectVolume | EnableVolume}, {Name: EnsureStagePodsFromRunning, Step: StepStageBackup, all: HasPVs | IndirectVolume}, {Name: EnsureStagePodsFromTemplates, Step: StepStageBackup, all: HasPVs | IndirectVolume}, {Name: EnsureStagePodsFromOrphanedPVCs, Step: StepStageBackup, all: HasPVs | IndirectVolume}, @@ -189,7 +204,7 @@ var StageItinerary = Itinerary{ } var FinalItinerary = Itinerary{ - Name: "Final", + Name: FinalItineraryName, Phases: []Phase{ {Name: Created, Step: StepPrepare}, {Name: Started, Step: StepPrepare}, @@ -207,10 +222,11 @@ var FinalItinerary = Itinerary{ {Name: EnsureCloudSecretPropagated, Step: StepPrepare}, {Name: PreBackupHooks, Step: PreBackupHooks, all: HasPreBackupHooks}, {Name: CreateDirectImageMigration, Step: StepBackup, all: DirectImage | EnableImage}, + {Name: CreateDirectVolumeMigrationStage, Step: StepStageBackup, all: DirectVolume | EnableVolume}, {Name: EnsureInitialBackup, Step: StepBackup}, {Name: InitialBackupCreated, Step: StepBackup}, - {Name: QuiesceApplications, Step: StepStageBackup, all: Quiesce}, - {Name: EnsureQuiesced, Step: StepStageBackup, all: Quiesce}, + {Name: QuiesceSourceApplications, Step: StepStageBackup, all: Quiesce}, + {Name: EnsureSrcQuiesced, Step: StepStageBackup, all: Quiesce}, {Name: EnsureStagePodsFromRunning, Step: StepStageBackup, all: HasPVs | IndirectVolume}, {Name: EnsureStagePodsFromTemplates, Step: StepStageBackup, all: HasPVs | IndirectVolume}, {Name: EnsureStagePodsFromOrphanedPVCs, Step: StepStageBackup, all: HasPVs | IndirectVolume}, @@ -218,7 +234,7 @@ var FinalItinerary = Itinerary{ {Name: RestartRestic, Step: StepStageBackup, all: HasStagePods}, {Name: AnnotateResources, Step: StepStageBackup, all: HasStageBackup}, {Name: WaitForResticReady, Step: StepStageBackup, any: HasPVs | HasStagePods}, - {Name: CreateDirectVolumeMigration, Step: StepStageBackup, all: DirectVolume | EnableVolume}, + {Name: CreateDirectVolumeMigrationFinal, Step: StepStageBackup, all: DirectVolume | EnableVolume}, {Name: EnsureStageBackup, Step: StepStageBackup, all: HasStageBackup}, {Name: StageBackupCreated, Step: StepStageBackup, all: HasStageBackup}, {Name: EnsureStageBackupReplicated, Step: StepStageBackup, all: HasStageBackup}, @@ -244,7 +260,7 @@ var FinalItinerary = Itinerary{ } var CancelItinerary = Itinerary{ - Name: "Cancel", + Name: CancelItineraryName, Phases: []Phase{ {Name: Canceling, Step: StepCleanupVelero}, {Name: DeleteBackups, Step: StepCleanupVelero}, @@ -255,13 +271,14 @@ var CancelItinerary = Itinerary{ {Name: DeleteDirectImageMigrationResources, Step: StepCleanupHelpers, all: DirectImage}, {Name: EnsureStagePodsDeleted, Step: StepCleanupHelpers, all: HasStagePods}, {Name: EnsureAnnotationsDeleted, Step: StepCleanupHelpers, all: HasStageBackup}, + {Name: UnQuiesceSrcApplications, Step: StepCleanupUnquiesce}, {Name: Canceled, Step: StepCleanup}, {Name: Completed, Step: StepCleanup}, }, } var FailedItinerary = Itinerary{ - Name: "Failed", + Name: FailedItineraryName, Phases: []Phase{ {Name: MigrationFailed, Step: StepCleanupHelpers}, {Name: DeleteRegistries, Step: StepCleanupHelpers}, @@ -271,7 +288,7 @@ var FailedItinerary = Itinerary{ } var RollbackItinerary = Itinerary{ - Name: "Rollback", + Name: RollbackItineraryName, Phases: []Phase{ {Name: Rollback, Step: StepCleanupVelero}, {Name: DeleteBackups, Step: StepCleanupVelero}, @@ -279,7 +296,11 @@ var RollbackItinerary = Itinerary{ {Name: DeleteRegistries, Step: StepCleanupHelpers}, {Name: EnsureStagePodsDeleted, Step: StepCleanupHelpers}, {Name: EnsureAnnotationsDeleted, Step: StepCleanupHelpers, any: HasPVs | HasISs}, + {Name: QuiesceDestinationApplications, Step: StepCleanupMigrated, any: DirectVolume}, + {Name: EnsureDestQuiesced, Step: StepCleanupMigrated}, {Name: SwapPVCReferences, Step: StepCleanupMigrated, all: StorageConversion}, + {Name: CreateDirectVolumeMigrationRollback, Step: StepRollbackLiveMigration, all: DirectVolume | EnableVolume}, + {Name: WaitForDirectVolumeMigrationRollbackToComplete, Step: StepRollbackLiveMigration, all: DirectVolume | EnableVolume}, {Name: DeleteMigrated, Step: StepCleanupMigrated}, {Name: EnsureMigratedDeleted, Step: StepCleanupMigrated}, {Name: UnQuiesceSrcApplications, Step: StepCleanupUnquiesce}, @@ -327,18 +348,20 @@ func (r Itinerary) progressReport(phaseName string) (string, int, int) { // Errors - Migration errors. // Failed - Task phase has failed. type Task struct { - Scheme *runtime.Scheme - Log logr.Logger - Client k8sclient.Client - Owner *migapi.MigMigration - PlanResources *migapi.PlanResources - Annotations map[string]string - BackupResources mapset.Set - Phase string - Requeue time.Duration - Itinerary Itinerary - Errors []string - Step string + Scheme *runtime.Scheme + Log logr.Logger + Client k8sclient.Client + destinationClient compat.Client + sourceClient compat.Client + Owner *migapi.MigMigration + PlanResources *migapi.PlanResources + Annotations map[string]string + BackupResources mapset.Set + Phase string + Requeue time.Duration + Itinerary Itinerary + Errors []string + Step string Tracer opentracing.Tracer ReconcileSpan opentracing.Span @@ -671,16 +694,38 @@ func (t *Task) Run(ctx context.Context) error { t.Log.Info("Velero Pod(s) are unready on the source or target cluster. Waiting.") t.Requeue = PollReQ } - case QuiesceApplications: - err := t.quiesceApplications() + case QuiesceSourceApplications: + err := t.quiesceSourceApplications() + if err != nil { + return liberr.Wrap(err) + } + if err = t.next(); err != nil { + return liberr.Wrap(err) + } + case QuiesceDestinationApplications: + err := t.quiesceDestinationApplications() if err != nil { return liberr.Wrap(err) } if err = t.next(); err != nil { return liberr.Wrap(err) } - case EnsureQuiesced: - quiesced, err := t.ensureQuiescedPodsTerminated() + case EnsureSrcQuiesced: + quiesced, err := t.ensureSourceQuiescedPodsTerminated() + if err != nil { + return liberr.Wrap(err) + } + if quiesced { + if err = t.next(); err != nil { + return liberr.Wrap(err) + } + } else { + t.Log.Info("Quiescing on source cluster is incomplete. " + + "Pods are not yet terminated, waiting.") + t.Requeue = PollReQ + } + case EnsureDestQuiesced: + quiesced, err := t.ensureDestinationQuiescedPodsTerminated() if err != nil { return liberr.Wrap(err) } @@ -721,9 +766,9 @@ func (t *Task) Run(ctx context.Context) error { if err = t.next(); err != nil { return liberr.Wrap(err) } - case CreateDirectVolumeMigration: + case CreateDirectVolumeMigrationStage, CreateDirectVolumeMigrationFinal: if t.hasDirectVolumes() { - err := t.createDirectVolumeMigration() + err := t.createDirectVolumeMigration(nil) if err != nil { return liberr.Wrap(err) } @@ -731,7 +776,16 @@ func (t *Task) Run(ctx context.Context) error { if err := t.next(); err != nil { return liberr.Wrap(err) } - case WaitForDirectVolumeMigrationToComplete: + case CreateDirectVolumeMigrationRollback: + rollback := migapi.MigrationTypeRollback + err := t.createDirectVolumeMigration(&rollback) + if err != nil { + return liberr.Wrap(err) + } + if err = t.next(); err != nil { + return liberr.Wrap(err) + } + case WaitForDirectVolumeMigrationToComplete, WaitForDirectVolumeMigrationRollbackToComplete: dvm, err := t.getDirectVolumeMigration() if err != nil { return liberr.Wrap(err) @@ -743,30 +797,8 @@ func (t *Task) Run(ctx context.Context) error { } break } - // Check if DVM is complete and report progress - completed, reasons, progress := t.hasDirectVolumeMigrationCompleted(dvm) - PhaseDescriptions[t.Phase] = dvm.Status.PhaseDescription - t.setProgress(progress) - if completed { - step := t.Owner.Status.FindStep(t.Step) - step.MarkCompleted() - if len(reasons) > 0 { - t.setDirectVolumeMigrationFailureWarning(dvm) - } - if err = t.next(); err != nil { - return liberr.Wrap(err) - } - } else { - t.Requeue = PollReQ - criticalWarning, err := t.getWarningForDVM(dvm) - if err != nil { - return liberr.Wrap(err) - } - if criticalWarning != nil { - t.Owner.Status.SetCondition(*criticalWarning) - return nil - } - t.Owner.Status.DeleteCondition(DirectVolumeMigrationBlocked) + if err := t.waitForDVMToComplete(dvm); err != nil { + return liberr.Wrap(err) } case EnsureStageBackup: _, err := t.ensureStageBackup() @@ -1516,9 +1548,7 @@ func (t *Task) failCurrentStep() { // Add errors. func (t *Task) addErrors(errors []string) { - for _, e := range errors { - t.Errors = append(t.Errors, e) - } + t.Errors = append(t.Errors, errors...) } // Migration UID. @@ -1604,7 +1634,14 @@ func (t *Task) keepAnnotations() bool { // Get a client for the source cluster. func (t *Task) getSourceClient() (compat.Client, error) { - return t.PlanResources.SrcMigCluster.GetClient(t.Client) + if t.sourceClient == nil { + c, err := t.PlanResources.SrcMigCluster.GetClient(t.Client) + if err != nil { + return nil, err + } + t.sourceClient = c + } + return t.sourceClient, nil } // Get a client for the source cluster. @@ -1614,7 +1651,14 @@ func (t *Task) getSourceRestConfig() (*rest.Config, error) { // Get a client for the destination cluster. func (t *Task) getDestinationClient() (compat.Client, error) { - return t.PlanResources.DestMigCluster.GetClient(t.Client) + if t.destinationClient == nil { + c, err := t.PlanResources.DestMigCluster.GetClient(t.Client) + if err != nil { + return nil, err + } + t.destinationClient = c + } + return t.destinationClient, nil } // Get a client for the source cluster. @@ -1852,3 +1896,33 @@ func (t *Task) hasPostRestoreHooks() bool { } return anyPostRestoreHooks } + +func (t *Task) waitForDVMToComplete(dvm *migapi.DirectVolumeMigration) error { + // Check if DVM is complete and report progress + completed, reasons, progress := t.hasDirectVolumeMigrationCompleted(dvm) + t.Log.V(3).Info("DVM status", "completed", completed, "reasons", reasons, "progress", progress) + PhaseDescriptions[t.Phase] = dvm.Status.PhaseDescription + t.setProgress(progress) + if completed { + step := t.Owner.Status.FindStep(t.Step) + if step == nil { + return fmt.Errorf("step %s not found in pipeline", t.Step) + } + step.MarkCompleted() + if len(reasons) > 0 { + t.setDirectVolumeMigrationFailureWarning(dvm) + } + if err := t.next(); err != nil { + return liberr.Wrap(err) + } + } else { + t.Requeue = PollReQ + criticalWarning := t.getWarningForDVM(dvm) + if criticalWarning != nil { + t.Owner.Status.SetCondition(*criticalWarning) + return nil + } + t.Owner.Status.DeleteCondition(DirectVolumeMigrationBlocked) + } + return nil +} diff --git a/pkg/controller/migmigration/task_test.go b/pkg/controller/migmigration/task_test.go index e539b2318a..f4c4f3e145 100644 --- a/pkg/controller/migmigration/task_test.go +++ b/pkg/controller/migmigration/task_test.go @@ -1,13 +1,16 @@ package migmigration import ( - "github.com/go-logr/logr" - migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" "reflect" "testing" + + "github.com/go-logr/logr" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + dvmc "github.com/konveyor/mig-controller/pkg/controller/directvolumemigration" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestTask_getStagePVs(t1 *testing.T) { +func TestTask_getStagePVs(t *testing.T) { type fields struct { Log logr.Logger PlanResources *migapi.PlanResources @@ -176,15 +179,189 @@ func TestTask_getStagePVs(t1 *testing.T) { }, } for _, tt := range tests { - t1.Run(tt.name, func(t1 *testing.T) { - t := &Task{ + t.Run(tt.name, func(t *testing.T) { + task := &Task{ Log: tt.fields.Log, PlanResources: tt.fields.PlanResources, Phase: tt.fields.Phase, Step: tt.fields.Step, } - if got := t.getStagePVs(); !reflect.DeepEqual(got, tt.want) { - t1.Errorf("getStagePVs() = %v, want %v", got, tt.want) + if got := task.getStagePVs(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getStagePVs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestTask_waitForDMVToComplete(t *testing.T) { + tests := []struct { + name string + step string + dvm *migapi.DirectVolumeMigration + initialConditions []migapi.Condition + expectedConditions []migapi.Condition + wantErr bool + }{ + { + name: "dvm uncompleted, no warnings", + dvm: &migapi.DirectVolumeMigration{}, + initialConditions: []migapi.Condition{ + { + Type: DirectVolumeMigrationBlocked, + }, + }, + expectedConditions: []migapi.Condition{}, + wantErr: false, + }, + { + name: "dvm uncompleted, warnings", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + Conditions: migapi.Conditions{ + List: []migapi.Condition{ + { + Category: migapi.Warn, + Message: "warning", + }, + }, + }, + }, + }, + initialConditions: []migapi.Condition{ + { + Type: DirectVolumeMigrationBlocked, + }, + }, + expectedConditions: []migapi.Condition{ + { + Type: DirectVolumeMigrationBlocked, + Status: True, + Reason: migapi.NotReady, + Category: migapi.Warn, + Message: "warning", + }, + }, + wantErr: false, + }, + { + name: "dvm completed, no warnings", + step: "test", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + Phase: dvmc.Completed, + Itinerary: dvmc.VolumeMigrationItinerary, + Conditions: migapi.Conditions{ + List: []migapi.Condition{ + { + Type: dvmc.Succeeded, + Status: True, + }, + }, + }, + }, + }, + initialConditions: []migapi.Condition{}, + expectedConditions: []migapi.Condition{}, + wantErr: false, + }, + { + name: "dvm completed, invalid next step", + step: "invalid", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + Phase: dvmc.Completed, + Itinerary: dvmc.VolumeMigrationItinerary, + Conditions: migapi.Conditions{ + List: []migapi.Condition{ + { + Type: dvmc.Succeeded, + Status: True, + }, + }, + }, + }, + }, + initialConditions: []migapi.Condition{}, + expectedConditions: []migapi.Condition{}, + wantErr: true, + }, + { + name: "dvm completed, warnings", + step: "test", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Status: migapi.DirectVolumeMigrationStatus{ + Phase: dvmc.Completed, + Itinerary: dvmc.VolumeMigrationItinerary, + Conditions: migapi.Conditions{ + List: []migapi.Condition{ + { + Type: dvmc.Failed, + Reason: "test failure", + Status: True, + }, + }, + }, + }, + }, + initialConditions: []migapi.Condition{}, + expectedConditions: []migapi.Condition{ + { + Type: DirectVolumeMigrationFailed, + Status: True, + Category: migapi.Warn, + Message: "DirectVolumeMigration (dvm): test/test failed. See in dvm status.Errors", + Durable: true, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{ + Step: tt.step, + Owner: &migapi.MigMigration{ + Status: migapi.MigMigrationStatus{ + Pipeline: []*migapi.Step{ + { + Name: "test", + }, + }, + Conditions: migapi.Conditions{ + List: tt.initialConditions, + }, + }, + }, + } + err := task.waitForDVMToComplete(tt.dvm) + if (err != nil) != tt.wantErr { + t.Errorf("waitForDMVToComplete() error = %v, wantErr %v", err, tt.wantErr) + t.FailNow() + } + if len(task.Owner.Status.Conditions.List) != len(tt.expectedConditions) { + t.Errorf("waitForDMVToComplete() = %v, want %v", task.Owner.Status.Conditions.List, tt.expectedConditions) + t.FailNow() + } + for i, c := range task.Owner.Status.Conditions.List { + if c.Category != tt.expectedConditions[i].Category { + t.Errorf("category = %s, want %s", c.Category, tt.expectedConditions[i].Category) + } + if c.Type != tt.expectedConditions[i].Type { + t.Errorf("type = %s, want %s", c.Type, tt.expectedConditions[i].Type) + } + if c.Status != tt.expectedConditions[i].Status { + t.Errorf("status = %s, want %s", c.Status, tt.expectedConditions[i].Status) + } + if c.Reason != tt.expectedConditions[i].Reason { + t.Errorf("reason = %s, want %s", c.Reason, tt.expectedConditions[i].Reason) + } + if c.Message != tt.expectedConditions[i].Message { + t.Errorf("message = %s, want %s", c.Message, tt.expectedConditions[i].Message) + } } }) } diff --git a/pkg/controller/migplan/validation.go b/pkg/controller/migplan/validation.go index 7897fce497..0b8a48d412 100644 --- a/pkg/controller/migplan/validation.go +++ b/pkg/controller/migplan/validation.go @@ -6,11 +6,14 @@ import ( "fmt" "net" "path" + "slices" "sort" + "strconv" "strings" liberr "github.com/konveyor/controller/pkg/error" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/compat" "github.com/konveyor/mig-controller/pkg/controller/migcluster" "github.com/konveyor/mig-controller/pkg/health" "github.com/konveyor/mig-controller/pkg/pods" @@ -27,6 +30,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/util/exec" + virtv1 "kubevirt.io/api/core/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -88,6 +92,9 @@ const ( HookPhaseUnknown = "HookPhaseUnknown" HookPhaseDuplicate = "HookPhaseDuplicate" IntraClusterMigration = "IntraClusterMigration" + KubeVirtNotInstalledSourceCluster = "KubeVirtNotInstalledSourceCluster" + KubeVirtVersionNotSupported = "KubeVirtVersionNotSupported" + KubeVirtStorageLiveMigrationNotEnabled = "KubeVirtStorageLiveMigrationNotEnabled" ) // Categories @@ -115,6 +122,14 @@ const ( DuplicateNs = "DuplicateNamespaces" ConflictingNamespaces = "ConflictingNamespaces" ConflictingPermissions = "ConflictingPermissions" + NotSupported = "NotSupported" +) + +// Messages +const ( + KubeVirtNotInstalledSourceClusterMessage = "KubeVirt is not installed on the source cluster" + KubeVirtVersionNotSupportedMessage = "KubeVirt version does not support storage live migration, Virtual Machines will be stopped instead" + KubeVirtStorageLiveMigrationNotEnabledMessage = "KubeVirt storage live migration is not enabled, Virtual Machines will be stopped instead" ) // Statuses @@ -130,6 +145,13 @@ const ( openShiftUIDRangeAnnotation = "openshift.io/sa.scc.uid-range" ) +// Valid kubevirt feature gates +const ( + VolumesUpdateStrategy = "VolumesUpdateStrategy" + VolumeMigrationConfig = "VolumeMigration" + VMLiveUpdateFeatures = "VMLiveUpdateFeatures" +) + // Valid AccessMode values var validAccessModes = []kapi.PersistentVolumeAccessMode{kapi.ReadWriteOnce, kapi.ReadOnlyMany, kapi.ReadWriteMany} @@ -224,6 +246,13 @@ func (r ReconcileMigPlan) validate(ctx context.Context, plan *migapi.MigPlan) er if err != nil { return liberr.Wrap(err) } + + if plan.LiveMigrationChecked() { + // Live migration possible + if err := r.validateLiveMigrationPossible(ctx, plan); err != nil { + return liberr.Wrap(err) + } + } return nil } @@ -1523,7 +1552,7 @@ func (r ReconcileMigPlan) validatePodHealth(ctx context.Context, plan *migapi.Mi return nil } -func (r ReconcileMigPlan) validateHooks(ctx context.Context, plan *migapi.MigPlan) error { +func (r *ReconcileMigPlan) validateHooks(ctx context.Context, plan *migapi.MigPlan) error { if opentracing.SpanFromContext(ctx) != nil { span, _ := opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validateHooks") defer span.Finish() @@ -1626,6 +1655,127 @@ func (r ReconcileMigPlan) validateHooks(ctx context.Context, plan *migapi.MigPla return nil } +func (r *ReconcileMigPlan) validateLiveMigrationPossible(ctx context.Context, plan *migapi.MigPlan) error { + // Check if kubevirt is installed, if not installed, return nil + srcCluster, err := plan.GetSourceCluster(r) + if err != nil { + return liberr.Wrap(err) + } + if err := r.validateCluster(ctx, srcCluster, plan); err != nil { + return liberr.Wrap(err) + } + dstCluster, err := plan.GetDestinationCluster(r) + if err != nil { + return liberr.Wrap(err) + } + return r.validateCluster(ctx, dstCluster, plan) +} + +func (r *ReconcileMigPlan) validateCluster(ctx context.Context, cluster *migapi.MigCluster, plan *migapi.MigPlan) error { + if cluster == nil || !cluster.Status.IsReady() { + return nil + } + srcClient, err := cluster.GetClient(r) + if err != nil { + return liberr.Wrap(err) + } + if err := r.validateKubeVirtInstalled(ctx, srcClient, plan); err != nil { + return err + } + return nil +} + +func (r *ReconcileMigPlan) validateKubeVirtInstalled(ctx context.Context, client compat.Client, plan *migapi.MigPlan) error { + if !plan.LiveMigrationChecked() { + return nil + } + kubevirtList := &virtv1.KubeVirtList{} + if err := client.List(ctx, kubevirtList); err != nil { + return liberr.Wrap(err) + } + if len(kubevirtList.Items) == 0 || len(kubevirtList.Items) > 1 { + plan.Status.SetCondition(migapi.Condition{ + Type: KubeVirtNotInstalledSourceCluster, + Status: True, + Reason: NotFound, + Category: Advisory, + Message: KubeVirtNotInstalledSourceClusterMessage, + }) + return nil + } + kubevirt := kubevirtList.Items[0] + operatorVersion := kubevirt.Status.OperatorVersion + major, minor, bugfix, err := parseKubeVirtOperatorSemver(operatorVersion) + if err != nil { + plan.Status.SetCondition(migapi.Condition{ + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }) + return nil + } + log.V(3).Info("KubeVirt operator version", "major", major, "minor", minor, "bugfix", bugfix) + // Check if kubevirt operator version is at least 1.3.0 if live migration is enabled. + if major < 1 || (major == 1 && minor < 3) { + plan.Status.SetCondition(migapi.Condition{ + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }) + return nil + } + // Check if the appropriate feature gates are enabled + if kubevirt.Spec.Configuration.VMRolloutStrategy == nil || + *kubevirt.Spec.Configuration.VMRolloutStrategy != virtv1.VMRolloutStrategyLiveUpdate || + kubevirt.Spec.Configuration.DeveloperConfiguration == nil || + !slices.Contains(kubevirt.Spec.Configuration.DeveloperConfiguration.FeatureGates, VolumesUpdateStrategy) || + !slices.Contains(kubevirt.Spec.Configuration.DeveloperConfiguration.FeatureGates, VolumeMigrationConfig) || + !slices.Contains(kubevirt.Spec.Configuration.DeveloperConfiguration.FeatureGates, VMLiveUpdateFeatures) { + plan.Status.SetCondition(migapi.Condition{ + Type: KubeVirtStorageLiveMigrationNotEnabled, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtStorageLiveMigrationNotEnabledMessage, + }) + return nil + } + return nil +} + +func parseKubeVirtOperatorSemver(operatorVersion string) (int, int, int, error) { + // example versions: v1.1.1-106-g0be1a2073, or: v1.3.0-beta.0.202+f8efa57713ba76-dirty + tokens := strings.Split(operatorVersion, ".") + if len(tokens) < 3 { + return -1, -1, -1, liberr.Wrap(fmt.Errorf("version string was not in semver format, != 3 tokens")) + } + + if tokens[0][0] == 'v' { + tokens[0] = tokens[0][1:] + } + major, err := strconv.Atoi(tokens[0]) + if err != nil { + return -1, -1, -1, liberr.Wrap(fmt.Errorf("major version could not be parsed as integer")) + } + + minor, err := strconv.Atoi(tokens[1]) + if err != nil { + return -1, -1, -1, liberr.Wrap(fmt.Errorf("minor version could not be parsed as integer")) + } + + bugfixTokens := strings.Split(tokens[2], "-") + bugfix, err := strconv.Atoi(bugfixTokens[0]) + if err != nil { + return -1, -1, -1, liberr.Wrap(fmt.Errorf("bugfix version could not be parsed as integer")) + } + + return major, minor, bugfix, nil +} + func containsAccessMode(modeList []kapi.PersistentVolumeAccessMode, accessMode kapi.PersistentVolumeAccessMode) bool { for _, mode := range modeList { if mode == accessMode { diff --git a/pkg/controller/migplan/validation_test.go b/pkg/controller/migplan/validation_test.go index ea86f6f03c..091ebb4b51 100644 --- a/pkg/controller/migplan/validation_test.go +++ b/pkg/controller/migplan/validation_test.go @@ -11,43 +11,49 @@ import ( "github.com/opentracing/opentracing-go/mocktracer" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func TestReconcileMigPlan_validatePossibleMigrationTypes(t *testing.T) { - getFakeClientWithObjs := func(obj ...k8sclient.Object) compat.Client { - client, _ := fakecompat.NewFakeClient(obj...) - return client - } - getTestMigCluster := func(name string, url string) *migapi.MigCluster { - return &migapi.MigCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: migapi.OpenshiftMigrationNamespace, - }, - Spec: migapi.MigClusterSpec{ - URL: url, - }, - } +func getFakeClientWithObjs(obj ...k8sclient.Object) compat.Client { + client, _ := fakecompat.NewFakeClient(obj...) + return client +} + +func getTestMigCluster(name string, url string) *migapi.MigCluster { + return &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: migapi.OpenshiftMigrationNamespace, + }, + Spec: migapi.MigClusterSpec{ + URL: url, + }, } - getTestMigPlan := func(srcCluster string, destCluster string, ns []string, conds []migapi.Condition) *migapi.MigPlan { - return &migapi.MigPlan{ - ObjectMeta: metav1.ObjectMeta{ - Name: "migplan", - Namespace: migapi.OpenshiftMigrationNamespace, - }, - Spec: migapi.MigPlanSpec{ - SrcMigClusterRef: &v1.ObjectReference{Name: srcCluster, Namespace: migapi.OpenshiftMigrationNamespace}, - DestMigClusterRef: &v1.ObjectReference{Name: destCluster, Namespace: migapi.OpenshiftMigrationNamespace}, - Namespaces: ns, - }, - Status: migapi.MigPlanStatus{ - Conditions: migapi.Conditions{ - List: conds, - }, +} + +func getTestMigPlan(srcCluster string, destCluster string, ns []string, conds []migapi.Condition) *migapi.MigPlan { + return &migapi.MigPlan{ + ObjectMeta: metav1.ObjectMeta{ + Name: "migplan", + Namespace: migapi.OpenshiftMigrationNamespace, + }, + Spec: migapi.MigPlanSpec{ + SrcMigClusterRef: &v1.ObjectReference{Name: srcCluster, Namespace: migapi.OpenshiftMigrationNamespace}, + DestMigClusterRef: &v1.ObjectReference{Name: destCluster, Namespace: migapi.OpenshiftMigrationNamespace}, + Namespaces: ns, + LiveMigrate: ptr.To[bool](true), + }, + Status: migapi.MigPlanStatus{ + Conditions: migapi.Conditions{ + List: conds, }, - } + }, } +} + +func TestReconcileMigPlan_validatePossibleMigrationTypes(t *testing.T) { tests := []struct { name string client k8sclient.Client @@ -273,3 +279,346 @@ func TestReconcileMigPlan_validatePossibleMigrationTypes(t *testing.T) { }) } } + +func TestReconcileMigPlan_validateparseKubeVirtOperatorSemver(t *testing.T) { + tests := []struct { + name string + input string + wantErr bool + major int + minor int + bugfix int + }{ + { + name: "given a valid semver string, should not return an error", + input: "v0.0.0", + wantErr: false, + major: 0, + minor: 0, + bugfix: 0, + }, + { + name: "given a valid semver string with extra info, should not return an error", + input: "v1.2.3-rc1", + wantErr: false, + major: 1, + minor: 2, + bugfix: 3, + }, + { + name: "given a valid semver string with extra info, should not return an error", + input: "v1.2.3-rc1.debug.1", + wantErr: false, + major: 1, + minor: 2, + bugfix: 3, + }, + { + name: "given a semver string with two dots, should return an error", + input: "v0.0", + wantErr: true, + }, + { + name: "given a semver string without a v should not return an error", + input: "1.1.1", + wantErr: false, + major: 1, + minor: 1, + bugfix: 1, + }, + { + name: "given a semver with an invalid major version, should return an error", + input: "va.1.1", + wantErr: true, + }, + { + name: "given a semver with an invalid minor version, should return an error", + input: "v4.b.1", + wantErr: true, + }, + { + name: "given a semver with an invalid bugfix version, should return an error", + input: "v2.1.c", + wantErr: true, + }, + { + name: "given a semver with an invalid bugfix version with dash, should return an error", + input: "v2.1.-", + wantErr: true, + }, + { + name: "given a semver with a dot instead of a valid bugfix version, should return an error", + input: "v2.1.-", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + major, minor, patch, err := parseKubeVirtOperatorSemver(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("parseKubeVirtOperatorSemver() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err == nil { + if major != tt.major { + t.Errorf("parseKubeVirtOperatorSemver() major = %v, want %v", major, tt.major) + } + if minor != tt.minor { + t.Errorf("parseKubeVirtOperatorSemver() minor = %v, want %v", minor, tt.minor) + } + if patch != tt.bugfix { + t.Errorf("parseKubeVirtOperatorSemver() patch = %v, want %v", patch, tt.bugfix) + } + } + }) + } +} + +func TestReconcileMigPlan_validateKubeVirtInstalled(t *testing.T) { + plan := getTestMigPlan("test-cluster", "test-cluster", []string{ + "ns-00:ns-00", + "ns-01:ns-02", + }, []migapi.Condition{}) + noLiveMigratePlan := plan.DeepCopy() + noLiveMigratePlan.Spec.LiveMigrate = ptr.To[bool](false) + tests := []struct { + name string + client compat.Client + plan *migapi.MigPlan + wantErr bool + wantConditions []migapi.Condition + dontWantConditions []migapi.Condition + }{ + { + name: "given a cluster without kubevirt installed, should return a warning condition", + client: getFakeClientWithObjs(), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtNotInstalledSourceCluster, + Status: True, + Reason: NotFound, + Category: Advisory, + Message: KubeVirtNotInstalledSourceClusterMessage, + }, + }, + }, + { + name: "given a cluster with multiple kubevirt CRs, should return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + }, + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt-two", + Namespace: "openshift-cnv", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtNotInstalledSourceCluster, + Status: True, + Reason: NotFound, + Category: Advisory, + Message: KubeVirtNotInstalledSourceClusterMessage, + }, + }, + }, + { + name: "given a cluster with kubevirt installed, but invalid version, should return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "a.b.c", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }, + }, + }, + { + name: "given a cluster with kubevirt installed, but older version, should return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "0.4.3", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }, + }, + }, + { + name: "given a cluster with kubevirt installed, but plan has no live migration, should not return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "1.4.3", + }, + }, + ), + plan: noLiveMigratePlan, + wantErr: false, + dontWantConditions: []migapi.Condition{ + { + Type: KubeVirtStorageLiveMigrationNotEnabled, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtStorageLiveMigrationNotEnabledMessage, + }, + { + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }, + { + Type: KubeVirtNotInstalledSourceCluster, + Status: True, + Reason: NotFound, + Category: Advisory, + Message: KubeVirtNotInstalledSourceClusterMessage, + }, + }, + }, + { + name: "given a cluster with new enough kubevirt installed, should not return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Spec: virtv1.KubeVirtSpec{ + Configuration: virtv1.KubeVirtConfiguration{ + VMRolloutStrategy: ptr.To[virtv1.VMRolloutStrategy](virtv1.VMRolloutStrategyLiveUpdate), + DeveloperConfiguration: &virtv1.DeveloperConfiguration{ + FeatureGates: []string{ + VMLiveUpdateFeatures, + VolumeMigrationConfig, + VolumesUpdateStrategy, + }, + }, + }, + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "1.4.3", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + dontWantConditions: []migapi.Condition{ + { + Type: KubeVirtStorageLiveMigrationNotEnabled, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtStorageLiveMigrationNotEnabledMessage, + }, + }, + }, + { + name: "given a cluster with new enough kubevirt installed, but not all featuregates should a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Spec: virtv1.KubeVirtSpec{ + Configuration: virtv1.KubeVirtConfiguration{ + VMRolloutStrategy: ptr.To[virtv1.VMRolloutStrategy](virtv1.VMRolloutStrategyLiveUpdate), + DeveloperConfiguration: &virtv1.DeveloperConfiguration{ + FeatureGates: []string{ + VolumeMigrationConfig, + VolumesUpdateStrategy, + }, + }, + }, + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "1.4.3", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtStorageLiveMigrationNotEnabled, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtStorageLiveMigrationNotEnabledMessage, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := ReconcileMigPlan{ + Client: tt.client, + tracer: mocktracer.New(), + } + if err := r.validateKubeVirtInstalled(context.TODO(), tt.client, tt.plan); (err != nil) != tt.wantErr { + t.Errorf("ReconcileMigPlan.validateKubeVirtInstalled() error = %v, wantErr %v", err, tt.wantErr) + } + for _, wantCond := range tt.wantConditions { + foundCond := tt.plan.Status.FindCondition(wantCond.Type) + if foundCond == nil { + t.Errorf("ReconcileMigPlan.validatePossibleMigrationTypes() wantCondition = %s, found nil", wantCond.Type) + } + if foundCond != nil && foundCond.Reason != wantCond.Reason { + t.Errorf("ReconcileMigPlan.validatePossibleMigrationTypes() want reason = %s, found %s", wantCond.Reason, foundCond.Reason) + } + } + for _, dontWantCond := range tt.dontWantConditions { + foundCond := tt.plan.Status.FindCondition(dontWantCond.Type) + if foundCond != nil { + t.Errorf("ReconcileMigPlan.validatePossibleMigrationTypes() dontWantCondition = %s, found = %s", dontWantCond.Type, foundCond.Type) + } + } + }) + } +}