diff --git a/bootstrap/config/rbac/role.yaml b/bootstrap/config/rbac/role.yaml index 9f2fa3df..50e1603a 100644 --- a/bootstrap/config/rbac/role.yaml +++ b/bootstrap/config/rbac/role.yaml @@ -47,11 +47,33 @@ rules: resources: - clusters - clusters/status + verbs: + - get + - list + - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + - clusters/status + - machines + - machines/status + verbs: + - get + - list + - watch +- apiGroups: + - cluster.x-k8s.io + resources: - machines - machines/status verbs: + - create + - delete - get - list + - patch + - update - watch - apiGroups: - exp.cluster.x-k8s.io diff --git a/bootstrap/controllers/upgrade_controller.go b/bootstrap/controllers/upgrade_controller.go index a72d39ec..4daaa296 100644 --- a/bootstrap/controllers/upgrade_controller.go +++ b/bootstrap/controllers/upgrade_controller.go @@ -66,6 +66,7 @@ type UpgradeScope struct { // +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=ck8sconfigs,verbs=get;list;watch // +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=ck8sconfigs/status,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch + func (r *InPlaceUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("namespace", req.Namespace, "machine", req.Name) diff --git a/hack/build-e2e-images.sh b/hack/build-e2e-images.sh index 85f12de0..3541e7d2 100755 --- a/hack/build-e2e-images.sh +++ b/hack/build-e2e-images.sh @@ -9,6 +9,6 @@ DIR="$(realpath "$(dirname "${0}")")" cd "${DIR}/../templates/docker" -sudo docker build . -t k8s-snap:dev-old --build-arg BRANCH=main --build-arg KUBERNETES_VERSION=v1.29.6 +sudo docker build . -t k8s-snap:dev-old --build-arg BRANCH=main --build-arg KUBERNETES_VERSION=v1.29.6 --build-arg KUBERNETES_VERSION_UPGRADE_TO=v1.30.4 sudo docker build . -t k8s-snap:dev-new --build-arg BRANCH=main --build-arg KUBERNETES_VERSION=v1.30.4 cd - diff --git a/templates/docker/Dockerfile b/templates/docker/Dockerfile index b6a3b08f..7a6b3d28 100644 --- a/templates/docker/Dockerfile +++ b/templates/docker/Dockerfile @@ -37,6 +37,8 @@ ARG REPO=https://github.com/canonical/k8s-snap ARG BRANCH=main ARG KUBERNETES_VERSION="" +# This is used to build the kubernetes version to upgrade when testing in place upgrades +ARG KUBERNETES_VERSION_UPGRADE_TO="" ## NOTE(neoaggelos): install dependencies needed to build the tools ## !!!IMPORTANT!!! Keep up to date with "snapcraft.yaml:parts.build-deps.build-packages" @@ -95,6 +97,15 @@ RUN if [ -n "$KUBERNETES_VERSION" ]; then \ fi RUN /src/k8s-snap/build-scripts/build-component.sh kubernetes +## kubernetes upgrade version build +FROM builder AS build-kubernetes-upgrade-to +ENV KUBERNETES_VERSION_UPGRADE_TO=${KUBERNETES_VERSION_UPGRADE_TO} +RUN if [ -n "$KUBERNETES_VERSION_UPGRADE_TO" ]; then \ + echo "Overwriting Kubernetes version with $KUBERNETES_VERSION_UPGRADE_TO"; \ + echo "$KUBERNETES_VERSION_UPGRADE_TO" > /src/k8s-snap/build-scripts/components/kubernetes/version; \ + fi +RUN /src/k8s-snap/build-scripts/build-component.sh kubernetes + ## runc build FROM builder AS build-runc RUN /src/k8s-snap/build-scripts/build-component.sh runc @@ -141,6 +152,7 @@ COPY --from=build-helm /out /snap/k8s/current COPY --from=build-containerd /out /snap/k8s/current COPY --from=build-cni /out /snap/k8s/current COPY --from=build-kubernetes /out /snap/k8s/current +COPY --from=build-kubernetes-upgrade-to /out /k8s/upgrade COPY --from=build-k8sd /out /snap/k8s/current COPY --from=build-pebble /out /snap/k8s/current COPY --from=build-preload-images /out/images /var/snap/k8s/common/images diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index f0f1984d..80dc7052 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -25,23 +25,6 @@ import ( ) var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() { - // Skipping this test as in-place upgrades are not supported yet. - // TODO(ben): Remove this skip when in-place upgrades are supported. - //Context("Upgrading a cluster with 1 control plane", func() { - /* ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { - return ClusterUpgradeSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To("docker"), - ControlPlaneMachineCount: ptr.To[int64](1), - WorkerMachineCount: ptr.To[int64](2), - } - }) */ - //}) - Context("Upgrading a cluster with HA control plane", func() { ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { return ClusterUpgradeSpecInput{ diff --git a/test/e2e/common.go b/test/e2e/common.go index ced12240..111228e4 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -41,6 +41,7 @@ const ( CPMachineTemplateUpgradeTo = "CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO" WorkersMachineTemplateUpgradeTo = "WORKERS_MACHINE_TEMPLATE_UPGRADE_TO" IPFamily = "IP_FAMILY" + InPlaceUpgradeOption = "IN_PLACE_UPGRADE_OPTION" KindImageVersion = "KIND_IMAGE_VERSION" ) diff --git a/test/e2e/config/ck8s-docker.yaml b/test/e2e/config/ck8s-docker.yaml index 06134e12..8dba00e0 100644 --- a/test/e2e/config/ck8s-docker.yaml +++ b/test/e2e/config/ck8s-docker.yaml @@ -89,6 +89,7 @@ variables: KUBERNETES_VERSION_UPGRADE_TO: "v1.30.3" IP_FAMILY: "IPv4" KIND_IMAGE_VERSION: "v1.28.0" + IN_PLACE_UPGRADE_OPTION: "localPath=/k8s/upgrade/bin/kubernetes" intervals: # The array is defined as [timeout, polling interval] diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 8c64ad84..a2d6e95f 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -40,6 +40,7 @@ import ( "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" + bootstrapv1 "github.com/canonical/cluster-api-k8s/bootstrap/api/v1beta2" controlplanev1 "github.com/canonical/cluster-api-k8s/controlplane/api/v1beta2" ) @@ -552,6 +553,135 @@ func WaitForControlPlaneAndMachinesReady(ctx context.Context, input WaitForContr }) } +type ApplyInPlaceUpgradeAndWaitInput struct { + Getter framework.Getter + Machine *clusterv1.Machine + ClusterProxy framework.ClusterProxy + UpgradeOption string + WaitForUpgradeIntervals []interface{} +} + +func ApplyInPlaceUpgradeAndWait(ctx context.Context, input ApplyInPlaceUpgradeAndWaitInput) { + Expect(ctx).NotTo(BeNil()) + Expect(input.Machine).ToNot(BeNil()) + Expect(input.ClusterProxy).ToNot(BeNil()) + Expect(input.UpgradeOption).ToNot(BeEmpty()) + + mgmtClient := input.ClusterProxy.GetClient() + + patchHelper, err := patch.NewHelper(input.Machine, mgmtClient) + Expect(err).ToNot(HaveOccurred()) + mAnnotations := input.Machine.GetAnnotations() + + if mAnnotations == nil { + mAnnotations = map[string]string{} + } + + mAnnotations[bootstrapv1.InPlaceUpgradeToAnnotation] = input.UpgradeOption + input.Machine.SetAnnotations(mAnnotations) + err = patchHelper.Patch(ctx, input.Machine) + Expect(err).ToNot(HaveOccurred()) + + By("Checking for in-place upgrade status to be equal to done") + + Eventually(func() (bool, error) { + um := &clusterv1.Machine{} + if err := input.Getter.Get(ctx, client.ObjectKey{Namespace: input.Machine.Namespace, Name: input.Machine.Name}, um); err != nil { + Byf("Failed to get the machine: %+v", err) + return false, err + } + + mAnnotations := um.GetAnnotations() + + status, ok := mAnnotations[bootstrapv1.InPlaceUpgradeStatusAnnotation] + if !ok { + return false, nil + } + + return status == bootstrapv1.InPlaceUpgradeDoneStatus, nil + }, input.WaitForUpgradeIntervals...).Should(BeTrue(), "In-place upgrade failed for %s", input.Machine.Name) +} + +type ApplyInPlaceUpgradeForControlPlaneInput struct { + Lister framework.Lister + Getter framework.Getter + ClusterProxy framework.ClusterProxy + Cluster *clusterv1.Cluster + UpgradeOption string + WaitForUpgradeIntervals []interface{} +} + +func ApplyInPlaceUpgradeForControlPlane(ctx context.Context, input ApplyInPlaceUpgradeForControlPlaneInput) { + Expect(ctx).NotTo(BeNil()) + Expect(input.ClusterProxy).ToNot(BeNil()) + Expect(input.Cluster).ToNot(BeNil()) + Expect(input.UpgradeOption).ToNot(BeEmpty()) + + // Look up all the control plane machines. + inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace) + matchClusterListOption := client.MatchingLabels{ + clusterv1.ClusterNameLabel: input.Cluster.Name, + clusterv1.MachineControlPlaneLabel: "", + } + + machineList := &clusterv1.MachineList{} + Eventually(func() error { + return input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Couldn't list control-plane machines for the cluster %q", input.Cluster.Name) + + for _, machine := range machineList.Items { + ApplyInPlaceUpgradeAndWait(ctx, ApplyInPlaceUpgradeAndWaitInput{ + Getter: input.Getter, + Machine: &machine, + ClusterProxy: input.ClusterProxy, + UpgradeOption: input.UpgradeOption, + WaitForUpgradeIntervals: input.WaitForUpgradeIntervals, + }) + } +} + +type ApplyInPlaceUpgradeForWorkerInput struct { + Lister framework.Lister + Getter framework.Getter + ClusterProxy framework.ClusterProxy + Cluster *clusterv1.Cluster + MachineDeployments []*clusterv1.MachineDeployment + UpgradeOption string + WaitForUpgradeIntervals []interface{} +} + +func ApplyInPlaceUpgradeForWorker(ctx context.Context, input ApplyInPlaceUpgradeForWorkerInput) { + Expect(ctx).NotTo(BeNil()) + Expect(input.ClusterProxy).ToNot(BeNil()) + Expect(input.Cluster).ToNot(BeNil()) + Expect(input.MachineDeployments).ToNot(BeNil()) + Expect(input.UpgradeOption).ToNot(BeEmpty()) + + for _, md := range input.MachineDeployments { + // Look up all the control plane machines. + inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace) + matchClusterListOption := client.MatchingLabels{ + clusterv1.ClusterNameLabel: input.Cluster.Name, + clusterv1.MachineDeploymentNameLabel: md.Name, + } + + machineList := &clusterv1.MachineList{} + Eventually(func() error { + return input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Couldn't list control-plane machines for the cluster %q", input.Cluster.Name) + + for _, machine := range machineList.Items { + ApplyInPlaceUpgradeAndWait(ctx, ApplyInPlaceUpgradeAndWaitInput{ + Getter: input.Getter, + Machine: &machine, + ClusterProxy: input.ClusterProxy, + UpgradeOption: input.UpgradeOption, + WaitForUpgradeIntervals: input.WaitForUpgradeIntervals, + }) + } + } +} + // UpgradeControlPlaneAndWaitForUpgradeInput is the input type for UpgradeControlPlaneAndWaitForUpgrade. type UpgradeControlPlaneAndWaitForUpgradeInput struct { ClusterProxy framework.ClusterProxy diff --git a/test/e2e/in_place_upgrade_test.go b/test/e2e/in_place_upgrade_test.go new file mode 100644 index 00000000..3d63471f --- /dev/null +++ b/test/e2e/in_place_upgrade_test.go @@ -0,0 +1,122 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +var _ = Describe("In place upgrade", func() { + var ( + ctx = context.TODO() + specName = "workload-cluster-inplace" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + result *ApplyClusterTemplateAndWaitResult + clusterName string + clusterctlLogFolder string + infrastructureProvider string + ) + + BeforeEach(func() { + Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) + + clusterName = fmt.Sprintf("capick8s-in-place-%s", util.RandomString(6)) + infrastructureProvider = clusterctl.DefaultInfrastructureProvider + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + + result = new(ApplyClusterTemplateAndWaitResult) + + clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) + }) + + AfterEach(func() { + cleanInput := cleanupInput{ + SpecName: specName, + Cluster: result.Cluster, + ClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + CancelWatches: cancelWatches, + IntervalsGetter: e2eConfig.GetIntervals, + SkipCleanup: skipCleanup, + ArtifactFolder: artifactFolder, + } + + dumpSpecResourcesAndCleanup(ctx, cleanInput) + }) + + Context("Performing in-place upgrades", func() { + It("Creating a workload cluster and applying in-place upgrade to control-plane and worker machines [PR-Blocking]", func() { + By("Creating a workload cluster of 1 control plane and 1 worker node") + ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: infrastructureProvider, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: ptr.To(int64(1)), + WorkerMachineCount: ptr.To(int64(1)), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + + bootstrapProxyClient := bootstrapClusterProxy.GetClient() + + By("Applying in place upgrade with local path for control plane nodes") + ApplyInPlaceUpgradeForControlPlane(ctx, ApplyInPlaceUpgradeForControlPlaneInput{ + Lister: bootstrapProxyClient, + Getter: bootstrapProxyClient, + ClusterProxy: bootstrapClusterProxy, + Cluster: result.Cluster, + UpgradeOption: e2eConfig.GetVariable(InPlaceUpgradeOption), + WaitForUpgradeIntervals: e2eConfig.GetIntervals(specName, "wait-machine-upgrade"), + }) + + By("Applying in place upgrade with local path for worker nodes") + ApplyInPlaceUpgradeForWorker(ctx, ApplyInPlaceUpgradeForWorkerInput{ + Lister: bootstrapProxyClient, + Getter: bootstrapProxyClient, + ClusterProxy: bootstrapClusterProxy, + Cluster: result.Cluster, + WaitForUpgradeIntervals: e2eConfig.GetIntervals(specName, "wait-machine-upgrade"), + UpgradeOption: e2eConfig.GetVariable(InPlaceUpgradeOption), + MachineDeployments: result.MachineDeployments, + }) + }) + }) + +})