Add rollout upgrade integration test

.github/workflows/e2e.yaml

@@ -11,6 +11,14 @@ jobs:
     name: Build & Run E2E Images
     runs-on: [self-hosted, linux, X64, jammy, large]
     steps:
+      -
+        name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          # We run into rate limiting issues if we don't authenticate
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Check out repo
         uses: actions/checkout@v4
       - name: Install requirements
@@ -22,29 +30,32 @@ jobs:
           sudo snap install kubectl --classic --channel=1.30/stable
       - name: Build provider images
         run: sudo make docker-build-e2e
-      - name: Build k8s-snap image
+      - name: Build k8s-snap images
+        working-directory: hack/
         run: |
-          cd templates/docker
-          sudo docker build . -t k8s-snap:dev
+          ./build-e2e-images.sh
       - name: Save provider image
         run: |
           sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev
           sudo chmod 775 provider-images.tar
       - name: Save k8s-snap image
         run: |
-          sudo docker save -o k8s-snap-image.tar k8s-snap:dev
-          sudo chmod 775 k8s-snap-image.tar
+          sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old
+          sudo docker save -o k8s-snap-image-new.tar k8s-snap:dev-new
+          sudo chmod 775 k8s-snap-image-old.tar
+          sudo chmod 775 k8s-snap-image-new.tar
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: e2e-images
           path: |
             provider-images.tar
-            k8s-snap-image.tar
+            k8s-snap-image-old.tar
+            k8s-snap-image-new.tar
 
   run-e2e-tests:
     name: Run E2E Tests
-    runs-on: [self-hosted, linux, X64, jammy, large]
+    runs-on: [self-hosted, linux, X64, jammy, xlarge]
     needs: build-e2e-images
     strategy:
       matrix:
@@ -54,7 +65,17 @@ jobs:
           - "Workload cluster creation"
           - "Workload cluster scaling"
           - "Workload cluster upgrade"
+      # TODO(ben): Remove once all tests are running stable.
+      fail-fast: false
     steps:
+      -
+        name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          # We run into rate limiting issues if we don't authenticate
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Check out repo
         uses: actions/checkout@v4
       - name: Install requirements
@@ -71,8 +92,13 @@ jobs:
           path: .
       - name: Load provider image
         run: sudo docker load -i provider-images.tar
-      - name: Load k8s-snap image
-        run: sudo docker load -i k8s-snap-image.tar
+      - name: Load k8s-snap old image
+        if: matrix.ginkgo_focus == 'Workload cluster upgrade'
+        run: |
+          sudo docker load -i k8s-snap-image-old.tar
+      - name: Load k8s-snap new image
+        run: |
+          sudo docker load -i k8s-snap-image-new.tar
       - name: Create docker network
         run: |
           sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true

hack/build-e2e-images.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Description:
+#   Build k8s-snap docker images required for e2e tests.
+#
+# Usage:
+#   ./build-e2e-images.sh
+
+DIR="$(realpath "$(dirname "${0}")")"
+
+cd "${DIR}/../templates/docker"
+sudo docker build . -t k8s-snap:dev-old --build-arg BRANCH=main --build-arg KUBERNETES_VERSION=v1.29.6
+sudo docker build . -t k8s-snap:dev-new --build-arg BRANCH=main
+cd -

pkg/ck8s/workload_cluster.go

@@ -290,7 +290,7 @@ func (w *Workload) doK8sdRequest(ctx context.Context, method, endpoint string, r
 		return fmt.Errorf("k8sd request failed: %s", responseBody.Error)
 	}
 	if responseBody.Metadata == nil || response == nil {
-		// Nothing to decode
+		// No response expected.
 		return nil
 	}
 	if err := json.Unmarshal(responseBody.Metadata, response); err != nil {

templates/docker/Dockerfile

@@ -36,6 +36,8 @@ FROM $BUILD_BASE AS builder
 ARG REPO=https://github.com/canonical/k8s-snap
 ARG BRANCH=main
 
+ARG KUBERNETES_VERSION=""
+
 ## NOTE(neoaggelos): install dependencies needed to build the tools
 ## !!!IMPORTANT!!! Keep up to date with "snapcraft.yaml:parts.build-deps.build-packages"
 RUN apt-get update \
@@ -86,7 +88,12 @@ RUN /src/k8s-snap/build-scripts/build-component.sh helm
 
 ## kubernetes build
 FROM builder AS build-kubernetes
-RUN /src/k8s-snap/build-scripts/build-component.sh kubernetes
+ENV KUBERNETES_VERSION=${KUBERNETES_VERSION}
+RUN if [ -n "$KUBERNETES_VERSION" ]; then \
+      echo "Overwriting Kubernetes version with $KUBERNETES_VERSION"; \
+      echo "$KUBERNETES_VERSION" > /src/k8s-snap/build-scripts/components/kubernetes/version; \
+    fi
+RUN  /src/k8s-snap/build-scripts/build-component.sh kubernetes
 
 ## runc build
 FROM builder AS build-runc
@@ -156,3 +163,7 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/k8s/
 
 ## NOTE(neoaggelos): Required for containerd to properly set up overlayfs for pods
 VOLUME ["/var/snap/k8s/common/var/lib/containerd"]
+
+## NOTE(ben): Remove existing kind image kubectl and kubelet binaries
+# to avoid version confusion.
+RUN rm -f /usr/bin/kubectl /usr/bin/kubelet

test/e2e/cluster_upgrade.go

@@ -139,7 +139,7 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp
 				ClusterctlConfigPath:     input.ClusterctlConfigPath,
 				KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
 				InfrastructureProvider:   *input.InfrastructureProvider,
-				Flavor:                   ptr.Deref(input.Flavor, ""),
+				Flavor:                   ptr.Deref(input.Flavor, "upgrades"),
 				Namespace:                namespace.Name,
 				ClusterName:              clusterName,
 				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
@@ -158,6 +158,7 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp
 			Cluster:                     result.Cluster,
 			ControlPlane:                result.ControlPlane,
 			KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+			UpgradeMachineTemplate:      ptr.To(fmt.Sprintf("%s-control-plane-old", clusterName)),
 			WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
 		})
 
@@ -167,6 +168,7 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp
 			Cluster:                     result.Cluster,
 			UpgradeVersion:              input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
 			MachineDeployments:          result.MachineDeployments,
+			UpgradeMachineTemplate:      ptr.To(fmt.Sprintf("%s-md-1.30-0", clusterName)),
 			WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
 		})
 

test/e2e/cluster_upgrade_test.go

@@ -25,13 +25,10 @@ import (
 )
 
 var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() {
-	BeforeEach(func() {
-		// TODO(bschimke): Remove once we find a way to run e2e tests with other infrastructure providers that support snap.
-		Skip("Skipping the upgrade tests as snap does not work on CAPD.")
-	})
-
-	Context("Upgrading a cluster with 1 control plane", func() {
-		ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput {
+	// Skipping this test as in-place upgrades are not supported yet.
+	// TODO(ben): Remove this skip when in-place upgrades are supported.
+	//Context("Upgrading a cluster with 1 control plane", func() {
+	/* 			ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput {
 			return ClusterUpgradeSpecInput{
 				E2EConfig:                e2eConfig,
 				ClusterctlConfigPath:     clusterctlConfigPath,
@@ -42,8 +39,8 @@ var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() {
 				ControlPlaneMachineCount: ptr.To[int64](1),
 				WorkerMachineCount:       ptr.To[int64](2),
 			}
-		})
-	})
+	}) */
+	//})
 
 	Context("Upgrading a cluster with HA control plane", func() {
 		ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput {

test/e2e/config/ck8s-docker.yaml

@@ -51,9 +51,10 @@ providers:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
     files:
-      - sourcePath: "../data/infrastructure-docker/cluster-template.yaml"
       - sourcePath: "../data/infrastructure-docker/cluster-template-kcp-remediation.yaml"
       - sourcePath: "../data/infrastructure-docker/cluster-template-md-remediation.yaml"
+      - sourcePath: "../data/infrastructure-docker/cluster-template-upgrades.yaml"
+      - sourcePath: "../data/infrastructure-docker/cluster-template.yaml"
   - name: ck8s
     type: BootstrapProvider
     versions:
@@ -84,8 +85,8 @@ providers:
 
 variables:
   KUBERNETES_VERSION_MANAGEMENT: "v1.28.0"
-  KUBERNETES_VERSION: "v1.30.0"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.30.1"
+  KUBERNETES_VERSION: "v1.29.6"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.30.3"
   IP_FAMILY: "IPv4"
   KIND_IMAGE_VERSION: "v1.28.0"
 

test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml

@@ -84,7 +84,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: k8s-snap:dev
+      customImage: k8s-snap:dev-new
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -127,7 +127,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: k8s-snap:dev
+      customImage: k8s-snap:dev-new
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate

test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml

@@ -57,7 +57,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: k8s-snap:dev
+      customImage: k8s-snap:dev-old
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -101,7 +101,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: k8s-snap:dev
+      customImage: k8s-snap:dev-old
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate

test/e2e/data/infrastructure-docker/cluster-template-upgrades.yaml

@@ -0,0 +1,135 @@
+# TODO: copied and modified from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca
+# should be the same as samples/docker/quickstart.yaml in the future
+# for testing the quickstart scenario
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 10.1.0.0/16
+    services:
+      cidrBlocks:
+      - 10.152.0.0/16
+    serviceDomain: cluster.local
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta2
+    kind: CK8sControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec: {}
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
+kind: CK8sControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+  namespace: ${NAMESPACE}
+spec:
+  machineTemplate:
+    infrastructureTemplate:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane-old
+  spec:
+    airGapped: true
+    controlPlane:
+      extraKubeAPIServerArgs:
+        --anonymous-auth: "true"
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  version: ${KUBERNETES_VERSION}
+# Initial template for the machine deployment
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane-old
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      customImage: k8s-snap:dev-old
+# After upgrade template for the machine deployment
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane-new
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      customImage: k8s-snap:dev-new
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: worker-md-0
+  namespace: ${NAMESPACE}
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+
+      # This label will be needed for upgrade test
+      # it will be used as a selector for only selecting
+      # machines belonging to this machine deployment
+      cluster.x-k8s.io/deployment-name: worker-md-0
+  template:
+    metadata:
+      labels:
+        cluster.x-k8s.io/deployment-name: worker-md-0
+    spec:
+      version: ${KUBERNETES_VERSION}
+      clusterName: ${CLUSTER_NAME}
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
+          kind: CK8sConfigTemplate
+          name: ${CLUSTER_NAME}-md-0
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: DockerMachineTemplate
+        name: ${CLUSTER_NAME}-md-old-0
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-old-0
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      customImage: k8s-snap:dev-old
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-new-0
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      customImage: k8s-snap:dev-new
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
+kind: CK8sConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      airGapped: true