From 82098f9a3461715672fe1d7fe1c8037460bb0c6f Mon Sep 17 00:00:00 2001 From: Matthias Teich Date: Tue, 27 Aug 2024 10:18:42 +0200 Subject: [PATCH] Add e2e test --- test/e2e/capic_test.go | 25 + test/e2e/config/ionoscloud.yaml | 14 + .../cluster-template-ipam.yaml | 465 ++++++++++++++++++ .../data/shared/capi-ipam/v0.1/metadata.yaml | 12 + test/e2e/env_test.go | 10 +- test/e2e/suite_test.go | 3 + 6 files changed, 527 insertions(+), 2 deletions(-) create mode 100644 test/e2e/data/infrastructure-ionoscloud/cluster-template-ipam.yaml create mode 100644 test/e2e/data/shared/capi-ipam/v0.1/metadata.yaml diff --git a/test/e2e/capic_test.go b/test/e2e/capic_test.go index 1295cc11..6e84b759 100644 --- a/test/e2e/capic_test.go +++ b/test/e2e/capic_test.go @@ -22,6 +22,8 @@ package e2e import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "os" + clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster" capie2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" @@ -95,3 +97,26 @@ var _ = Describe("Should be able to create a cluster with 1 control-plane and 1 } }) }) + +var _ = Describe("Should be able to create a cluster with 1 control-plane using an IP from the IPAddressPool", func() { + capie2e.QuickStartSpec(ctx, func() capie2e.QuickStartSpecInput { + return capie2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ControlPlaneMachineCount: ptr.To(int64(1)), + WorkerMachineCount: ptr.To(int64(0)), + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + Flavor: ptr.To("ipam"), + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + PostNamespaceCreated: cloudEnv.createCredentialsSecretPNC, + PostMachinesProvisioned: func(managementClusterProxy framework.ClusterProxy, namespace, clusterName string) { + machines := &infrav1.IonosCloudMachineList{} + Expect(managementClusterProxy.GetClient().List(ctx, machines, runtimeclient.InNamespace(namespace))).NotTo(HaveOccurred()) + nic := machines.Items[0].Status.MachineNetworkInfo.NICInfo[0] + desired := os.Getenv("ADDITIONAL_IPS") + Expect(nic.IPv4Addresses).To(ContainElement(desired)) + }, + } + }) +}) diff --git a/test/e2e/config/ionoscloud.yaml b/test/e2e/config/ionoscloud.yaml index bbd49a2d..49208d8a 100644 --- a/test/e2e/config/ionoscloud.yaml +++ b/test/e2e/config/ionoscloud.yaml @@ -38,6 +38,19 @@ providers: new: --metrics-addr=:8443 files: - sourcePath: "../data/shared/v1.7/metadata.yaml" + - name: in-cluster + type: IPAMProvider + versions: + - name: "{go://sigs.k8s.io/cluster-api-ipam-provider-in-cluster@v0.1}" # supported release in the v1alpha2 series + # Use manifest from source files + value: "https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/releases/download/{go://sigs.k8s.io/cluster-api-ipam-provider-in-cluster@v0.1}/ipam-components.yaml" + type: url + contract: v1beta1 + files: + - sourcePath: "../data/shared/capi-ipam/v0.1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" - name: ionoscloud type: InfrastructureProvider versions: @@ -51,6 +64,7 @@ providers: files: - sourcePath: "../../../metadata.yaml" - sourcePath: "../data/infrastructure-ionoscloud/cluster-template.yaml" + - sourcePath: "../data/infrastructure-ionoscloud/cluster-template-ipam.yaml" variables: # Default variables for the e2e test; those values could be overridden via env variables, thus # allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation. diff --git a/test/e2e/data/infrastructure-ionoscloud/cluster-template-ipam.yaml b/test/e2e/data/infrastructure-ionoscloud/cluster-template-ipam.yaml new file mode 100644 index 00000000..7d1f28cf --- /dev/null +++ b/test/e2e/data/infrastructure-ionoscloud/cluster-template-ipam.yaml @@ -0,0 +1,465 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + cni: "${CLUSTER_NAME}-crs-0" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: IonosCloudCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + controlPlaneEndpoint: + host: ${CONTROL_PLANE_ENDPOINT_HOST:-${CONTROL_PLANE_ENDPOINT_IP}} + port: ${CONTROL_PLANE_ENDPOINT_PORT:-6443} + location: ${CONTROL_PLANE_ENDPOINT_LOCATION} + credentialsRef: + name: "ionoscloud-credentials" +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: IonosCloudMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + users: + - name: root + sshAuthorizedKeys: [${IONOSCLOUD_MACHINE_SSH_KEYS}] + ntp: + enabled: true + servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + - 3.de.pool.ntp.org + files: + - path: /etc/ssh/sshd_config.d/ssh-audit_hardening.conf + owner: root:root + permissions: '0644' + content: | + # Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com + # hardening guide. + KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + - path: /etc/sysctl.d/k8s.conf + content: | + fs.inotify.max_user_watches = 65536 + net.netfilter.nf_conntrack_max = 1000000 + - path: /etc/modules-load.d/k8s.conf + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + ip_vs_sed + # Crictl config + - path: /etc/crictl.yaml + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock + timeout: 10 + - path: /etc/kubernetes/manifests/kube-vip.yaml + owner: root:root + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: cp_enable + value: "true" + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE=""} + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: port + value: "${CONTROL_PLANE_ENDPOINT_PORT:-6443}" + - name: vip_arp + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + image: ghcr.io/kube-vip/kube-vip:v0.7.1 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + - localhost + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + type: FileOrCreate + name: kubeconfig + status: {} + - path: /etc/kube-vip-prepare.sh + content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + permissions: "0700" + + # CSI Metadata config + - content: | + { + "datacenter-id": "${IONOSCLOUD_DATACENTER_ID}" + } + owner: root:root + path: /etc/ie-csi/cfg.json + permissions: '0644' + + - content: | + #!/bin/bash + set -e + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + NODE_IPv4_ADDRESS=$(ip -j addr show dev ens6 | jq -r '.[].addr_info[] | select(.family == "inet") | select(.scope=="global") | select(.dynamic) | .local') + if [[ $NODE_IPv4_ADDRESS ]]; then + sed -i '$ s/$/ --node-ip '"$NODE_IPv4_ADDRESS"'/' /etc/default/kubelet + fi + # IPv6 currently not set, the ip is not set then this runs. Needs to be waited for. + NODE_IPv6_ADDRESS=$(ip -j addr show dev ens6 | jq -r '.[].addr_info[] | select(.family == "inet6") | select(.scope=="global") | .local') + if [[ $NODE_IPv6_ADDRESS ]]; then + sed -i '$ s/$/ --node-ip '"$NODE_IPv6_ADDRESS"'/' /etc/default/kubelet + fi + owner: root:root + path: /etc/set-node-ip.sh + permissions: '0700' + + preKubeadmCommands: + - systemctl restart systemd-networkd.service systemd-modules-load.service systemd-journald containerd + # disable swap + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab + - sysctl --system + - /etc/kube-vip-prepare.sh + # workaround 1.29 IP issue + - /etc/set-node-ip.sh + postKubeadmCommands: + - > + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + - > + systemctl disable --now udisks2 multipathd motd-news.timer fwupd-refresh.timer + packagekit ModemManager snapd snapd.socket snapd.apparmor snapd.seeded + # INFO(schegi-ionos): We decided to not remove this for now, since removing this would require the ccm to be installed for cluster-api + # to continue after the first node. + - export system_uuid=$(kubectl --kubeconfig /etc/kubernetes/kubelet.conf get node $(hostname) -ojsonpath='{..systemUUID }') + - > + kubectl --kubeconfig /etc/kubernetes/kubelet.conf + patch node $(hostname) + --type strategic -p '{"spec": {"providerID": "ionos://'$${system_uuid}'"}}' + - rm /etc/ssh/ssh_host_* + - ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N "" + - ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" + - sed -i 's/^\#HostKey \/etc\/ssh\/ssh_host_\(rsa\|ed25519\)_key$/HostKey \/etc\/ssh\/ssh_host_\1_key/g' /etc/ssh/sshd_config + - awk '$5 >= 3071' /etc/ssh/moduli > /etc/ssh/moduli.safe + - mv /etc/ssh/moduli.safe /etc/ssh/moduli + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - apt-get update + - DEBIAN_FRONTEND=noninteractive apt-get install -q -y netfilter-persistent iptables-persistent + - service netfilter-persistent save + - systemctl restart sshd + initConfiguration: + localAPIEndpoint: + bindPort: ${CONTROL_PLANE_ENDPOINT_PORT:-6443} + nodeRegistration: + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + joinConfiguration: + nodeRegistration: + criSocket: unix:///run/containerd/containerd.sock + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + version: "${KUBERNETES_VERSION}" +--- +kind: IonosCloudMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + ipv4PoolRef: + apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: ${CLUSTER_NAME} + datacenterID: ${IONOSCLOUD_DATACENTER_ID} + numCores: ${IONOSCLOUD_MACHINE_NUM_CORES:-4} + memoryMB: ${IONOSCLOUD_MACHINE_MEMORY_MB:-8192} + disk: + image: + id: ${IONOSCLOUD_MACHINE_IMAGE_ID} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-workers" + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + node-role.kubernetes.io/node: "" + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-worker" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-worker" + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: IonosCloudMachineTemplate +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + template: + spec: + ipv4PoolRef: + apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: ${CLUSTER_NAME} + datacenterID: ${IONOSCLOUD_DATACENTER_ID} + numCores: ${IONOSCLOUD_MACHINE_NUM_CORES:-2} + memoryMB: ${IONOSCLOUD_MACHINE_MEMORY_MB:-4096} + disk: + image: + id: ${IONOSCLOUD_MACHINE_IMAGE_ID} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + template: + spec: + users: + - name: root + sshAuthorizedKeys: [${IONOSCLOUD_MACHINE_SSH_KEYS}] + ntp: + enabled: true + servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + - 3.de.pool.ntp.org + files: + - path: /etc/ssh/sshd_config.d/ssh-audit_hardening.conf + owner: root:root + permissions: '0644' + content: | + # Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com + # hardening guide. + KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + - path: /etc/sysctl.d/k8s.conf + content: | + fs.inotify.max_user_watches = 65536 + net.netfilter.nf_conntrack_max = 1000000 + - path: /etc/modules-load.d/k8s.conf + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + ip_vs_sed + # Crictl config + - path: /etc/crictl.yaml + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock + timeout: 10 + # CSI Metadata config + - content: | + { + "datacenter-id": "${IONOSCLOUD_DATACENTER_ID}" + } + owner: root:root + path: /etc/ie-csi/cfg.json + permissions: '0644' + preKubeadmCommands: + - systemctl restart systemd-networkd.service systemd-modules-load.service systemd-journald containerd + # disable swap + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab + - sysctl --system + postKubeadmCommands: + - > + systemctl disable --now udisks2 multipathd motd-news.timer fwupd-refresh.timer + packagekit ModemManager snapd snapd.socket snapd.apparmor snapd.seeded + # INFO(schegi-ionos): We decided to not remove this for now, since removing this would require the ccm to be + # installed for cluster-api to continue after the first node. + - export system_uuid=$(kubectl --kubeconfig /etc/kubernetes/kubelet.conf get node $(hostname) -ojsonpath='{..systemUUID }') + - > + kubectl --kubeconfig /etc/kubernetes/kubelet.conf + patch node $(hostname) + --type strategic -p '{"spec": {"providerID": "ionos://'$${system_uuid}'"}}' + - rm /etc/ssh/ssh_host_* + - ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N "" + - ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" + - sed -i 's/^\#HostKey \/etc\/ssh\/ssh_host_\(rsa\|ed25519\)_key$/HostKey \/etc\/ssh\/ssh_host_\1_key/g' /etc/ssh/sshd_config + - awk '$5 >= 3071' /etc/ssh/moduli > /etc/ssh/moduli.safe + - mv /etc/ssh/moduli.safe /etc/ssh/moduli + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - apt-get update + - DEBIAN_FRONTEND=noninteractive apt-get install -q -y netfilter-persistent iptables-persistent + - service netfilter-persistent save + - systemctl restart sshd + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + criSocket: unix:///run/containerd/containerd.sock +--- +# ConfigMap object referenced by the ClusterResourceSet object and with +# the CNI resource defined in the test config file +apiVersion: v1 +kind: ConfigMap +metadata: + name: "cni-${CLUSTER_NAME}-crs-0" +data: ${CNI_RESOURCES} +--- +# ClusterResourceSet object with +# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0 +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + labels: + cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' + name: "${CLUSTER_NAME}-crs-0" +spec: + strategy: ApplyOnce + clusterSelector: + matchLabels: + cni: "${CLUSTER_NAME}-crs-0" + resources: + - name: "cni-${CLUSTER_NAME}-crs-0" + kind: ConfigMap +--- +apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +kind: InClusterIPPool +metadata: + name: ${CLUSTER_NAME} +spec: + prefix: ${IPAM_PREFIX:-24} + addresses: + - ${ADDITIONAL_IPS} diff --git a/test/e2e/data/shared/capi-ipam/v0.1/metadata.yaml b/test/e2e/data/shared/capi-ipam/v0.1/metadata.yaml new file mode 100644 index 00000000..56d61ac6 --- /dev/null +++ b/test/e2e/data/shared/capi-ipam/v0.1/metadata.yaml @@ -0,0 +1,12 @@ +# maps release series of major.minor to cluster-api contract version +# the contract version may change between minor or major versions, but *not* +# between patch versions. +# +# update this file only when a new major or minor version is released +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 0 + minor: 1 + contract: v1beta1 + diff --git a/test/e2e/env_test.go b/test/e2e/env_test.go index 3a5844c1..f6e4f105 100644 --- a/test/e2e/env_test.go +++ b/test/e2e/env_test.go @@ -23,6 +23,7 @@ import ( "context" "fmt" "os" + "strings" "github.com/google/uuid" corev1 "k8s.io/api/core/v1" @@ -64,7 +65,7 @@ func (e *ionosCloudEnv) setup() { dcRequest := e.createDatacenter(ctx, location) By("Requesting an IP block") - ipbRequest := e.reserveIPBlock(ctx, location, 1) + ipbRequest := e.reserveIPBlock(ctx, location, 2) By("Waiting for requests to complete") e.waitForCreationRequests(ctx, dcRequest, ipbRequest) @@ -134,7 +135,12 @@ func (e *ionosCloudEnv) reserveIPBlock(ctx context.Context, location string, siz if os.Getenv("CI") == "true" { e.writeToGithubOutput("IP_BLOCK_ID", *e.ipBlock.Id) } - Expect(os.Setenv("CONTROL_PLANE_ENDPOINT_IP", (*e.ipBlock.Properties.Ips)[0])).ToNot(HaveOccurred(), "Failed setting datacenter ID in environment variable") + + ips := (*e.ipBlock.Properties.Ips) + Expect(os.Setenv("CONTROL_PLANE_ENDPOINT_IP", ips[0])).ToNot(HaveOccurred(), "Failed setting control plane endpoint ID in environment variable") + if len(ips) > 1 { + Expect(os.Setenv("ADDITIONAL_IPS", strings.Join(ips[1:], ","))).ToNot(HaveOccurred(), "Failed setting additional IPs in environment variable") + } return res.Header.Get(apiLocationHeaderKey) } diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index a205b516..44ebc644 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" @@ -193,6 +194,7 @@ var _ = SynchronizedAfterSuite(func() { func initScheme() *runtime.Scheme { s := runtime.NewScheme() framework.TryAddDefaultSchemes(s) + Expect(ipamv1.AddToScheme(s)).To(Succeed()) Expect(infrav1.AddToScheme(s)).To(Succeed()) return s } @@ -248,6 +250,7 @@ func initBootstrapCluster() { clusterctl.InitManagementClusterAndWatchControllerLogs(watchesCtx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{ ClusterProxy: bootstrapClusterProxy, ClusterctlConfigPath: clusterctlConfigPath, + IPAMProviders: e2eConfig.IPAMProviders(), InfrastructureProviders: e2eConfig.InfrastructureProviders(), LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), }, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)