From 7fae14766ba500cb9c0d8d377f0f0a7821eeaaf8 Mon Sep 17 00:00:00 2001 From: Jan Jansen Date: Fri, 21 Jun 2024 15:28:23 +0200 Subject: [PATCH] :sparkles: Add preparation of clusterclass (#141) **What is the purpose of this pull request/Why do we need it?** **Issue #, if available:** **Description of changes:** **Special notes for your reviewer:** **Checklist:** - [x] Documentation updated - [ ] Unit Tests added - [ ] E2E Tests added - [x] Includes [emojis](https://github.com/kubernetes-sigs/kubebuilder-release-tools?tab=readme-ov-file#kubebuilder-project-versioning) --------- Signed-off-by: Jan Jansen --- .gitignore | 6 + .../ionoscloudclustertemplate_types.go | 54 ++ api/v1alpha1/zz_generated.deepcopy.go | 90 +++ ...r.x-k8s.io_ionoscloudclustertemplates.yaml | 112 +++ config/crd/kustomization.yaml | 1 + envfile.example | 9 +- hack/test-cluster-class.sh | 39 ++ .../cluster-template-topology-calico.yaml | 70 ++ templates/clusterclass-template.yaml | 647 ++++++++++++++++++ 9 files changed, 1027 insertions(+), 1 deletion(-) create mode 100644 api/v1alpha1/ionoscloudclustertemplate_types.go create mode 100644 config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudclustertemplates.yaml create mode 100755 hack/test-cluster-class.sh create mode 100644 templates/cluster-template-topology-calico.yaml create mode 100644 templates/clusterclass-template.yaml diff --git a/.gitignore b/.gitignore index 8e991497..db4731f5 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,9 @@ _artifacts/ # crs templates/crs/cni/calico.yaml + +#testing stuff +clusterctl-settings.json +templates/clusterclass-template-replaced.yaml +templates/cluster-template-topology-replaced.yaml +output/ diff --git a/api/v1alpha1/ionoscloudclustertemplate_types.go b/api/v1alpha1/ionoscloudclustertemplate_types.go new file mode 100644 index 00000000..cbce26a8 --- /dev/null +++ b/api/v1alpha1/ionoscloudclustertemplate_types.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 IONOS Cloud. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IonosCloudClusterTemplateSpec defines the desired state of IonosCloudClusterTemplate. +type IonosCloudClusterTemplateSpec struct { + Template IonosCloudClusterTemplateResource `json:"template"` +} + +// IonosCloudClusterTemplateResource describes the data for creating a IonosCloudCluster from a template. +type IonosCloudClusterTemplateResource struct { + Spec IonosCloudClusterSpec `json:"spec"` +} + +//+kubebuilder:object:root=true + +// IonosCloudClusterTemplate is the Schema for the ionoscloudclustertemplates API. +type IonosCloudClusterTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec IonosCloudClusterTemplateSpec `json:"spec,omitempty"` +} + +//+kubebuilder:object:root=true + +// IonosCloudClusterTemplateList contains a list of IonosCloudClusterTemplate. +type IonosCloudClusterTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []IonosCloudCluster `json:"items"` +} + +func init() { + objectTypes = append(objectTypes, &IonosCloudClusterTemplate{}, &IonosCloudClusterTemplateList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c7240835..a78b38e3 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -177,6 +177,96 @@ func (in *IonosCloudClusterStatus) DeepCopy() *IonosCloudClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IonosCloudClusterTemplate) DeepCopyInto(out *IonosCloudClusterTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IonosCloudClusterTemplate. +func (in *IonosCloudClusterTemplate) DeepCopy() *IonosCloudClusterTemplate { + if in == nil { + return nil + } + out := new(IonosCloudClusterTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IonosCloudClusterTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IonosCloudClusterTemplateList) DeepCopyInto(out *IonosCloudClusterTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IonosCloudCluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IonosCloudClusterTemplateList. +func (in *IonosCloudClusterTemplateList) DeepCopy() *IonosCloudClusterTemplateList { + if in == nil { + return nil + } + out := new(IonosCloudClusterTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IonosCloudClusterTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IonosCloudClusterTemplateResource) DeepCopyInto(out *IonosCloudClusterTemplateResource) { + *out = *in + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IonosCloudClusterTemplateResource. +func (in *IonosCloudClusterTemplateResource) DeepCopy() *IonosCloudClusterTemplateResource { + if in == nil { + return nil + } + out := new(IonosCloudClusterTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IonosCloudClusterTemplateSpec) DeepCopyInto(out *IonosCloudClusterTemplateSpec) { + *out = *in + out.Template = in.Template +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IonosCloudClusterTemplateSpec. +func (in *IonosCloudClusterTemplateSpec) DeepCopy() *IonosCloudClusterTemplateSpec { + if in == nil { + return nil + } + out := new(IonosCloudClusterTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IonosCloudMachine) DeepCopyInto(out *IonosCloudMachine) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudclustertemplates.yaml new file mode 100644 index 00000000..542cc7b0 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudclustertemplates.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: ionoscloudclustertemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + kind: IonosCloudClusterTemplate + listKind: IonosCloudClusterTemplateList + plural: ionoscloudclustertemplates + singular: ionoscloudclustertemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: IonosCloudClusterTemplate is the Schema for the ionoscloudclustertemplates + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: IonosCloudClusterTemplateSpec defines the desired state of + IonosCloudClusterTemplate. + properties: + template: + description: IonosCloudClusterTemplateResource describes the data + for creating a IonosCloudCluster from a template. + properties: + spec: + description: IonosCloudClusterSpec defines the desired state of + IonosCloudCluster. + properties: + controlPlaneEndpoint: + description: |- + ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + + + TODO(gfariasalves): as of now, IP must be provided by the user as we still don't insert the + provider-provided block IP into the kube-vip manifest. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + x-kubernetes-validations: + - message: control plane endpoint host cannot be updated + rule: self.host == oldSelf.host || oldSelf.host == '' + - message: control plane endpoint port cannot be updated + rule: self.port == oldSelf.port || oldSelf.port == 0 + credentialsRef: + description: CredentialsRef is a reference to the secret containing + the credentials to access the IONOS Cloud API. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: credentialsRef.name must be provided + rule: has(self.name) && self.name != '' + location: + description: Location is the location where the data centers + should be located. + example: de/txl + minLength: 1 + type: string + x-kubernetes-validations: + - message: location is immutable + rule: self == oldSelf + required: + - credentialsRef + - location + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index b4b34cab..59e9227f 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/infrastructure.cluster.x-k8s.io_ionoscloudclusters.yaml +- bases/infrastructure.cluster.x-k8s.io_ionoscloudclustertemplates.yaml - bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml - bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/envfile.example b/envfile.example index a5c89eaa..99dd2924 100644 --- a/envfile.example +++ b/envfile.example @@ -4,12 +4,16 @@ export IONOS_TOKEN="token" export IONOS_API_URL="https://api.ionos.com/cloudapi/v6" +# ClusterClass and Cluster API related environment variables +export NAMESPACE="default" # Cluster API related environment variables export CONTROL_PLANE_ENDPOINT_HOST="example.org" export CONTROL_PLANE_ENDPOINT_IP="192.168.0.1" export CONTROL_PLANE_ENDPOINT_PORT=6443 export CONTROL_PLANE_ENDPOINT_LOCATION="de/txl" +export CONTROL_PLANE_MACHINE_COUNT=3 +export WORKER_MACHINE_COUNT=1 export CLUSTER_NAME="cluster-name" export KUBERNETES_VERSION="1.29.2" @@ -17,5 +21,8 @@ export KUBERNETES_VERSION="1.29.2" export IONOSCLOUD_DATACENTER_ID="12345" export IONOSCLOUD_MACHINE_NUM_CORES=2 export IONOSCLOUD_MACHINE_MEMORY_MB=4096 -export IONOSCLOUD_MACHINE_IMAGE_ID=123456 +export IONOSCLOUD_MACHINE_IMAGE_ID="123456" export IONOSCLOUD_MACHINE_SSH_KEYS="ssh-ed25519 ..., ssh-ed25519 ..." + +# ClusterClass related environment variables +export CLUSTER_CLASS_NAME="testclass" diff --git a/hack/test-cluster-class.sh b/hack/test-cluster-class.sh new file mode 100755 index 00000000..f4b9a2cc --- /dev/null +++ b/hack/test-cluster-class.sh @@ -0,0 +1,39 @@ +#/bin/bash + +ROOT_DIR=$(cd $(realpath $(dirname $0))/.. && pwd) + +if [[ ! -d "${ROOT_DIR}/output" ]]; then + echo "create output directory" + mkdir "${ROOT_DIR}/output" +fi + +if [[ ! -f "${ROOT_DIR}/.envfile" ]]; then + echo "create .envfile from envfile.example" + exit 1 +fi +source ${ROOT_DIR}/.envfile + +echo "Setup Cluster API with ClusterResourceSet and ClusterTopology" +echo "export EXP_CLUSTER_RESOURCE_SET=\"true\"" +echo "export CLUSTER_TOPOLOGY=\"true\"" +echo "clusterctl init --infrastructure=ionoscloud-ionoscloud" + +GENERATED_CLUSTER_CLASS_FILE="${ROOT_DIR}/output/generated-clusterclass-template.yaml" +GENERATED_CLUSTER_FILE="${ROOT_DIR}/output/generated-cluster-template-topology-calico.yaml" + +clusterctl generate yaml --from "${ROOT_DIR}/templates/clusterclass-template.yaml" > $GENERATED_CLUSTER_CLASS_FILE +clusterctl generate cluster $CLUSTER_NAME -n $NAMESPACE --from "${ROOT_DIR}/templates/cluster-template-topology-calico.yaml" > $GENERATED_CLUSTER_FILE + +echo "Validate generated resources" +clusterctl alpha topology plan -f $GENERATED_CLUSTER_CLASS_FILE -f $GENERATED_CLUSTER_FILE -o "${ROOT_DIR}/output/" + +echo "Apply cluster class resources" +echo "kubectl apply -f $GENERATED_CLUSTER_CLASS_FILE" +echo "Apply cluster which uses the cluster class resources" +echo "kubectl apply -f $GENERATED_CLUSTER_FILE" + +echo "Installing calico cni" +echo "make crs-calico" +echo "kubectl apply -f ${ROOT_DIR}/templates/crs/cni/calico.yaml" + +echo "Done" diff --git a/templates/cluster-template-topology-calico.yaml b/templates/cluster-template-topology-calico.yaml new file mode 100644 index 00000000..87bdeb7b --- /dev/null +++ b/templates/cluster-template-topology-calico.yaml @@ -0,0 +1,70 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' + name: '${CLUSTER_NAME}' + namespace: '${NAMESPACE}' +spec: + topology: + class: '${CLUSTER_CLASS_NAME}' + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: sshKeys + value: '${IONOSCLOUD_MACHINE_SSH_KEYS}' + - name: controlPlaneIpAddr + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: controlPlanePort + value: ${CONTROL_PLANE_ENDPOINT_PORT:-6443} + - name: credsSecretName + value: '${CLUSTER_NAME}-credentials' + - name: datacenterID + value: '${IONOSCLOUD_DATACENTER_ID}' + - name: location + value: '${CONTROL_PLANE_ENDPOINT_LOCATION}' + - name: controlPlaneNumCores + value: ${IONOSCLOUD_MACHINE_NUM_CORES} + - name: controlPlaneMemoryMB + value: ${IONOSCLOUD_MACHINE_MEMORY_MB} + - name: controlPlaneImageID + value: '${IONOSCLOUD_MACHINE_IMAGE_ID}' + - name: workerNumCores + value: ${IONOSCLOUD_MACHINE_NUM_CORES} + - name: workerMemoryMB + value: ${IONOSCLOUD_MACHINE_MEMORY_MB} + - name: workerImageID + value: '${IONOSCLOUD_MACHINE_IMAGE_ID}' + version: '${KUBERNETES_VERSION}' + workers: + machineDeployments: + - class: worker + metadata: {} + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} +--- +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: '${NAMESPACE}' +type: Opaque +stringData: + token: "${IONOS_TOKEN}" + apiURL: "${IONOS_API_URL:-https://api.ionos.com/cloudapi/v6}" +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + labels: + cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' + name: ${CLUSTER_NAME}-crs-0 + namespace: '${NAMESPACE}' +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' + resources: + - kind: ConfigMap + name: calico diff --git a/templates/clusterclass-template.yaml b/templates/clusterclass-template.yaml new file mode 100644 index 00000000..2518262d --- /dev/null +++ b/templates/clusterclass-template.yaml @@ -0,0 +1,647 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: IonosCloudClusterTemplate +metadata: + name: '${CLUSTER_CLASS_NAME}' + namespace: '${NAMESPACE}' +spec: + template: + spec: + credentialsRef: + name: "place-holder" + location: "place-holder" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: '${CLUSTER_CLASS_NAME}' + namespace: '${NAMESPACE}' +spec: + controlPlane: + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate + name: ${CLUSTER_CLASS_NAME}-template + namespace: '${NAMESPACE}' + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: ${CLUSTER_CLASS_NAME}-controlplane + namespace: '${NAMESPACE}' + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudClusterTemplate + name: '${CLUSTER_CLASS_NAME}' + namespace: '${NAMESPACE}' + patches: + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/users + valueFrom: + template: | + - name: root + sshAuthorizedKeys: [{{ .sshKeys }}] + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/users + valueFrom: + template: | + - name: root + sshAuthorizedKeys: [{{ .sshKeys }}] + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - worker + enabledIf: '{{ if .sshKeys }}true{{end}}' + name: enableSSHIntoNodes + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + # CSI Metadata config + content: | + { + "datacenter-id": "{{ .datacenterID }}" + } + owner: root:root + path: /etc/ie-csi/cfg.json + permissions: '0644' + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/files/- + valueFrom: + template: | + # CSI Metadata config + content: | + { + "datacenter-id": "{{ .datacenterID }}" + } + owner: root:root + path: /etc/ie-csi/cfg.json + permissions: '0644' + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - worker + - jsonPatches: + - op: add + path: /spec/template/spec/datacenterID + valueFrom: + variable: datacenterID + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/datacenterID + valueFrom: + variable: datacenterID + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate + matchResources: + machineDeploymentClass: + names: + - worker + name: datacenterIdSubstitutionsOnKubeadm + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/initConfiguration/localAPIEndpoint/bindPort + valueFrom: + variable: controlPlanePort + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/controlPlaneEndpoint + valueFrom: + template: | + host: '{{ .controlPlaneHost | default .controlPlaneIpAddr }}' + port: {{ .controlPlanePort }} + - op: add + path: /spec/template/spec/credentialsRef/name + valueFrom: + variable: credsSecretName + - op: add + path: /spec/template/spec/location + valueFrom: + variable: location + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudClusterTemplate + matchResources: + infrastructureCluster: true + name: infraClusterSubstitutions + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/numCores + valueFrom: + variable: workerNumCores + - op: add + path: /spec/template/spec/memoryMB + valueFrom: + variable: workerMemoryMB + - op: add + path: /spec/template/spec/disk/image/id + valueFrom: + variable: workerImageID + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate + matchResources: + machineDeploymentClass: + names: + - worker + name: workerMachineType + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/numCores + valueFrom: + variable: controlPlaneNumCores + - op: add + path: /spec/template/spec/memoryMB + valueFrom: + variable: controlPlaneMemoryMB + - op: add + path: /spec/template/spec/disk/image/id + valueFrom: + variable: controlPlaneImageID + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate + matchResources: + controlPlane: true + name: controlPlaneMachineType + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: |- + owner: "root:root" + path: "/etc/kubernetes/manifests/kube-vip.yaml" + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: cp_enable + value: "true" + - name: vip_interface + value: "" + - name: address + value: "{{ .controlPlaneIpAddr }}" + - name: port + value: "{{ .controlPlanePort }}" + - name: vip_arp + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + image: ghcr.io/kube-vip/kube-vip:v0.7.1 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + - localhost + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + type: FileOrCreate + name: kubeconfig + status: {} + permissions: "0644" + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + name: kubeVipPodManifest + variables: + - name: sshKeys + required: false + schema: + openAPIV3Schema: + description: Public key to SSH onto the cluster nodes. + type: string + - name: datacenterID + required: true + schema: + openAPIV3Schema: + description: datacenterID for IONOS + type: string + - name: controlPlaneIpAddr + required: true + schema: + openAPIV3Schema: + description: Floating VIP for the control plane. + type: string + - name: controlPlaneHost + required: false + schema: + openAPIV3Schema: + description: FQDN for the control plane. + type: string + - name: controlPlanePort + required: true + schema: + openAPIV3Schema: + description: Port for the control plane endpoint. + type: integer + default: 6443 + - name: controlPlaneNumCores + required: true + schema: + openAPIV3Schema: + description: NumCores for the control plane machines. + type: integer + - name: controlPlaneMemoryMB + required: true + schema: + openAPIV3Schema: + description: MemoryMB for the control plane machines. + type: integer + - name: controlPlaneImageID + required: true + schema: + openAPIV3Schema: + description: ImageID for the control plane machines. + type: string + - name: workerNumCores + required: true + schema: + openAPIV3Schema: + description: NumCores for the worker machines. + type: integer + - name: workerMemoryMB + required: true + schema: + openAPIV3Schema: + description: MemoryMB for the worker machines. + type: integer + - name: workerImageID + required: true + schema: + openAPIV3Schema: + description: ImageID for the worker machines. + type: string + - name: credsSecretName + required: true + schema: + openAPIV3Schema: + description: Secret containing the credentials for the infra cluster. + type: string + - name: location + required: true + schema: + openAPIV3Schema: + description: cluster location on IONOS + type: string + workers: + machineDeployments: + - class: worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_CLASS_NAME}-worker + namespace: '${NAMESPACE}' + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate + name: ${CLUSTER_CLASS_NAME}-worker-machinetemplate + namespace: '${NAMESPACE}' + metadata: {} +--- +kind: KubeadmControlPlaneTemplate +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_CLASS_NAME}-controlplane" + namespace: '${NAMESPACE}' +spec: + template: + spec: + kubeadmConfigSpec: + ntp: + enabled: true + servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + - 3.de.pool.ntp.org + files: + - path: /etc/ssh/sshd_config.d/ssh-audit_hardening.conf + owner: root:root + permissions: '0644' + content: | + # Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com + # hardening guide. + KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + - path: /etc/sysctl.d/k8s.conf + content: | + fs.inotify.max_user_watches = 65536 + net.netfilter.nf_conntrack_max = 1000000 + - path: /etc/modules-load.d/k8s.conf + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + ip_vs_sed + # Crictl config + - path: /etc/crictl.yaml + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock + timeout: 10 + - content: | + #!/bin/bash + set -e + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + NODE_IPv4_ADDRESS=$(ip -j addr show dev ens6 | jq -r '.[].addr_info[] | select(.family == "inet") | select(.scope=="global") | select(.dynamic) | .local') + if [[ $NODE_IPv4_ADDRESS ]]; then + sed -i '$ s/$/ --node-ip '"$NODE_IPv4_ADDRESS"'/' /etc/default/kubelet + fi + # IPv6 currently not set, the ip is not set then this runs. Needs to be waited for. + NODE_IPv6_ADDRESS=$(ip -j addr show dev ens6 | jq -r '.[].addr_info[] | select(.family == "inet6") | select(.scope=="global") | .local') + if [[ $NODE_IPv6_ADDRESS ]]; then + sed -i '$ s/$/ --node-ip '"$NODE_IPv6_ADDRESS"'/' /etc/default/kubelet + fi + owner: root:root + path: /etc/set-node-ip.sh + permissions: '0700' + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh + permissions: "0700" + preKubeadmCommands: + - systemctl restart systemd-networkd.service systemd-modules-load.service systemd-journald containerd + # disable swap + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab + - sysctl --system + - /etc/kube-vip-prepare.sh + # workaround 1.29 IP issue + - /etc/set-node-ip.sh + postKubeadmCommands: + - > + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + - > + systemctl disable --now udisks2 multipathd motd-news.timer fwupd-refresh.timer + packagekit ModemManager snapd snapd.socket snapd.apparmor snapd.seeded + # TODO(jriedel-ionos): remove that if we have a CCM + - export system_uuid=$(kubectl --kubeconfig /etc/kubernetes/kubelet.conf get node $(hostname) -ojsonpath='{..systemUUID }') + - > + kubectl --kubeconfig /etc/kubernetes/kubelet.conf + patch node $(hostname) + --type strategic -p '{"spec": {"providerID": "ionos://'$${system_uuid}'"}}' + - rm /etc/ssh/ssh_host_* + - ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N "" + - ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" + - sed -i 's/^\#HostKey \/etc\/ssh\/ssh_host_\(rsa\|ed25519\)_key$/HostKey \/etc\/ssh\/ssh_host_\1_key/g' /etc/ssh/sshd_config + - awk '$5 >= 3071' /etc/ssh/moduli > /etc/ssh/moduli.safe + - mv /etc/ssh/moduli.safe /etc/ssh/moduli + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - apt-get update + - DEBIAN_FRONTEND=noninteractive apt-get install -q -y netfilter-persistent iptables-persistent + - service netfilter-persistent save + - systemctl restart sshd + initConfiguration: + localAPIEndpoint: {} + nodeRegistration: + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + joinConfiguration: + nodeRegistration: + criSocket: unix:///run/containerd/containerd.sock + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" +--- +kind: IonosCloudMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +metadata: + name: "${CLUSTER_CLASS_NAME}-template" + namespace: '${NAMESPACE}' +spec: + template: + spec: + numCores: 2 + memoryMB: 2048 + datacenterID: "00000000-0000-0000-0000-000000000000" + disk: + image: + id: "placeholder" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: IonosCloudMachineTemplate +metadata: + name: "${CLUSTER_CLASS_NAME}-worker-machinetemplate" + namespace: '${NAMESPACE}' +spec: + template: + spec: + numCores: 2 + memoryMB: 2048 + datacenterID: "00000000-0000-0000-0000-000000000000" + disk: + image: + id: "placeholder" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_CLASS_NAME}-worker" + namespace: '${NAMESPACE}' +spec: + template: + spec: + ntp: + enabled: true + servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + - 3.de.pool.ntp.org + files: + - path: /etc/ssh/sshd_config.d/ssh-audit_hardening.conf + owner: root:root + permissions: '0644' + content: | + # Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com + # hardening guide. + KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + - path: /etc/sysctl.d/k8s.conf + content: | + fs.inotify.max_user_watches = 65536 + net.netfilter.nf_conntrack_max = 1000000 + - path: /etc/modules-load.d/k8s.conf + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + ip_vs_sed + # Crictl config + - path: /etc/crictl.yaml + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock + timeout: 10 + preKubeadmCommands: + - systemctl restart systemd-networkd.service systemd-modules-load.service systemd-journald containerd + # disable swap + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab + - sysctl --system + postKubeadmCommands: + - > + systemctl disable --now udisks2 multipathd motd-news.timer fwupd-refresh.timer + packagekit ModemManager snapd snapd.socket snapd.apparmor snapd.seeded + # INFO(schegi-ionos): We decided to not remove this for now, since removing this would require the ccm to be + # installed for cluster-api to continue after the first node. + - export system_uuid=$(kubectl --kubeconfig /etc/kubernetes/kubelet.conf get node $(hostname) -ojsonpath='{..systemUUID }') + - > + kubectl --kubeconfig /etc/kubernetes/kubelet.conf + patch node $(hostname) + --type strategic -p '{"spec": {"providerID": "ionos://'$${system_uuid}'"}}' + - rm /etc/ssh/ssh_host_* + - ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N "" + - ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" + - sed -i 's/^\#HostKey \/etc\/ssh\/ssh_host_\(rsa\|ed25519\)_key$/HostKey \/etc\/ssh\/ssh_host_\1_key/g' /etc/ssh/sshd_config + - awk '$5 >= 3071' /etc/ssh/moduli > /etc/ssh/moduli.safe + - mv /etc/ssh/moduli.safe /etc/ssh/moduli + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - apt-get update + - DEBIAN_FRONTEND=noninteractive apt-get install -q -y netfilter-persistent iptables-persistent + - service netfilter-persistent save + - systemctl restart sshd + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + criSocket: unix:///run/containerd/containerd.sock