From 233a807a72248443de536dcc1103784ff366e14b Mon Sep 17 00:00:00 2001 From: Philipp Born Date: Thu, 5 Dec 2024 10:22:40 +0100 Subject: [PATCH 1/2] samples: add proxmox Signed-off-by: Philipp Born --- samples/proxmox/cluster-template-k3s.yaml | 289 ++++++++++++++++++++++ samples/proxmox/setup.sh | 83 +++++++ 2 files changed, 372 insertions(+) create mode 100644 samples/proxmox/cluster-template-k3s.yaml create mode 100644 samples/proxmox/setup.sh diff --git a/samples/proxmox/cluster-template-k3s.yaml b/samples/proxmox/cluster-template-k3s.yaml new file mode 100644 index 00000000..5643c437 --- /dev/null +++ b/samples/proxmox/cluster-template-k3s.yaml @@ -0,0 +1,289 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.42.0.0/16 + services: + cidrBlocks: + - 10.43.0.0/16 + serviceDomain: cluster.local + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: KThreesControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: ProxmoxCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + controlPlaneEndpoint: + host: ${CONTROL_PLANE_ENDPOINT_IP} + port: 6443 + ipv4Config: + addresses: ${NODE_IP_RANGES} + prefix: ${IP_PREFIX} + gateway: ${GATEWAY} + dnsServers: ${DNS_SERVERS} + allowedNodes: ${ALLOWED_NODES:=[]} +--- +kind: ProxmoxMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + sourceNode: "${PROXMOX_SOURCENODE}" + templateID: ${PROXMOX_TEMPLATE_VMID} + format: "qcow2" + full: true + numSockets: ${NUM_SOCKETS:=1} + numCores: ${NUM_CORES:=2} + memoryMiB: ${MEMORY_MIB:=2048} + disks: + bootVolume: + disk: ${BOOT_VOLUME_DEVICE:=scsi0} + sizeGb: ${BOOT_VOLUME_SIZE:=32} + network: + default: + bridge: ${BRIDGE} + model: virtio + vmIDRange: + start: 2000 + end: 3000 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: ProxmoxMachineTemplate +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + template: + spec: + sourceNode: "${PROXMOX_SOURCENODE}" + templateID: ${PROXMOX_TEMPLATE_VMID} + format: "qcow2" + full: true + numSockets: ${NUM_SOCKETS:=1} + numCores: ${NUM_CORES:=1} + memoryMiB: ${MEMORY_MIB:=2048} + disks: + bootVolume: + disk: ${BOOT_VOLUME_DEVICE:=scsi0} + sizeGb: ${BOOT_VOLUME_SIZE:=32} + network: + default: + bridge: ${BRIDGE} + model: virtio + vmIDRange: + start: 2000 + end: 3000 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KThreesControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxMachineTemplate + name: "${CLUSTER_NAME}-control-plane" + kthreesConfigSpec: + serverConfig: + # cloudProviderName: "external" + disableCloudController: false + disableComponents: ${K3S_DISABLE_COMPONENTS:=[]} + agentConfig: + nodeName: "{{ ds.meta_data.local_hostname }}" + kubeletArgs: + - "provider-id=proxmox://{{ ds.meta_data.instance_id }}" + files: + - path: /var/lib/rancher/k3s/server/manifests/kube-vip.yaml + owner: root:root + content: | + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-vip + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role + rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:kube-vip-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role + subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.8.7 + name: kube-vip-ds + namespace: kube-system + spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.8.7 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: "" + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: port + value: ${CONTROL_PLANE_ENDPOINT_PORT="6443"} + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: prometheus_server + value: :2112 + - name: enableUPNP + value: "false" + image: ghcr.io/kube-vip/kube-vip:v0.8.7 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} + preK3sCommands: + - mkdir -p /root/.ssh + - chmod 700 /root/.ssh + - echo "${VM_SSH_KEYS}" > /root/.ssh/authorized_keys + - chmod 600 /root/.ssh/authorized_keys + replicas: ${CONTROL_PLANE_MACHINE_COUNT=1} + version: "${KUBERNETES_VERSION}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KThreesConfigTemplate +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + template: + spec: + preK3sCommands: + - mkdir -p /root/.ssh + - chmod 700 /root/.ssh + - echo "${VM_SSH_KEYS}" > /root/.ssh/authorized_keys + - chmod 600 /root/.ssh/authorized_keys +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT=1} + selector: + matchLabels: {} + template: + metadata: + labels: + node-role.kubernetes.io/node: "" + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KThreesConfigTemplate + name: "${CLUSTER_NAME}-worker" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxMachineTemplate + name: "${CLUSTER_NAME}-worker" diff --git a/samples/proxmox/setup.sh b/samples/proxmox/setup.sh new file mode 100644 index 00000000..157a5260 --- /dev/null +++ b/samples/proxmox/setup.sh @@ -0,0 +1,83 @@ +## Configure your Proxmox parameters + +if [ -z "${CLUSTER_NAME}" ]; then + echo "Please set CLUSTER_NAME" + exit 0 +fi + +if [ -z "${KUBERNETES_VERSION}" ]; then + echo "Please set KUBERNETES_VERSION. For ex. v1.31.2+k3s1" + exit 0 +fi + +if [ -z "${CONTROL_PLANE_ENDPOINT_IP}" ]; then + echo "Please set CONTROL_PLANE_ENDPOINT_IP. For ex. '10.10.10.4'" + exit 0 +fi + +if [ -z "${NODE_IP_RANGES}" ] || [ -z "${GATEWAY}" ] || [ -z "${IP_PREFIX}" ] || [ -z "${DNS_SERVERS}" ] || [ -z "${BRIDGE}" ]; then + echo "Please set NODE_IP_RANGES. For ex. '[10.10.10.5-10.10.10.50]'" + echo "Please set GATEWAY. For ex. '10.10.10.1'" + echo "Please set IP_PREFIX. For ex. '24'" + echo "Please set DNS_SERVERS. For ex. '[8.8.8.8,8.8.4.4]'" + echo "Please set BRIDGE. For ex. 'vmbr0'" + exit 0 +fi + +if [ -z "${PROXMOX_URL}" ] || [ -z "${PROXMOX_TOKEN}" ] || [ -z "${PROXMOX_SECRET}" ] || [ -z "${PROXMOX_SOURCENODE}" ] || [ -z "${PROXMOX_TEMPLATE_VMID}" ]; then + echo "Please set PROXMOX_URL, PROXMOX_TOKEN, PROXMOX_SECRET, PROXMOX_SOURCENODE, PROXMOX_TEMPLATE_VMID" + echo "- See https://github.com/ionos-cloud/cluster-api-provider-proxmox/blob/main/docs/Usage.md" + exit 0 +fi + +# The device used for the boot disk. +export BOOT_VOLUME_DEVICE="scsi0" +# The size of the boot disk in GB. +export BOOT_VOLUME_SIZE="32" +# The number of sockets for the VMs. +export NUM_SOCKETS="1" +# The number of cores for the VMs. +export NUM_CORES="1" +# The memory size for the VMs. +export MEMORY_MIB="4069" + +# K3s components to disable +# For example because you plan to use MetalLB over ServiceLB, or Longhorn over local-storage, or... +# export K3S_DISABLE_COMPONENTS="[servicelb,local-storage,traefik,metrics-server,helm-controller]" + +## Install your cluser-api-k3s provider correctly +mkdir -p ~/.cluster-api +cat samples/clusterctl.yaml | envsubst > ~/.cluster-api/clusterctl.yaml + +cat >> ~/.cluster-api/clusterctl.yaml < k3s.yaml' to retrieve your kubeconfig" +echo "- Run 'kubectl scale kthreescontrolplane $CLUSTER_NAME-control-plane --replicas 3' to enable HA for your control-planes" +echo "- or run 'kubectl scale machinedeployment $CLUSTER_NAME-worker --replicas 3' to deploy worker nodes" +echo "- or to just use the single node cluster, you might need to also run the following commands:" +echo " kubectl taint nodes --all node-role.kubernetes.io/control-plane-" +echo " kubectl taint nodes --all node-role.kubernetes.io/master-" From 6f33d5bc2e369bb19af680137aa0fd7d8d76c31a Mon Sep 17 00:00:00 2001 From: Philipp Born Date: Wed, 12 Feb 2025 11:15:06 +0100 Subject: [PATCH 2/2] add README with template import instructions; remove vmIDRange Signed-off-by: Philipp Born --- samples/proxmox/README.md | 26 +++++++++++++++++++++++ samples/proxmox/cluster-template-k3s.yaml | 6 ------ 2 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 samples/proxmox/README.md diff --git a/samples/proxmox/README.md b/samples/proxmox/README.md new file mode 100644 index 00000000..bd792a3a --- /dev/null +++ b/samples/proxmox/README.md @@ -0,0 +1,26 @@ +# importing a vm template into proxmox + +A standard KVM optimized Ubuntu 22.04 image can be imported via + +``` +TEMPLATE_URL=https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-disk-kvm.img +TEMPLATE_VMID=10000 +TEMPLATE_NAME=ubuntu-22.04 +TEMPLATE_STORAGE=tank +TEMPLATE_DISK_OPTIONS="discard=on,iothread=1,ssd=1" + +curl -o template.img ${TEMPLATE_URL} + +qm create "${TEMPLATE_VMID}" --name ${TEMPLATE_NAME} --memory 16 +qm importdisk "${TEMPLATE_VMID}" template.img "${TEMPLATE_STORAGE}" +qm set "${TEMPLATE_VMID}" \ + --scsihw virtio-scsi-single \ + --scsi0 ${TEMPLATE_STORAGE}:vm-${TEMPLATE_VMID}-disk-0,${TEMPLATE_DISK_OPTIONS} \ + --boot order=scsi0 \ + --cpu host \ + --rng0 source=/dev/urandom \ + --template 1 \ + --agent 1 \ + --onboot 1 +``` +Then for Ubuntu to work properly, you have to extend the `preK3sCommands` of both `KThreesConfigTemplate` and `KThreesControlPlane` with `apt update && apt -y install qemu-guest-agent && systemctl enable --now qemu-guest-agent` diff --git a/samples/proxmox/cluster-template-k3s.yaml b/samples/proxmox/cluster-template-k3s.yaml index 5643c437..85c7947c 100644 --- a/samples/proxmox/cluster-template-k3s.yaml +++ b/samples/proxmox/cluster-template-k3s.yaml @@ -58,9 +58,6 @@ spec: default: bridge: ${BRIDGE} model: virtio - vmIDRange: - start: 2000 - end: 3000 --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: ProxmoxMachineTemplate @@ -84,9 +81,6 @@ spec: default: bridge: ${BRIDGE} model: virtio - vmIDRange: - start: 2000 - end: 3000 --- apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: KThreesControlPlane