diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml new file mode 100644 index 00000000..fc932d77 --- /dev/null +++ b/templates/vsphere/cluster-template.yaml @@ -0,0 +1,1540 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-vsphere-proxy-env +stringData: + proxy-env: | + HTTP_PROXY=${HTTP_PROXY} + HTTPS_PROXY=${HTTPS_PROXY} + NO_PROXY=${NO_PROXY} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-vsphere-proxy-config +stringData: + proxy-config: | + ${PROXY_DISABLE}[Service] + ${PROXY_DISABLE}EnvironmentFile=/etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.env +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-kube-vip-rbac +stringData: + kube-vip-rbac: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-vip + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role + rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:kube-vip-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role + subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-kube-vip-manifest +stringData: + kube-vip-manifest: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + selector: + matchLabels: + name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + name: kube-vip-ds + spec: + serviceAccountName: kube-vip + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: "${VIP_NETWORK_INTERFACE}" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: vip_address + value: "${CONTROL_PLANE_ENDPOINT_IP}" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/hosts + name: etchosts + hostNetwork: true + volumes: + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} + status: {} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.1.0.0/16 + services: + cidrBlocks: + - 10.152.183.0/24 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereCluster +metadata: + name: ${CLUSTER_NAME} +spec: + controlPlaneEndpoint: + host: ${CONTROL_PLANE_ENDPOINT_IP} + port: 6443 + identityRef: + kind: Secret + name: ${CLUSTER_NAME} + server: ${VSPHERE_SERVER} + thumbprint: ${VSPHERE_TLS_THUMBPRINT} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: ${CLUSTER_NAME} +spec: + template: + spec: + cloneMode: linkedClone + datacenter: ${VSPHERE_DATACENTER} + datastore: ${VSPHERE_DATASTORE} + diskGiB: ${CONTROL_PLANE_MACHINE_DISK_GB} + folder: ${VSPHERE_FOLDER} + memoryMiB: ${CONTROL_PLANE_MACHINE_MEMORY_MB} + network: + devices: + - dhcp4: true + networkName: ${VSPHERE_NETWORK} + numCPUs: ${CONTROL_PLANE_MACHINE_CPU_COUNT} + os: Linux + powerOffMode: trySoft + resourcePool: '${VSPHERE_RESOURCE_POOL}' + server: ${VSPHERE_SERVER} + storagePolicyName: "" + template: ${VSPHERE_TEMPLATE} + thumbprint: ${VSPHERE_TLS_THUMBPRINT} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: ${CLUSTER_NAME}-worker +spec: + template: + spec: + cloneMode: linkedClone + datacenter: ${VSPHERE_DATACENTER} + datastore: ${VSPHERE_DATASTORE} + diskGiB: ${WORKER_MACHINE_DISK_GB} + folder: ${VSPHERE_FOLDER} + memoryMiB: ${WORKER_MACHINE_MEMORY_MB} + network: + devices: + - dhcp4: true + networkName: ${VSPHERE_NETWORK} + numCPUs: ${WORKER_MACHINE_CPU_COUNT} + os: Linux + powerOffMode: trySoft + resourcePool: '${VSPHERE_RESOURCE_POOL}' + server: ${VSPHERE_SERVER} + storagePolicyName: "" + template: ${VSPHERE_TEMPLATE} + thumbprint: ${VSPHERE_TLS_THUMBPRINT} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME} +spec: + spec: + controlPlane: + cloudProvider: external + files: + - content: | + ${VSPHERE_SSH_AUTHORIZED_KEY} + owner: root:root + path: /root/.ssh/authorized_keys + permissions: "600" + - contentFrom: + secret: + key: proxy-env + name: ${CLUSTER_NAME}-vsphere-proxy-env + owner: root:root + path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.env + permissions: "0644" + - contentFrom: + secret: + key: proxy-config + name: ${CLUSTER_NAME}-vsphere-proxy-config + owner: root:root + path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf + permissions: "0644" + - contentFrom: + secret: + key: kube-vip-rbac + name: ${CLUSTER_NAME}-kube-vip-rbac + owner: root:root + path: /capi/manifests/kube-vip-rbac.yaml + permissions: "0644" + - contentFrom: + secret: + key: kube-vip-manifest + name: ${CLUSTER_NAME}-kube-vip-manifest + owner: root:root + path: /capi/manifests/kube-vip.yaml + permissions: "0644" + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + preRunCommands: + - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost + localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereMachineTemplate + name: ${CLUSTER_NAME} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + controlPlane: + cloudProvider: external + preRunCommands: + - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost + localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + files: + - content: | + ${VSPHERE_SSH_AUTHORIZED_KEY} + owner: root:root + path: /root/.ssh/authorized_keys + permissions: "600" + - contentFrom: + secret: + key: proxy-config + name: ${CLUSTER_NAME}-vsphere-proxy-config + owner: root:root + path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf + permissions: "0644" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: {} + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereMachineTemplate + name: ${CLUSTER_NAME}-worker + version: ${KUBERNETES_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME}-crs-0 +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: Secret + name: vsphere-config-secret + - kind: ConfigMap + name: csi-manifests + - kind: Secret + name: cloud-provider-vsphere-credentials + - kind: ConfigMap + name: cpi-manifests +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME} +stringData: + password: ${VSPHERE_PASSWORD} + username: ${VSPHERE_USERNAME} +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-config-secret +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + name: vsphere-config-secret + namespace: vmware-system-csi + stringData: + csi-vsphere.conf: |+ + [Global] + thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + insecure-flag = "true" # todo - set to false causes in csi-controller pod "error thumbprint does not match". + # Investigate: https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-BFF39F1D-F70A-4360-ABC9-85BDAFBE8864.html + # and https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-BFF39F1D-F70A-4360-ABC9-85BDAFBE8864.html#GUID-D1F411D8-FE2E-466D-B85B-EEC847385E61 + + [VirtualCenter "${VSPHERE_SERVER}"] + user = "${VSPHERE_USERNAME}" + password = "${VSPHERE_PASSWORD}" + datacenters = "${VSPHERE_DATACENTER}" + + [Network] + public-network = "${VSPHERE_NETWORK}" + type: Opaque +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + --- + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + name: csi.vsphere.vmware.com + spec: + attachRequired: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-controller + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-controller-role + rules: + - apiGroups: + - "" + resources: + - nodes + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - patch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: + - create + - get + - update + - watch + - list + - apiGroups: + - cns.vmware.com + resources: + - cnsvspherevolumemigrations + verbs: + - create + - get + - list + - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get + - list + - watch + - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create + - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: + - create + - get + - list + - update + - delete + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-controller-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-controller-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-controller + namespace: vmware-system-csi + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + replicas: 1 + selector: + matchLabels: + app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: vsphere-csi-controller + role: vsphere-csi + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname + containers: + - args: + + - --logtostderr + - -v=6 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 + env: + - name: ADDRESS + value: /csi/csi.sock + image: ${CSI_ATTACHER_IMAGE} + name: csi-attacher + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: ${CSI_RESIZER_IMAGE} + name: csi-resizer + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: X_CSI_MODE + value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m + - name: VSPHERE_CSI_CONFIG + value: /etc/cloud/csi-vsphere.conf + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ${CSI_DRIVER_IMAGE} + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 + name: vsphere-csi-controller + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - mountPath: /csi + name: socket-dir + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: ${CSI_LIVENESS_PROBE_IMAGE} + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: FULL_SYNC_INTERVAL_MINUTES + value: "30" + - name: VSPHERE_CSI_CONFIG + value: /etc/cloud/csi-vsphere.conf + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ${CSI_SYNCER_IMAGE} + imagePullPolicy: Always + name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - args: + - --v=4 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /csi/csi.sock + image: ${CSI_PROVISIONER_IMAGE} + name: csi-provisioner + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: ${CSI_SNAPSHOTTER_IMAGE} + name: csi-snapshotter + volumeMounts: + - mountPath: /csi + name: socket-dir + dnsPolicy: Default + priorityClassName: system-cluster-critical + serviceAccountName: vsphere-csi-controller + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + #https://github.com/kubernetes-sigs/vsphere-csi-driver/issues/2284 + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + volumes: + - name: vsphere-config-volume + secret: + secretName: vsphere-config-secret + - emptyDir: {} + name: socket-dir + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi + spec: + selector: + matchLabels: + app: vsphere-csi-node + template: + metadata: + labels: + app: vsphere-csi-node + role: vsphere-csi + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + image: ${CSI_REGISTRAR_IMAGE} + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" + - name: X_CSI_MODE + value: node + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: PRODUCTION + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: ${CSI_DRIVER_IMAGE} + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + name: vsphere-csi-node + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: ${CSI_LIVENESS_PROBE_IMAGE} + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: vsphere-csi-node-windows + namespace: vmware-system-csi + spec: + selector: + matchLabels: + app: vsphere-csi-node-windows + template: + metadata: + labels: + app: vsphere-csi-node-windows + role: vsphere-csi-windows + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: unix://C:\\csi\\csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock + image: ${CSI_REGISTRAR_IMAGE} + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" + - name: X_CSI_MODE + value: node + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: PRODUCTION + - name: X_CSI_LOG_LEVEL + value: DEBUG + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: ${CSI_DRIVER_IMAGE} + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + name: vsphere-csi-node + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: C:\csi + name: plugin-dir + - mountPath: C:\var\lib\kubelet + name: pods-mount-dir + - mountPath: \\.\pipe\csi-proxy-volume-v1 + name: csi-proxy-volume-v1 + - mountPath: \\.\pipe\csi-proxy-filesystem-v1 + name: csi-proxy-filesystem-v1 + - mountPath: \\.\pipe\csi-proxy-disk-v1 + name: csi-proxy-disk-v1 + - mountPath: \\.\pipe\csi-proxy-system-v1alpha1 + name: csi-proxy-system-v1alpha1 + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: ${CSI_LIVENESS_PROBE_IMAGE} + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: windows + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: C:\var\lib\kubelet\plugins_registry\ + type: Directory + name: registration-dir + - hostPath: + path: C:\var\lib\kubelet\plugins\csi.vsphere.vmware.com\ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: \var\lib\kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + type: "" + name: csi-proxy-disk-v1 + - hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + type: "" + name: csi-proxy-volume-v1 + - hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + type: "" + name: csi-proxy-filesystem-v1 + - hostPath: + path: \\.\pipe\csi-proxy-system-v1alpha1 + type: "" + name: csi-proxy-system-v1alpha1 + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +kind: ConfigMap +metadata: + name: csi-manifests +--- +apiVersion: v1 +kind: Secret +metadata: + name: cloud-provider-vsphere-credentials +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + labels: + component: cloud-controller-manager + vsphere-cpi-infra: secret + name: cloud-provider-vsphere-credentials + namespace: kube-system + stringData: + ${VSPHERE_SERVER}.password: ${VSPHERE_PASSWORD} + ${VSPHERE_SERVER}.username: ${VSPHERE_USERNAME} + type: Opaque +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + --- + # Source: vsphere-cpi/templates/service-account.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + labels: + app: vsphere-cpi + vsphere-cpi-infra: service-account + component: cloud-controller-manager + namespace: kube-system + --- + # Source: vsphere-cpi/templates/role.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: cloud-controller-manager + labels: + app: vsphere-cpi + vsphere-cpi-infra: role + component: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + # Source: vsphere-cpi/templates/daemonset.yaml + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: vsphere-cpi + labels: + app: vsphere-cpi + vsphere-cpi-infra: daemonset + component: cloud-controller-manager + tier: control-plane + namespace: kube-system + annotations: + spec: + selector: + matchLabels: + app: vsphere-cpi + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + tier: control-plane + release: release-name + vsphere-cpi-infra: daemonset + spec: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + effect: NoExecute + operator: Exists + securityContext: + fsGroup: 1001 + runAsUser: 1001 + serviceAccountName: cloud-controller-manager + hostNetwork: true + dnsPolicy: ClusterFirst + priorityClassName: system-node-critical + containers: + - name: vsphere-cpi + image: ${CPI_MANAGER_IMAGE} + imagePullPolicy: IfNotPresent + args: + - --cloud-provider=vsphere + - --v=2 + - --cloud-config=/etc/cloud/vsphere.conf + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + volumes: + - name: vsphere-config-volume + configMap: + name: cloud-config + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + vsphere-cpi-infra: role-binding + name: servicecatalog.k8s.io:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + vsphere-cpi-infra: cluster-role-binding + name: cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + data: + vsphere.conf: | + global: + port: 443 + secretName: cloud-provider-vsphere-credentials + secretNamespace: kube-system + thumbprint: '${VSPHERE_TLS_THUMBPRINT}' + insecure-flag: 'true' + vcenter: + ${VSPHERE_SERVER}: + datacenters: + - '${VSPHERE_DATACENTER}' + server: '${VSPHERE_SERVER}' + kind: ConfigMap + metadata: + name: cloud-config + namespace: kube-system +kind: ConfigMap +metadata: + name: cpi-manifests diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc new file mode 100644 index 00000000..e44feeaf --- /dev/null +++ b/templates/vsphere/template-variables.rc @@ -0,0 +1,55 @@ +export KUBERNETES_VERSION= # K8s Version of the cluster, e.g. "v1.30.0" + +export PROXY_DISABLE= # Set to "#" to disable, or "" to enable the proxy configuration + # If enabled, make sure to configure the proxy in the following section. + +if [ "$PROXY_DISABLE" == "#" ]; then + export HTTP_PROXY= + export HTTPS_PROXY= + export NO_PROXY= +else + export HTTP_PROXY=http://squid.internal:3128 + export HTTPS_PROXY=http://squid.internal:3128 + export NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0 +fi + +export CONTROL_PLANE_MACHINE_COUNT=1 +export WORKER_MACHINE_COUNT=1 + +export CONTROL_PLANE_MACHINE_CPU_COUNT=4 +export CONTROL_PLANE_MACHINE_MEMORY_MB=4096 +export CONTROL_PLANE_MACHINE_DISK_GB=25 + +export WORKER_MACHINE_CPU_COUNT=2 +export WORKER_MACHINE_MEMORY_MB=2048 +export WORKER_MACHINE_DISK_GB=25 + +# Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md + +export VSPHERE_USERNAME= # The username used to access the remote vSphere endpoint +export VSPHERE_PASSWORD= # The password used to access the remote vSphere endpoint + +export VSPHERE_SERVER= # The vCenter server IP or FQDN, e.g. "10.0.0.1" +export VSPHERE_DATACENTER= # The vSphere datacenter to deploy the management cluster on, e.g. "SDDC-Datacenter" +export VSPHERE_DATASTORE= # The vSphere datastore to deploy the management cluster on, e.g. "DefaultDatastore" +export VSPHERE_NETWORK= # The VM network to deploy the management cluster on, e.g. "VM Network" +export VSPHERE_RESOURCE_POOL= # The vSphere resource pool for your VMs, e.g. "*/Resources" +export VSPHERE_FOLDER= # The VM folder for your VMs. Set to "" to use the root vSphere folder, e.g. "vm" +export VSPHERE_TEMPLATE= # The VM template to use for your management cluster, e.g. "ubuntu-1804-kube-v1.17.3" +export CONTROL_PLANE_ENDPOINT_IP= # The IP that kube-vip is going to use as a control plane endpoint, e.g. "192.168.1.130" +export VIP_NETWORK_INTERFACE= # The interface that kube-vip should apply the IP to. Omit to autodetect the interface, e.g. "ens192" +export VSPHERE_TLS_THUMBPRINT= # SHA1 thumbprint of the vCenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout +export EXP_CLUSTER_RESOURCE_SET="true" # Enables the ClusterResourceSet feature used to deploy CSI +export VSPHERE_SSH_AUTHORIZED_KEY= # The public SSH authorized key on all machines in this cluster. + # Set to "" if you don't want to enable SSH or are using another solution. +export VSPHERE_STORAGE_POLICY= # The vSphere storage policy. Set to "" if you don't want to use a storage policy. + +export CPI_MANAGER_IMAGE="gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.30.0" # The version of the vSphere CPI image to be used by the CPI workloads +export CSI_DRIVER_IMAGE="gcr.io/cloud-provider-vsphere/csi/release/driver:v3.3.0" +export CSI_SYNCER_IMAGE="gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.3.0" +export CSI_LIVENESS_PROBE_IMAGE="registry.k8s.io/sig-storage/livenessprobe:v2.10.0" +export CSI_ATTACHER_IMAGE="registry.k8s.io/sig-storage/csi-attacher:v4.3.0" +export CSI_RESIZER_IMAGE="registry.k8s.io/sig-storage/csi-resizer:v1.8.0" +export CSI_PROVISIONER_IMAGE="registry.k8s.io/sig-storage/csi-provisioner:v3.5.0" +export CSI_SNAPSHOTTER_IMAGE="registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2" +export CSI_REGISTRAR_IMAGE="registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0"