feat(t8s-cluster/clusterClass)!: configure static machineDeploymentCl…

…asses instead of one per computePlane (#594)

charts/t8s-cluster/ci/cidrs-values.yaml

@@ -1,5 +1,4 @@
 controlPlane:
   flavor: standard
-  nodeDrainTimeout: 5m
   allowedCIDRs:
     - 1.2.3.4

charts/t8s-cluster/ci/gpu-flavor-values.yaml

@@ -1,4 +1,4 @@
-workers:
+nodePools:
   test:
     replicas: 1
     availabilityZone: Zone1

charts/t8s-cluster/ci/injectedCertificateAuthorities-values.yaml

@@ -46,7 +46,7 @@ global:
     -----END CERTIFICATE-----
 controlPlane:
   flavor: standard.2.1905
-workers:
+nodePools:
   test:
     replicas: 1
     availabilityZone: Zone1

charts/t8s-cluster/ci/securityGroups-values.yaml

@@ -3,10 +3,8 @@ controlPlane:
   additionalSecurityGroups:
     - kuchen
     - pizza
-workers:
+nodePools:
   test:
     replicas: 1
     availabilityZone: Zone1
     flavor: standard.2.1905
-    additionalSecurityGroups:
-      - kuchen

charts/t8s-cluster/templates/_helpers.tpl

@@ -9,7 +9,8 @@
 {{- end -}}
 
 {{- define "t8s-cluster.helm.chartVersion" -}}
-{{- dig .repo "charts" .chart nil .context.Values.global.helmRepositories | required (printf "The repo '%s' is either missing or doesn't contain the chart '%s'" .repo .chart) -}}
+  {{- $_ := set . "Values" .context.Values -}}
+  {{- dig .repo "charts" .chart nil .Values.global.helmRepositories | required (printf "The repo '%s' is either missing or doesn't contain the chart '%s'" .repo .chart) -}}
 {{- end -}}
 
 {{- define "t8s-cluster.helm.chartSpec" -}}
@@ -22,11 +23,12 @@ sourceRef:
 {{- end -}}
 
 {{- define "t8s-cluster.hasGPUNodes" -}}
-{{- $hasGPUFlavor := false -}}
-{{- range $name, $machineDeploymentClass := .Values.workers -}}
-  {{- if contains "gpu" (lower $machineDeploymentClass.flavor) -}}
-    {{- $hasGPUFlavor = true -}}
+  {{- $_ := set . "Values" .context.Values -}}
+  {{- $hasGPUFlavor := false -}}
+  {{- range $name, $machineDeploymentClass := .Values.nodePools -}}
+    {{- if contains "gpu" (lower $machineDeploymentClass.flavor) -}}
+      {{- $hasGPUFlavor = true -}}
+    {{- end -}}
   {{- end -}}
-{{- end -}}
-{{- $hasGPUFlavor -}}
+  {{- $hasGPUFlavor -}}
 {{- end -}}

charts/t8s-cluster/templates/management-cluster/cluster.yaml

@@ -1,4 +1,4 @@
-apiVersion: t8s-engine.teuto.net/v1alpha1
+apiVersion: t8s-engine.teuto.net/v1beta1
 kind: TeutonetesCluster
 metadata:
   name: {{ .Release.Name }}
@@ -28,7 +28,7 @@ spec:
   version: {{ printf "v%d.%d.%d" (.major | int) (.minor | int) (.patch | int) }}
   {{- end }}
   clusterClass: {{ $.Release.Name }}
-  workers: {{- range $name, $machineDeploymentClass := .Values.workers }}
+  nodePools: {{- range $name, $machineDeploymentClass := .Values.nodePools }}
     {{ $name -}}:
       {{- if $machineDeploymentClass.availabilityZone }}
       availabilityZone: {{ $machineDeploymentClass.availabilityZone | quote }}

charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl

@@ -7,6 +7,7 @@ openstack
 {{- end -}}
 
 {{- define "t8s-cluster.clusterClass.getIdentityRefSecretName" -}}
+  {{- $_ := set . "Release" .context.Release -}}
   {{- printf "cloud-config-%s" .Release.Name -}}
 {{- end -}}
 
@@ -16,6 +17,7 @@ openstack
 {{- end -}}
 
 {{- define "t8s-cluster.clusterClass.preKubeadmCommands" -}}
+  {{- $_ := set . "Values" .context.Values -}}
   {{- $commands := list -}}
   {{- $commands = append $commands "bash /etc/kube-proxy-patch.sh" }}
   {{- if .Values.global.injectedCertificateAuthorities -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml

@@ -1,3 +1,7 @@
+{{- $machineDeploymentClasses := dict "compute-plane" false "gpu-compute-plane" true -}}
+{{- range $name, $spec := .Values.nodePools -}}
+  {{- $machineDeploymentClasses = set $machineDeploymentClasses $name ($spec.flavor | contains "gpu") -}}
+{{- end -}}
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: ClusterClass
 metadata:
@@ -16,123 +20,126 @@ spec:
         - status: 'False'
           timeout: 600s
           type: Ready
-    nodeDrainTimeout: {{ .Values.controlPlane.nodeDrainTimeout | default "8m" }}
+    nodeDrainTimeout: "8m"
     machineInfrastructure:
       ref:
-        apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+        apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
         kind: OpenStackMachineTemplate
         name: {{ printf "%s-control-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfControlPlane" (dict "context" $)) }}
     ref:
       apiVersion: controlplane.cluster.x-k8s.io/v1beta1
       kind: KubeadmControlPlaneTemplate
-      # the full context is needed for .Files.Get
-      name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }}
+      name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }}
   infrastructure:
     ref:
-      apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+      apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
       kind: OpenStackClusterTemplate
       name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackClusterTemplate.specHash" (dict "context" $)) }}
   variables:
     - name: controlPlaneServerGroupID
       required: true
       schema:
         openAPIV3Schema:
-          description: |-
-            OpenStack Server Group to use for Control Plane machines.
-            Field is optional, but must be set for HA clusters.
           type: string
     - name: machineDeploymentServerGroupID
       required: false
       schema:
         openAPIV3Schema:
-          description: |-
-            OpenStack Server Group to use for MachineDeployment machines.
-            Field is optional, but should be set for HA clusters.
           type: string
     - name: dnsNameservers
       required: true
       schema:
         openAPIV3Schema:
-          description: |
-            [Optional] Can be specified to set a list of DNS Nameservers for the servers.
           type: array
           minItems: 1
           uniqueItems: true
           items:
             type: string
             format: ipv4
-            example: "203.0.113.123"
     - name: controlPlaneAvailabilityZones
       required: false
       schema:
         openAPIV3Schema:
           type: array
-          description: |
-            [Optional] Can be specified to restrict the Zones used for
-                  ControlPlane Machines (default is all available Zones)
           minItems: 1
           uniqueItems: true
           items:
             type: string
-            example: Zone1
+    - name: machineDeploymentFlavor
+      required: true
+      schema:
+        openAPIV3Schema:
+          type: string
+          default: compute-plane-placeholder
   patches:
-    - definitions:
+    - name: controlPlaneServerGroupID
+      description: Sets the ServerGroupID for Control Plane machines.
+      definitions:
         - jsonPatches:
             - op: add
               path: /spec/template/spec/serverGroupID
               valueFrom:
                 variable: controlPlaneServerGroupID
           selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackMachineTemplate
             matchResources:
               controlPlane: true
-      description: Sets the ServerGroupID for Control Plane machines.
-      name: controlPlaneServerGroupID
-    - definitions:
+    - name: machineDeploymentServerGroupID
+      description: Sets the ServerGroupID for MachineDeployment machines.
+      definitions:
         - jsonPatches:
             - op: add
               path: /spec/template/spec/serverGroupID
               valueFrom:
                 variable: machineDeploymentServerGroupID
           selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackMachineTemplate
             matchResources:
               machineDeploymentClass:
-                names: {{- .Values.workers | keys | sortAlpha | toYaml | nindent 18 }}
-      description: Sets the ServerGroupID for MachineDeployment machines.
-      name: machineDeploymentServerGroupID
-    - definitions:
+                names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }}
+    - name: machineDeploymentFlavour
+      definitions:
         - jsonPatches:
             - op: add
-              path: /spec/template/spec/image
+              path: /spec/template/spec/flavor
               valueFrom:
-                template: {{ .Values.imageNameTemplate.workers }}
+                variable: machineDeploymentFlavor
           selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackMachineTemplate
             matchResources:
               machineDeploymentClass:
-                names: {{- .Values.workers | keys | sortAlpha | toYaml | nindent 18 }}
+                names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }}
+    - name: imageVersion
+      description: Sets the image version for machines.
+      definitions:
         - jsonPatches:
-            - op: add
+            - &imagePatch
+              op: add
               path: /spec/template/spec/image
+              valueFrom:
+                template: {{ .Values.imageNameTemplate.computePlane }}
+          selector: &imagePatchSelector
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
+            kind: OpenStackMachineTemplate
+            matchResources:
+              machineDeploymentClass:
+                names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }}
+        - jsonPatches:
+            - <<: *imagePatch
               valueFrom:
                 template: {{ .Values.imageNameTemplate.controlPlane }}
           selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
-            kind: OpenStackMachineTemplate
+            <<: *imagePatchSelector
             matchResources:
               controlPlane: true
-
-      description: Sets the image version for machines.
-      name: imageVersion
     - name: controlPlaneAvailabilityZones
       enabledIf: {{ `{{ if .controlPlaneAvailabilityZones }}true{{ end }}` | quote }}
       definitions:
         - selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackClusterTemplate
             matchResources:
               infrastructureCluster: true
@@ -144,7 +151,7 @@ spec:
     - name: dnsNameservers
       definitions:
         - selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackClusterTemplate
             matchResources:
               infrastructureCluster: true
@@ -155,8 +162,7 @@ spec:
                 variable: dnsNameservers
   workers:
     machineDeployments:
-      {{- range $name, $machineDeploymentClass := .Values.workers }}
-        {{- $isGpuDeploymentClass := $machineDeploymentClass.flavor | contains "gpu" }}
+      {{- range $name, $isGpuDeploymentClass := $machineDeploymentClasses }}
       - class: {{ $name }}
         machineHealthCheck:
           nodeStartupTimeout: 8m
@@ -167,11 +173,12 @@ spec:
             - status: 'False'
               timeout: 300s
               type: Ready
-        nodeDrainTimeout: {{ $machineDeploymentClass.nodeDrainTimeout | default "8m" }}
+        nodeDrainTimeout: "8m"
         strategy:
           type: RollingUpdate
           rollingUpdate:
             deletePolicy: Oldest
+            {{/* TODO: this is only for our beta phase */}}
             {{- if $isGpuDeploymentClass }}
             maxSurge: 0
             maxUnavailable: 1
@@ -184,10 +191,10 @@ spec:
             ref:
               apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
               kind: KubeadmConfigTemplate
-              name: {{ printf "%s-%s-worker" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }}
+              name: {{ printf "%s-%s-compute-plane" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }}
           infrastructure:
             ref:
-              apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+              apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
               kind: OpenStackMachineTemplate
-              name: {{ printf "%s-%s-%s" $.Release.Name $name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfWorkers" (dict "context" $ "worker" $machineDeploymentClass)) }}
+              name: {{ printf "%s-compute-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfNodePools" (dict "context" $)) }}
       {{- end }}

charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml

@@ -2,11 +2,11 @@
   {{- $_ := set . "Values" .context.Values -}}
 joinConfiguration:
   nodeRegistration:
-    kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" .context | nindent 6 }}
+    kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }}
     name: '{{ `{{ local_hostname }}` }}'
   patches:
     directory: /etc/kubernetes/patches
-files: {{- include "t8s-cluster.patches.kubelet.patches" .context | nindent 2 }}
+files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | nindent 2 }}
     {{- if .Values.containerRegistryMirror.mirrorEndpoint }}
       {{- include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | nindent 2 }}
     {{- end }}

charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_workerKubeadmConfigTemplate.yaml → charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_nodePoolKubeadmConfigTemplate.yaml

@@ -2,7 +2,7 @@
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
 metadata:
-  name: {{ printf "%s-%s-worker" .context.Release.Name (.gpu | ternary "gpu" "standard") }}
+  name: {{ printf "%s-%s-compute-plane" .context.Release.Name (.gpu | ternary "gpu" "standard") }}
   namespace: {{ .context.Release.Namespace }}
   labels: {{- include "common.labels.standard" .context | nindent 4 }}
 spec:

charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuNodePoolKubeadmConfigTemplate.yaml

@@ -0,0 +1 @@
+{{- include "t8s-cluster.clusterClass.kubeadmConfigTemplate" (dict "gpu" true "context" $) -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardNodePoolKubeadmConfigTemplate.yaml

@@ -0,0 +1 @@
+{{- include "t8s-cluster.clusterClass.kubeadmConfigTemplate" (dict "gpu" false "context" $) -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl

@@ -1,6 +1,7 @@
 {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" -}}
+  {{/* the full context is needed for .Files.Get */}}
   {{- $inputs := (dict
-    "spec" (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" .)
+    "spec" (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" $)
     ) -}}
   {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}}
 {{- end -}}