diff --git a/aws/deployments/cilium.yaml b/aws/deployments/cilium.yaml deleted file mode 100644 index 8ffaf05..0000000 --- a/aws/deployments/cilium.yaml +++ /dev/null @@ -1,77 +0,0 @@ ---- - -k8sServiceHost: "api.cluster.local" -k8sServicePort: "6443" - -operator: - enabled: true - rollOutPods: true - replicas: 1 - prometheus: - enabled: false - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - operator: Exists - effect: NoSchedule - -identityAllocationMode: crd -kubeProxyReplacement: strict -enableK8sEndpointSlice: true -localRedirectPolicy: true - -tunnel: "vxlan" -autoDirectNodeRoutes: false -devices: [eth+] - -healthChecking: true - -cni: - install: true - -ipam: - mode: "kubernetes" -k8s: - requireIPv4PodCIDR: true - requireIPv6PodCIDR: true - -bpf: - masquerade: false -ipv4: - enabled: true -ipv6: - enabled: true -hostServices: - enabled: true -hostPort: - enabled: true -nodePort: - enabled: true -externalIPs: - enabled: true -hostFirewall: - enabled: true -ingressController: - enabled: false - -securityContext: - privileged: true - -hubble: - enabled: false - -prometheus: - enabled: true - -cgroup: - autoMount: - enabled: false - hostRoot: /sys/fs/cgroup - -resources: - limits: - cpu: 2 - memory: 1Gi - requests: - cpu: 100m - memory: 128Mi diff --git a/aws/deployments/coredns-local.yaml b/aws/deployments/coredns-local.yaml deleted file mode 100644 index e702d9b..0000000 --- a/aws/deployments/coredns-local.yaml +++ /dev/null @@ -1,153 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns-local - namespace: kube-system -data: - empty.db: | - @ 60 IN SOA localnet. root.localnet. ( - 1 ; serial - 60 ; refresh - 60 ; retry - 60 ; expiry - 60 ) ; minimum - ; - @ IN NS localnet. - - hosts: | - # static hosts - 169.254.2.53 dns.local - - Corefile.local: | - (empty) { - file /etc/coredns/empty.db - } - - .:53 { - errors - bind 169.254.2.53 - - health 127.0.0.1:8091 { - lameduck 5s - } - - hosts /etc/coredns/hosts { - reload 60s - fallthrough - } - - kubernetes cluster.local in-addr.arpa ip6.arpa { - endpoint https://api.cluster.local:6443 - kubeconfig /etc/coredns/kubeconfig.conf coredns - pods insecure - ttl 60 - } - prometheus :9153 - - forward . /etc/resolv.conf { - policy sequential - expire 30s - } - - cache 300 - loop - reload - loadbalance - } - kubeconfig.conf: |- - apiVersion: v1 - kind: Config - clusters: - - cluster: - certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - server: https://api.cluster.local:6443 - name: default - contexts: - - context: - cluster: default - namespace: kube-system - user: coredns - name: coredns - current-context: coredns - users: - - name: coredns - user: - tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: coredns-local - namespace: kube-system - labels: - k8s-app: kube-dns-local - kubernetes.io/name: CoreDNS -spec: - updateStrategy: - type: RollingUpdate - minReadySeconds: 15 - selector: - matchLabels: - k8s-app: kube-dns-local - kubernetes.io/name: CoreDNS - template: - metadata: - labels: - k8s-app: kube-dns-local - kubernetes.io/name: CoreDNS - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9153" - spec: - priorityClassName: system-node-critical - serviceAccount: coredns - serviceAccountName: coredns - enableServiceLinks: false - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists - hostNetwork: true - containers: - - name: coredns - image: coredns/coredns:1.9.4 - imagePullPolicy: IfNotPresent - resources: - limits: - cpu: 100m - memory: 128Mi - requests: - cpu: 50m - memory: 64Mi - args: [ "-conf", "/etc/coredns/Corefile.local" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /health - port: 8091 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - all - readOnlyRootFilesystem: true - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns-local diff --git a/aws/deployments/ingress-ns.yaml b/aws/deployments/ingress-ns.yaml deleted file mode 100644 index 6878f0b..0000000 --- a/aws/deployments/ingress-ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx diff --git a/aws/deployments/ingress.yaml b/aws/deployments/ingress.yaml deleted file mode 100644 index 0528956..0000000 --- a/aws/deployments/ingress.yaml +++ /dev/null @@ -1,116 +0,0 @@ - -controller: - kind: DaemonSet - - hostNetwork: true - hostPort: - enabled: false - ports: - http: 80 - https: 443 - - dnsPolicy: ClusterFirstWithHostNet - - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - - publishService: - enabled: false - - config: - worker-processes: "auto" - worker-cpu-affinity: "auto" - error-log-level: "error" - - server-tokens: "false" - http-redirect-code: "301" - - use-gzip: "true" - use-geoip: "false" - use-geoip2: "false" - - use-forwarded-headers: "true" - # curl https://www.cloudflare.com/ips-v4 2>/dev/null | tr '\n' ',' - proxy-real-ip-cidr: "173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,172.64.0.0/13,131.0.72.0/22,104.16.0.0/13,104.24.0.0/14,172.16.0.0/12" - - enable-access-log-for-default-backend: "true" - log-format-escape-json: "true" - log-format-upstream: '{"ip":"$remote_addr", "ssl":"$ssl_protocol", "method":"$request_method", "proto":"$scheme", "host":"$host", "uri":"$request_uri", "status":$status, "size":$bytes_sent, "agent":"$http_user_agent", "referer":"$http_referer", "namespace":"$namespace"}' - - upstream-keepalive-connections: "32" - proxy-connect-timeout: "10" - proxy-read-timeout: "60" - proxy-send-timeout: "60" - - ssl-protocols: "TLSv1.3" - hsts: "true" - hsts-max-age: "31536000" - hsts-include-subdomains: "true" - hsts-preload: "true" - proxy-hide-headers: "strict-transport-security" - proxy-headers-hash-bucket-size: "128" - - server-name-hash-bucket-size: "64" - server-name-hash-max-size: "512" - - limit-req-status-code: "429" - - client-header-timeout: "30" - client-body-timeout: "30" - - minReadySeconds: 15 - - podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/port: "10254" - - extraEnvs: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - livenessProbe: - initialDelaySeconds: 15 - periodSeconds: 30 - readinessProbe: - periodSeconds: 30 - - resources: - limits: - cpu: 1 - memory: 1Gi - requests: - cpu: 100m - memory: 128Mi - - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: project.io/node-pool - operator: In - values: - - web - - service: - enabled: true - type: ClusterIP - clusterIP: None - ipFamilyPolicy: "RequireDualStack" - ipFamilies: - - IPv4 - - IPv6 - - admissionWebhooks: - enabled: false - metrics: - enabled: false - -revisionHistoryLimit: 2 - -defaultBackend: - enabled: false diff --git a/aws/deployments/kubelet-serving-cert-approver.yaml b/aws/deployments/kubelet-serving-cert-approver.yaml deleted file mode 100644 index 7ef7eca..0000000 --- a/aws/deployments/kubelet-serving-cert-approver.yaml +++ /dev/null @@ -1,250 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: certificates:kubelet-serving-cert-approver -rules: -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - verbs: - - get - - list - - watch -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests/approval - verbs: - - update -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - certificates.k8s.io - resourceNames: - - kubernetes.io/kubelet-serving - resources: - - signers - verbs: - - approve ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: events:kubelet-serving-cert-approver -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: psp:kubelet-serving-cert-approver -rules: -- apiGroups: - - policy - resourceNames: - - kubelet-serving-cert-approver - resources: - - podsecuritypolicies - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: events:kubelet-serving-cert-approver - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: events:kubelet-serving-cert-approver -subjects: -- kind: ServiceAccount - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: psp:kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:kubelet-serving-cert-approver -subjects: -- kind: ServiceAccount - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: certificates:kubelet-serving-cert-approver -subjects: -- kind: ServiceAccount - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver -spec: - ports: - - name: metrics - port: 9090 - protocol: TCP - targetPort: metrics - selector: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - template: - metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - spec: - tolerations: - - key: "node.cloudprovider.kubernetes.io/uninitialized" - value: "true" - effect: NoSchedule - - key: "CriticalAddonsOnly" - operator: Exists - - key: "node-role.kubernetes.io/master" - effect: NoSchedule - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: DoesNotExist - - key: node-role.kubernetes.io/control-plane - operator: DoesNotExist - weight: 100 - containers: - - args: - - serve - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: ghcr.io/alex1989hu/kubelet-serving-cert-approver:main - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /healthz - port: health - initialDelaySeconds: 6 - name: cert-approver - ports: - - containerPort: 8080 - name: health - - containerPort: 9090 - name: metrics - readinessProbe: - httpGet: - path: /readyz - port: health - initialDelaySeconds: 3 - resources: - limits: - cpu: 250m - memory: 32Mi - requests: - cpu: 10m - memory: 16Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true - runAsNonRoot: true - priorityClassName: system-cluster-critical - securityContext: - fsGroup: 65534 - runAsGroup: 65534 - runAsUser: 65534 - serviceAccountName: kubelet-serving-cert-approver - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists diff --git a/aws/deployments/metrics-server.yaml b/aws/deployments/metrics-server.yaml deleted file mode 100644 index f259001..0000000 --- a/aws/deployments/metrics-server.yaml +++ /dev/null @@ -1,197 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader -rules: -- apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -rules: -- apiGroups: - - "" - resources: - - pods - - nodes - - nodes/stats - - namespaces - - configmaps - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - nodeSelector: - kubernetes.io/os: linux - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: "node-role.kubernetes.io/control-plane" - effect: NoSchedule - containers: - - args: - - --cert-dir=/tmp - - --secure-port=6443 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - - --authorization-always-allow-paths=/metrics - image: k8s.gcr.io/metrics-server/metrics-server:v0.5.0 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 6443 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 200Mi - securityContext: - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /tmp - name: tmp-dir - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - volumes: - - emptyDir: {} - name: tmp-dir ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io -spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100 diff --git a/aws/deployments/test-as.yaml b/aws/deployments/test-as.yaml deleted file mode 100644 index c6c89b9..0000000 --- a/aws/deployments/test-as.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: overprovisioning -value: -1 -globalDefault: false -description: "Priority class used by overprovisioning." ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: overprovisioning - namespace: default -spec: - replicas: 1 - selector: - matchLabels: - run: overprovisioning - template: - metadata: - labels: - run: overprovisioning - spec: - nodeSelector: - project.io/node-pool: web - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node.kubernetes.io/instance-type - operator: Exists - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - topologyKey: kubernetes.io/hostname - labelSelector: - matchExpressions: - - key: run - operator: In - values: - - overprovisioning - priorityClassName: overprovisioning - containers: - - name: reserve-resources - image: k8s.gcr.io/pause - resources: - requests: - cpu: "700m" diff --git a/hetzner/Makefile b/hetzner/Makefile index 4b6493d..30bb660 100644 --- a/hetzner/Makefile +++ b/hetzner/Makefile @@ -76,8 +76,11 @@ system-static: helm template --namespace=kube-system -f deployments/hcloud-ccm.yaml \ hcloud-cloud-controller-manager hcloud/hcloud-cloud-controller-manager > deployments/hcloud-cloud-controller-manager-result.yaml - # helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \ - # autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml + helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \ + autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml + + helm template --namespace=kube-system -f deployments/hcloud-csi.yaml hcloud-csi \ + hcloud/hcloud-csi > deployments/hcloud-csi-result.yaml system: helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system --version=1.15.7 -f deployments/cilium.yaml \ @@ -95,7 +98,12 @@ system: talos-cloud-controller-manager \ oci://ghcr.io/siderolabs/charts/talos-cloud-controller-manager + base64 -i _cfgs/worker-as.yaml > _cfgs/worker-as.yaml.base64 + kubectl --kubeconfig=kubeconfig -n kube-system create secret generic hcloud-init --from-file=worker=_cfgs/worker-as.yaml.base64 + deploy-csi: dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret rm -f hcloud-csi-secret.secret + + kubectl --kubeconfig=kubeconfig apply -f deployments/hcloud-csi-result.yaml diff --git a/hetzner/deployments/cluster-autoscaler-hcloud-result.yaml b/hetzner/deployments/hcloud-autoscaler-result.yaml similarity index 91% rename from hetzner/deployments/cluster-autoscaler-hcloud-result.yaml rename to hetzner/deployments/hcloud-autoscaler-result.yaml index 1da6e1b..3cfb211 100644 --- a/hetzner/deployments/cluster-autoscaler-hcloud-result.yaml +++ b/hetzner/deployments/hcloud-autoscaler-result.yaml @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" app.kubernetes.io/name: "hetzner-cluster-autoscaler" app.kubernetes.io/managed-by: "Helm" - helm.sh/chart: "cluster-autoscaler-9.29.3" + helm.sh/chart: "cluster-autoscaler-9.37.0" name: cluster-autoscaler-hcloud namespace: kube-system spec: @@ -26,7 +26,7 @@ metadata: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" app.kubernetes.io/name: "hetzner-cluster-autoscaler" app.kubernetes.io/managed-by: "Helm" - helm.sh/chart: "cluster-autoscaler-9.29.3" + helm.sh/chart: "cluster-autoscaler-9.37.0" name: cluster-autoscaler-hcloud namespace: kube-system automountServiceAccountToken: true @@ -39,7 +39,7 @@ metadata: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" app.kubernetes.io/name: "hetzner-cluster-autoscaler" app.kubernetes.io/managed-by: "Helm" - helm.sh/chart: "cluster-autoscaler-9.29.3" + helm.sh/chart: "cluster-autoscaler-9.37.0" name: cluster-autoscaler-hcloud rules: - apiGroups: @@ -78,6 +78,8 @@ rules: verbs: - watch - list + - create + - delete - get - update - apiGroups: @@ -156,6 +158,7 @@ rules: verbs: - list - watch + - get - apiGroups: - coordination.k8s.io resources: @@ -180,7 +183,7 @@ metadata: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" app.kubernetes.io/name: "hetzner-cluster-autoscaler" app.kubernetes.io/managed-by: "Helm" - helm.sh/chart: "cluster-autoscaler-9.29.3" + helm.sh/chart: "cluster-autoscaler-9.37.0" name: cluster-autoscaler-hcloud roleRef: apiGroup: rbac.authorization.k8s.io @@ -199,7 +202,7 @@ metadata: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" app.kubernetes.io/name: "hetzner-cluster-autoscaler" app.kubernetes.io/managed-by: "Helm" - helm.sh/chart: "cluster-autoscaler-9.29.3" + helm.sh/chart: "cluster-autoscaler-9.37.0" name: cluster-autoscaler-hcloud namespace: kube-system rules: @@ -228,7 +231,7 @@ metadata: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" app.kubernetes.io/name: "hetzner-cluster-autoscaler" app.kubernetes.io/managed-by: "Helm" - helm.sh/chart: "cluster-autoscaler-9.29.3" + helm.sh/chart: "cluster-autoscaler-9.37.0" name: cluster-autoscaler-hcloud namespace: kube-system roleRef: @@ -248,7 +251,7 @@ metadata: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" app.kubernetes.io/name: "hetzner-cluster-autoscaler" app.kubernetes.io/managed-by: "Helm" - helm.sh/chart: "cluster-autoscaler-9.29.3" + helm.sh/chart: "cluster-autoscaler-9.37.0" name: cluster-autoscaler-hcloud namespace: kube-system spec: @@ -272,11 +275,12 @@ metadata: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" app.kubernetes.io/name: "hetzner-cluster-autoscaler" app.kubernetes.io/managed-by: "Helm" - helm.sh/chart: "cluster-autoscaler-9.29.3" + helm.sh/chart: "cluster-autoscaler-9.37.0" name: cluster-autoscaler-hcloud namespace: kube-system spec: replicas: 1 + revisionHistoryLimit: 10 selector: matchLabels: app.kubernetes.io/instance: "cluster-autoscaler-hcloud" @@ -291,15 +295,13 @@ spec: dnsPolicy: "ClusterFirst" containers: - name: hetzner-cluster-autoscaler - image: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.27.3" + image: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.0" imagePullPolicy: "IfNotPresent" command: - ./cluster-autoscaler - --cloud-provider=hetzner - --namespace=kube-system - - --nodes=0:2:CPX31:NBG1:worker-nbg1 - --nodes=0:2:CPX31:FSN1:worker-fsn1 - - --nodes=0:2:CPX31:HEL1:worker-hel1 - --logtostderr=true - --node-deletion-delay-timeout=10m0s - --regional=true @@ -307,10 +309,18 @@ spec: - --stderrthreshold=info - --v=4 env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName - name: HCLOUD_CLOUD_INIT valueFrom: secretKeyRef: - name: hcloud + name: hcloud-init key: worker - name: HCLOUD_IMAGE valueFrom: diff --git a/hetzner/deployments/hcloud-autoscaler.yaml b/hetzner/deployments/hcloud-autoscaler.yaml index c133f32..b19e58f 100644 --- a/hetzner/deployments/hcloud-autoscaler.yaml +++ b/hetzner/deployments/hcloud-autoscaler.yaml @@ -1,20 +1,26 @@ fullnameOverride: cluster-autoscaler-hcloud -image: - tag: v1.27.3 +# image: +# tag: v1.27.3 cloudProvider: hetzner autoscalingGroups: - - name: CPX31:NBG1:worker-nbg1 - maxSize: 2 - minSize: 0 - - name: CPX31:FSN1:worker-fsn1 - maxSize: 2 - minSize: 0 - - name: CPX31:HEL1:worker-hel1 + # - name: worker-nbg1 + # maxSize: 2 + # minSize: 0 + # instanceType: CPX31 + # region: FSN1 + - name: worker-fsn1 maxSize: 2 minSize: 0 + instanceType: CPX31 + region: FSN1 + # - name: worker-hel1 + # maxSize: 2 + # minSize: 0 + # instanceType: CPX31 + # region: FSN1 extraEnvSecrets: HCLOUD_TOKEN: @@ -31,7 +37,7 @@ extraEnvSecrets: key: image HCLOUD_CLOUD_INIT: name: hcloud - key: worker + key: init containerSecurityContext: allowPrivilegeEscalation: false diff --git a/hetzner/deployments/hcloud-csi-result.yaml b/hetzner/deployments/hcloud-csi-result.yaml new file mode 100644 index 0000000..104d254 --- /dev/null +++ b/hetzner/deployments/hcloud-csi-result.yaml @@ -0,0 +1,380 @@ +--- +# Source: hcloud-csi/templates/controller/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hcloud-csi-controller + namespace: "kube-system" + labels: + app.kubernetes.io/name: hcloud-csi + helm.sh/chart: hcloud-csi-2.9.0 + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller +automountServiceAccountToken: true +--- +# Source: hcloud-csi/templates/core/storageclass.yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: hcloud-volumes + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: csi.hetzner.cloud +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +reclaimPolicy: "Delete" +--- +# Source: hcloud-csi/templates/controller/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hcloud-csi-controller + labels: + app.kubernetes.io/name: hcloud-csi + helm.sh/chart: hcloud-csi-2.9.0 + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller +rules: +# attacher +- apiGroups: [""] + resources: [persistentvolumes] + verbs: [get, list, watch, update, patch] +- apiGroups: [""] + resources: [nodes] + verbs: [get, list, watch] +- apiGroups: [csi.storage.k8s.io] + resources: [csinodeinfos] + verbs: [get, list, watch] +- apiGroups: [storage.k8s.io] + resources: [csinodes] + verbs: [get, list, watch] +- apiGroups: [storage.k8s.io] + resources: [volumeattachments] + verbs: [get, list, watch, update, patch] +- apiGroups: [storage.k8s.io] + resources: [volumeattachments/status] + verbs: [patch] +# provisioner +- apiGroups: [""] + resources: [secrets] + verbs: [get, list] +- apiGroups: [""] + resources: [persistentvolumes] + verbs: [get, list, watch, create, delete, patch] +- apiGroups: [""] + resources: [persistentvolumeclaims, persistentvolumeclaims/status] + verbs: [get, list, watch, update, patch] +- apiGroups: [storage.k8s.io] + resources: [storageclasses] + verbs: [get, list, watch] +- apiGroups: [""] + resources: [events] + verbs: [list, watch, create, update, patch] +- apiGroups: [snapshot.storage.k8s.io] + resources: [volumesnapshots] + verbs: [get, list] +- apiGroups: [snapshot.storage.k8s.io] + resources: [volumesnapshotcontents] + verbs: [get, list] +# resizer +- apiGroups: [""] + resources: [pods] + verbs: [get, list, watch] +# node +- apiGroups: [""] + resources: [events] + verbs: [get, list, watch, create, update, patch] +--- +# Source: hcloud-csi/templates/controller/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hcloud-csi-controller + labels: + app.kubernetes.io/name: hcloud-csi + helm.sh/chart: hcloud-csi-2.9.0 + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hcloud-csi-controller +subjects: + - kind: ServiceAccount + name: hcloud-csi-controller + namespace: "kube-system" +--- +# Source: hcloud-csi/templates/node/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: hcloud-csi-node + namespace: "kube-system" + labels: + app.kubernetes.io/name: hcloud-csi + helm.sh/chart: hcloud-csi-2.9.0 + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: node + app: hcloud-csi +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: hcloud-csi + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/component: node + template: + metadata: + labels: + app.kubernetes.io/name: hcloud-csi + helm.sh/chart: hcloud-csi-2.9.0 + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: node + spec: + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: instance.hetzner.cloud/is-root-server + operator: NotIn + values: + - "true" + nodeSelector: + node.cloudprovider.kubernetes.io/platform: hcloud + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + priorityClassName: "system-node-critical" + securityContext: + fsGroup: 1001 + initContainers: + containers: + - name: csi-node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1 + imagePullPolicy: IfNotPresent + args: + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket + volumeMounts: + - name: plugin-dir + mountPath: /run/csi + - name: registration-dir + mountPath: /registration + resources: + limits: {} + requests: {} + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.13.1 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /run/csi + name: plugin-dir + resources: + limits: {} + requests: {} + - name: hcloud-csi-driver + image: docker.io/hetznercloud/hcloud-csi-driver:v2.9.0 # x-release-please-version + imagePullPolicy: IfNotPresent + command: [/bin/hcloud-csi-driver-node] + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /run/csi + - name: device-dir + mountPath: /dev + securityContext: + privileged: true + env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: ENABLE_METRICS + value: "false" + ports: + - name: healthz + protocol: TCP + containerPort: 9808 + resources: + limits: {} + requests: {} + livenessProbe: + failureThreshold: 5 + initialDelaySeconds: 10 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 3 + httpGet: + path: /healthz + port: healthz + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory +--- +# Source: hcloud-csi/templates/controller/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hcloud-csi-controller + namespace: "kube-system" + labels: + app.kubernetes.io/name: hcloud-csi + helm.sh/chart: hcloud-csi-2.9.0 + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + app: hcloud-csi-controller +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: hcloud-csi + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/component: controller + template: + metadata: + labels: + app.kubernetes.io/name: hcloud-csi + helm.sh/chart: hcloud-csi-2.9.0 + app.kubernetes.io/instance: hcloud-csi + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + spec: + serviceAccountName: hcloud-csi-controller + + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + priorityClassName: "system-cluster-critical" + securityContext: + fsGroup: 1001 + initContainers: + containers: + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.6.1 + imagePullPolicy: IfNotPresent + resources: + limits: {} + requests: {} + args: + - --default-fstype=ext4 + volumeMounts: + - name: socket-dir + mountPath: /run/csi + + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.11.2 + imagePullPolicy: IfNotPresent + resources: + limits: {} + requests: {} + volumeMounts: + - name: socket-dir + mountPath: /run/csi + + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v5.0.2 + imagePullPolicy: IfNotPresent + resources: + limits: {} + requests: {} + args: + - --feature-gates=Topology=true + - --default-fstype=ext4 + volumeMounts: + - name: socket-dir + mountPath: /run/csi + + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.13.1 + imagePullPolicy: IfNotPresent + resources: + limits: {} + requests: {} + volumeMounts: + - mountPath: /run/csi + name: socket-dir + + - name: hcloud-csi-driver + image: docker.io/hetznercloud/hcloud-csi-driver:v2.9.0 # x-release-please-version + imagePullPolicy: IfNotPresent + command: [/bin/hcloud-csi-driver-controller] + env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: ENABLE_METRICS + value: "false" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: hcloud + key: token + resources: + limits: {} + requests: {} + ports: + - name: healthz + protocol: TCP + containerPort: 9808 + livenessProbe: + failureThreshold: 5 + initialDelaySeconds: 10 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 3 + httpGet: + path: /healthz + port: healthz + volumeMounts: + - name: socket-dir + mountPath: /run/csi + + volumes: + - name: socket-dir + emptyDir: {} +--- +# Source: hcloud-csi/templates/core/csidriver.yaml +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.hetzner.cloud +spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: true + volumeLifecycleModes: + - Persistent diff --git a/hetzner/deployments/hcloud-csi.yaml b/hetzner/deployments/hcloud-csi.yaml index 991efd2..0202272 100644 --- a/hetzner/deployments/hcloud-csi.yaml +++ b/hetzner/deployments/hcloud-csi.yaml @@ -1,411 +1,24 @@ ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: hcloud-volumes - annotations: - storageclass.kubernetes.io/is-default-class: "false" -provisioner: csi.hetzner.cloud -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: hcloud-volumes-enc - annotations: - storageclass.kubernetes.io/is-default-class: "false" -provisioner: csi.hetzner.cloud -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true -parameters: - csi.storage.k8s.io/node-publish-secret-name: hcloud-csi-secret - csi.storage.k8s.io/node-publish-secret-namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: hcloud-csi-controller - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: hcloud-csi-controller -rules: -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - csi.storage.k8s.io - resources: - - csinodeinfos - verbs: - - get - - list - - watch -- apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments/status - verbs: - - patch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - create - - delete - - patch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - - persistentvolumeclaims/status - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshots - verbs: - - get - - list -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - get - - list -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - get - - list - - watch - - create - - update - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hcloud-csi-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: hcloud-csi-controller -subjects: -- kind: ServiceAccount - name: hcloud-csi-controller - namespace: kube-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: hcloud-csi-controller - name: hcloud-csi-controller-metrics - namespace: kube-system -spec: - ports: - - name: metrics - port: 9189 - targetPort: metrics - selector: - app: hcloud-csi-controller ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: hcloud-csi - name: hcloud-csi-node-metrics - namespace: kube-system -spec: - ports: - - name: metrics - port: 9189 - targetPort: metrics - selector: - app: hcloud-csi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-csi-controller - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - app: hcloud-csi-controller - template: - metadata: - labels: - app: hcloud-csi-controller - spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" - node.cloudprovider.kubernetes.io/platform: hcloud - tolerations: - - key: "node-role.kubernetes.io/control-plane" - effect: NoSchedule - containers: - - args: - - --default-fstype=ext4 - image: registry.k8s.io/sig-storage/csi-attacher:v4.1.0 - name: csi-attacher - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 - name: csi-resizer - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - args: - - --feature-gates=Topology=true - - --default-fstype=ext4 - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 - name: csi-provisioner - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - command: - - /bin/hcloud-csi-driver-controller - env: - - name: CSI_ENDPOINT - value: unix:///run/csi/socket - - name: METRICS_ENDPOINT - value: 0.0.0.0:9189 - - name: ENABLE_METRICS - value: "true" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - key: token - name: hcloud - image: hetznercloud/hcloud-csi-driver:v2.3.2 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 2 - timeoutSeconds: 3 - name: hcloud-csi-driver - ports: - - containerPort: 9189 - name: metrics - - containerPort: 9808 - name: healthz - protocol: TCP - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - image: registry.k8s.io/sig-storage/livenessprobe:v2.9.0 - imagePullPolicy: Always - name: liveness-probe - volumeMounts: - - mountPath: /run/csi - name: socket-dir - serviceAccountName: hcloud-csi-controller - volumes: - - emptyDir: {} - name: socket-dir ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: hcloud-csi - name: hcloud-csi-node - namespace: kube-system -spec: - selector: - matchLabels: - app: hcloud-csi - template: - metadata: - labels: - app: hcloud-csi - spec: - nodeSelector: - node.cloudprovider.kubernetes.io/platform: hcloud - tolerations: - - effect: NoSchedule - operator: Exists - containers: - - args: - - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 - name: csi-node-driver-registrar - volumeMounts: - - mountPath: /run/csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - - command: - - /bin/hcloud-csi-driver-node - env: - - name: CSI_ENDPOINT - value: unix:///run/csi/socket - - name: METRICS_ENDPOINT - value: 0.0.0.0:9189 - - name: ENABLE_METRICS - value: "true" - image: hetznercloud/hcloud-csi-driver:v2.3.2 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 2 - timeoutSeconds: 3 - name: hcloud-csi-driver - ports: - - containerPort: 9189 - name: metrics - - containerPort: 9808 - name: healthz - protocol: TCP - securityContext: - privileged: true - volumeMounts: - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: kubelet-dir - - mountPath: /run/csi - name: plugin-dir - - mountPath: /dev - name: device-dir - - image: registry.k8s.io/sig-storage/livenessprobe:v2.9.0 - imagePullPolicy: Always - name: liveness-probe - volumeMounts: - - mountPath: /run/csi - name: plugin-dir - tolerations: - - effect: NoExecute - operator: Exists - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - volumes: - - hostPath: - path: /var/lib/kubelet - type: Directory - name: kubelet-dir - - hostPath: - path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - name: registration-dir - - hostPath: - path: /dev - type: Directory - name: device-dir ---- -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: csi.hetzner.cloud -spec: - attachRequired: true - fsGroupPolicy: File - podInfoOnMount: true - volumeLifecycleModes: - - Persistent + +controller: + hcloudToken: + existingSecret: + name: hcloud + key: token + + priorityClassName: system-cluster-critical + + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +node: + priorityClassName: system-node-critical + nodeSelector: + node.cloudprovider.kubernetes.io/platform: hcloud + +storageClasses: + - name: hcloud-volumes + defaultStorageClass: false + reclaimPolicy: Delete diff --git a/hetzner/deployments/test-as.yaml b/hetzner/deployments/test-as.yaml index 3ba39e6..3c96711 100644 --- a/hetzner/deployments/test-as.yaml +++ b/hetzner/deployments/test-as.yaml @@ -24,18 +24,18 @@ spec: # nodeSelector: # node.cloudprovider.kubernetes.io/platform: hcloud affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - # - key: node.kubernetes.io/instance-type - # operator: Exists - # - key: instance.hetzner.cloud/is-root-server - # operator: NotIn - # values: - # - "true" - - key: hcloud/node-group - operator: Exists + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # # - key: node.kubernetes.io/instance-type + # # operator: Exists + # # - key: instance.hetzner.cloud/is-root-server + # # operator: NotIn + # # values: + # # - "true" + # - key: hcloud/node-group + # operator: Exists podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: kubernetes.io/hostname diff --git a/hetzner/deployments/test-csi.yaml b/hetzner/deployments/test-csi.yaml index ebb30ed..e417675 100644 --- a/hetzner/deployments/test-csi.yaml +++ b/hetzner/deployments/test-csi.yaml @@ -15,8 +15,8 @@ apiVersion: v1 metadata: name: csi-app spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" + # nodeSelector: + # node-role.kubernetes.io/control-plane: "" tolerations: - key: "node-role.kubernetes.io/control-plane" effect: NoSchedule diff --git a/hetzner/instances-controlplane.tf b/hetzner/instances-controlplane.tf index 535208b..4804ba3 100644 --- a/hetzner/instances-controlplane.tf +++ b/hetzner/instances-controlplane.tf @@ -74,6 +74,13 @@ resource "local_sensitive_file" "controlplane" { hcloud_token = var.hcloud_token hcloud_image = data.hcloud_image.talos["amd64"].id hcloud_sshkey = hcloud_ssh_key.infra.id + hcloud_init = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl", + merge(local.kubernetes, try(var.instances["all"], {}), { + lbv4 = local.ipv4_vip + nodeSubnets = var.vpc_main_cidr + labels = "${local.worker_labels},hcloud/node-group=worker-as" + }) + )) robot_user = var.robot_user robot_password = var.robot_password }) diff --git a/hetzner/network-lb.tf b/hetzner/network-lb.tf index e20f213..19ba690 100644 --- a/hetzner/network-lb.tf +++ b/hetzner/network-lb.tf @@ -18,11 +18,17 @@ resource "hcloud_floating_ip" "api" { labels = merge(var.tags, { type = "infra" }) } -# resource "hcloud_floating_ip_assignment" "api" { -# count = local.lb_enable ? 0 : 1 -# floating_ip_id = hcloud_floating_ip.api[0].id -# server_id = one(hcloud_server.controlplane).id -# } +resource "hcloud_floating_ip_assignment" "api" { + count = local.lb_enable && length(local.controlplanes) > 0 ? 0 : 1 + floating_ip_id = hcloud_floating_ip.api[0].id + server_id = hcloud_server.controlplane[keys(local.controlplanes)[0]].id + + lifecycle { + ignore_changes = [ + server_id, + ] + } +} resource "hcloud_load_balancer" "api" { count = local.lb_enable ? 1 : 0 diff --git a/hetzner/templates/controlplane.yaml.tpl b/hetzner/templates/controlplane.yaml.tpl index 3d6b79c..57b8b12 100644 --- a/hetzner/templates/controlplane.yaml.tpl +++ b/hetzner/templates/controlplane.yaml.tpl @@ -116,8 +116,9 @@ cluster: token: ${base64encode(hcloud_token)} user: ${base64encode(robot_user)} password: ${base64encode(robot_password)} - image: ${base64encode(hcloud_image)} sshkey: ${base64encode(hcloud_sshkey)} + image: ${base64encode(hcloud_image)} + init: ${base64encode(hcloud_init)} externalCloudProvider: enabled: true manifests: diff --git a/hetzner/variables.tf b/hetzner/variables.tf index fbd05f7..9a9ab93 100644 --- a/hetzner/variables.tf +++ b/hetzner/variables.tf @@ -66,15 +66,15 @@ variable "controlplane" { }, "nbg1" = { count = 0, - type = "cpx11", + type = "cax21", }, "fsn1" = { - count = 0, - type = "cpx11", + count = 1, + type = "cax21", }, "hel1" = { count = 0, - type = "cax11", + type = "cax21", } } }