diff --git a/charts/index.yaml b/charts/index.yaml index 2eb0c12b3..3cf133be2 100644 --- a/charts/index.yaml +++ b/charts/index.yaml @@ -235,6 +235,15 @@ entries: urls: - https://raw.githubusercontent.com/avoltz/blob-csi-driver/staging/charts/v4.0.0/blob-csi-driver-v4.0.0.tgz version: v4.0.0 + - apiVersion: v1 + appVersion: v4.1.0 + created: "2023-07-05T15:32:39.254303884Z" + description: Azure Blob Storage CSI driver + digest: 9b663f91d4dff55080ab21a719eb60d70ae71cee2df493927dc483e96c85b41e + name: blob-csi-driver + urls: + - https://raw.githubusercontent.com/avoltz/blob-csi-driver/staging/charts/v4.1.0/blob-csi-driver-v4.1.0.tgz + version: v4.1.0 - apiVersion: v1 appVersion: latest created: "2023-06-05T13:16:16.079514405Z" diff --git a/charts/v4.1.0/blob-csi-driver-v4.1.0.tgz b/charts/v4.1.0/blob-csi-driver-v4.1.0.tgz new file mode 100644 index 000000000..560162d63 Binary files /dev/null and b/charts/v4.1.0/blob-csi-driver-v4.1.0.tgz differ diff --git a/charts/v4.1.0/blob-csi-driver/Chart.yaml b/charts/v4.1.0/blob-csi-driver/Chart.yaml new file mode 100644 index 000000000..77f6c8ee2 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: latest +description: Azure Blob Storage CSI driver +name: blob-csi-driver +version: v4.1.0 diff --git a/charts/v4.1.0/blob-csi-driver/templates/NOTES.txt b/charts/v4.1.0/blob-csi-driver/templates/NOTES.txt new file mode 100644 index 000000000..9ad135dd4 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/NOTES.txt @@ -0,0 +1,5 @@ +The Azure Blob Storage CSI driver is getting deployed to your cluster. + +To check Azure Blob Storage CSI driver pods status, please run: + + kubectl --namespace={{ .Release.Namespace }} get pods --selector="release={{ .Release.Name }}" --watch diff --git a/charts/v4.1.0/blob-csi-driver/templates/_helpers.tpl b/charts/v4.1.0/blob-csi-driver/templates/_helpers.tpl new file mode 100644 index 000000000..d99392f32 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/_helpers.tpl @@ -0,0 +1,49 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* Expand the name of the chart.*/}} +{{- define "blob.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "blob.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common selectors. +*/}} +{{- define "blob.selectorLabels" -}} +app.kubernetes.io/name: {{ template "blob.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Common labels. +*/}} +{{- define "blob.labels" -}} +{{- include "blob.selectorLabels" . }} +app.kubernetes.io/component: csi-driver +app.kubernetes.io/part-of: {{ template "blob.name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +helm.sh/chart: {{ template "blob.chart" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end -}} + + +{{/* pull secrets for containers */}} +{{- define "blob.pullSecrets" -}} +{{- if .Values.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/v4.1.0/blob-csi-driver/templates/csi-blob-controller.yaml b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-controller.yaml new file mode 100644 index 000000000..50afbb969 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-controller.yaml @@ -0,0 +1,216 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Values.controller.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.controller.name }} + {{- include "blob.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controller.replicas }} + selector: + matchLabels: + app: {{ .Values.controller.name }} + {{- include "blob.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ .Values.controller.name }} + {{- include "blob.labels" . | nindent 8 }} + {{- if .Values.workloadIdentity.clientID }} + azure.workload.identity/use: "true" + {{- end }} + {{- if .Values.podLabels }} +{{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} +{{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + spec: +{{- with .Values.controller.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.controller.hostNetwork }} + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ .Values.serviceAccount.controller }} + nodeSelector: + kubernetes.io/os: linux + {{- if .Values.controller.runOnMaster}} + node-role.kubernetes.io/master: "" + {{- end}} + {{- if .Values.controller.runOnControlPlane}} + node-role.kubernetes.io/control-plane: "" + {{- end}} +{{- with .Values.controller.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + priorityClassName: {{ .Values.priorityClassName | quote }} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- with .Values.controller.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} + containers: + - name: csi-provisioner +{{- if hasPrefix "/" .Values.image.csiProvisioner.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" +{{- else }} + image: "{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" +{{- end }} + args: + - "-v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + - "--timeout=120s" + - "--extra-create-metadata=true" + - "--kube-api-qps=50" + - "--kube-api-burst=100" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiProvisioner.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: {{- toYaml .Values.controller.resources.csiProvisioner | nindent 12 }} + - name: liveness-probe +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port={{ .Values.controller.livenessProbe.healthPort }} + imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.livenessProbe | nindent 12 }} + - name: blob +{{- if hasPrefix "/" .Values.image.blob.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- else }} + image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- end }} + args: + - "--v={{ .Values.controller.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:{{ .Values.controller.metricsPort }}" + - "--drivername={{ .Values.driver.name }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--cloud-config-secret-name={{ .Values.controller.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.controller.cloudConfigSecretNamespace }}" + - "--allow-empty-cloud-config={{ .Values.controller.allowEmptyCloudConfig }}" + ports: + - containerPort: {{ .Values.controller.livenessProbe.healthPort }} + name: healthz + protocol: TCP + - containerPort: {{ .Values.controller.metricsPort }} + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + {{- if ne .Values.driver.httpsProxy "" }} + - name: HTTPS_PROXY + value: {{ .Values.driver.httpsProxy }} + {{- end }} + {{- if ne .Values.driver.httpProxy "" }} + - name: HTTP_PROXY + value: {{ .Values.driver.httpProxy }} + {{- end }} + - name: AZURE_GO_SDK_LOG_LEVEL + value: {{ .Values.driver.azureGoSDKLogLevel }} + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + {{- end }} + imagePullPolicy: {{ .Values.image.blob.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + - name: ssl-pki + mountPath: /etc/pki/ca-trust/extracted + readOnly: true + {{- end }} + resources: {{- toYaml .Values.controller.resources.blob | nindent 12 }} + - name: csi-resizer +{{- if hasPrefix "/" .Values.image.csiResizer.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- else }} + image: "{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- end }} + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + - '-handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiResizer.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.csiResizer | nindent 12 }} + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + - name: ssl-pki + hostPath: + path: /etc/pki/ca-trust/extracted + {{- end }} + {{- if .Values.securityContext }} + securityContext: {{- toYaml .Values.securityContext | nindent 8 }} + {{- end }} diff --git a/charts/v4.1.0/blob-csi-driver/templates/csi-blob-driver.yaml b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-driver.yaml new file mode 100644 index 000000000..9a6aea64a --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-driver.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.driver.name }} + labels: + {{- include "blob.labels" . | nindent 4 }} +spec: + attachRequired: false + podInfoOnMount: true + fsGroupPolicy: {{ .Values.feature.fsGroupPolicy }} + volumeLifecycleModes: + - Persistent + - Ephemeral diff --git a/charts/v4.1.0/blob-csi-driver/templates/csi-blob-node.yaml b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-node.yaml new file mode 100644 index 000000000..13e4291ed --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-node.yaml @@ -0,0 +1,288 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .Values.node.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.node.name }} + {{- include "blob.labels" . | nindent 4 }} +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: {{ .Values.node.maxUnavailable }} + type: RollingUpdate + selector: + matchLabels: + app: {{ .Values.node.name }} + {{- include "blob.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ .Values.node.name }} + {{- include "blob.labels" . | nindent 8 }} + {{- if .Values.workloadIdentity.clientID }} + azure.workload.identity/use: "true" + {{- end }} + {{- if .Values.podLabels }} +{{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} +{{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} +{{- if .Values.node.enableBlobfuseProxy }} + hostPID: true +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ .Values.serviceAccount.node }} + nodeSelector: + kubernetes.io/os: linux +{{- with .Values.node.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + {{- if .Values.node.affinity }} +{{- toYaml .Values.node.affinity | nindent 8 }} + {{- end }} + priorityClassName: {{ .Values.priorityClassName | quote }} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- with .Values.node.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- if .Values.node.enableBlobfuseProxy }} + initContainers: + - name: install-blobfuse-proxy +{{- if hasPrefix "/" .Values.image.blob.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- else }} + image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- end }} + imagePullPolicy: IfNotPresent + command: + - "/blobfuse-proxy/init.sh" + securityContext: + privileged: true + env: + - name: DEBIAN_FRONTEND + value: "noninteractive" + - name: INSTALL_BLOBFUSE + value: "{{ .Values.node.blobfuseProxy.installBlobfuse }}" + - name: BLOBFUSE_VERSION + value: "{{ .Values.node.blobfuseProxy.blobfuseVersion }}" + - name: INSTALL_BLOBFUSE2 + value: "{{ .Values.node.blobfuseProxy.installBlobfuse2 }}" + - name: BLOBFUSE2_VERSION + value: "{{ .Values.node.blobfuseProxy.blobfuse2Version }}" + - name: SET_MAX_OPEN_FILE_NUM + value: "{{ .Values.node.blobfuseProxy.setMaxOpenFileNum }}" + - name: MAX_FILE_NUM + value: "{{ .Values.node.blobfuseProxy.maxOpenFileNum }}" + - name: DISABLE_UPDATEDB + value: "{{ .Values.node.blobfuseProxy.disableUpdateDB }}" + volumeMounts: + - name: host-usr + mountPath: /host/usr + - name: host-etc + mountPath: /host/etc +{{- end }} + containers: + - name: liveness-probe + imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port={{ .Values.node.livenessProbe.healthPort }} + - --v=2 + resources: {{- toYaml .Values.node.resources.livenessProbe | nindent 12 }} + - name: node-driver-registrar +{{- if hasPrefix "/" .Values.image.nodeDriverRegistrar.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- else }} + image: "{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- end }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }}/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: {{- toYaml .Values.node.resources.nodeDriverRegistrar | nindent 12 }} + - name: blob +{{- if hasPrefix "/" .Values.image.blob.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- else }} + image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- end }} + args: + - "--v={{ .Values.node.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--blobfuse-proxy-endpoint=$(BLOBFUSE_PROXY_ENDPOINT)" + - "--edgecache-mount-endpoint=$(EDGECACHE_MOUNT_ENDPOINT)" + - "--enable-blobfuse-proxy={{ .Values.node.enableBlobfuseProxy }}" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--drivername={{ .Values.driver.name }}" + - "--cloud-config-secret-name={{ .Values.node.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.node.cloudConfigSecretNamespace }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--allow-empty-cloud-config={{ .Values.node.allowEmptyCloudConfig }}" + - "--enable-get-volume-stats={{ .Values.feature.enableGetVolumeStats }}" + - "--append-timestamp-cache-dir={{ .Values.node.appendTimeStampInCacheDir }}" + - "--mount-permissions={{ .Values.node.mountPermissions }}" + - "--allow-inline-volume-key-access-with-idenitity={{ .Values.node.allowInlineVolumeKeyAccessWithIdentity }}" + ports: + - containerPort: {{ .Values.node.livenessProbe.healthPort }} + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: EDGECACHE_MOUNT_ENDPOINT + value: unix:///csi/csi_mounts.sock + - name: BLOBFUSE_PROXY_ENDPOINT + value: unix:///csi/blobfuse-proxy.sock + {{- if ne .Values.driver.httpsProxy "" }} + - name: HTTPS_PROXY + value: {{ .Values.driver.httpsProxy }} + {{- end }} + {{- if ne .Values.driver.httpProxy "" }} + - name: HTTP_PROXY + value: {{ .Values.driver.httpProxy }} + {{- end }} + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AZURE_GO_SDK_LOG_LEVEL + value: {{ .Values.driver.azureGoSDKLogLevel }} + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + {{- end }} + imagePullPolicy: {{ .Values.image.blob.pullPolicy }} + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: {{ .Values.linux.kubelet }}/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /mnt + name: blob-cache + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + - name: ssl-pki + mountPath: /etc/pki/ca-trust/extracted + readOnly: true + {{- end }} + resources: {{- toYaml .Values.node.resources.blob | nindent 12 }} + volumes: +{{- if .Values.node.enableBlobfuseProxy }} + - name: host-usr + hostPath: + path: /usr + - name: host-etc + hostPath: + path: /etc +{{- end }} + - hostPath: + path: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }} + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: {{ .Values.linux.kubelet }}/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: {{ .Values.linux.kubelet }}/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred + - hostPath: + path: {{ .Values.node.blobfuseCachePath }} + name: blob-cache + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + - name: ssl-pki + hostPath: + path: /etc/pki/ca-trust/extracted + {{- end }} + {{- if .Values.securityContext }} + securityContext: {{- toYaml .Values.securityContext | nindent 8 }} + {{- end }} diff --git a/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-controller.yaml b/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-controller.yaml new file mode 100644 index 000000000..833dcc640 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-controller.yaml @@ -0,0 +1,115 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-provisioner-role + labels: + {{- include "blob.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-provisioner-binding + labels: + {{- include "blob.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-resizer-role + labels: + {{- include "blob.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-resizer-role + labels: + {{- include "blob.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-controller-secret-role + labels: + {{- include "blob.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-controller-secret-binding + labels: + {{- include "blob.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-controller-secret-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-node.yaml b/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-node.yaml new file mode 100644 index 000000000..f4eb48e93 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-node.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.create -}} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-role + labels: + {{- include "blob.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + + # the node plugin must apply annotations to the PVC for edgecache volumes + # it gets the PVC's through the PV's + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-binding + labels: + {{- include "blob.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-node-secret-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-controller.yaml b/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-controller.yaml new file mode 100644 index 000000000..7433bccf1 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-controller.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "blob.labels" . | nindent 4 }} +{{- if .Values.workloadIdentity.clientID }} + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: {{ .Values.workloadIdentity.clientID }} +{{- if .Values.workloadIdentity.tenantID }} + azure.workload.identity/tenant-id: {{ .Values.workloadIdentity.tenantID }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-node.yaml b/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-node.yaml new file mode 100644 index 000000000..a25090e30 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-node.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "blob.labels" . | nindent 4 }} +{{- if .Values.workloadIdentity.clientID }} + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: {{ .Values.workloadIdentity.clientID }} +{{- if .Values.workloadIdentity.tenantID }} + azure.workload.identity/tenant-id: {{ .Values.workloadIdentity.tenantID }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/v4.1.0/blob-csi-driver/values.yaml b/charts/v4.1.0/blob-csi-driver/values.yaml new file mode 100644 index 000000000..1ff9bbfaf --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/values.yaml @@ -0,0 +1,173 @@ +image: + baseRepo: mcr.microsoft.com + blob: + repository: /k8s/csi/blob-csi + tag: latest + pullPolicy: IfNotPresent + csiProvisioner: + repository: /oss/kubernetes-csi/csi-provisioner + tag: v3.5.0 + pullPolicy: IfNotPresent + livenessProbe: + repository: /oss/kubernetes-csi/livenessprobe + tag: v2.10.0 + pullPolicy: IfNotPresent + nodeDriverRegistrar: + repository: /oss/kubernetes-csi/csi-node-driver-registrar + tag: v2.8.0 + pullPolicy: IfNotPresent + csiResizer: + repository: /oss/kubernetes-csi/csi-resizer + tag: v1.8.0 + pullPolicy: IfNotPresent + +cloud: AzurePublicCloud + +## Reference to one or more secrets to be used when pulling images +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# - name: myRegistryKeySecretName + +serviceAccount: + create: true # When true, service accounts will be created for you. Set to false if you want to use your own. + controller: csi-blob-controller-sa # Name of Service Account to be created or used + node: csi-blob-node-sa # Name of Service Account to be created or used + +rbac: + create: true + name: blob + +## Collection of annotations to add to all the pods +podAnnotations: {} +## Collection of labels to add to all the pods +podLabels: {} +# -- Custom labels to add into metadata +customLabels: {} + # k8s-app: blob-csi-driver + +## Leverage a PriorityClass to ensure your pods survive resource shortages +## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: system-cluster-critical +## Security context give the opportunity to run container as nonroot by setting a securityContext +## by example : +## securityContext: { runAsUser: 1001 } +securityContext: {} + +controller: + name: csi-blob-controller + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: kube-system + allowEmptyCloudConfig: true + hostNetwork: true # this setting could be disabled if controller does not depend on MSI setting + metricsPort: 29634 + livenessProbe: + healthPort: 29632 + replicas: 2 + runOnMaster: false + runOnControlPlane: false + logLevel: 5 + resources: + csiProvisioner: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + livenessProbe: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + blob: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + csiResizer: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + affinity: {} + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + +node: + name: csi-blob-node + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: kube-system + allowEmptyCloudConfig: true + allowInlineVolumeKeyAccessWithIdentity: false + maxUnavailable: 1 + livenessProbe: + healthPort: 29633 + logLevel: 5 + enableBlobfuseProxy: false + blobfuseProxy: + installBlobfuse: true + blobfuseVersion: "1.4.5" + installBlobfuse2: true + blobfuse2Version: "2.0.3" + setMaxOpenFileNum: true + maxOpenFileNum: "9000000" + disableUpdateDB: true + blobfuseCachePath: /mnt + appendTimeStampInCacheDir: false + mountPermissions: 0777 + resources: + livenessProbe: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + nodeDriverRegistrar: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + blob: + limits: + memory: 2100Mi + requests: + cpu: 10m + memory: 20Mi + affinity: {} + nodeSelector: {} + tolerations: + - operator: "Exists" + +feature: + fsGroupPolicy: ReadWriteOnceWithFSType + enableGetVolumeStats: false + +driver: + name: blob.csi.azure.com + customUserAgent: "" + userAgentSuffix: "OSS-helm" + azureGoSDKLogLevel: "" # available values: ""(no logs), DEBUG, INFO, WARNING, ERROR + httpsProxy: "" + httpProxy: "" + +linux: + kubelet: /var/lib/kubelet + distro: debian + +workloadIdentity: + clientID: "" + # [optional] If the AAD application or user-assigned managed identity is not in the same tenant as the cluster + # then set tenantID with the application or user-assigned managed identity tenant ID + tenantID: "" diff --git a/go.mod b/go.mod index e839efaad..6bad0e8be 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( k8s.io/client-go v0.27.1 k8s.io/component-base v0.27.1 k8s.io/klog/v2 v2.90.1 - k8s.io/kubernetes v1.27.1 + k8s.io/kubernetes v1.27.3 k8s.io/mount-utils v0.27.1 k8s.io/utils v0.0.0-20230209194617-a36077c30491 sigs.k8s.io/cloud-provider-azure v1.27.1-0.20230423180712-b979bea29b6a @@ -146,10 +146,10 @@ require ( k8s.io/component-helpers v0.27.1 // indirect k8s.io/controller-manager v0.27.1 // indirect k8s.io/kms v0.27.1 // indirect - k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect + k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/kubectl v0.0.0 // indirect k8s.io/kubelet v0.27.1 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.1 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index ccddbe63d..a6ec66363 100644 --- a/go.sum +++ b/go.sum @@ -841,14 +841,14 @@ k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kms v0.27.1 h1:JTSQbJb+mcobScQwF0bOmZhIwP17k8GvBsiLlA6SQqw= k8s.io/kms v0.27.1/go.mod h1:VuTsw0uHlSycKLCkypCGxfFCjLfzf/5YMeATECd/zJA= -k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= -k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/kubectl v0.27.1 h1:9T5c5KdpburYiW8XKQSH0Uly1kMNE90aGSnbYUZNdcA= k8s.io/kubectl v0.27.1/go.mod h1:QsAkSmrRsKTPlAFzF8kODGDl4p35BIwQnc9XFhkcsy8= k8s.io/kubelet v0.27.1 h1:IkfZ0N9CX/g6EDis7nJw8ZsOuHcpFA6cm0pXQx0g5TY= k8s.io/kubelet v0.27.1/go.mod h1:g3cIhpZPawo/MvsdnmcLmqDJvDPdbUFkzfyLNz03nQg= -k8s.io/kubernetes v1.27.1 h1:DFeW4Lv+kh5DyYcezOzwmQAbC3VqXAxnMyZabALiRSc= -k8s.io/kubernetes v1.27.1/go.mod h1:TTwPjSCKQ+a/NTiFKRGjvOnEaQL8wIG40nsYH8Er4bA= +k8s.io/kubernetes v1.27.3 h1:gwufSj7y6X18Q2Gl8v4Ev+AJHdzWkG7A8VNFffS9vu0= +k8s.io/kubernetes v1.27.3/go.mod h1:U8ZXeKBAPxeb4J4/HOaxjw1A9K6WfSH+fY2SS7CR6IM= k8s.io/mount-utils v0.27.1 h1:RSd0wslbIuwLRaGGNAGMZ3m9FLcvukxJ3FWlOm76W2A= k8s.io/mount-utils v0.27.1/go.mod h1:vmcjYdi2Vg1VTWY7KkhvwJVY6WDHxb/QQhiQKkR8iNs= k8s.io/pod-security-admission v0.27.1 h1:if4d1zzcpNOZNvljvJ0nTCshFPUmnkIsy7KYJg7FP08= @@ -858,8 +858,8 @@ k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.1 h1:MB1zkK+WMOmfLxEpjr1wEmkpcIhZC7kfTkZ0stg5bog= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.1/go.mod h1:/4NLd21PQY0B+H+X0aDZdwUiVXYJQl/2NXA5KVtDiP4= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= sigs.k8s.io/cloud-provider-azure v1.27.1-0.20230423180712-b979bea29b6a h1:wscyx3uWXA8OiV8v8x18ak1lIK8U3q6yS2laIHfe8RQ= sigs.k8s.io/cloud-provider-azure v1.27.1-0.20230423180712-b979bea29b6a/go.mod h1:vRG0GaFye/VlBpSNUGxYHdC3ZTBwFUnc5ZLng+YoE9g= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/pkg/blob/azure.go b/pkg/blob/azure.go index 359fcc6cc..751fcc9dd 100644 --- a/pkg/blob/azure.go +++ b/pkg/blob/azure.go @@ -89,21 +89,21 @@ func getCloudProvider(kubeconfig, nodeID, secretName, secretNamespace, userAgent config, err = az.GetConfigFromSecret() if err == nil && config != nil { fromSecret = true + + if tenantID := os.Getenv("AZURE_TENANT_ID"); tenantID != "" { + config.TenantID = tenantID + } + if clientID := os.Getenv("AZURE_CLIENT_ID"); clientID != "" { + config.AADClientID = clientID + } + if federatedTokenFile := os.Getenv("AZURE_FEDERATED_TOKEN_FILE"); federatedTokenFile != "" { + config.AADFederatedTokenFile = federatedTokenFile + config.UseFederatedWorkloadIdentityExtension = true + } } if err != nil { klog.V(2).Infof("InitializeCloudFromSecret: failed to get cloud config from secret %s/%s: %v", az.SecretNamespace, az.SecretName, err) } - - if tenantID := os.Getenv("AZURE_TENANT_ID"); tenantID != "" { - config.TenantID = tenantID - } - if clientID := os.Getenv("AZURE_CLIENT_ID"); clientID != "" { - config.AADClientID = clientID - } - if federatedTokenFile := os.Getenv("AZURE_FEDERATED_TOKEN_FILE"); federatedTokenFile != "" { - config.AADFederatedTokenFile = federatedTokenFile - config.UseFederatedWorkloadIdentityExtension = true - } } if config == nil { diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go index f427c5118..2d53f76e8 100644 --- a/pkg/blob/blob.go +++ b/pkg/blob/blob.go @@ -48,65 +48,64 @@ import ( const ( // DefaultDriverName holds the name of the csi-driver - DefaultDriverName = "blob.csi.azure.com" - blobCSIDriverName = "blob_csi_driver" - separator = "#" - volumeIDTemplate = "%s#%s#%s#%s#%s#%s" - secretNameTemplate = "azure-storage-account-%s-secret" - serverNameField = "server" - storageEndpointSuffixField = "storageendpointsuffix" - tagsField = "tags" - matchTagsField = "matchtags" - protocolField = "protocol" - accountNameField = "accountname" - accountKeyField = "accountkey" - storageAccountField = "storageaccount" - storageAccountTypeField = "storageaccounttype" - skuNameField = "skuname" - subscriptionIDField = "subscriptionid" - resourceGroupField = "resourcegroup" - locationField = "location" - secretNameField = "secretname" - secretNamespaceField = "secretnamespace" - containerNameField = "containername" - containerNamePrefixField = "containernameprefix" - storeAccountKeyField = "storeaccountkey" - isHnsEnabledField = "ishnsenabled" - softDeleteBlobsField = "softdeleteblobs" - softDeleteContainersField = "softdeletecontainers" - enableBlobVersioningField = "enableblobversioning" - getAccountKeyFromSecretField = "getaccountkeyfromsecret" - storageSPNClientIDField = "azurestoragespnclientid" - storageSPNTenantIDField = "azurestoragespntenantid" - keyVaultURLField = "keyvaulturl" - keyVaultSecretNameField = "keyvaultsecretname" - keyVaultSecretVersionField = "keyvaultsecretversion" - storageAccountNameField = "storageaccountname" - allowBlobPublicAccessField = "allowblobpublicaccess" - requireInfraEncryptionField = "requireinfraencryption" - ephemeralField = "csi.storage.k8s.io/ephemeral" - podNamespaceField = "csi.storage.k8s.io/pod.namespace" - mountOptionsField = "mountoptions" - falseValue = "false" - trueValue = "true" - defaultSecretAccountName = "azurestorageaccountname" - defaultSecretAccountKey = "azurestorageaccountkey" - accountSasTokenField = "azurestorageaccountsastoken" - msiSecretField = "msisecret" - storageSPNClientSecretField = "azurestoragespnclientsecret" - EcProtocol = "edgecache" - Fuse = "fuse" - Fuse2 = "fuse2" - NFS = "nfs" - vnetResourceGroupField = "vnetresourcegroup" - vnetNameField = "vnetname" - subnetNameField = "subnetname" - accessTierField = "accesstier" - networkEndpointTypeField = "networkendpointtype" - mountPermissionsField = "mountpermissions" - useDataPlaneAPIField = "usedataplaneapi" - provisionerSecretNameField = "volume.kubernetes.io/provisioner-deletion-secret-name" - provisionerSecretNamespaceField = "volume.kubernetes.io/provisioner-deletion-secret-namespace" + DefaultDriverName = "blob.csi.azure.com" + blobCSIDriverName = "blob_csi_driver" + separator = "#" + volumeIDTemplate = "%s#%s#%s#%s#%s#%s" + secretNameTemplate = "azure-storage-account-%s-secret" + serverNameField = "server" + storageEndpointSuffixField = "storageendpointsuffix" + tagsField = "tags" + matchTagsField = "matchtags" + protocolField = "protocol" + accountNameField = "accountname" + accountKeyField = "accountkey" + storageAccountField = "storageaccount" + storageAccountTypeField = "storageaccounttype" + skuNameField = "skuname" + subscriptionIDField = "subscriptionid" + resourceGroupField = "resourcegroup" + locationField = "location" + secretNameField = "secretname" + secretNamespaceField = "secretnamespace" + containerNameField = "containername" + containerNamePrefixField = "containernameprefix" + storeAccountKeyField = "storeaccountkey" + isHnsEnabledField = "ishnsenabled" + softDeleteBlobsField = "softdeleteblobs" + softDeleteContainersField = "softdeletecontainers" + enableBlobVersioningField = "enableblobversioning" + getAccountKeyFromSecretField = "getaccountkeyfromsecret" + storageSPNClientIDField = "azurestoragespnclientid" + storageSPNTenantIDField = "azurestoragespntenantid" + keyVaultURLField = "keyvaulturl" + keyVaultSecretNameField = "keyvaultsecretname" + keyVaultSecretVersionField = "keyvaultsecretversion" + storageAccountNameField = "storageaccountname" + allowBlobPublicAccessField = "allowblobpublicaccess" + requireInfraEncryptionField = "requireinfraencryption" + ephemeralField = "csi.storage.k8s.io/ephemeral" + podNamespaceField = "csi.storage.k8s.io/pod.namespace" + mountOptionsField = "mountoptions" + falseValue = "false" + trueValue = "true" + defaultSecretAccountName = "azurestorageaccountname" + defaultSecretAccountKey = "azurestorageaccountkey" + accountSasTokenField = "azurestorageaccountsastoken" + msiSecretField = "msisecret" + storageSPNClientSecretField = "azurestoragespnclientsecret" + EcProtocol = "edgecache" + Fuse = "fuse" + Fuse2 = "fuse2" + NFS = "nfs" + vnetResourceGroupField = "vnetresourcegroup" + vnetNameField = "vnetname" + subnetNameField = "subnetname" + accessTierField = "accesstier" + networkEndpointTypeField = "networkendpointtype" + mountPermissionsField = "mountpermissions" + useDataPlaneAPIField = "usedataplaneapi" + EcStrgAuthenticationField = "edgecache-storage-auth" // See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names containerNameMinLength = 3 diff --git a/pkg/blob/controllerserver.go b/pkg/blob/controllerserver.go index 2326d931f..ed1f552a4 100644 --- a/pkg/blob/controllerserver.go +++ b/pkg/blob/controllerserver.go @@ -166,6 +166,8 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) accessTier = v case networkEndpointTypeField: networkEndpointType = v + case EcStrgAuthenticationField: + containerNameReplaceMap[EcStrgAuthenticationField] = v case mountPermissionsField: // only do validations here, used in NodeStageVolume, NodePublishVolume if v != "" { diff --git a/pkg/blob/nodeserver.go b/pkg/blob/nodeserver.go index 28f5aec67..927c4c907 100644 --- a/pkg/blob/nodeserver.go +++ b/pkg/blob/nodeserver.go @@ -27,6 +27,7 @@ import ( "time" "sigs.k8s.io/blob-csi-driver/pkg/edgecache" + cv "sigs.k8s.io/blob-csi-driver/pkg/edgecache/cachevolume" blobcsiutil "sigs.k8s.io/blob-csi-driver/pkg/util" "github.com/Azure/azure-sdk-for-go/storage" @@ -42,7 +43,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - maps "golang.org/x/exp/maps" "golang.org/x/net/context" "google.golang.org/grpc" mount_azure_blob "sigs.k8s.io/blob-csi-driver/pkg/blobfuse-proxy/pb" @@ -336,6 +336,14 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } if protocol == EcProtocol { + // get authentication method + storageAuthType, storageAuthTypeOk := attrib[EcStrgAuthenticationField] + if !storageAuthTypeOk { + err = fmt.Errorf("could not determine storage authentication for edgecache, missing key is %s", EcStrgAuthenticationField) + klog.Error(err) + return nil, err + } + klog.V(2).Infof("edgecache will be used for volume %s", volumeID) klog.V(3).Infof("edgecache attrib %v", attrib) pvName, exists := attrib[pvNameKey] @@ -350,41 +358,9 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe return nil, err } - // attempt to figure out the name of the kube secret for the storage account key - if len(secretName) == 0 { // if the keyName wasn't already figured out by 'GetAuthEnv' - secretName, exists = pv.ObjectMeta.Annotations[provisionerSecretNameField] - secretNamespace = pv.ObjectMeta.Annotations[provisionerSecretNamespaceField] - if !exists { // if keyName doesn't exist in the PV annotations - klog.Errorf("Failed to discover storage account key name.") - return nil, fmt.Errorf("failed to discover storage account key name") - } - } - - var storageAuthType string - if d.cloud.Config.AzureAuthConfig.UseFederatedWorkloadIdentityExtension { - storageAuthType = "WorkloadIdentity" - } else if d.cloud.Config.AzureAuthConfig.UseManagedIdentityExtension { - storageAuthType = "ManagedIdentity" - } else { - return nil, fmt.Errorf("unable to detect authentication type, cannot continue") - } - - // Pass this to RetryUpdatePVC to confidently add these annotations - var addAnnotations = func(inpvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { - pvcClone := inpvc.DeepCopy() - annotations := map[string]string{ - "external/edgecache-create-volume": "yes", - "external/edgecache-secret-name": secretName, - "external/edgecache-secret-namespace": secretNamespace, - "external/edgecache-account": accountName, - "external/edgecache-container": containerName, - "external/edgecache-authentication": storageAuthType, - } - maps.Copy(pvcClone.ObjectMeta.Annotations, annotations) - return pvcClone - } - err = blobcsiutil.RetryUpdatePVC(d.cloud.KubeClient, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name, addAnnotations) - if err != nil { + annotator := cv.NewPVCAnnotator(d.cloud.KubeClient) + providedAuth := cv.NewBlobAuth(accountName, containerName, secretName, secretNamespace, storageAuthType) + if err := annotator.SendProvisionVolume(pv, d.cloud.Config.AzureAuthConfig, providedAuth); err != nil { return nil, err } diff --git a/pkg/edgecache/cachevolume/pvc_annotator.go b/pkg/edgecache/cachevolume/pvc_annotator.go new file mode 100644 index 000000000..150dbf2ba --- /dev/null +++ b/pkg/edgecache/cachevolume/pvc_annotator.go @@ -0,0 +1,170 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cachevolume + +import ( + "fmt" + + "golang.org/x/exp/maps" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" + blobcsiutil "sigs.k8s.io/blob-csi-driver/pkg/util" + "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" + + v1 "k8s.io/api/core/v1" +) + +const ( + accountAnnotation string = "external/edgecache-account" + containerAnnotation string = "external/edgecache-container" + createVolumeAnnotation string = "external/edgecache-create-volume" + secretNameAnnotation string = "external/edgecache-secret-name" + secretNamespaceAnnotation string = "external/edgecache-secret-namespace" + storageAuthenticationAnnotation string = "external/edgecache-authentication" + provisionerSecretNameField string = "volume.kubernetes.io/provisioner-deletion-secret-name" + provisionerSecretNamespaceField string = "volume.kubernetes.io/provisioner-deletion-secret-namespace" +) + +var ( + validStorageAuthentications = []string{"WorkloadIdentity", "AccountKey"} +) + +type BlobAuth struct { + account string + container string + secretName string + secretNamespace string + authType string +} + +func NewBlobAuth(account, container, secretName, secretNamespace, authType string) BlobAuth { + return BlobAuth{ + account: account, + container: container, + secretName: secretName, + secretNamespace: secretNamespace, + authType: authType, + } +} + +type PVCAnnotator struct { + client clientset.Interface +} + +func NewPVCAnnotator(client clientset.Interface) *PVCAnnotator { + return &PVCAnnotator{ + client: client, + } +} + +type PVCAnnotatorInterface interface { + SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig config.AzureAuthConfig, strgAuthentication, acct, container string) error +} + +func (c *PVCAnnotator) requestAuthIsValid(auth string) bool { + return slices.Contains(validStorageAuthentications, auth) +} + +func (c *PVCAnnotator) needsToBeProvisioned(pvc *v1.PersistentVolumeClaim) bool { + // check if pv connected to the pvc has already been passed to be created + pvState, pvStateOk := pvc.ObjectMeta.Annotations[createVolumeAnnotation] + if pvStateOk && pvState == "no" { + return false + } + + return true +} + +func (c *PVCAnnotator) buildAnnotations(pv *v1.PersistentVolume, cfg config.AzureAuthConfig, providedAuth BlobAuth) (map[string]string, error) { + annotations := map[string]string{ + createVolumeAnnotation: "yes", + accountAnnotation: providedAuth.account, + containerAnnotation: providedAuth.container, + storageAuthenticationAnnotation: providedAuth.authType, + } + + // check if authentication is possible + if providedAuth.authType == "WorkloadIdentity" && !cfg.UseFederatedWorkloadIdentityExtension { + err := fmt.Errorf("workload identity was requested by the csi driver didn't initialize with the workload identity env vars") + klog.Error(err) + return nil, err + + } else if providedAuth.authType == "AccountKey" { + secretName := providedAuth.secretName + secretNamespace := providedAuth.secretNamespace + if len(secretName) == 0 { + // attempt to figure out the name of the kube secret for the storage account key + var secretNameOk bool + var secretNamespaceOk bool + + secretName, secretNameOk = pv.ObjectMeta.Annotations[provisionerSecretNameField] + secretNamespace, secretNamespaceOk = pv.ObjectMeta.Annotations[provisionerSecretNamespaceField] + if !secretNameOk || !secretNamespaceOk { // if keyName doesn't exist in the PV annotations + err := fmt.Errorf("failed to discover storage account key secret; name: '%s' ns: '%s'", secretName, secretNamespace) + klog.Error(err) + return nil, err + } + } + + secretHintAnno := map[string]string{ + secretNameAnnotation: secretName, + secretNamespaceAnnotation: secretNamespace, + } + maps.Copy(annotations, secretHintAnno) + + } + + return annotations, nil +} + +func (c *PVCAnnotator) SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig config.AzureAuthConfig, providedAuth BlobAuth) error { + pvc, err := blobcsiutil.GetPVCByName(c.client, pv.Spec.ClaimRef.Name, pv.Spec.ClaimRef.Namespace) + if err != nil { + return err + } + + if valid := c.requestAuthIsValid(providedAuth.authType); !valid { + err := fmt.Errorf("requested storage auth %s is not a member of valid auths %+v", providedAuth.authType, validStorageAuthentications) + klog.Error(err) + return err + } + + if prepare := c.needsToBeProvisioned(pvc); !prepare { + err := fmt.Errorf("pv is already being provisioned") + klog.Error(err) + return err + } + + annotations, err := c.buildAnnotations(pv, cloudConfig, providedAuth) + if err != nil { + return err + } + + // Pass this to RetryUpdatePVC to confidently add these annotations + var addAnnotations = func(inpvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { + pvcClone := inpvc.DeepCopy() + maps.Copy(pvcClone.ObjectMeta.Annotations, annotations) + return pvcClone + } + err = blobcsiutil.RetryUpdatePVC(c.client, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name, addAnnotations) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/edgecache/cachevolume/pvc_annotator_test.go b/pkg/edgecache/cachevolume/pvc_annotator_test.go new file mode 100644 index 000000000..8d72fe7e7 --- /dev/null +++ b/pkg/edgecache/cachevolume/pvc_annotator_test.go @@ -0,0 +1,209 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cachevolume + +import ( + "context" + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" +) + +const ( + defaultPVCName string = "pvc" + defaultPVCNamespace string = "pvcspace" + defaultPVName string = "pv" + defaultVolumeID string = "volumeid" +) + +func pvc() *v1.PersistentVolumeClaim { + return &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultPVCName, + Namespace: defaultPVCNamespace, + Finalizers: []string{}, + Annotations: make(map[string]string), + }, + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: defaultPVName, + }, + } +} + +func pv() *v1.PersistentVolume { + return &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultPVName, + Finalizers: []string{}, + Annotations: make(map[string]string), + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{CSI: &v1.CSIPersistentVolumeSource{ + VolumeHandle: defaultVolumeID, + }}, + ClaimRef: &v1.ObjectReference{ + Namespace: defaultPVCNamespace, + Name: defaultPVCName, + }, + }, + } +} + +func pvcWithVolume(p *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { + p.Spec = v1.PersistentVolumeClaimSpec{ + VolumeName: defaultPVName, + } + return p +} + +func pvcWithAnnotations(p *v1.PersistentVolumeClaim, annotations map[string]string) *v1.PersistentVolumeClaim { + p.SetAnnotations(annotations) + return p +} + +func pvWithAnnotations(p *v1.PersistentVolume, annotations map[string]string) *v1.PersistentVolume { + p.SetAnnotations(annotations) + return p +} + +type FailureTestCase struct { + name string + annotations map[string]string + config config.AzureAuthConfig + blobAuth BlobAuth +} + +func TestSendProvisionVolumeFailures(t *testing.T) { + testcases := []FailureTestCase{ + { + name: "InvalidAuthenticationType", + annotations: map[string]string{}, + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth("", "", "", "", "SomethingInvalid"), + }, + { + name: "VolumeIsAlreadyBeingProvisioned", + annotations: map[string]string{createVolumeAnnotation: "no"}, + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth("", "", "", "", "WorkloadIdentity"), + }, + { + name: "ConfigDoesntHaveWIEnabled", + annotations: map[string]string{}, + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth("", "", "", "", "WorkloadIdentity"), + }, + { + name: "ProvisionerFieldsAreEmpty", + annotations: map[string]string{}, + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth("", "", "", "", "AccountKey"), + }, + } + + for _, testcase := range testcases { + pvc := pvcWithAnnotations(pvcWithVolume(pvc()), testcase.annotations) + pv := pv() + + client := fake.NewSimpleClientset(pvc, pv) + fakeAnnotator := NewPVCAnnotator(client) + res := fakeAnnotator.SendProvisionVolume(pv, testcase.config, testcase.blobAuth) + assert.NotNil(t, res) + + pvcAfter, _ := client.CoreV1().PersistentVolumeClaims(defaultPVCNamespace).Get(context.TODO(), defaultPVCName, metav1.GetOptions{}) + assert.Equal(t, pvcAfter.GetAnnotations(), testcase.annotations) + } +} + +type SuccessTestCase struct { + name string + config config.AzureAuthConfig + blobAuth BlobAuth + expectedAnnotations map[string]string + pvAnnotations map[string]string +} + +func TestSendProvisionVolumeSuccess(t *testing.T) { + acct := "iamaccount" + container := "iamcontainer" + secret := "shhhhhhhhh" + secretNamespace := "shhhhhhh " + + testcases := []SuccessTestCase{ + { + name: "ValidWIAuth", + config: config.AzureAuthConfig{UseFederatedWorkloadIdentityExtension: true}, + blobAuth: NewBlobAuth(acct, container, "", "", "WorkloadIdentity"), + expectedAnnotations: map[string]string{ + createVolumeAnnotation: "yes", + accountAnnotation: acct, + containerAnnotation: container, + storageAuthenticationAnnotation: "WorkloadIdentity", + }, + pvAnnotations: map[string]string{}, + }, + { + name: "ProvidedAuthWithAccountKey", + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth(acct, container, secret, secretNamespace, "AccountKey"), + expectedAnnotations: map[string]string{ + createVolumeAnnotation: "yes", + accountAnnotation: acct, + containerAnnotation: container, + secretNamespaceAnnotation: secretNamespace, + secretNameAnnotation: secret, + storageAuthenticationAnnotation: "AccountKey", + }, + pvAnnotations: map[string]string{}, + }, + { + name: "ProvisionerFieldsExist", + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth(acct, container, "", "", "AccountKey"), + expectedAnnotations: map[string]string{ + createVolumeAnnotation: "yes", + accountAnnotation: acct, + containerAnnotation: container, + secretNamespaceAnnotation: secretNamespace, + secretNameAnnotation: secret, + storageAuthenticationAnnotation: "AccountKey", + }, + pvAnnotations: map[string]string{ + provisionerSecretNameField: secret, + provisionerSecretNamespaceField: secretNamespace, + }, + }, + } + + for _, testcase := range testcases { + pvc := pvcWithVolume(pvc()) + pv := pvWithAnnotations(pv(), testcase.pvAnnotations) + + client := fake.NewSimpleClientset(pvc, pv) + fakeAnnotator := NewPVCAnnotator(client) + res := fakeAnnotator.SendProvisionVolume(pv, testcase.config, testcase.blobAuth) + assert.Nil(t, res) + + pvcAfter, _ := client.CoreV1().PersistentVolumeClaims(defaultPVCNamespace).Get(context.TODO(), defaultPVCName, metav1.GetOptions{}) + assert.Equal(t, testcase.expectedAnnotations, pvcAfter.GetAnnotations()) + } +} diff --git a/pkg/util/pvutil.go b/pkg/util/pvutil.go index fa2eaeffc..877ed3f83 100644 --- a/pkg/util/pvutil.go +++ b/pkg/util/pvutil.go @@ -59,6 +59,15 @@ func GetPVByName(client clientset.Interface, pvName string) (*v1.PersistentVolum return pv, nil } +func GetPVCByName(client clientset.Interface, pvcName string, namespace string) (*v1.PersistentVolumeClaim, error) { + pvc, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(context.TODO(), pvcName, metav1.GetOptions{}) + if err != nil { + klog.Errorf("unable to get PVC %s", pvcName) + return nil, err + } + return pvc, nil +} + var CustomRetry = wait.Backoff{ Steps: 5, Duration: 10 * time.Millisecond, diff --git a/vendor/k8s.io/kube-openapi/pkg/handler/handler.go b/vendor/k8s.io/kube-openapi/pkg/handler/handler.go index 84e902646..37cb96f1b 100644 --- a/vendor/k8s.io/kube-openapi/pkg/handler/handler.go +++ b/vendor/k8s.io/kube-openapi/pkg/handler/handler.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "strconv" + "sync" "time" "github.com/NYTimes/gziphandler" @@ -98,16 +99,6 @@ func NewOpenAPIServiceLazy(swagger cached.Data[*spec.Swagger]) *OpenAPIService { return o } -func (o *OpenAPIService) getSwaggerBytes() (timedSpec, string, error) { - result := o.jsonCache.Get() - return result.Data, result.Etag, result.Err -} - -func (o *OpenAPIService) getSwaggerPbBytes() (timedSpec, string, error) { - result := o.protoCache.Get() - return result.Data, result.Etag, result.Err -} - func (o *OpenAPIService) UpdateSpec(swagger *spec.Swagger) error { o.UpdateSpecLazy(cached.NewResultOK(swagger, uuid.New().String())) return nil @@ -135,6 +126,9 @@ func RegisterOpenAPIVersionedService(spec *spec.Swagger, servePath string, handl // RegisterOpenAPIVersionedService registers a handler to provide access to provided swagger spec. func (o *OpenAPIService) RegisterOpenAPIVersionedService(servePath string, handler common.PathHandler) error { + // Mutex protects the cache chain + var mutex sync.Mutex + accepted := []struct { Type string SubType string @@ -163,7 +157,9 @@ func (o *OpenAPIService) RegisterOpenAPIVersionedService(servePath string, handl continue } // serve the first matching media type in the sorted clause list + mutex.Lock() result := accepts.GetDataAndEtag.Get() + mutex.Unlock() if result.Err != nil { klog.Errorf("Error in OpenAPI handler: %s", result.Err) // only return a 503 if we have no older cache data to serve diff --git a/vendor/k8s.io/kubernetes/pkg/apis/core/types.go b/vendor/k8s.io/kubernetes/pkg/apis/core/types.go index d8f657b74..5d5b51b1b 100644 --- a/vendor/k8s.io/kubernetes/pkg/apis/core/types.go +++ b/vendor/k8s.io/kubernetes/pkg/apis/core/types.go @@ -2037,7 +2037,8 @@ type SecretEnvSource struct { // HTTPHeader describes a custom header to be used in HTTP probes type HTTPHeader struct { - // The header field name + // The header field name. + // This will be canonicalized upon output, so case-variant names will be understood as the same header. Name string // The header field value Value string diff --git a/vendor/k8s.io/kubernetes/pkg/features/kube_features.go b/vendor/k8s.io/kubernetes/pkg/features/kube_features.go index dae179775..bf641bfa8 100644 --- a/vendor/k8s.io/kubernetes/pkg/features/kube_features.go +++ b/vendor/k8s.io/kubernetes/pkg/features/kube_features.go @@ -1078,7 +1078,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS NetworkPolicyStatus: {Default: false, PreRelease: featuregate.Alpha}, - NewVolumeManagerReconstruction: {Default: true, PreRelease: featuregate.Beta}, + NewVolumeManagerReconstruction: {Default: false, PreRelease: featuregate.Beta}, // disabled for https://github.com/kubernetes/kubernetes/issues/117745 NodeLogQuery: {Default: false, PreRelease: featuregate.Alpha}, @@ -1160,7 +1160,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS NodeInclusionPolicyInPodTopologySpread: {Default: true, PreRelease: featuregate.Beta}, - SELinuxMountReadWriteOncePod: {Default: true, PreRelease: featuregate.Beta}, + SELinuxMountReadWriteOncePod: {Default: false, PreRelease: featuregate.Beta}, // disabled for https://github.com/kubernetes/kubernetes/issues/117745 InPlacePodVerticalScaling: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go b/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go index 57c028150..ec7f6da4b 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go @@ -40,22 +40,22 @@ const ( // SetVolumeOwnership modifies the given volume to be owned by // fsGroup, and sets SetGid so that newly created files are owned by // fsGroup. If fsGroup is nil nothing is done. -func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error { +func SetVolumeOwnership(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error { if fsGroup == nil { return nil } timer := time.AfterFunc(30*time.Second, func() { - klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", mounter.GetPath()) + klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", dir) }) defer timer.Stop() - if skipPermissionChange(mounter, fsGroup, fsGroupChangePolicy) { - klog.V(3).InfoS("Skipping permission and ownership change for volume", "path", mounter.GetPath()) + if skipPermissionChange(mounter, dir, fsGroup, fsGroupChangePolicy) { + klog.V(3).InfoS("Skipping permission and ownership change for volume", "path", dir) return nil } - err := walkDeep(mounter.GetPath(), func(path string, info os.FileInfo, err error) error { + err := walkDeep(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -104,14 +104,12 @@ func changeFilePermission(filename string, fsGroup *int64, readonly bool, info o return nil } -func skipPermissionChange(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool { - dir := mounter.GetPath() - +func skipPermissionChange(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool { if fsGroupChangePolicy == nil || *fsGroupChangePolicy != v1.FSGroupChangeOnRootMismatch { klog.V(4).InfoS("Perform recursive ownership change for directory", "path", dir) return false } - return !requiresPermissionChange(mounter.GetPath(), fsGroup, mounter.GetAttributes().ReadOnly) + return !requiresPermissionChange(dir, fsGroup, mounter.GetAttributes().ReadOnly) } func requiresPermissionChange(rootDir string, fsGroup *int64, readonly bool) bool { diff --git a/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go b/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go index 20c56d4b6..3b5a200a6 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go @@ -24,6 +24,6 @@ import ( "k8s.io/kubernetes/pkg/volume/util/types" ) -func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error { +func SetVolumeOwnership(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error { return nil } diff --git a/vendor/k8s.io/kubernetes/test/e2e/framework/get.go b/vendor/k8s.io/kubernetes/test/e2e/framework/get.go index ffa26d307..1e1a83856 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/framework/get.go +++ b/vendor/k8s.io/kubernetes/test/e2e/framework/get.go @@ -100,7 +100,10 @@ func ShouldRetry(err error) (retry bool, retryAfter time.Duration) { } // these errors indicate a transient error that should be retried. - if apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) || errors.As(err, &transientError{}) { + if apierrors.IsTimeout(err) || + apierrors.IsTooManyRequests(err) || + apierrors.IsServiceUnavailable(err) || + errors.As(err, &transientError{}) { return true, 0 } diff --git a/vendor/k8s.io/kubernetes/test/e2e/framework/kubectl/builder.go b/vendor/k8s.io/kubernetes/test/e2e/framework/kubectl/builder.go index 916389589..3811fa572 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/framework/kubectl/builder.go +++ b/vendor/k8s.io/kubernetes/test/e2e/framework/kubectl/builder.go @@ -22,6 +22,7 @@ import ( "io" "net" "net/url" + "os" "os/exec" "strings" "syscall" @@ -48,9 +49,12 @@ func NewKubectlCommand(namespace string, args ...string) *KubectlBuilder { return b } -// WithEnv sets the given environment and returns itself. -func (b *KubectlBuilder) WithEnv(env []string) *KubectlBuilder { - b.cmd.Env = env +// WithEnv appends the given environment and returns itself. +func (b *KubectlBuilder) AppendEnv(env []string) *KubectlBuilder { + if b.cmd.Env == nil { + b.cmd.Env = os.Environ() + } + b.cmd.Env = append(b.cmd.Env, env...) return b } diff --git a/vendor/modules.txt b/vendor/modules.txt index fdd1e5fc5..e21035c06 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1366,8 +1366,8 @@ k8s.io/kms/apis/v1beta1 k8s.io/kms/apis/v2 k8s.io/kms/pkg/service k8s.io/kms/pkg/util -# k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a -## explicit; go 1.18 +# k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f +## explicit; go 1.19 k8s.io/kube-openapi/pkg/builder k8s.io/kube-openapi/pkg/builder3 k8s.io/kube-openapi/pkg/builder3/util @@ -1396,7 +1396,7 @@ k8s.io/kubectl/pkg/util/podutils ## explicit; go 1.20 k8s.io/kubelet/pkg/apis k8s.io/kubelet/pkg/apis/stats/v1alpha1 -# k8s.io/kubernetes v1.27.1 +# k8s.io/kubernetes v1.27.3 ## explicit; go 1.20 k8s.io/kubernetes/pkg/api/legacyscheme k8s.io/kubernetes/pkg/api/service @@ -1493,7 +1493,7 @@ k8s.io/utils/pointer k8s.io/utils/strings k8s.io/utils/strings/slices k8s.io/utils/trace -# sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.1 +# sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 ## explicit; go 1.17 sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go index 68a3ebf12..541e6a5aa 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go @@ -132,6 +132,11 @@ type grpcTunnel struct { // serving. done chan struct{} + // started is an atomic bool represented as a 0 or 1, and set to true when a single-use tunnel has been started (dialed). + // started should only be accessed through atomic methods. + // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+ + started uint32 + // closing is an atomic bool represented as a 0 or 1, and set to true when the tunnel is being closed. // closing should only be accessed through atomic methods. // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+ @@ -197,6 +202,7 @@ func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *g conns: connectionManager{conns: make(map[int64]*conn)}, readTimeoutSeconds: 10, done: make(chan struct{}), + started: 0, } s := metrics.ClientConnectionStatusCreated t.prevStatus.Store(s) @@ -393,6 +399,11 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s } func (t *grpcTunnel) dialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) { + prevStarted := atomic.SwapUint32(&t.started, 1) + if prevStarted != 0 { + return nil, &dialFailure{"single-use dialer already dialed", metrics.DialFailureAlreadyStarted} + } + select { case <-t.done: return nil, errors.New("tunnel is closed") @@ -515,11 +526,11 @@ func (t *grpcTunnel) Recv() (*client.Packet, error) { const segment = commonmetrics.SegmentToClient pkt, err := t.stream.Recv() - if err != nil && err != io.EOF { - metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) - } if err != nil { - return pkt, err + if err != io.EOF { + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) + } + return nil, err } metrics.Metrics.ObservePacket(segment, pkt.Type) return pkt, nil diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go index 03e9d94da..ce942e3eb 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go @@ -62,6 +62,8 @@ const ( // DialFailureTunnelClosed indicates that the client connection was closed before the dial could // complete. DialFailureTunnelClosed DialFailureReason = "tunnelclosed" + // DialFailureAlreadyStarted indicates that a single-use tunnel dialer was already used once. + DialFailureAlreadyStarted DialFailureReason = "tunnelstarted" ) type ClientConnectionStatus string