From a39cf5feb28d711e25ee917490090cb5a8fcca65 Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Tue, 27 Jun 2023 17:21:34 -0400 Subject: [PATCH 01/10] azure: only check for wi creds if config is valid --- pkg/blob/azure.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/blob/azure.go b/pkg/blob/azure.go index 359fcc6cc..751fcc9dd 100644 --- a/pkg/blob/azure.go +++ b/pkg/blob/azure.go @@ -89,21 +89,21 @@ func getCloudProvider(kubeconfig, nodeID, secretName, secretNamespace, userAgent config, err = az.GetConfigFromSecret() if err == nil && config != nil { fromSecret = true + + if tenantID := os.Getenv("AZURE_TENANT_ID"); tenantID != "" { + config.TenantID = tenantID + } + if clientID := os.Getenv("AZURE_CLIENT_ID"); clientID != "" { + config.AADClientID = clientID + } + if federatedTokenFile := os.Getenv("AZURE_FEDERATED_TOKEN_FILE"); federatedTokenFile != "" { + config.AADFederatedTokenFile = federatedTokenFile + config.UseFederatedWorkloadIdentityExtension = true + } } if err != nil { klog.V(2).Infof("InitializeCloudFromSecret: failed to get cloud config from secret %s/%s: %v", az.SecretNamespace, az.SecretName, err) } - - if tenantID := os.Getenv("AZURE_TENANT_ID"); tenantID != "" { - config.TenantID = tenantID - } - if clientID := os.Getenv("AZURE_CLIENT_ID"); clientID != "" { - config.AADClientID = clientID - } - if federatedTokenFile := os.Getenv("AZURE_FEDERATED_TOKEN_FILE"); federatedTokenFile != "" { - config.AADFederatedTokenFile = federatedTokenFile - config.UseFederatedWorkloadIdentityExtension = true - } } if config == nil { From a58b4f815349704923ba6910819e61fbb2dfbef1 Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Tue, 27 Jun 2023 17:22:07 -0400 Subject: [PATCH 02/10] blob: allow for an authentication type parameter for storage classes --- pkg/blob/blob.go | 1 + pkg/blob/controllerserver.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go index f427c5118..f3f867e21 100644 --- a/pkg/blob/blob.go +++ b/pkg/blob/blob.go @@ -107,6 +107,7 @@ const ( useDataPlaneAPIField = "usedataplaneapi" provisionerSecretNameField = "volume.kubernetes.io/provisioner-deletion-secret-name" provisionerSecretNamespaceField = "volume.kubernetes.io/provisioner-deletion-secret-namespace" + EcStrgAuthenticationField = "edgecache-storage-auth" // See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names containerNameMinLength = 3 diff --git a/pkg/blob/controllerserver.go b/pkg/blob/controllerserver.go index 2326d931f..ed1f552a4 100644 --- a/pkg/blob/controllerserver.go +++ b/pkg/blob/controllerserver.go @@ -166,6 +166,8 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) accessTier = v case networkEndpointTypeField: networkEndpointType = v + case EcStrgAuthenticationField: + containerNameReplaceMap[EcStrgAuthenticationField] = v case mountPermissionsField: // only do validations here, used in NodeStageVolume, NodePublishVolume if v != "" { From e1960aeaefc6455e6a5701860951ee1d36111ab5 Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Tue, 27 Jun 2023 17:44:43 -0400 Subject: [PATCH 03/10] nodeserver: pass different annotations depending on authentication --- pkg/blob/nodeserver.go | 55 ++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/pkg/blob/nodeserver.go b/pkg/blob/nodeserver.go index 28f5aec67..cf7dc2d56 100644 --- a/pkg/blob/nodeserver.go +++ b/pkg/blob/nodeserver.go @@ -360,26 +360,51 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } } - var storageAuthType string - if d.cloud.Config.AzureAuthConfig.UseFederatedWorkloadIdentityExtension { - storageAuthType = "WorkloadIdentity" - } else if d.cloud.Config.AzureAuthConfig.UseManagedIdentityExtension { - storageAuthType = "ManagedIdentity" - } else { - return nil, fmt.Errorf("unable to detect authentication type, cannot continue") + + // get authentication method + storageAuthType, storageAuthTypeOk := attrib[EcStrgAuthenticationField] + if !storageAuthTypeOk { + err = fmt.Errorf("could not determine storage authentication for edgecache, missing key is %s.", EcStrgAuthenticationField) + klog.Error(err) + return nil, err + } + + // initialize the annotations for the pvc + annotations := map[string]string{ + "external/edgecache-create-volume": "yes", + "external/edgecache-account": accountName, + "external/edgecache-container": containerName, + "external/edgecache-authentication": storageAuthType, + } + + // check if authentication is possible + if storageAuthType == "WorkloadIdentity" && !d.cloud.Config.AzureAuthConfig.UseFederatedWorkloadIdentityExtension { + err = fmt.Errorf("workload identity was requested by the csi driver didn't initialize with the workload identity env vars") + klog.Error(err) + return nil, err + + } else if storageAuthType == "AccountKey" { + if len(secretName) == 0 { + // attempt to figure out the name of the kube secret for the storage account key + var secretNameOk bool + var secretNamespaceOk bool + + secretName, secretNameOk = pv.ObjectMeta.Annotations[provisionerSecretNameField] + secretNamespace, secretNamespaceOk = pv.ObjectMeta.Annotations[provisionerSecretNamespaceField] + if !secretNameOk || !secretNamespaceOk { // if keyName doesn't exist in the PV annotations + err = fmt.Errorf("failed to discover storage account key secret name or namespace") + klog.Error(err) + return nil, err + } + } + + annotations["external/edgecache-secret-name"] = secretName + annotations["external/edgecache-secret-namespace"] = secretNamespace } // Pass this to RetryUpdatePVC to confidently add these annotations var addAnnotations = func(inpvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { pvcClone := inpvc.DeepCopy() - annotations := map[string]string{ - "external/edgecache-create-volume": "yes", - "external/edgecache-secret-name": secretName, - "external/edgecache-secret-namespace": secretNamespace, - "external/edgecache-account": accountName, - "external/edgecache-container": containerName, - "external/edgecache-authentication": storageAuthType, - } maps.Copy(pvcClone.ObjectMeta.Annotations, annotations) return pvcClone } From 00e210f57f7e46244d7454141f9fa4d89edda925 Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Tue, 27 Jun 2023 17:47:58 -0400 Subject: [PATCH 04/10] nodeserver: only try to mount volume if provisioning hasnt started already --- pkg/blob/nodeserver.go | 18 ++++++++++-------- pkg/util/pvutil.go | 9 +++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/blob/nodeserver.go b/pkg/blob/nodeserver.go index cf7dc2d56..12f900b14 100644 --- a/pkg/blob/nodeserver.go +++ b/pkg/blob/nodeserver.go @@ -350,16 +350,18 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe return nil, err } - // attempt to figure out the name of the kube secret for the storage account key - if len(secretName) == 0 { // if the keyName wasn't already figured out by 'GetAuthEnv' - secretName, exists = pv.ObjectMeta.Annotations[provisionerSecretNameField] - secretNamespace = pv.ObjectMeta.Annotations[provisionerSecretNamespaceField] - if !exists { // if keyName doesn't exist in the PV annotations - klog.Errorf("Failed to discover storage account key name.") - return nil, fmt.Errorf("failed to discover storage account key name") - } + pvc, err := blobcsiutil.GetPVCByName(d.cloud.KubeClient, pv.Spec.ClaimRef.Name, pv.Spec.ClaimRef.Namespace) + if err != nil { + return nil, err } + // check if this pv is already trying to be mounted + pvState, pvStateOk := pvc.ObjectMeta.Annotations["external/edgecache-create-volume"] + if pvStateOk && pvState == "no" { + err = fmt.Errorf("pv is already being provisioned") + klog.Error(err) + return nil, err + } // get authentication method storageAuthType, storageAuthTypeOk := attrib[EcStrgAuthenticationField] diff --git a/pkg/util/pvutil.go b/pkg/util/pvutil.go index fa2eaeffc..877ed3f83 100644 --- a/pkg/util/pvutil.go +++ b/pkg/util/pvutil.go @@ -59,6 +59,15 @@ func GetPVByName(client clientset.Interface, pvName string) (*v1.PersistentVolum return pv, nil } +func GetPVCByName(client clientset.Interface, pvcName string, namespace string) (*v1.PersistentVolumeClaim, error) { + pvc, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(context.TODO(), pvcName, metav1.GetOptions{}) + if err != nil { + klog.Errorf("unable to get PVC %s", pvcName) + return nil, err + } + return pvc, nil +} + var CustomRetry = wait.Backoff{ Steps: 5, Duration: 10 * time.Millisecond, From d568fc85d7488a173c76bbf22aa4e65686df5141 Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Thu, 29 Jun 2023 09:18:39 -0400 Subject: [PATCH 05/10] edgecache: move annotation creation to separate module --- pkg/blob/blob.go | 2 - pkg/blob/nodeserver.go | 75 +++---------- pkg/edgecache/cachevolume/helper.go | 159 ++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 65 deletions(-) create mode 100644 pkg/edgecache/cachevolume/helper.go diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go index f3f867e21..7c212e2e9 100644 --- a/pkg/blob/blob.go +++ b/pkg/blob/blob.go @@ -105,8 +105,6 @@ const ( networkEndpointTypeField = "networkendpointtype" mountPermissionsField = "mountpermissions" useDataPlaneAPIField = "usedataplaneapi" - provisionerSecretNameField = "volume.kubernetes.io/provisioner-deletion-secret-name" - provisionerSecretNamespaceField = "volume.kubernetes.io/provisioner-deletion-secret-namespace" EcStrgAuthenticationField = "edgecache-storage-auth" // See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names diff --git a/pkg/blob/nodeserver.go b/pkg/blob/nodeserver.go index 12f900b14..e0d076614 100644 --- a/pkg/blob/nodeserver.go +++ b/pkg/blob/nodeserver.go @@ -27,6 +27,7 @@ import ( "time" "sigs.k8s.io/blob-csi-driver/pkg/edgecache" + cv "sigs.k8s.io/blob-csi-driver/pkg/edgecache/cachevolume" blobcsiutil "sigs.k8s.io/blob-csi-driver/pkg/util" "github.com/Azure/azure-sdk-for-go/storage" @@ -42,7 +43,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - maps "golang.org/x/exp/maps" "golang.org/x/net/context" "google.golang.org/grpc" mount_azure_blob "sigs.k8s.io/blob-csi-driver/pkg/blobfuse-proxy/pb" @@ -336,6 +336,14 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } if protocol == EcProtocol { + // get authentication method + storageAuthType, storageAuthTypeOk := attrib[EcStrgAuthenticationField] + if !storageAuthTypeOk { + err = fmt.Errorf("could not determine storage authentication for edgecache, missing key is %s", EcStrgAuthenticationField) + klog.Error(err) + return nil, err + } + klog.V(2).Infof("edgecache will be used for volume %s", volumeID) klog.V(3).Infof("edgecache attrib %v", attrib) pvName, exists := attrib[pvNameKey] @@ -350,68 +358,9 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe return nil, err } - pvc, err := blobcsiutil.GetPVCByName(d.cloud.KubeClient, pv.Spec.ClaimRef.Name, pv.Spec.ClaimRef.Namespace) - if err != nil { - return nil, err - } - - // check if this pv is already trying to be mounted - pvState, pvStateOk := pvc.ObjectMeta.Annotations["external/edgecache-create-volume"] - if pvStateOk && pvState == "no" { - err = fmt.Errorf("pv is already being provisioned") - klog.Error(err) - return nil, err - } - - // get authentication method - storageAuthType, storageAuthTypeOk := attrib[EcStrgAuthenticationField] - if !storageAuthTypeOk { - err = fmt.Errorf("could not determine storage authentication for edgecache, missing key is %s.", EcStrgAuthenticationField) - klog.Error(err) - return nil, err - } - - // initialize the annotations for the pvc - annotations := map[string]string{ - "external/edgecache-create-volume": "yes", - "external/edgecache-account": accountName, - "external/edgecache-container": containerName, - "external/edgecache-authentication": storageAuthType, - } - - // check if authentication is possible - if storageAuthType == "WorkloadIdentity" && !d.cloud.Config.AzureAuthConfig.UseFederatedWorkloadIdentityExtension { - err = fmt.Errorf("workload identity was requested by the csi driver didn't initialize with the workload identity env vars") - klog.Error(err) - return nil, err - - } else if storageAuthType == "AccountKey" { - if len(secretName) == 0 { - // attempt to figure out the name of the kube secret for the storage account key - var secretNameOk bool - var secretNamespaceOk bool - - secretName, secretNameOk = pv.ObjectMeta.Annotations[provisionerSecretNameField] - secretNamespace, secretNamespaceOk = pv.ObjectMeta.Annotations[provisionerSecretNamespaceField] - if !secretNameOk || !secretNamespaceOk { // if keyName doesn't exist in the PV annotations - err = fmt.Errorf("failed to discover storage account key secret name or namespace") - klog.Error(err) - return nil, err - } - } - - annotations["external/edgecache-secret-name"] = secretName - annotations["external/edgecache-secret-namespace"] = secretNamespace - } - - // Pass this to RetryUpdatePVC to confidently add these annotations - var addAnnotations = func(inpvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { - pvcClone := inpvc.DeepCopy() - maps.Copy(pvcClone.ObjectMeta.Annotations, annotations) - return pvcClone - } - err = blobcsiutil.RetryUpdatePVC(d.cloud.KubeClient, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name, addAnnotations) - if err != nil { + cvHelper := cv.NewCVHelper(d.cloud.KubeClient) + providedAuth := cv.NewBlobAuth(accountName, containerName, secretName, secretNamespace, storageAuthType) + if err := cvHelper.SendProvisionVolume(pv, d.cloud.Config.AzureAuthConfig, providedAuth); err != nil { return nil, err } diff --git a/pkg/edgecache/cachevolume/helper.go b/pkg/edgecache/cachevolume/helper.go new file mode 100644 index 000000000..d59d13b15 --- /dev/null +++ b/pkg/edgecache/cachevolume/helper.go @@ -0,0 +1,159 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cachevolume + +import ( + "fmt" + + "golang.org/x/exp/maps" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + blobcsiutil "sigs.k8s.io/blob-csi-driver/pkg/util" + "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" + + v1 "k8s.io/api/core/v1" +) + +const ( + accountAnnotation string = "external/edgecache-account" + containerAnnotation string = "external/edgecache-container" + createVolumeAnnotation string = "external/edgecache-create-volume" + secretNameAnnotation string = "external/edgecache-secret-name" + secretNamespaceAnnotation string = "external/edgecache-secret-namespace" + storageAuthenticationAnnotation string = "external/edgecache-authentication" + provisionerSecretNameField string = "volume.kubernetes.io/provisioner-deletion-secret-name" + provisionerSecretNamespaceField string = "volume.kubernetes.io/provisioner-deletion-secret-namespace" +) + +type BlobAuth struct { + account string + container string + secretName string + secretNamespace string + authType string +} + +func NewBlobAuth(account, container, secretName, secretNamespace, authType string) BlobAuth { + return BlobAuth{ + account: account, + container: container, + secretName: secretName, + secretNamespace: secretNamespace, + authType: authType, + } +} + +type CVHelper struct { + client clientset.Interface +} + +func NewCVHelper(client clientset.Interface) *CVHelper { + return &CVHelper{ + client: client, + } +} + +type CVHelperInterface interface { + SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig config.AzureAuthConfig, strgAuthentication, acct, container string) error +} + +func (c *CVHelper) needsToBeProvisioned(pvc *v1.PersistentVolumeClaim) bool { + // check if pv connected to the pvc has already been passed to be created + pvState, pvStateOk := pvc.ObjectMeta.Annotations[createVolumeAnnotation] + if pvStateOk && pvState == "no" { + return false + } + + return true +} + +func (c *CVHelper) buildAnnotations(pv *v1.PersistentVolume, cfg config.AzureAuthConfig, providedAuth BlobAuth) (map[string]string, error) { + annotations := map[string]string{ + createVolumeAnnotation: "yes", + accountAnnotation: providedAuth.account, + containerAnnotation: providedAuth.container, + storageAuthenticationAnnotation: providedAuth.authType, + } + + // check if authentication is possible + if providedAuth.authType == "WorkloadIdentity" && !cfg.UseFederatedWorkloadIdentityExtension { + err := fmt.Errorf("workload identity was requested by the csi driver didn't initialize with the workload identity env vars") + klog.Error(err) + return nil, err + + } else if providedAuth.authType == "AccountKey" { + secretName := providedAuth.secretName + secretNamespace := providedAuth.secretNamespace + if len(secretName) == 0 { + // attempt to figure out the name of the kube secret for the storage account key + var secretNameOk bool + var secretNamespaceOk bool + + secretName, secretNameOk = pv.ObjectMeta.Annotations[provisionerSecretNameField] + secretNamespace, secretNamespaceOk = pv.ObjectMeta.Annotations[provisionerSecretNamespaceField] + if !secretNameOk || !secretNamespaceOk { // if keyName doesn't exist in the PV annotations + err := fmt.Errorf("failed to discover storage account key secret; name: '%s' ns: '%s'", secretName, secretNamespace) + klog.Error(err) + return nil, err + } + } + + secretHintAnno := map[string]string{ + secretNameAnnotation: secretName, + secretNamespaceAnnotation: secretNamespace, + } + maps.Copy(annotations, secretHintAnno) + + } else { + err := fmt.Errorf("requested authentication method: '%s' is unknown, cannot continue", providedAuth.authType) + klog.Error(err) + return nil, err + } + + return annotations, nil +} + +func (c *CVHelper) SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig config.AzureAuthConfig, providedAuth BlobAuth) error { + pvc, err := blobcsiutil.GetPVCByName(c.client, pv.Spec.ClaimRef.Name, pv.Spec.ClaimRef.Namespace) + if err != nil { + return err + } + + if prepare := c.needsToBeProvisioned(pvc); !prepare { + err := fmt.Errorf("pv is already being provisioned") + klog.Error(err) + return err + } + + annotations, err := c.buildAnnotations(pv, cloudConfig, providedAuth) + if err != nil { + return err + } + + // Pass this to RetryUpdatePVC to confidently add these annotations + var addAnnotations = func(inpvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { + pvcClone := inpvc.DeepCopy() + maps.Copy(pvcClone.ObjectMeta.Annotations, annotations) + return pvcClone + } + err = blobcsiutil.RetryUpdatePVC(c.client, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name, addAnnotations) + if err != nil { + return err + } + + return nil +} From b913aa3800604db07cbc56a23deb943b0762f198 Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Thu, 29 Jun 2023 10:33:25 -0400 Subject: [PATCH 06/10] cachevolume: add UTs for helper code --- pkg/edgecache/cachevolume/helper.go | 19 ++- pkg/edgecache/cachevolume/helper_test.go | 209 +++++++++++++++++++++++ 2 files changed, 224 insertions(+), 4 deletions(-) create mode 100644 pkg/edgecache/cachevolume/helper_test.go diff --git a/pkg/edgecache/cachevolume/helper.go b/pkg/edgecache/cachevolume/helper.go index d59d13b15..5c873bd13 100644 --- a/pkg/edgecache/cachevolume/helper.go +++ b/pkg/edgecache/cachevolume/helper.go @@ -22,6 +22,7 @@ import ( "golang.org/x/exp/maps" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" blobcsiutil "sigs.k8s.io/blob-csi-driver/pkg/util" "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" @@ -39,6 +40,10 @@ const ( provisionerSecretNamespaceField string = "volume.kubernetes.io/provisioner-deletion-secret-namespace" ) +var ( + validStorageAuthentications = []string{"WorkloadIdentity", "AccountKey"} +) + type BlobAuth struct { account string container string @@ -71,6 +76,10 @@ type CVHelperInterface interface { SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig config.AzureAuthConfig, strgAuthentication, acct, container string) error } +func (c *CVHelper) requestAuthIsValid(auth string) bool { + return slices.Contains(validStorageAuthentications, auth) +} + func (c *CVHelper) needsToBeProvisioned(pvc *v1.PersistentVolumeClaim) bool { // check if pv connected to the pvc has already been passed to be created pvState, pvStateOk := pvc.ObjectMeta.Annotations[createVolumeAnnotation] @@ -118,10 +127,6 @@ func (c *CVHelper) buildAnnotations(pv *v1.PersistentVolume, cfg config.AzureAut } maps.Copy(annotations, secretHintAnno) - } else { - err := fmt.Errorf("requested authentication method: '%s' is unknown, cannot continue", providedAuth.authType) - klog.Error(err) - return nil, err } return annotations, nil @@ -133,6 +138,12 @@ func (c *CVHelper) SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig conf return err } + if valid := c.requestAuthIsValid(providedAuth.authType); !valid { + err := fmt.Errorf("requested storage auth %s is not a member of valid auths %+v", providedAuth.authType, validStorageAuthentications) + klog.Error(err) + return err + } + if prepare := c.needsToBeProvisioned(pvc); !prepare { err := fmt.Errorf("pv is already being provisioned") klog.Error(err) diff --git a/pkg/edgecache/cachevolume/helper_test.go b/pkg/edgecache/cachevolume/helper_test.go new file mode 100644 index 000000000..b347fd588 --- /dev/null +++ b/pkg/edgecache/cachevolume/helper_test.go @@ -0,0 +1,209 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cachevolume + +import ( + "context" + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" +) + +const ( + defaultPVCName string = "pvc" + defaultPVCNamespace string = "pvcspace" + defaultPVName string = "pv" + defaultVolumeID string = "volumeid" +) + +func pvc() *v1.PersistentVolumeClaim { + return &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultPVCName, + Namespace: defaultPVCNamespace, + Finalizers: []string{}, + Annotations: make(map[string]string), + }, + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: defaultPVName, + }, + } +} + +func pv() *v1.PersistentVolume { + return &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultPVName, + Finalizers: []string{}, + Annotations: make(map[string]string), + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{CSI: &v1.CSIPersistentVolumeSource{ + VolumeHandle: defaultVolumeID, + }}, + ClaimRef: &v1.ObjectReference{ + Namespace: defaultPVCNamespace, + Name: defaultPVCName, + }, + }, + } +} + +func pvcWithVolume(p *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { + p.Spec = v1.PersistentVolumeClaimSpec{ + VolumeName: defaultPVName, + } + return p +} + +func pvcWithAnnotations(p *v1.PersistentVolumeClaim, annotations map[string]string) *v1.PersistentVolumeClaim { + p.SetAnnotations(annotations) + return p +} + +func pvWithAnnotations(p *v1.PersistentVolume, annotations map[string]string) *v1.PersistentVolume { + p.SetAnnotations(annotations) + return p +} + +type FailureTestCase struct { + name string + annotations map[string]string + config config.AzureAuthConfig + blobAuth BlobAuth +} + +func TestSendProvisionVolumeFailures(t *testing.T) { + testcases := []FailureTestCase{ + { + name: "InvalidAuthenticationType", + annotations: map[string]string{}, + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth("", "", "", "", "SomethingInvalid"), + }, + { + name: "VolumeIsAlreadyBeingProvisioned", + annotations: map[string]string{createVolumeAnnotation: "no"}, + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth("", "", "", "", "WorkloadIdentity"), + }, + { + name: "ConfigDoesntHaveWIEnabled", + annotations: map[string]string{}, + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth("", "", "", "", "WorkloadIdentity"), + }, + { + name: "ProvisionerFieldsAreEmpty", + annotations: map[string]string{}, + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth("", "", "", "", "AccountKey"), + }, + } + + for _, testcase := range testcases { + pvc := pvcWithAnnotations(pvcWithVolume(pvc()), testcase.annotations) + pv := pv() + + client := fake.NewSimpleClientset(pvc, pv) + fkHelper := NewCVHelper(client) + res := fkHelper.SendProvisionVolume(pv, testcase.config, testcase.blobAuth) + assert.NotNil(t, res) + + pvcAfter, _ := client.CoreV1().PersistentVolumeClaims(defaultPVCNamespace).Get(context.TODO(), defaultPVCName, metav1.GetOptions{}) + assert.Equal(t, pvcAfter.GetAnnotations(), testcase.annotations) + } +} + +type SuccessTestCase struct { + name string + config config.AzureAuthConfig + blobAuth BlobAuth + expectedAnnotations map[string]string + pvAnnotations map[string]string +} + +func TestSendProvisionVolumeSuccess(t *testing.T) { + acct := "iamaccount" + container := "iamcontainer" + secret := "shhhhhhhhh" + secretNamespace := "shhhhhhh " + + testcases := []SuccessTestCase{ + { + name: "ValidWIAuth", + config: config.AzureAuthConfig{UseFederatedWorkloadIdentityExtension: true}, + blobAuth: NewBlobAuth(acct, container, "", "", "WorkloadIdentity"), + expectedAnnotations: map[string]string{ + createVolumeAnnotation: "yes", + accountAnnotation: acct, + containerAnnotation: container, + storageAuthenticationAnnotation: "WorkloadIdentity", + }, + pvAnnotations: map[string]string{}, + }, + { + name: "ProvidedAuthWithAccountKey", + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth(acct, container, secret, secretNamespace, "AccountKey"), + expectedAnnotations: map[string]string{ + createVolumeAnnotation: "yes", + accountAnnotation: acct, + containerAnnotation: container, + secretNamespaceAnnotation: secretNamespace, + secretNameAnnotation: secret, + storageAuthenticationAnnotation: "AccountKey", + }, + pvAnnotations: map[string]string{}, + }, + { + name: "ProvisionerFieldsExist", + config: config.AzureAuthConfig{}, + blobAuth: NewBlobAuth(acct, container, "", "", "AccountKey"), + expectedAnnotations: map[string]string{ + createVolumeAnnotation: "yes", + accountAnnotation: acct, + containerAnnotation: container, + secretNamespaceAnnotation: secretNamespace, + secretNameAnnotation: secret, + storageAuthenticationAnnotation: "AccountKey", + }, + pvAnnotations: map[string]string{ + provisionerSecretNameField: secret, + provisionerSecretNamespaceField: secretNamespace, + }, + }, + } + + for _, testcase := range testcases { + pvc := pvcWithVolume(pvc()) + pv := pvWithAnnotations(pv(), testcase.pvAnnotations) + + client := fake.NewSimpleClientset(pvc, pv) + fkHelper := NewCVHelper(client) + res := fkHelper.SendProvisionVolume(pv, testcase.config, testcase.blobAuth) + assert.Nil(t, res) + + pvcAfter, _ := client.CoreV1().PersistentVolumeClaims(defaultPVCNamespace).Get(context.TODO(), defaultPVCName, metav1.GetOptions{}) + assert.Equal(t, testcase.expectedAnnotations, pvcAfter.GetAnnotations()) + } +} From 3e31985bd433393512ec89a5687a0d85dc093ab6 Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Thu, 29 Jun 2023 10:37:39 -0400 Subject: [PATCH 07/10] pls pls precommits --- pkg/blob/blob.go | 116 +++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go index 7c212e2e9..2d53f76e8 100644 --- a/pkg/blob/blob.go +++ b/pkg/blob/blob.go @@ -48,64 +48,64 @@ import ( const ( // DefaultDriverName holds the name of the csi-driver - DefaultDriverName = "blob.csi.azure.com" - blobCSIDriverName = "blob_csi_driver" - separator = "#" - volumeIDTemplate = "%s#%s#%s#%s#%s#%s" - secretNameTemplate = "azure-storage-account-%s-secret" - serverNameField = "server" - storageEndpointSuffixField = "storageendpointsuffix" - tagsField = "tags" - matchTagsField = "matchtags" - protocolField = "protocol" - accountNameField = "accountname" - accountKeyField = "accountkey" - storageAccountField = "storageaccount" - storageAccountTypeField = "storageaccounttype" - skuNameField = "skuname" - subscriptionIDField = "subscriptionid" - resourceGroupField = "resourcegroup" - locationField = "location" - secretNameField = "secretname" - secretNamespaceField = "secretnamespace" - containerNameField = "containername" - containerNamePrefixField = "containernameprefix" - storeAccountKeyField = "storeaccountkey" - isHnsEnabledField = "ishnsenabled" - softDeleteBlobsField = "softdeleteblobs" - softDeleteContainersField = "softdeletecontainers" - enableBlobVersioningField = "enableblobversioning" - getAccountKeyFromSecretField = "getaccountkeyfromsecret" - storageSPNClientIDField = "azurestoragespnclientid" - storageSPNTenantIDField = "azurestoragespntenantid" - keyVaultURLField = "keyvaulturl" - keyVaultSecretNameField = "keyvaultsecretname" - keyVaultSecretVersionField = "keyvaultsecretversion" - storageAccountNameField = "storageaccountname" - allowBlobPublicAccessField = "allowblobpublicaccess" - requireInfraEncryptionField = "requireinfraencryption" - ephemeralField = "csi.storage.k8s.io/ephemeral" - podNamespaceField = "csi.storage.k8s.io/pod.namespace" - mountOptionsField = "mountoptions" - falseValue = "false" - trueValue = "true" - defaultSecretAccountName = "azurestorageaccountname" - defaultSecretAccountKey = "azurestorageaccountkey" - accountSasTokenField = "azurestorageaccountsastoken" - msiSecretField = "msisecret" - storageSPNClientSecretField = "azurestoragespnclientsecret" - EcProtocol = "edgecache" - Fuse = "fuse" - Fuse2 = "fuse2" - NFS = "nfs" - vnetResourceGroupField = "vnetresourcegroup" - vnetNameField = "vnetname" - subnetNameField = "subnetname" - accessTierField = "accesstier" - networkEndpointTypeField = "networkendpointtype" - mountPermissionsField = "mountpermissions" - useDataPlaneAPIField = "usedataplaneapi" - EcStrgAuthenticationField = "edgecache-storage-auth" + DefaultDriverName = "blob.csi.azure.com" + blobCSIDriverName = "blob_csi_driver" + separator = "#" + volumeIDTemplate = "%s#%s#%s#%s#%s#%s" + secretNameTemplate = "azure-storage-account-%s-secret" + serverNameField = "server" + storageEndpointSuffixField = "storageendpointsuffix" + tagsField = "tags" + matchTagsField = "matchtags" + protocolField = "protocol" + accountNameField = "accountname" + accountKeyField = "accountkey" + storageAccountField = "storageaccount" + storageAccountTypeField = "storageaccounttype" + skuNameField = "skuname" + subscriptionIDField = "subscriptionid" + resourceGroupField = "resourcegroup" + locationField = "location" + secretNameField = "secretname" + secretNamespaceField = "secretnamespace" + containerNameField = "containername" + containerNamePrefixField = "containernameprefix" + storeAccountKeyField = "storeaccountkey" + isHnsEnabledField = "ishnsenabled" + softDeleteBlobsField = "softdeleteblobs" + softDeleteContainersField = "softdeletecontainers" + enableBlobVersioningField = "enableblobversioning" + getAccountKeyFromSecretField = "getaccountkeyfromsecret" + storageSPNClientIDField = "azurestoragespnclientid" + storageSPNTenantIDField = "azurestoragespntenantid" + keyVaultURLField = "keyvaulturl" + keyVaultSecretNameField = "keyvaultsecretname" + keyVaultSecretVersionField = "keyvaultsecretversion" + storageAccountNameField = "storageaccountname" + allowBlobPublicAccessField = "allowblobpublicaccess" + requireInfraEncryptionField = "requireinfraencryption" + ephemeralField = "csi.storage.k8s.io/ephemeral" + podNamespaceField = "csi.storage.k8s.io/pod.namespace" + mountOptionsField = "mountoptions" + falseValue = "false" + trueValue = "true" + defaultSecretAccountName = "azurestorageaccountname" + defaultSecretAccountKey = "azurestorageaccountkey" + accountSasTokenField = "azurestorageaccountsastoken" + msiSecretField = "msisecret" + storageSPNClientSecretField = "azurestoragespnclientsecret" + EcProtocol = "edgecache" + Fuse = "fuse" + Fuse2 = "fuse2" + NFS = "nfs" + vnetResourceGroupField = "vnetresourcegroup" + vnetNameField = "vnetname" + subnetNameField = "subnetname" + accessTierField = "accesstier" + networkEndpointTypeField = "networkendpointtype" + mountPermissionsField = "mountpermissions" + useDataPlaneAPIField = "usedataplaneapi" + EcStrgAuthenticationField = "edgecache-storage-auth" // See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names containerNameMinLength = 3 From 6dfef9fa2d5de879dcf98268e4f2ce1ae92bbc4e Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Wed, 5 Jul 2023 17:09:32 -0400 Subject: [PATCH 08/10] cachevolume: rename cvhelper to pvc annotator --- pkg/blob/nodeserver.go | 4 ++-- .../cachevolume/{helper.go => pvc_annotator.go} | 16 ++++++++-------- .../{helper_test.go => pvc_annotator_test.go} | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) rename pkg/edgecache/cachevolume/{helper.go => pvc_annotator.go} (89%) rename pkg/edgecache/cachevolume/{helper_test.go => pvc_annotator_test.go} (96%) diff --git a/pkg/blob/nodeserver.go b/pkg/blob/nodeserver.go index e0d076614..927c4c907 100644 --- a/pkg/blob/nodeserver.go +++ b/pkg/blob/nodeserver.go @@ -358,9 +358,9 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe return nil, err } - cvHelper := cv.NewCVHelper(d.cloud.KubeClient) + annotator := cv.NewPVCAnnotator(d.cloud.KubeClient) providedAuth := cv.NewBlobAuth(accountName, containerName, secretName, secretNamespace, storageAuthType) - if err := cvHelper.SendProvisionVolume(pv, d.cloud.Config.AzureAuthConfig, providedAuth); err != nil { + if err := annotator.SendProvisionVolume(pv, d.cloud.Config.AzureAuthConfig, providedAuth); err != nil { return nil, err } diff --git a/pkg/edgecache/cachevolume/helper.go b/pkg/edgecache/cachevolume/pvc_annotator.go similarity index 89% rename from pkg/edgecache/cachevolume/helper.go rename to pkg/edgecache/cachevolume/pvc_annotator.go index 5c873bd13..150dbf2ba 100644 --- a/pkg/edgecache/cachevolume/helper.go +++ b/pkg/edgecache/cachevolume/pvc_annotator.go @@ -62,25 +62,25 @@ func NewBlobAuth(account, container, secretName, secretNamespace, authType strin } } -type CVHelper struct { +type PVCAnnotator struct { client clientset.Interface } -func NewCVHelper(client clientset.Interface) *CVHelper { - return &CVHelper{ +func NewPVCAnnotator(client clientset.Interface) *PVCAnnotator { + return &PVCAnnotator{ client: client, } } -type CVHelperInterface interface { +type PVCAnnotatorInterface interface { SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig config.AzureAuthConfig, strgAuthentication, acct, container string) error } -func (c *CVHelper) requestAuthIsValid(auth string) bool { +func (c *PVCAnnotator) requestAuthIsValid(auth string) bool { return slices.Contains(validStorageAuthentications, auth) } -func (c *CVHelper) needsToBeProvisioned(pvc *v1.PersistentVolumeClaim) bool { +func (c *PVCAnnotator) needsToBeProvisioned(pvc *v1.PersistentVolumeClaim) bool { // check if pv connected to the pvc has already been passed to be created pvState, pvStateOk := pvc.ObjectMeta.Annotations[createVolumeAnnotation] if pvStateOk && pvState == "no" { @@ -90,7 +90,7 @@ func (c *CVHelper) needsToBeProvisioned(pvc *v1.PersistentVolumeClaim) bool { return true } -func (c *CVHelper) buildAnnotations(pv *v1.PersistentVolume, cfg config.AzureAuthConfig, providedAuth BlobAuth) (map[string]string, error) { +func (c *PVCAnnotator) buildAnnotations(pv *v1.PersistentVolume, cfg config.AzureAuthConfig, providedAuth BlobAuth) (map[string]string, error) { annotations := map[string]string{ createVolumeAnnotation: "yes", accountAnnotation: providedAuth.account, @@ -132,7 +132,7 @@ func (c *CVHelper) buildAnnotations(pv *v1.PersistentVolume, cfg config.AzureAut return annotations, nil } -func (c *CVHelper) SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig config.AzureAuthConfig, providedAuth BlobAuth) error { +func (c *PVCAnnotator) SendProvisionVolume(pv *v1.PersistentVolume, cloudConfig config.AzureAuthConfig, providedAuth BlobAuth) error { pvc, err := blobcsiutil.GetPVCByName(c.client, pv.Spec.ClaimRef.Name, pv.Spec.ClaimRef.Namespace) if err != nil { return err diff --git a/pkg/edgecache/cachevolume/helper_test.go b/pkg/edgecache/cachevolume/pvc_annotator_test.go similarity index 96% rename from pkg/edgecache/cachevolume/helper_test.go rename to pkg/edgecache/cachevolume/pvc_annotator_test.go index b347fd588..8d72fe7e7 100644 --- a/pkg/edgecache/cachevolume/helper_test.go +++ b/pkg/edgecache/cachevolume/pvc_annotator_test.go @@ -125,8 +125,8 @@ func TestSendProvisionVolumeFailures(t *testing.T) { pv := pv() client := fake.NewSimpleClientset(pvc, pv) - fkHelper := NewCVHelper(client) - res := fkHelper.SendProvisionVolume(pv, testcase.config, testcase.blobAuth) + fakeAnnotator := NewPVCAnnotator(client) + res := fakeAnnotator.SendProvisionVolume(pv, testcase.config, testcase.blobAuth) assert.NotNil(t, res) pvcAfter, _ := client.CoreV1().PersistentVolumeClaims(defaultPVCNamespace).Get(context.TODO(), defaultPVCName, metav1.GetOptions{}) @@ -199,8 +199,8 @@ func TestSendProvisionVolumeSuccess(t *testing.T) { pv := pvWithAnnotations(pv(), testcase.pvAnnotations) client := fake.NewSimpleClientset(pvc, pv) - fkHelper := NewCVHelper(client) - res := fkHelper.SendProvisionVolume(pv, testcase.config, testcase.blobAuth) + fakeAnnotator := NewPVCAnnotator(client) + res := fakeAnnotator.SendProvisionVolume(pv, testcase.config, testcase.blobAuth) assert.Nil(t, res) pvcAfter, _ := client.CoreV1().PersistentVolumeClaims(defaultPVCNamespace).Get(context.TODO(), defaultPVCName, metav1.GetOptions{}) From 08abe4ffe5526ae15b990f9216984100472b7f7d Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Wed, 5 Jul 2023 17:17:25 -0400 Subject: [PATCH 09/10] charts: add a new minor version for these changes --- charts/index.yaml | 9 + charts/v4.1.0/blob-csi-driver-v4.1.0.tgz | Bin 0 -> 5821 bytes charts/v4.1.0/blob-csi-driver/Chart.yaml | 5 + .../blob-csi-driver/templates/NOTES.txt | 5 + .../blob-csi-driver/templates/_helpers.tpl | 49 +++ .../templates/csi-blob-controller.yaml | 216 +++++++++++++ .../templates/csi-blob-driver.yaml | 14 + .../templates/csi-blob-node.yaml | 288 ++++++++++++++++++ .../templates/rbac-csi-blob-controller.yaml | 115 +++++++ .../templates/rbac-csi-blob-node.yaml | 38 +++ .../serviceaccount-csi-blob-controller.yaml | 17 ++ .../serviceaccount-csi-blob-node.yaml | 17 ++ charts/v4.1.0/blob-csi-driver/values.yaml | 173 +++++++++++ 13 files changed, 946 insertions(+) create mode 100644 charts/v4.1.0/blob-csi-driver-v4.1.0.tgz create mode 100644 charts/v4.1.0/blob-csi-driver/Chart.yaml create mode 100644 charts/v4.1.0/blob-csi-driver/templates/NOTES.txt create mode 100644 charts/v4.1.0/blob-csi-driver/templates/_helpers.tpl create mode 100644 charts/v4.1.0/blob-csi-driver/templates/csi-blob-controller.yaml create mode 100644 charts/v4.1.0/blob-csi-driver/templates/csi-blob-driver.yaml create mode 100644 charts/v4.1.0/blob-csi-driver/templates/csi-blob-node.yaml create mode 100644 charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-controller.yaml create mode 100644 charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-node.yaml create mode 100644 charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-controller.yaml create mode 100644 charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-node.yaml create mode 100644 charts/v4.1.0/blob-csi-driver/values.yaml diff --git a/charts/index.yaml b/charts/index.yaml index 2eb0c12b3..3cf133be2 100644 --- a/charts/index.yaml +++ b/charts/index.yaml @@ -235,6 +235,15 @@ entries: urls: - https://raw.githubusercontent.com/avoltz/blob-csi-driver/staging/charts/v4.0.0/blob-csi-driver-v4.0.0.tgz version: v4.0.0 + - apiVersion: v1 + appVersion: v4.1.0 + created: "2023-07-05T15:32:39.254303884Z" + description: Azure Blob Storage CSI driver + digest: 9b663f91d4dff55080ab21a719eb60d70ae71cee2df493927dc483e96c85b41e + name: blob-csi-driver + urls: + - https://raw.githubusercontent.com/avoltz/blob-csi-driver/staging/charts/v4.1.0/blob-csi-driver-v4.1.0.tgz + version: v4.1.0 - apiVersion: v1 appVersion: latest created: "2023-06-05T13:16:16.079514405Z" diff --git a/charts/v4.1.0/blob-csi-driver-v4.1.0.tgz b/charts/v4.1.0/blob-csi-driver-v4.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..560162d63bde24c2cca1f76ac296fd10087dbec7 GIT binary patch literal 5821 zcmV;u7DDMCiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH<&bK5r3{j6VsQF_zfJ)z$GlAdP#!LgK76GzrqPI~urIv$8z zNjQ@L3xJAM*Z=()0DOOuWLZ{neZ+i-MJ{$1;O=6vy8tIHn^?9Wmcz*cc=KY8xoj-4 z=YF@dtJP|?PLGfE|E*Rl|9`7}dh}iU`1quKbaHxf^5(l%`?&q)=sVQf9ShZ0ASCAB zwVq8Y-?{&i3;X0Va6uUDqD8xgeLw!`VhKXl91u1qzSQ^5{|Gpsw=SEYv1A<209}j+ z$Ouo3Vh_3~CsMUVjQ_aNZnSD&yUT|wb>;Pcf!zSaZVkX{{cpEUPu}G9|M<9b(Eoj0 z#KSY_))1Ot0oUNOF7j;N@QBTsU{l$!nWvt*Y@(nL;@~quB;(62YTo@MnzkU#NQyLs z#Ix9VsNa6zx+CTid)Y;U>5$102LV()v;`S)wjjzr052iHglOIc6W|mi2t`8Cv;3q{ zLUM7`IBB%jk#xxdC`P`xoMCh)m~4kX1gqK!dYSzOk84Ls5`aVt5@h6roe&PfFwhV z0%9C_i~|y2bC5#ao&X92IOxk9s3t?`3~3P+3I&I07tN*gMYow^xYl3ZJEsSw1HVEFl`wWwWO5j{?^n8*aMj*WYTdkcG^g<}JgQ zxBLK012V)O)M^6wg4l3w+bp0`jjzoCOXwoW13<6PFRCl*@7E}VMc4of^o6*t>a%I$ zpef^M$$|zNgH*#Y#jXHErs^j8f~me42vEN{Vqa)O(PLC{=DNVU>P=fl7vr0jz~~iH z%px{LepOk-qO=pqo3dAW9kN`^d;*ufIl zYz>lXh|nu^_mi-&?-v~(#nUA$K)Xp8jW}VP$mNBLg~kt5C~r!?OvMxu0bh^>AP#~B z+y*4(jLS`>-)7VXUnUBmN2jTjzz<1VJ_qo$FW0R3X?}+Kjfka?;90gC2npP>Ig!N1 zuA)8$J5bPC-r!!Mnc`H9C-Z&AWkA(K$q)}Hij@vwfheP#F^MKiBtS}~(pV|IMt5Pb zDGHM%f_v=wE}*XZodfBj2Zj4wd=g63+pSg`J=TgqkF{D#C1PPKQ4J-R$~;}@@k+G1 zi?A|amVRjF8mAgs_}GRnQZK|Z<5xpU*4UT6=d00;+r;8OXF?7^eqsDBwi9wrL_9t= z3tUGjltTobxZo%wkT8xM25MX-a=-`bAV$&0@gQnmLrP*fvBijS*+rdyo*o^SPknR1 zuAC=eotmB=Tw-IUmwtxEb^O|1P)Vu^=TO{4Qn8u^L* zrG^eqr$ni9XyTOhAA>TjjEA=*bAgA}S7Wiz9W1-3u4f9)Txeu1l&2-A9#Uk!YOS!< z`}agh5j}wE6l~c=^&uPEb8rF|>Kow?mBUxU_~KNoXOOAsskLa9)a|ckko405bqR&f z%ndyF0LydR1|fbSaz1c?N-YkNl6(066U7TmTxFfQsJ)qWIV#u(pbEiTmrbUDfDvc+ zDJql9ZHPU>aNIwkEoNyAQ~)i^Q@1;~&1{>lgXj<_%k zyi`js_n`hyv-C49L(Bcicd&%s+qhjs?*iL%7-2c@qNbtwS1KeoUd$bpGRaX8(3lrYE!^6;QB30#e2>IKUzCo z19pDlM8XxdK`Gk1@y*hQSS{XzG}<_pSglBm6x57&+0D4o4O@_gHWJ~2-FT|XRXU#m zRYpjOWf1)y2h%CJ@1pwEcx=tV_0&_{{yiJ_K3v91TCby5DAsgj!b9rzkCZ807Jq+@ zdi}Sb-oHkJ;k&EX=$G^B;b8dwHR@kqUtKF_M|rHFW@{$YjpTRDB{aBKy-^0s1=&T- z1?EkcOq$`Hn${-CSr<7lA(+-`%5`!Xc4E_{jUfRl2fdKYEA%VVo-THOLxZVyRnO0R zO3jpeW0nbeox&|F1esBAkcTOrDR(Qxi-_^F2-WR{TK!0%YAJ?=z7_h)nxIr`J)zca z8f4HzUx=J*u%}TCNep^Q%tBncTRV^c;c5?B* zlhZe+hxp$~7%-O>OYFvYcYs9_{dVEwY8Dy3vLbH5U=LWLS3Ozp7k!5|s z(w-mE51Ff4|8M8u`oKj)`p--StkM6@(dltc|4&a2`M>>K4-d_Mpat=|dfu867v$21 zGfxeRJ%=v(NAvNqR?}epdmmFrPiHg?1L_%2r;Ih+!-It!m=X%8uH6y^S4URz2nVi2 z4U=Wos@eE#GENHeY5|-R2hg9&Tc$QTJ<@*?&-g2-ZY9WoI{I(TTkS#z7!5CmJH=|4 zdQ@|@jw?@=3CAYT*Qk{uYe zrubA33ANq8sp?GyXEyVEMzv3yinSKue9WcArcds9=!{vDWw!58#5>}B8ol}RIA>mt zFwTD~Y=tI&qTd51Go$+WSW_-}L(H3649mI@%4Z|ZPN+%gAhmgnB>Ok^+!YWp;mn&g zYVQZG8z*U|j3?1ufy&>rpX5}4_MBTTqltwwFRk{ zRd|~Af}_`+&Fjak3$WD+V8oePtS0vnQVZ8Ex(=u$9`wvob7lWh0sMq;CaV909w$z5 z!jqLs@}*1G!q+^%OO1VI7yeH60aiT8-{>6c2oXFyOU-(8nh?_Ynh_a z4KzNJv)!%2uRvYYy)5vS>b@(An<#r$t*ZpmK{|5-fSi2n)!S68)mY$5&`Zr4h`uI$`E=Jh&cr=niOSd=#nv!!5fhkvs<%UV>4 zT$qk8pG1Hy3NJ!eZ&~2^a+#r5fxfg9EQeXyzk-CuF0QHQdX9NS@%(Hc{>VM^Ig`w0 zZWrBLjJ9DTE4!tXLy!$AmzGqSYl?{*aJZRs5Od}_rnD`&=2H6aL3Z<$jA^bI0NuB; zXM_?-u-gL{FUMdr>WD5nYGq);2`+xz&OY{m6XujRu66_e_tSO%_TswV>kn@R=a;wd z2ABOZr|=zTUcRq-eKaMrkJ$eJ%WIex0ixFumO659EVaTbfzvgbYV=B;MI}F}I=y^H z(nhk}sSPN(?>3v&u0%MsQOH}Hq}U17>v_S882I_-W;DJXU0?m{-{tg+RxE`#hqQEl zS;OHb3_hK|r?8h5s`pp7|9ZISz;enYf_3?9UyF&l0;OWg2Oo8 zFRLl!mVcLjiFpG49ckLwk~|QysbdPZ%)ODP56l*82qm*?mk#O8hnvO8!YvUykCBz) zEsa%0klV+`o=I!7^~I*quhQ7rJ|bd2SaVDr7c67IEJ6bT7I4n^nJ#|*;p@h$qOt$h z7wZYD73yXfEs0c@@~2l8dLP0=QC~|<{G18XnXKEne&R4O{2t*t>9D-wrmic7*=pt} zH^pmtEZXJjgymCahn=xgJqxTwwq{CLmto9hXR6aSWg?qzT$;K~aqzWEB>vLZ2K7Io zv%US1AnWS?kKg3#|2wUA`%wSCm&>xOOv#ThHT#XbpG@syyJjkcF2)11$7^*lP&W3L zEAiQ)b_Yv|?fEtQ9uN*rvJuW_&VWwY71fh%?6c-JSfv;XcVRU$5QarF8D5ep*h|}m zkIaF1L)S=+i;(6V4RxdM&%pzZ-M@%g+Ccv)bKqGA0p5mxX^hgj<=7K<~O8OO7SX!mY!~OD1$?HsnT*QGO$Ho{#MFh@-I5 zB#m&J%E3ZUze>$u2^WWI!LL>=SlY4u7WCzaYxyYIr=P9;8x4BlEWEljFcn;Sq0&Fa z@3uvMg=#UQ9NX8yCFPtEX@%^(aM$cmS3XWJ;DpY?^CAeH4Q7FU@Lb>?T{iPPOr1iwwwWF3rg}pSF;B-#D@8FB^aXKYrj|{)kbY(8wAX(-I3M1=yS^IU z^oPCj5Ne$=N~i>mZAlh|QbdDkFdW~UUtZq6y}Ww+?$fw5QJfunEu)zuUmCe%TS_9v zhFC=8_H+MwJh&QeMJ#uKOeL{=&pNy4T4!zFIy>oHXI<~c{hQm5=l{CB8uf=d)pYag`;ySX)1cpBIRvHr#Bd%zrF0=ej4@8Z~DErThYp$>ysn4J_De^P8WG7_u_9qTUbgla+dOm3nao^O}R4ciWK&i^m)|D@T6Y;VNEdCcL;}tXY;4cL(n6NT((k?RUz}h3+`s*J^=Wvs8jTJn zh;;bZhTXrgD1*7^XRqdDxi9O#k67LXRWVPzl)pE>6 zY5e2})&j!(8Lg{^lISCMqmJLf(!%=GUF!>xbBj2jW@cH`tIf{qecqmqC;8~x?dCY_ zsW|MZD4?pEg*HNatF=#5W3QMg*CWr)bcZb=8}YZzmXNFmu(>i!3Aj#$7UaHiRs_(E zjl(T`u(k2iA9yL_@UUs+B{r>Wo6Ok9W|vjV%hIJAXFi@`^Ub`s#L@N4+J=_0pxbCK z{N?uCWD$9p6jajKl!GY`Xtxr1*s$}BZ`jEN3|4H_iT4gXImf&*39_uwWM$R3ZW&IK7TinV6uap&^7MIz+wULY zforM|Ew%R3TRub={aUa8rZJ^_pQnq^tFqPD&z)FE{C5=xsp)6$xRq_*Gk6-@pE zwhXn_bY+|=?XD%R(o;@;-DFFOvNmDtsA%$@W4|J?STXKZFYiT7ZFt1Oe}kFbz(nlW z%07!i(|q12PiV_tl-t$k8PvU7CnewewoSVc@V|gj$_>QE25G1ENWGEDx0U)}a^CU{HP1pa=*#y4s`){pQJAeMe z@$upNZ+p4ctpE2q5$GD(Z)pnnt)}n2GM~I-4j>cCMqVJK{-omw@y!T^Z#+{5dYuLRY-JY9sR@;$dyo)TF8|v^fhaU zrym^i1nd6Pvq?5Pmn3ue$Es!qy+xeNfUM+V9mnw(=GSm+;MS5(qkCBQ{mjFh@z=SY zZvUUY5?pKlPu}G0f9K@%u>RZ2_0{Zu;wc|2|H1MfuETY>4%gw@^YwoL00960mov8Z H0HOc@<@=QW literal 0 HcmV?d00001 diff --git a/charts/v4.1.0/blob-csi-driver/Chart.yaml b/charts/v4.1.0/blob-csi-driver/Chart.yaml new file mode 100644 index 000000000..77f6c8ee2 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: latest +description: Azure Blob Storage CSI driver +name: blob-csi-driver +version: v4.1.0 diff --git a/charts/v4.1.0/blob-csi-driver/templates/NOTES.txt b/charts/v4.1.0/blob-csi-driver/templates/NOTES.txt new file mode 100644 index 000000000..9ad135dd4 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/NOTES.txt @@ -0,0 +1,5 @@ +The Azure Blob Storage CSI driver is getting deployed to your cluster. + +To check Azure Blob Storage CSI driver pods status, please run: + + kubectl --namespace={{ .Release.Namespace }} get pods --selector="release={{ .Release.Name }}" --watch diff --git a/charts/v4.1.0/blob-csi-driver/templates/_helpers.tpl b/charts/v4.1.0/blob-csi-driver/templates/_helpers.tpl new file mode 100644 index 000000000..d99392f32 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/_helpers.tpl @@ -0,0 +1,49 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* Expand the name of the chart.*/}} +{{- define "blob.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "blob.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common selectors. +*/}} +{{- define "blob.selectorLabels" -}} +app.kubernetes.io/name: {{ template "blob.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Common labels. +*/}} +{{- define "blob.labels" -}} +{{- include "blob.selectorLabels" . }} +app.kubernetes.io/component: csi-driver +app.kubernetes.io/part-of: {{ template "blob.name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +helm.sh/chart: {{ template "blob.chart" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end -}} + + +{{/* pull secrets for containers */}} +{{- define "blob.pullSecrets" -}} +{{- if .Values.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/v4.1.0/blob-csi-driver/templates/csi-blob-controller.yaml b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-controller.yaml new file mode 100644 index 000000000..50afbb969 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-controller.yaml @@ -0,0 +1,216 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Values.controller.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.controller.name }} + {{- include "blob.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controller.replicas }} + selector: + matchLabels: + app: {{ .Values.controller.name }} + {{- include "blob.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ .Values.controller.name }} + {{- include "blob.labels" . | nindent 8 }} + {{- if .Values.workloadIdentity.clientID }} + azure.workload.identity/use: "true" + {{- end }} + {{- if .Values.podLabels }} +{{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} +{{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + spec: +{{- with .Values.controller.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.controller.hostNetwork }} + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ .Values.serviceAccount.controller }} + nodeSelector: + kubernetes.io/os: linux + {{- if .Values.controller.runOnMaster}} + node-role.kubernetes.io/master: "" + {{- end}} + {{- if .Values.controller.runOnControlPlane}} + node-role.kubernetes.io/control-plane: "" + {{- end}} +{{- with .Values.controller.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + priorityClassName: {{ .Values.priorityClassName | quote }} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- with .Values.controller.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} + containers: + - name: csi-provisioner +{{- if hasPrefix "/" .Values.image.csiProvisioner.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" +{{- else }} + image: "{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" +{{- end }} + args: + - "-v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + - "--timeout=120s" + - "--extra-create-metadata=true" + - "--kube-api-qps=50" + - "--kube-api-burst=100" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiProvisioner.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: {{- toYaml .Values.controller.resources.csiProvisioner | nindent 12 }} + - name: liveness-probe +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port={{ .Values.controller.livenessProbe.healthPort }} + imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.livenessProbe | nindent 12 }} + - name: blob +{{- if hasPrefix "/" .Values.image.blob.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- else }} + image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- end }} + args: + - "--v={{ .Values.controller.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:{{ .Values.controller.metricsPort }}" + - "--drivername={{ .Values.driver.name }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--cloud-config-secret-name={{ .Values.controller.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.controller.cloudConfigSecretNamespace }}" + - "--allow-empty-cloud-config={{ .Values.controller.allowEmptyCloudConfig }}" + ports: + - containerPort: {{ .Values.controller.livenessProbe.healthPort }} + name: healthz + protocol: TCP + - containerPort: {{ .Values.controller.metricsPort }} + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + {{- if ne .Values.driver.httpsProxy "" }} + - name: HTTPS_PROXY + value: {{ .Values.driver.httpsProxy }} + {{- end }} + {{- if ne .Values.driver.httpProxy "" }} + - name: HTTP_PROXY + value: {{ .Values.driver.httpProxy }} + {{- end }} + - name: AZURE_GO_SDK_LOG_LEVEL + value: {{ .Values.driver.azureGoSDKLogLevel }} + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + {{- end }} + imagePullPolicy: {{ .Values.image.blob.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + - name: ssl-pki + mountPath: /etc/pki/ca-trust/extracted + readOnly: true + {{- end }} + resources: {{- toYaml .Values.controller.resources.blob | nindent 12 }} + - name: csi-resizer +{{- if hasPrefix "/" .Values.image.csiResizer.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- else }} + image: "{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- end }} + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + - '-handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiResizer.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.csiResizer | nindent 12 }} + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + - name: ssl-pki + hostPath: + path: /etc/pki/ca-trust/extracted + {{- end }} + {{- if .Values.securityContext }} + securityContext: {{- toYaml .Values.securityContext | nindent 8 }} + {{- end }} diff --git a/charts/v4.1.0/blob-csi-driver/templates/csi-blob-driver.yaml b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-driver.yaml new file mode 100644 index 000000000..9a6aea64a --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-driver.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.driver.name }} + labels: + {{- include "blob.labels" . | nindent 4 }} +spec: + attachRequired: false + podInfoOnMount: true + fsGroupPolicy: {{ .Values.feature.fsGroupPolicy }} + volumeLifecycleModes: + - Persistent + - Ephemeral diff --git a/charts/v4.1.0/blob-csi-driver/templates/csi-blob-node.yaml b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-node.yaml new file mode 100644 index 000000000..13e4291ed --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/csi-blob-node.yaml @@ -0,0 +1,288 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .Values.node.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.node.name }} + {{- include "blob.labels" . | nindent 4 }} +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: {{ .Values.node.maxUnavailable }} + type: RollingUpdate + selector: + matchLabels: + app: {{ .Values.node.name }} + {{- include "blob.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ .Values.node.name }} + {{- include "blob.labels" . | nindent 8 }} + {{- if .Values.workloadIdentity.clientID }} + azure.workload.identity/use: "true" + {{- end }} + {{- if .Values.podLabels }} +{{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} +{{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} +{{- if .Values.node.enableBlobfuseProxy }} + hostPID: true +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ .Values.serviceAccount.node }} + nodeSelector: + kubernetes.io/os: linux +{{- with .Values.node.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + {{- if .Values.node.affinity }} +{{- toYaml .Values.node.affinity | nindent 8 }} + {{- end }} + priorityClassName: {{ .Values.priorityClassName | quote }} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- with .Values.node.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- if .Values.node.enableBlobfuseProxy }} + initContainers: + - name: install-blobfuse-proxy +{{- if hasPrefix "/" .Values.image.blob.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- else }} + image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- end }} + imagePullPolicy: IfNotPresent + command: + - "/blobfuse-proxy/init.sh" + securityContext: + privileged: true + env: + - name: DEBIAN_FRONTEND + value: "noninteractive" + - name: INSTALL_BLOBFUSE + value: "{{ .Values.node.blobfuseProxy.installBlobfuse }}" + - name: BLOBFUSE_VERSION + value: "{{ .Values.node.blobfuseProxy.blobfuseVersion }}" + - name: INSTALL_BLOBFUSE2 + value: "{{ .Values.node.blobfuseProxy.installBlobfuse2 }}" + - name: BLOBFUSE2_VERSION + value: "{{ .Values.node.blobfuseProxy.blobfuse2Version }}" + - name: SET_MAX_OPEN_FILE_NUM + value: "{{ .Values.node.blobfuseProxy.setMaxOpenFileNum }}" + - name: MAX_FILE_NUM + value: "{{ .Values.node.blobfuseProxy.maxOpenFileNum }}" + - name: DISABLE_UPDATEDB + value: "{{ .Values.node.blobfuseProxy.disableUpdateDB }}" + volumeMounts: + - name: host-usr + mountPath: /host/usr + - name: host-etc + mountPath: /host/etc +{{- end }} + containers: + - name: liveness-probe + imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port={{ .Values.node.livenessProbe.healthPort }} + - --v=2 + resources: {{- toYaml .Values.node.resources.livenessProbe | nindent 12 }} + - name: node-driver-registrar +{{- if hasPrefix "/" .Values.image.nodeDriverRegistrar.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- else }} + image: "{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- end }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }}/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: {{- toYaml .Values.node.resources.nodeDriverRegistrar | nindent 12 }} + - name: blob +{{- if hasPrefix "/" .Values.image.blob.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- else }} + image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- end }} + args: + - "--v={{ .Values.node.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--blobfuse-proxy-endpoint=$(BLOBFUSE_PROXY_ENDPOINT)" + - "--edgecache-mount-endpoint=$(EDGECACHE_MOUNT_ENDPOINT)" + - "--enable-blobfuse-proxy={{ .Values.node.enableBlobfuseProxy }}" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--drivername={{ .Values.driver.name }}" + - "--cloud-config-secret-name={{ .Values.node.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.node.cloudConfigSecretNamespace }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--allow-empty-cloud-config={{ .Values.node.allowEmptyCloudConfig }}" + - "--enable-get-volume-stats={{ .Values.feature.enableGetVolumeStats }}" + - "--append-timestamp-cache-dir={{ .Values.node.appendTimeStampInCacheDir }}" + - "--mount-permissions={{ .Values.node.mountPermissions }}" + - "--allow-inline-volume-key-access-with-idenitity={{ .Values.node.allowInlineVolumeKeyAccessWithIdentity }}" + ports: + - containerPort: {{ .Values.node.livenessProbe.healthPort }} + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: EDGECACHE_MOUNT_ENDPOINT + value: unix:///csi/csi_mounts.sock + - name: BLOBFUSE_PROXY_ENDPOINT + value: unix:///csi/blobfuse-proxy.sock + {{- if ne .Values.driver.httpsProxy "" }} + - name: HTTPS_PROXY + value: {{ .Values.driver.httpsProxy }} + {{- end }} + {{- if ne .Values.driver.httpProxy "" }} + - name: HTTP_PROXY + value: {{ .Values.driver.httpProxy }} + {{- end }} + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AZURE_GO_SDK_LOG_LEVEL + value: {{ .Values.driver.azureGoSDKLogLevel }} + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + {{- end }} + imagePullPolicy: {{ .Values.image.blob.pullPolicy }} + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: {{ .Values.linux.kubelet }}/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /mnt + name: blob-cache + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + - name: ssl-pki + mountPath: /etc/pki/ca-trust/extracted + readOnly: true + {{- end }} + resources: {{- toYaml .Values.node.resources.blob | nindent 12 }} + volumes: +{{- if .Values.node.enableBlobfuseProxy }} + - name: host-usr + hostPath: + path: /usr + - name: host-etc + hostPath: + path: /etc +{{- end }} + - hostPath: + path: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }} + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: {{ .Values.linux.kubelet }}/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: {{ .Values.linux.kubelet }}/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred + - hostPath: + path: {{ .Values.node.blobfuseCachePath }} + name: blob-cache + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + - name: ssl-pki + hostPath: + path: /etc/pki/ca-trust/extracted + {{- end }} + {{- if .Values.securityContext }} + securityContext: {{- toYaml .Values.securityContext | nindent 8 }} + {{- end }} diff --git a/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-controller.yaml b/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-controller.yaml new file mode 100644 index 000000000..833dcc640 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-controller.yaml @@ -0,0 +1,115 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-provisioner-role + labels: + {{- include "blob.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-provisioner-binding + labels: + {{- include "blob.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-resizer-role + labels: + {{- include "blob.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-resizer-role + labels: + {{- include "blob.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-controller-secret-role + labels: + {{- include "blob.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-controller-secret-binding + labels: + {{- include "blob.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-controller-secret-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-node.yaml b/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-node.yaml new file mode 100644 index 000000000..f4eb48e93 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/rbac-csi-blob-node.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.create -}} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-role + labels: + {{- include "blob.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + + # the node plugin must apply annotations to the PVC for edgecache volumes + # it gets the PVC's through the PV's + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-binding + labels: + {{- include "blob.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-node-secret-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-controller.yaml b/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-controller.yaml new file mode 100644 index 000000000..7433bccf1 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-controller.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "blob.labels" . | nindent 4 }} +{{- if .Values.workloadIdentity.clientID }} + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: {{ .Values.workloadIdentity.clientID }} +{{- if .Values.workloadIdentity.tenantID }} + azure.workload.identity/tenant-id: {{ .Values.workloadIdentity.tenantID }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-node.yaml b/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-node.yaml new file mode 100644 index 000000000..a25090e30 --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/templates/serviceaccount-csi-blob-node.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "blob.labels" . | nindent 4 }} +{{- if .Values.workloadIdentity.clientID }} + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: {{ .Values.workloadIdentity.clientID }} +{{- if .Values.workloadIdentity.tenantID }} + azure.workload.identity/tenant-id: {{ .Values.workloadIdentity.tenantID }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/v4.1.0/blob-csi-driver/values.yaml b/charts/v4.1.0/blob-csi-driver/values.yaml new file mode 100644 index 000000000..1ff9bbfaf --- /dev/null +++ b/charts/v4.1.0/blob-csi-driver/values.yaml @@ -0,0 +1,173 @@ +image: + baseRepo: mcr.microsoft.com + blob: + repository: /k8s/csi/blob-csi + tag: latest + pullPolicy: IfNotPresent + csiProvisioner: + repository: /oss/kubernetes-csi/csi-provisioner + tag: v3.5.0 + pullPolicy: IfNotPresent + livenessProbe: + repository: /oss/kubernetes-csi/livenessprobe + tag: v2.10.0 + pullPolicy: IfNotPresent + nodeDriverRegistrar: + repository: /oss/kubernetes-csi/csi-node-driver-registrar + tag: v2.8.0 + pullPolicy: IfNotPresent + csiResizer: + repository: /oss/kubernetes-csi/csi-resizer + tag: v1.8.0 + pullPolicy: IfNotPresent + +cloud: AzurePublicCloud + +## Reference to one or more secrets to be used when pulling images +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# - name: myRegistryKeySecretName + +serviceAccount: + create: true # When true, service accounts will be created for you. Set to false if you want to use your own. + controller: csi-blob-controller-sa # Name of Service Account to be created or used + node: csi-blob-node-sa # Name of Service Account to be created or used + +rbac: + create: true + name: blob + +## Collection of annotations to add to all the pods +podAnnotations: {} +## Collection of labels to add to all the pods +podLabels: {} +# -- Custom labels to add into metadata +customLabels: {} + # k8s-app: blob-csi-driver + +## Leverage a PriorityClass to ensure your pods survive resource shortages +## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: system-cluster-critical +## Security context give the opportunity to run container as nonroot by setting a securityContext +## by example : +## securityContext: { runAsUser: 1001 } +securityContext: {} + +controller: + name: csi-blob-controller + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: kube-system + allowEmptyCloudConfig: true + hostNetwork: true # this setting could be disabled if controller does not depend on MSI setting + metricsPort: 29634 + livenessProbe: + healthPort: 29632 + replicas: 2 + runOnMaster: false + runOnControlPlane: false + logLevel: 5 + resources: + csiProvisioner: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + livenessProbe: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + blob: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + csiResizer: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + affinity: {} + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + +node: + name: csi-blob-node + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: kube-system + allowEmptyCloudConfig: true + allowInlineVolumeKeyAccessWithIdentity: false + maxUnavailable: 1 + livenessProbe: + healthPort: 29633 + logLevel: 5 + enableBlobfuseProxy: false + blobfuseProxy: + installBlobfuse: true + blobfuseVersion: "1.4.5" + installBlobfuse2: true + blobfuse2Version: "2.0.3" + setMaxOpenFileNum: true + maxOpenFileNum: "9000000" + disableUpdateDB: true + blobfuseCachePath: /mnt + appendTimeStampInCacheDir: false + mountPermissions: 0777 + resources: + livenessProbe: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + nodeDriverRegistrar: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + blob: + limits: + memory: 2100Mi + requests: + cpu: 10m + memory: 20Mi + affinity: {} + nodeSelector: {} + tolerations: + - operator: "Exists" + +feature: + fsGroupPolicy: ReadWriteOnceWithFSType + enableGetVolumeStats: false + +driver: + name: blob.csi.azure.com + customUserAgent: "" + userAgentSuffix: "OSS-helm" + azureGoSDKLogLevel: "" # available values: ""(no logs), DEBUG, INFO, WARNING, ERROR + httpsProxy: "" + httpProxy: "" + +linux: + kubelet: /var/lib/kubelet + distro: debian + +workloadIdentity: + clientID: "" + # [optional] If the AAD application or user-assigned managed identity is not in the same tenant as the cluster + # then set tenantID with the application or user-assigned managed identity tenant ID + tenantID: "" From 1a74ffca56ce6d9fd9ce35785d9655917639831f Mon Sep 17 00:00:00 2001 From: estevan <50840058+chaireze@users.noreply.github.com> Date: Wed, 5 Jul 2023 17:31:37 -0400 Subject: [PATCH 10/10] these vulnerabilities --- go.mod | 6 +++--- go.sum | 12 ++++++------ .../kube-openapi/pkg/handler/handler.go | 16 ++++++---------- .../k8s.io/kubernetes/pkg/apis/core/types.go | 3 ++- .../kubernetes/pkg/features/kube_features.go | 4 ++-- .../kubernetes/pkg/volume/volume_linux.go | 16 +++++++--------- .../pkg/volume/volume_unsupported.go | 2 +- .../kubernetes/test/e2e/framework/get.go | 5 ++++- .../test/e2e/framework/kubectl/builder.go | 10 +++++++--- vendor/modules.txt | 8 ++++---- .../konnectivity-client/pkg/client/client.go | 19 +++++++++++++++---- .../pkg/client/metrics/metrics.go | 2 ++ 12 files changed, 59 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index e839efaad..6bad0e8be 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( k8s.io/client-go v0.27.1 k8s.io/component-base v0.27.1 k8s.io/klog/v2 v2.90.1 - k8s.io/kubernetes v1.27.1 + k8s.io/kubernetes v1.27.3 k8s.io/mount-utils v0.27.1 k8s.io/utils v0.0.0-20230209194617-a36077c30491 sigs.k8s.io/cloud-provider-azure v1.27.1-0.20230423180712-b979bea29b6a @@ -146,10 +146,10 @@ require ( k8s.io/component-helpers v0.27.1 // indirect k8s.io/controller-manager v0.27.1 // indirect k8s.io/kms v0.27.1 // indirect - k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect + k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/kubectl v0.0.0 // indirect k8s.io/kubelet v0.27.1 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.1 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index ccddbe63d..a6ec66363 100644 --- a/go.sum +++ b/go.sum @@ -841,14 +841,14 @@ k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kms v0.27.1 h1:JTSQbJb+mcobScQwF0bOmZhIwP17k8GvBsiLlA6SQqw= k8s.io/kms v0.27.1/go.mod h1:VuTsw0uHlSycKLCkypCGxfFCjLfzf/5YMeATECd/zJA= -k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= -k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/kubectl v0.27.1 h1:9T5c5KdpburYiW8XKQSH0Uly1kMNE90aGSnbYUZNdcA= k8s.io/kubectl v0.27.1/go.mod h1:QsAkSmrRsKTPlAFzF8kODGDl4p35BIwQnc9XFhkcsy8= k8s.io/kubelet v0.27.1 h1:IkfZ0N9CX/g6EDis7nJw8ZsOuHcpFA6cm0pXQx0g5TY= k8s.io/kubelet v0.27.1/go.mod h1:g3cIhpZPawo/MvsdnmcLmqDJvDPdbUFkzfyLNz03nQg= -k8s.io/kubernetes v1.27.1 h1:DFeW4Lv+kh5DyYcezOzwmQAbC3VqXAxnMyZabALiRSc= -k8s.io/kubernetes v1.27.1/go.mod h1:TTwPjSCKQ+a/NTiFKRGjvOnEaQL8wIG40nsYH8Er4bA= +k8s.io/kubernetes v1.27.3 h1:gwufSj7y6X18Q2Gl8v4Ev+AJHdzWkG7A8VNFffS9vu0= +k8s.io/kubernetes v1.27.3/go.mod h1:U8ZXeKBAPxeb4J4/HOaxjw1A9K6WfSH+fY2SS7CR6IM= k8s.io/mount-utils v0.27.1 h1:RSd0wslbIuwLRaGGNAGMZ3m9FLcvukxJ3FWlOm76W2A= k8s.io/mount-utils v0.27.1/go.mod h1:vmcjYdi2Vg1VTWY7KkhvwJVY6WDHxb/QQhiQKkR8iNs= k8s.io/pod-security-admission v0.27.1 h1:if4d1zzcpNOZNvljvJ0nTCshFPUmnkIsy7KYJg7FP08= @@ -858,8 +858,8 @@ k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.1 h1:MB1zkK+WMOmfLxEpjr1wEmkpcIhZC7kfTkZ0stg5bog= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.1/go.mod h1:/4NLd21PQY0B+H+X0aDZdwUiVXYJQl/2NXA5KVtDiP4= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= sigs.k8s.io/cloud-provider-azure v1.27.1-0.20230423180712-b979bea29b6a h1:wscyx3uWXA8OiV8v8x18ak1lIK8U3q6yS2laIHfe8RQ= sigs.k8s.io/cloud-provider-azure v1.27.1-0.20230423180712-b979bea29b6a/go.mod h1:vRG0GaFye/VlBpSNUGxYHdC3ZTBwFUnc5ZLng+YoE9g= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/vendor/k8s.io/kube-openapi/pkg/handler/handler.go b/vendor/k8s.io/kube-openapi/pkg/handler/handler.go index 84e902646..37cb96f1b 100644 --- a/vendor/k8s.io/kube-openapi/pkg/handler/handler.go +++ b/vendor/k8s.io/kube-openapi/pkg/handler/handler.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "strconv" + "sync" "time" "github.com/NYTimes/gziphandler" @@ -98,16 +99,6 @@ func NewOpenAPIServiceLazy(swagger cached.Data[*spec.Swagger]) *OpenAPIService { return o } -func (o *OpenAPIService) getSwaggerBytes() (timedSpec, string, error) { - result := o.jsonCache.Get() - return result.Data, result.Etag, result.Err -} - -func (o *OpenAPIService) getSwaggerPbBytes() (timedSpec, string, error) { - result := o.protoCache.Get() - return result.Data, result.Etag, result.Err -} - func (o *OpenAPIService) UpdateSpec(swagger *spec.Swagger) error { o.UpdateSpecLazy(cached.NewResultOK(swagger, uuid.New().String())) return nil @@ -135,6 +126,9 @@ func RegisterOpenAPIVersionedService(spec *spec.Swagger, servePath string, handl // RegisterOpenAPIVersionedService registers a handler to provide access to provided swagger spec. func (o *OpenAPIService) RegisterOpenAPIVersionedService(servePath string, handler common.PathHandler) error { + // Mutex protects the cache chain + var mutex sync.Mutex + accepted := []struct { Type string SubType string @@ -163,7 +157,9 @@ func (o *OpenAPIService) RegisterOpenAPIVersionedService(servePath string, handl continue } // serve the first matching media type in the sorted clause list + mutex.Lock() result := accepts.GetDataAndEtag.Get() + mutex.Unlock() if result.Err != nil { klog.Errorf("Error in OpenAPI handler: %s", result.Err) // only return a 503 if we have no older cache data to serve diff --git a/vendor/k8s.io/kubernetes/pkg/apis/core/types.go b/vendor/k8s.io/kubernetes/pkg/apis/core/types.go index d8f657b74..5d5b51b1b 100644 --- a/vendor/k8s.io/kubernetes/pkg/apis/core/types.go +++ b/vendor/k8s.io/kubernetes/pkg/apis/core/types.go @@ -2037,7 +2037,8 @@ type SecretEnvSource struct { // HTTPHeader describes a custom header to be used in HTTP probes type HTTPHeader struct { - // The header field name + // The header field name. + // This will be canonicalized upon output, so case-variant names will be understood as the same header. Name string // The header field value Value string diff --git a/vendor/k8s.io/kubernetes/pkg/features/kube_features.go b/vendor/k8s.io/kubernetes/pkg/features/kube_features.go index dae179775..bf641bfa8 100644 --- a/vendor/k8s.io/kubernetes/pkg/features/kube_features.go +++ b/vendor/k8s.io/kubernetes/pkg/features/kube_features.go @@ -1078,7 +1078,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS NetworkPolicyStatus: {Default: false, PreRelease: featuregate.Alpha}, - NewVolumeManagerReconstruction: {Default: true, PreRelease: featuregate.Beta}, + NewVolumeManagerReconstruction: {Default: false, PreRelease: featuregate.Beta}, // disabled for https://github.com/kubernetes/kubernetes/issues/117745 NodeLogQuery: {Default: false, PreRelease: featuregate.Alpha}, @@ -1160,7 +1160,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS NodeInclusionPolicyInPodTopologySpread: {Default: true, PreRelease: featuregate.Beta}, - SELinuxMountReadWriteOncePod: {Default: true, PreRelease: featuregate.Beta}, + SELinuxMountReadWriteOncePod: {Default: false, PreRelease: featuregate.Beta}, // disabled for https://github.com/kubernetes/kubernetes/issues/117745 InPlacePodVerticalScaling: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go b/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go index 57c028150..ec7f6da4b 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/volume_linux.go @@ -40,22 +40,22 @@ const ( // SetVolumeOwnership modifies the given volume to be owned by // fsGroup, and sets SetGid so that newly created files are owned by // fsGroup. If fsGroup is nil nothing is done. -func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error { +func SetVolumeOwnership(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error { if fsGroup == nil { return nil } timer := time.AfterFunc(30*time.Second, func() { - klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", mounter.GetPath()) + klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", dir) }) defer timer.Stop() - if skipPermissionChange(mounter, fsGroup, fsGroupChangePolicy) { - klog.V(3).InfoS("Skipping permission and ownership change for volume", "path", mounter.GetPath()) + if skipPermissionChange(mounter, dir, fsGroup, fsGroupChangePolicy) { + klog.V(3).InfoS("Skipping permission and ownership change for volume", "path", dir) return nil } - err := walkDeep(mounter.GetPath(), func(path string, info os.FileInfo, err error) error { + err := walkDeep(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -104,14 +104,12 @@ func changeFilePermission(filename string, fsGroup *int64, readonly bool, info o return nil } -func skipPermissionChange(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool { - dir := mounter.GetPath() - +func skipPermissionChange(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool { if fsGroupChangePolicy == nil || *fsGroupChangePolicy != v1.FSGroupChangeOnRootMismatch { klog.V(4).InfoS("Perform recursive ownership change for directory", "path", dir) return false } - return !requiresPermissionChange(mounter.GetPath(), fsGroup, mounter.GetAttributes().ReadOnly) + return !requiresPermissionChange(dir, fsGroup, mounter.GetAttributes().ReadOnly) } func requiresPermissionChange(rootDir string, fsGroup *int64, readonly bool) bool { diff --git a/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go b/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go index 20c56d4b6..3b5a200a6 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/volume_unsupported.go @@ -24,6 +24,6 @@ import ( "k8s.io/kubernetes/pkg/volume/util/types" ) -func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error { +func SetVolumeOwnership(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error { return nil } diff --git a/vendor/k8s.io/kubernetes/test/e2e/framework/get.go b/vendor/k8s.io/kubernetes/test/e2e/framework/get.go index ffa26d307..1e1a83856 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/framework/get.go +++ b/vendor/k8s.io/kubernetes/test/e2e/framework/get.go @@ -100,7 +100,10 @@ func ShouldRetry(err error) (retry bool, retryAfter time.Duration) { } // these errors indicate a transient error that should be retried. - if apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) || errors.As(err, &transientError{}) { + if apierrors.IsTimeout(err) || + apierrors.IsTooManyRequests(err) || + apierrors.IsServiceUnavailable(err) || + errors.As(err, &transientError{}) { return true, 0 } diff --git a/vendor/k8s.io/kubernetes/test/e2e/framework/kubectl/builder.go b/vendor/k8s.io/kubernetes/test/e2e/framework/kubectl/builder.go index 916389589..3811fa572 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/framework/kubectl/builder.go +++ b/vendor/k8s.io/kubernetes/test/e2e/framework/kubectl/builder.go @@ -22,6 +22,7 @@ import ( "io" "net" "net/url" + "os" "os/exec" "strings" "syscall" @@ -48,9 +49,12 @@ func NewKubectlCommand(namespace string, args ...string) *KubectlBuilder { return b } -// WithEnv sets the given environment and returns itself. -func (b *KubectlBuilder) WithEnv(env []string) *KubectlBuilder { - b.cmd.Env = env +// WithEnv appends the given environment and returns itself. +func (b *KubectlBuilder) AppendEnv(env []string) *KubectlBuilder { + if b.cmd.Env == nil { + b.cmd.Env = os.Environ() + } + b.cmd.Env = append(b.cmd.Env, env...) return b } diff --git a/vendor/modules.txt b/vendor/modules.txt index fdd1e5fc5..e21035c06 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1366,8 +1366,8 @@ k8s.io/kms/apis/v1beta1 k8s.io/kms/apis/v2 k8s.io/kms/pkg/service k8s.io/kms/pkg/util -# k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a -## explicit; go 1.18 +# k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f +## explicit; go 1.19 k8s.io/kube-openapi/pkg/builder k8s.io/kube-openapi/pkg/builder3 k8s.io/kube-openapi/pkg/builder3/util @@ -1396,7 +1396,7 @@ k8s.io/kubectl/pkg/util/podutils ## explicit; go 1.20 k8s.io/kubelet/pkg/apis k8s.io/kubelet/pkg/apis/stats/v1alpha1 -# k8s.io/kubernetes v1.27.1 +# k8s.io/kubernetes v1.27.3 ## explicit; go 1.20 k8s.io/kubernetes/pkg/api/legacyscheme k8s.io/kubernetes/pkg/api/service @@ -1493,7 +1493,7 @@ k8s.io/utils/pointer k8s.io/utils/strings k8s.io/utils/strings/slices k8s.io/utils/trace -# sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.1 +# sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 ## explicit; go 1.17 sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go index 68a3ebf12..541e6a5aa 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go @@ -132,6 +132,11 @@ type grpcTunnel struct { // serving. done chan struct{} + // started is an atomic bool represented as a 0 or 1, and set to true when a single-use tunnel has been started (dialed). + // started should only be accessed through atomic methods. + // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+ + started uint32 + // closing is an atomic bool represented as a 0 or 1, and set to true when the tunnel is being closed. // closing should only be accessed through atomic methods. // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+ @@ -197,6 +202,7 @@ func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *g conns: connectionManager{conns: make(map[int64]*conn)}, readTimeoutSeconds: 10, done: make(chan struct{}), + started: 0, } s := metrics.ClientConnectionStatusCreated t.prevStatus.Store(s) @@ -393,6 +399,11 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s } func (t *grpcTunnel) dialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) { + prevStarted := atomic.SwapUint32(&t.started, 1) + if prevStarted != 0 { + return nil, &dialFailure{"single-use dialer already dialed", metrics.DialFailureAlreadyStarted} + } + select { case <-t.done: return nil, errors.New("tunnel is closed") @@ -515,11 +526,11 @@ func (t *grpcTunnel) Recv() (*client.Packet, error) { const segment = commonmetrics.SegmentToClient pkt, err := t.stream.Recv() - if err != nil && err != io.EOF { - metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) - } if err != nil { - return pkt, err + if err != io.EOF { + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) + } + return nil, err } metrics.Metrics.ObservePacket(segment, pkt.Type) return pkt, nil diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go index 03e9d94da..ce942e3eb 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go @@ -62,6 +62,8 @@ const ( // DialFailureTunnelClosed indicates that the client connection was closed before the dial could // complete. DialFailureTunnelClosed DialFailureReason = "tunnelclosed" + // DialFailureAlreadyStarted indicates that a single-use tunnel dialer was already used once. + DialFailureAlreadyStarted DialFailureReason = "tunnelstarted" ) type ClientConnectionStatus string