[cinder-csi-plugin] ephemeral volume removal (#2602)

Remove openstack credits from node plugin

Signed-off-by: Serge Logvinov <[email protected]>

charts/cinder-csi-plugin/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: v1.31.0
 description: Cinder CSI Chart for OpenStack
 name: openstack-cinder-csi
-version: 2.31.0
+version: 2.31.2
 home: https://github.com/kubernetes/cloud-provider-openstack
 icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png
 maintainers:

charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml

@@ -173,6 +173,7 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--cloud-config=$(CLOUD_CONFIG)"
             - "--cluster=$(CLUSTER_NAME)"
+            - "--provide-node-service=false"
             {{- if .Values.csi.plugin.httpEndpoint.enabled }}
             - "--http-endpoint=:{{ .Values.csi.plugin.httpEndpoint.port }}"
             {{- end }}

charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml

@@ -91,7 +91,8 @@ spec:
             - /bin/cinder-csi-plugin
             - "-v={{ .Values.logVerbosityLevel }}"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--cloud-config=$(CLOUD_CONFIG)"
+            - "--provide-controller-service=false"
+            # - "--cloud-config=$(CLOUD_CONFIG)"
             {{- if .Values.csi.plugin.extraArgs }}
             {{- with .Values.csi.plugin.extraArgs }}
             {{- tpl . $ | trim | nindent 12 }}
@@ -100,8 +101,8 @@ spec:
           env:
             - name: CSI_ENDPOINT
               value: unix://csi/csi.sock
-            - name: CLOUD_CONFIG
-              value: /etc/kubernetes/{{ .Values.secret.filename }}
+            # - name: CLOUD_CONFIG
+            #   value: /etc/kubernetes/{{ .Values.secret.filename }}
             {{- if .Values.csi.plugin.extraEnv }}
               {{- toYaml .Values.csi.plugin.extraEnv | nindent 12 }}
             {{- end }}

cmd/cinder-csi-plugin/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/spf13/cobra"
@@ -31,13 +32,18 @@ import (
 )
 
 var (
-	endpoint                 string
-	nodeID                   string
-	cloudConfig              []string
-	cloudNames               []string
-	additionalTopologies     map[string]string
-	cluster                  string
-	httpEndpoint             string
+	endpoint             string
+	nodeID               string
+	cloudConfig          []string
+	cloudNames           []string
+	additionalTopologies map[string]string
+	cluster              string
+	httpEndpoint         string
+
+	searchOrder           string
+	rescanOnResize        bool
+	nodeVolumeAttachLimit int64
+
 	provideControllerService bool
 	provideNodeService       bool
 	noClient                 bool
@@ -50,6 +56,24 @@ func main() {
 		Run: func(cmd *cobra.Command, args []string) {
 			handle()
 		},
+		PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
+			f := cmd.Flags()
+
+			if !provideControllerService {
+				return nil
+			}
+
+			configs, err := f.GetStringSlice("cloud-config")
+			if err != nil {
+				return err
+			}
+
+			if len(configs) == 0 {
+				return fmt.Errorf("unable to mark flag cloud-config to be required")
+			}
+
+			return nil
+		},
 		Version: version.Version,
 	}
 
@@ -63,10 +87,7 @@ func main() {
 		klog.Fatalf("Unable to mark flag endpoint to be required: %v", err)
 	}
 
-	cmd.PersistentFlags().StringSliceVar(&cloudConfig, "cloud-config", nil, "CSI driver cloud config. This option can be given multiple times")
-	if err := cmd.MarkPersistentFlagRequired("cloud-config"); err != nil {
-		klog.Fatalf("Unable to mark flag cloud-config to be required: %v", err)
-	}
+	cmd.Flags().StringSliceVar(&cloudConfig, "cloud-config", nil, "CSI driver cloud config. This option can be given multiple times")
 
 	cmd.PersistentFlags().StringSliceVar(&cloudNames, "cloud-name", []string{""}, "Cloud name to instruct CSI driver to read additional OpenStack cloud credentials from the configuration subsections. This option can be specified multiple times to manage multiple OpenStack clouds.")
 	cmd.PersistentFlags().StringToStringVar(&additionalTopologies, "additional-topology", map[string]string{}, "Additional CSI driver topology keys, for example topology.kubernetes.io/region=REGION1. This option can be specified multiple times to add multiple additional topology keys.")
@@ -77,6 +98,13 @@ func main() {
 	cmd.PersistentFlags().BoolVar(&provideControllerService, "provide-controller-service", true, "If set to true then the CSI driver does provide the controller service (default: true)")
 	cmd.PersistentFlags().BoolVar(&provideNodeService, "provide-node-service", true, "If set to true then the CSI driver does provide the node service (default: true)")
 	cmd.PersistentFlags().BoolVar(&noClient, "node-service-no-os-client", false, "If set to true then the CSI driver node service will not use the OpenStack client (default: false)")
+	cmd.PersistentFlags().MarkDeprecated("node-service-no-os-client", "This flag is deprecated and will be removed in the future. Node service do not use OpenStack credentials anymore.")
+
+	cmd.PersistentFlags().StringVar(&searchOrder, "search-order", "configDrive,metadataService", "The search order for metadata service")
+
+	// Node specific flags
+	cmd.PersistentFlags().BoolVar(&rescanOnResize, "rescan-on-resize", false, "If set to true then the CSI driver will rescan the device on volume resize")
+	cmd.PersistentFlags().Int64Var(&nodeVolumeAttachLimit, "node-volume-attach-limit", 256, "The maximum number of volumes that can be attached to a node")
 
 	openstack.AddExtraFlags(pflag.CommandLine)
 
@@ -94,7 +122,7 @@ func handle() {
 		var err error
 		clouds := make(map[string]openstack.IOpenStack)
 		for _, cloudName := range cloudNames {
-			clouds[cloudName], err = openstack.GetOpenStackProvider(cloudName, false)
+			clouds[cloudName], err = openstack.GetOpenStackProvider(cloudName)
 			if err != nil {
 				klog.Warningf("Failed to GetOpenStackProvider %s: %v", cloudName, err)
 				return
@@ -105,23 +133,39 @@ func handle() {
 	}
 
 	if provideNodeService {
-		var err error
-		clouds := make(map[string]openstack.IOpenStack)
-		for _, cloudName := range cloudNames {
-			clouds[cloudName], err = openstack.GetOpenStackProvider(cloudName, noClient)
-			if err != nil {
-				klog.Warningf("Failed to GetOpenStackProvider %s: %v", cloudName, err)
+		//Initialize mount
+		mount := mount.GetMountProvider()
+
+		//Backward compatibility, read [BlockStorage] parameters from cloud config
+		cfg, err := openstack.GetConfigFromFiles(cloudConfig)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				klog.Warningf("Failed to GetConfigFromFiles: %v", err)
 				return
 			}
 		}
 
-		//Initialize mount
-		mount := mount.GetMountProvider()
+		if cfg.Metadata.SearchOrder != "" {
+			searchOrder = cfg.Metadata.SearchOrder
+		}
+
+		opts := openstack.BlockStorageOpts{
+			RescanOnResize:        rescanOnResize,
+			NodeVolumeAttachLimit: nodeVolumeAttachLimit,
+		}
+
+		if cfg.BlockStorage.RescanOnResize {
+			opts.RescanOnResize = cfg.BlockStorage.RescanOnResize
+		}
+
+		if cfg.BlockStorage.NodeVolumeAttachLimit < nodeVolumeAttachLimit {
+			opts.NodeVolumeAttachLimit = cfg.BlockStorage.NodeVolumeAttachLimit
+		}
 
 		//Initialize Metadata
-		metadata := metadata.GetMetadataProvider(clouds[cloudNames[0]].GetMetadataOpts().SearchOrder)
+		metadata := metadata.GetMetadataProvider(searchOrder)
 
-		d.SetupNodeService(clouds[cloudNames[0]], mount, metadata, additionalTopologies)
+		d.SetupNodeService(mount, metadata, cfg.BlockStorage, additionalTopologies)
 	}
 
 	d.Run()

docs/cinder-csi-plugin/features.md

@@ -24,7 +24,7 @@
 
 Dynamic Provisioning uses persistence volume claim (PVC) to request the Kubernetes to create the Cinder volume on behalf of user and consumes the volume from inside container.
 
-For usage, refer [sample app](./examples.md#dynamic-volume-provisioning)  
+For usage, refer [sample app](./examples.md#dynamic-volume-provisioning)
 
 ## Topology
 
@@ -35,7 +35,7 @@ This feature enables driver to consider the topology constraints while creating
   `topology.cinder.csi.openstack.org/zone` : Availability by Zone
 * `allowedTopologies` can be specified in storage class to restrict the topology of provisioned volumes to specific zones and should be used as replacement of `availability` parameter.
 * To disable: set `--feature-gates=Topology=false` in external-provisioner (container `csi-provisioner` of `csi-cinder-controllerplugin`).
-  * If using Helm, it can be disabled by setting `Values.csi.provisioner.topology: "false"` 
+  * If using Helm, it can be disabled by setting `Values.csi.provisioner.topology: "false"`
 
 For usage, refer [sample app](./examples.md#use-topology)
 
@@ -51,7 +51,7 @@ For usage, refer [sample app](./examples.md#using-block-volume)
 
 ## Volume Expansion
 
-Driver supports both `Offline` and `Online` resize of cinder volumes. Cinder online resize support is available since cinder 3.42 microversion. 
+Driver supports both `Offline` and `Online` resize of cinder volumes. Cinder online resize support is available since cinder 3.42 microversion.
 The same should be supported by underlying OpenStack Cloud to avail the feature.
 
 * As of kubernetes v1.16, Volume Expansion is a beta feature and enabled by default.
@@ -60,7 +60,7 @@ The same should be supported by underlying OpenStack Cloud to avail the feature.
 
 ### Rescan on in-use volume resize
 
-Some hypervizors (like VMware) don't automatically send a new volume size to a Linux kernel, when a volume is in-use. Sending a "1" to `/sys/class/block/XXX/device/rescan` is telling the SCSI block device to refresh it's information about where it's ending boundary is (among other things) to give the kernel information about it's updated size. When a `rescan-on-resize` flag is set in a CSI node driver cloud-config `[BlockStorage]` section, a CSI node driver will rescan block device and verify its size before expanding the filesystem. CSI driver will raise an error, when expected volume size cannot be detected.
+Some hypervizors (like VMware) don't automatically send a new volume size to a Linux kernel, when a volume is in-use. Sending a "1" to `/sys/class/block/XXX/device/rescan` is telling the SCSI block device to refresh it's information about where it's ending boundary is (among other things) to give the kernel information about it's updated size. When a `rescan-on-resize` flag is set in a CSI node driver cloud-config `[BlockStorage]` section or through the CLI parameter `--rescan-on-resize`, a CSI node driver will rescan block device and verify its size before expanding the filesystem. CSI driver will raise an error, when expected volume size cannot be detected.
 
 Not all hypervizors have a `/sys/class/block/XXX/device/rescan` location, therefore if you enable this option and your hypervizor doesn't support this, you'll get a warning log on resize event. It is recommended to disable this option in this case.
 
@@ -81,7 +81,7 @@ Two different Kubernetes features allow volumes to follow the Pod's lifecycle: C
 
 This feature allows CSI volumes to be directly embedded in the Pod specification instead of a PersistentVolume. Volumes specified in this way are ephemeral and do not persist across Pod restarts.
 
-* As of Kubernetes v1.16 this feature is beta so enabled by default. 
+* As of Kubernetes v1.16 this feature is beta so enabled by default.
 * To enable this feature for CSI Driver, `volumeLifecycleModes` needs to be specified in [CSIDriver](../../manifests/cinder-csi-plugin/csi-cinder-driver.yaml) object. The driver can run in `Persistent` mode, `Ephemeral` or in both modes.
 * `podInfoOnMount` must be `true` to use this feature.
 * For usage, refer [sample app](./examples.md#deploy-app-using-inline-volumes)

docs/cinder-csi-plugin/using-cinder-csi-plugin.md

@@ -111,13 +111,6 @@ In addition to the standard set of klog flags, `cinder-csi-plugin` accepts the f
 
   The default is to provide the node service.
   </dd>
-
-  <dt>--node-service-no-os-client &lt;disabled&gt;</dt>
-  <dd>
-  If set to true then the CSI driver does not provide the OpenStack client in the node service.
-
-  The default is to provide the OpenStack client in the node service.
-  </dd>
 </dl>
 
 ## Driver Config
@@ -138,7 +131,7 @@ The following sections are supported in configuration file.
 For Cinder CSI Plugin to authenticate with OpenStack Keystone, required parameters needs to be passed in `[Global]` section of the file. For all supported parameters, please refer [Global](../openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md#global) section.
 
 ### Block Storage
-These configuration options pertain to block storage and should appear in the `[BlockStorage]` section of the `$CLOUD_CONFIG` file.
+These configuration options pertain to block storage and should appear in the `[BlockStorage]` section of the `$CLOUD_CONFIG` file. Some parameters can be passed as command line arguments as well.
 
 * `node-volume-attach-limit`
   Optional. To configure maximum volumes that can be attached to the node. Its default value is `256`.
@@ -277,7 +270,7 @@ helm install --namespace kube-system --name cinder-csi ./charts/cinder-csi-plugi
 | VolumeSnapshotClass `parameters` | `type`            | Empty String    | `snapshot` creates a VolumeSnapshot object linked to a Cinder volume snapshot. `backup` creates a VolumeSnapshot object linked to a cinder volume backup. Defaults to `snapshot` if not defined |
 | VolumeSnapshotClass `parameters` | `backup-max-duration-seconds-per-gb`  | `20`    | Defines the amount of time to wait for a backup to complete in seconds per GB of volume size |
 | VolumeSnapshotClass `parameters`  | `availability`          | Same as volume | String. Backup Availability Zone |
-| Inline Volume `volumeAttributes`   | `capacity`              | `1Gi`       | volume size for creating inline volumes| 
+| Inline Volume `volumeAttributes`   | `capacity`              | `1Gi`       | volume size for creating inline volumes|
 | Inline Volume `VolumeAttributes`   | `type`              | Empty String  | Name/ID of Volume type. Corresponding volume type should exist in cinder |
 
 ## Local Development

manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml

@@ -57,13 +57,14 @@ spec:
           args:
             - /bin/cinder-csi-plugin
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--cloud-config=$(CLOUD_CONFIG)"
+            - "--provide-controller-service=false"
+            # - "--cloud-config=$(CLOUD_CONFIG)"
             - "--v=1"
           env:
             - name: CSI_ENDPOINT
               value: unix://csi/csi.sock
-            - name: CLOUD_CONFIG
-              value: /etc/config/cloud.conf
+            # - name: CLOUD_CONFIG
+            #   value: /etc/config/cloud.conf
           imagePullPolicy: "IfNotPresent"
           ports:
             - containerPort: 9808
@@ -87,9 +88,9 @@ spec:
             - name: pods-probe-dir
               mountPath: /dev
               mountPropagation: "HostToContainer"
-            - name: secret-cinderplugin
-              mountPath: /etc/config
-              readOnly: true
+            # - name: secret-cinderplugin
+            #   mountPath: /etc/config
+            #   readOnly: true
             # - name: cacert
             #   mountPath: /etc/cacert
             #   readOnly: true
@@ -110,9 +111,9 @@ spec:
           hostPath:
             path: /dev
             type: Directory
-        - name: secret-cinderplugin
-          secret:
-            secretName: cloud-config
+        # - name: secret-cinderplugin
+        #   secret:
+        #     secretName: cloud-config
         # - name: cacert
         #   hostPath:
         #     path: /etc/cacert

pkg/csi/cinder/controllerserver.go

@@ -1061,8 +1061,11 @@ func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi
 func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibleTopologyReq *csi.TopologyRequirement) *csi.CreateVolumeResponse {
 
 	var volsrc *csi.VolumeContentSource
+	volCnx := map[string]string{}
 
 	if vol.SnapshotID != "" {
+		volCnx[ResizeRequired] = "true"
+
 		volsrc = &csi.VolumeContentSource{
 			Type: &csi.VolumeContentSource_Snapshot{
 				Snapshot: &csi.VolumeContentSource_SnapshotSource{
@@ -1073,6 +1076,8 @@ func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibl
 	}
 
 	if vol.SourceVolID != "" {
+		volCnx[ResizeRequired] = "true"
+
 		volsrc = &csi.VolumeContentSource{
 			Type: &csi.VolumeContentSource_Volume{
 				Volume: &csi.VolumeContentSource_VolumeSource{
@@ -1083,6 +1088,8 @@ func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibl
 	}
 
 	if vol.BackupID != nil && *vol.BackupID != "" {
+		volCnx[ResizeRequired] = "true"
+
 		volsrc = &csi.VolumeContentSource{
 			Type: &csi.VolumeContentSource_Snapshot{
 				Snapshot: &csi.VolumeContentSource_SnapshotSource{
@@ -1113,9 +1120,9 @@ func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibl
 			CapacityBytes:      int64(vol.Size * 1024 * 1024 * 1024),
 			AccessibleTopology: accessibleTopology,
 			ContentSource:      volsrc,
+			VolumeContext:      volCnx,
 		},
 	}
 
 	return resp
-
 }

pkg/csi/cinder/driver.go

@@ -32,6 +32,12 @@ import (
 const (
 	driverName  = "cinder.csi.openstack.org"
 	topologyKey = "topology." + driverName + "/zone"
+
+	// maxVolumesPerNode is the maximum number of volumes that can be attached to a node
+	maxVolumesPerNode = 256
+
+	// ResizeRequired parameter, if set to true, will trigger a resize on mount operation
+	ResizeRequired = "resizeRequired"
 )
 
 var (
@@ -177,9 +183,9 @@ func (d *Driver) SetupControllerService(clouds map[string]openstack.IOpenStack)
 	d.cs = NewControllerServer(d, clouds)
 }
 
-func (d *Driver) SetupNodeService(cloud openstack.IOpenStack, mount mount.IMount, metadata metadata.IMetadata, topologies map[string]string) {
+func (d *Driver) SetupNodeService(mount mount.IMount, metadata metadata.IMetadata, opts openstack.BlockStorageOpts, topologies map[string]string) {
 	klog.Info("Providing node service")
-	d.ns = NewNodeServer(d, mount, metadata, cloud, topologies)
+	d.ns = NewNodeServer(d, mount, metadata, opts, topologies)
 }
 
 func (d *Driver) Run() {