diff --git a/CHANGELOG.md b/CHANGELOG.md index c34af433..dd5567c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## unreleased +## v1.3.1 - 2021.03.31 + +* Support all formats of files in `/dev/disk/by-id` that were observed across different Linux distributions. + ## v1.3.0 - 2020.10.01 * Increase the default value of CLOUDSCALE_MAX_CSI_VOLUMES_PER_NODE to 125. diff --git a/README.md b/README.md index 5fe815c0..4567ed10 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ secret `my-pvc-luks-key`. ## Releases The cloudscale.ch CSI plugin follows [semantic versioning](https://semver.org/). -The current version is: **`v1.3.0`**. The project is still under active development and may not be +The current version is: **`v1.3.1`**. The project is still under active development and may not be production ready. * Bug fixes will be released as a `PATCH` update. @@ -103,10 +103,10 @@ cloudscale Opaque 1 18h Before you continue, be sure to checkout to a [tagged release](https://github.com/cloudscale-ch/csi-cloudscale/releases). Always use the [latest stable version](https://github.com/cloudscale-ch/csi-cloudscale/releases/latest) -For example, to use the latest stable version (`v1.3.0`) you can execute the following command: +For example, to use the latest stable version (`v1.3.1`) you can execute the following command: ``` -$ kubectl apply -f https://raw.githubusercontent.com/cloudscale-ch/csi-cloudscale/master/deploy/kubernetes/releases/csi-cloudscale-v1.3.0.yaml +$ kubectl apply -f https://raw.githubusercontent.com/cloudscale-ch/csi-cloudscale/master/deploy/kubernetes/releases/csi-cloudscale-v1.3.1.yaml ``` There are also `dev` images available: @@ -202,7 +202,7 @@ Please use the following options with care. ### Max. Number of CSI Volumes per Node -In the `v1.3.0` release the default CSI volumes per node limit of has been increased +In the `v1.3.1` release the default CSI volumes per node limit of has been increased to 125 (previously 23). To take advantage of the higher CSI limit you must ensure that all your cluster nodes are using `virtio-scsi` devices (i.e. `/dev/sdX` devices are used). This is the default for servers created after October 1st, 2020. @@ -271,15 +271,15 @@ $ git push origin After it's merged to master, [create a new Github release](https://github.com/cloudscale-ch/csi-cloudscale/releases/new) from -master with the version `v1.3.0` and then publish a new docker build: +master with the version `v1.3.1` and then publish a new docker build: ``` $ git checkout master $ make publish ``` -This will create a binary with version `v1.3.0` and docker image pushed to -`cloudscalech/cloudscale-csi-plugin:v1.3.0` +This will create a binary with version `v1.3.1` and docker image pushed to +`cloudscalech/cloudscale-csi-plugin:v1.3.1` ## Contributing diff --git a/VERSION b/VERSION index 18fa8e74..75740798 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.3.0 +v1.3.1 diff --git a/deploy/kubernetes/releases/csi-cloudscale-v1.3.1.yaml b/deploy/kubernetes/releases/csi-cloudscale-v1.3.1.yaml new file mode 100644 index 00000000..137e8aaf --- /dev/null +++ b/deploy/kubernetes/releases/csi-cloudscale-v1.3.1.yaml @@ -0,0 +1,526 @@ +# Copyright cloudscale.ch +# Copyright 2018 DigitalOcean +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration to deploy release version of the CSI cloudscale.ch +# plugin (https://github.com/cloudscale-ch/csi-cloudscale) compatible with +# Kubernetes >=v1.13.0 +# +# example usage: kubectl create -f + +#################################################### +########### ############ +########### CSI Node and Driver CRDs ############ +########### ############ +#################################################### +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: csinodeinfos.csi.storage.k8s.io +spec: + group: csi.storage.k8s.io + names: + kind: CSINodeInfo + plural: csinodeinfos + scope: Cluster + validation: + openAPIV3Schema: + properties: + csiDrivers: + description: List of CSI drivers running on the node and their properties. + items: + properties: + driver: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + type: array + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- + +########################################## +########### ############ +########### Storage Class ############ +########### ############ +########################################## + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: cloudscale-volume-ssd + namespace: kube-system + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: ch.cloudscale.csi +allowVolumeExpansion: true +parameters: + csi.cloudscale.ch/volume-type: ssd + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: cloudscale-volume-bulk + namespace: kube-system +provisioner: ch.cloudscale.csi +allowVolumeExpansion: true +parameters: + csi.cloudscale.ch/volume-type: bulk + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: cloudscale-volume-ssd-luks + namespace: kube-system +provisioner: ch.cloudscale.csi +allowVolumeExpansion: true +parameters: + csi.cloudscale.ch/volume-type: ssd + csi.cloudscale.ch/luks-encrypted: "true" + csi.cloudscale.ch/luks-cipher: "aes-xts-plain64" + csi.cloudscale.ch/luks-key-size: "512" + csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} + csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: cloudscale-volume-bulk-luks + namespace: kube-system +provisioner: ch.cloudscale.csi +allowVolumeExpansion: true +parameters: + csi.cloudscale.ch/volume-type: bulk + csi.cloudscale.ch/luks-encrypted: "true" + csi.cloudscale.ch/luks-cipher: "aes-xts-plain64" + csi.cloudscale.ch/luks-key-size: "512" + csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} + csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}-luks-key + +--- + +############################################## +########### ############ +########### Controller plugin ############ +########### ############ +############################################## + +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-cloudscale-controller + namespace: kube-system +spec: + serviceName: "csi-cloudscale" + selector: + matchLabels: + app: csi-cloudscale-controller + replicas: 1 + template: + metadata: + labels: + app: csi-cloudscale-controller + role: csi-cloudscale + spec: + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: csi-cloudscale-controller-sa + containers: + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.0.1 + imagePullPolicy: "Always" + args: + - "--provisioner=ch.cloudscale.csi" + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.1 + imagePullPolicy: "Always" + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.3.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--csiTimeout=30s" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-cluster-driver-registrar + image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 + args: + - "--v=5" + - "--pod-info-mount-version=\"v1\"" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-cloudscale-plugin + image: cloudscalech/cloudscale-csi-plugin:v1.3.1 + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--url=$(CLOUDSCALE_API_URL)" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CLOUDSCALE_API_URL + value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: cloudscale + key: access-token + imagePullPolicy: "Always" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cloudscale-controller-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-provisioner-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + # cluster-driver-registrar currently needs permissions to create the CSIDriver CRD + # see https://github.com/kubernetes-csi/cluster-driver-registrar/issues/3 + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "create"] + + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cloudscale-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +# Attacher must be able to work with PVs, nodes and VolumeAttachments +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- + +# Resizer must be able to work with PVCs, PVs, SCs. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-resizer-binding +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cloudscale-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cloudscale-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-cluster-driver-registrar-role +rules: + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["create", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-cluster-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: csi-cloudscale-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cloudscale-cluster-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + + +######################################## +########### ############ +########### Node plugin ############ +########### ############ +######################################## + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-cloudscale-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-cloudscale-node + template: + metadata: + labels: + app: csi-cloudscale-node + role: csi-cloudscale + spec: + priorityClassName: system-node-critical + serviceAccount: csi-cloudscale-node-sa + hostNetwork: true + containers: + - name: driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.cloudscale.ch /registration/csi.cloudscale.ch-reg.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.cloudscale.ch/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: csi-cloudscale-plugin + image: cloudscalech/cloudscale-csi-plugin:v1.3.1 + imagePullPolicy: "Always" + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--url=$(CLOUDSCALE_API_URL)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: CLOUDSCALE_API_URL + value: https://api.cloudscale.ch/ + - name: CLOUDSCALE_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: cloudscale + key: access-token + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + - name: device-dir + mountPath: /dev + - name: tmpfs + mountPath: /tmp + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.cloudscale.ch + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + # to make sure temporary stored luks keys never touch a disk + - name: tmpfs + emptyDir: + medium: Memory + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cloudscale-node-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-driver-registrar-role + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cloudscale-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: csi-cloudscale-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cloudscale-driver-registrar-role + apiGroup: rbac.authorization.k8s.io diff --git a/driver/mounter.go b/driver/mounter.go index e97637d1..ce09ac30 100644 --- a/driver/mounter.go +++ b/driver/mounter.go @@ -75,7 +75,7 @@ type Mounter interface { // Used to find a path in /dev/disk/by-id with a serial that we have from // the cloudscale API. - FinalizeVolumeAttachmentAndFindPath(logger *logrus.Entry, linuxSerial string) (*string, error) + FinalizeVolumeAttachmentAndFindPath(logger *logrus.Entry, VolumeId string) (*string, error) } // TODO(arslan): this is Linux only for now. Refactor this into a package with @@ -407,21 +407,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -func (m *mounter) FinalizeVolumeAttachmentAndFindPath(logger *logrus.Entry, linuxSerial string) (*string, error) { +func (m *mounter) FinalizeVolumeAttachmentAndFindPath(logger *logrus.Entry, volumeID string) (*string, error) { numTries := 0 for { probeAttachedVolume(logger) - sourcePathPrefixes := []string{"virtio-", "scsi-", "scsi-0QEMU_QEMU_HARDDISK_"} - for _, prefix := range sourcePathPrefixes { - source := filepath.Join(diskIDPath, prefix+linuxSerial) - _, err := os.Stat(source) - if err == nil { - return &source, nil - } - if !os.IsNotExist(err) { - return nil, err - } + // Get the first part of the UUID. + // The linux kernel limits volume serials to 20 bytes: + // include/uapi/linux/virtio_blk.h:#define VIRTIO_BLK_ID_BYTES 20 /* ID string length */ + linuxSerial := volumeID[:20] + + globExpr := diskIDPath + "/*" + linuxSerial + "*" + matches, _ := filepath.Glob(globExpr) + if len(matches) > 0 { + return &matches[0], nil } numTries++ diff --git a/driver/node.go b/driver/node.go index 89701b6a..93e80815 100644 --- a/driver/node.go +++ b/driver/node.go @@ -64,14 +64,10 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume Capability must be provided") } - // Get the first part of the UUID. - // The linux kernel limits volume serials to 20 bytes: - // include/uapi/linux/virtio_blk.h:#define VIRTIO_BLK_ID_BYTES 20 /* ID string length */ - linuxSerial := req.VolumeId[:20] // Apparently sometimes we need to call udevadm trigger to get the volume // properly registered in /dev/disk. More information can be found here: // https://github.com/cloudscale-ch/csi-cloudscale/issues/9 - sourcePtr, err := d.mounter.FinalizeVolumeAttachmentAndFindPath(d.log.WithFields(logrus.Fields{"volume_id": req.VolumeId}), linuxSerial) + sourcePtr, err := d.mounter.FinalizeVolumeAttachmentAndFindPath(d.log.WithFields(logrus.Fields{"volume_id": req.VolumeId}), req.VolumeId) if err != nil { return nil, err }