feat: regional block devices

Support shared block devices.
If storage is shared, pvc will have only regional affinity.

docker-compose.yml

@@ -24,7 +24,7 @@ services:
         source: ./
         target: /src
   csi-attacher:
-    image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0
+    image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0
     network_mode: "service:base"
     command:
       - "--v=5"
@@ -40,7 +40,7 @@ services:
         source: ./hack
         target: /etc/kubernetes
   csi-resizer:
-    image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0
+    image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0
     network_mode: "service:base"
     command:
       - "--v=5"
@@ -56,7 +56,7 @@ services:
         source: ./hack
         target: /etc/kubernetes
   csi-provisioner:
-    image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0
+    image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
     network_mode: "service:base"
     command:
       - "--v=5"

docs/deploy/test-pod-ephemeral.yaml

@@ -7,8 +7,8 @@ spec:
   tolerations:
     - effect: NoSchedule
       key: node-role.kubernetes.io/control-plane
-  # nodeSelector:
-  #    kubernetes.io/hostname: worker-11
+  nodeSelector:
+    kubernetes.io/hostname: kube-11
   containers:
     - name: alpine
       image: alpine
@@ -38,7 +38,7 @@ spec:
               type: pvc-volume
           spec:
             accessModes: [ "ReadWriteOnce" ]
-            storageClassName: proxmox-data-xfs
+            storageClassName: proxmox-rbd
             resources:
               requests:
                 storage: 1Gi

docs/deploy/test-statefulset.yaml

@@ -8,7 +8,7 @@ metadata:
 spec:
   podManagementPolicy: Parallel  # default is OrderedReady
   serviceName: test
-  replicas: 4
+  replicas: 1
   template:
     metadata:
       labels:
@@ -18,8 +18,9 @@ spec:
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/control-plane
-      # nodeSelector:
-      #   topology.kubernetes.io/zone: worker-metal-1
+      nodeSelector:
+        # kubernetes.io/hostname: kube-21
+        # topology.kubernetes.io/zone: hvm-1
       containers:
         - name: alpine
           image: alpine
@@ -45,4 +46,4 @@ spec:
         resources:
           requests:
             storage: 1Gi
-        storageClassName: proxmox-data
+        storageClassName: proxmox-rbd

docs/proxmox-lvm.yaml

@@ -0,0 +1,11 @@
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: proxmox-lvm
+parameters:
+  csi.storage.k8s.io/fstype: xfs
+  storage: lvm
+provisioner: csi.proxmox.sinextra.dev
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

docs/proxmox-rbd.yaml

@@ -0,0 +1,11 @@
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: proxmox-rbd
+parameters:
+  csi.storage.k8s.io/fstype: xfs
+  storage: rbd
+provisioner: csi.proxmox.sinextra.dev
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

pkg/csi/controller.go

@@ -122,25 +122,53 @@ func (d *ControllerService) CreateVolume(_ context.Context, request *csi.CreateV
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
+	storageConfig, err := cl.GetStorageConfig(params[StorageIDKey])
+	if err != nil {
+		klog.Errorf("failed to get proxmox storage config: %v", err)
+
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.V(4).Infof("CreateVolume: storage config: %+v", storageConfig)
+
 	vol := volume.NewVolume(region, zone, params[StorageIDKey], fmt.Sprintf("vm-%d-%s", vmID, pvc))
+	if storageConfig["path"] != nil && storageConfig["path"].(string) != "" {
+		vol = volume.NewVolume(region, zone, params[StorageIDKey], fmt.Sprintf("%d/vm-%d-%s.raw", vmID, vmID, pvc))
+	}
 
 	err = createVolume(cl, vol, volSizeGB)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
+	topology := &csi.Topology{
+		Segments: map[string]string{
+			corev1.LabelTopologyRegion: region,
+			corev1.LabelTopologyZone:   zone,
+		},
+	}
+
+	if storageConfig["shared"] != nil && int(storageConfig["shared"].(float64)) == 1 {
+		// https://pve.proxmox.com/wiki/Storage only block/local storage are supported
+		switch storageConfig["type"].(string) {
+		case "nfs", "cifs", "pbs":
+			return nil, status.Error(codes.InvalidArgument, "error: shared storage type nfs,cifs,pbs are not supported")
+		}
+
+		topology = &csi.Topology{
+			Segments: map[string]string{
+				corev1.LabelTopologyRegion: region,
+			},
+		}
+	}
+
 	volume := csi.Volume{
 		VolumeId:      vol.VolumeID(),
 		VolumeContext: params,
 		ContentSource: request.GetVolumeContentSource(),
 		CapacityBytes: int64(volSizeGB * 1024 * 1024 * 1024),
 		AccessibleTopology: []*csi.Topology{
-			{
-				Segments: map[string]string{
-					corev1.LabelTopologyRegion: region,
-					corev1.LabelTopologyZone:   zone,
-				},
-			},
+			topology,
 		},
 	}
 

pkg/csi/controller_test.go

@@ -141,6 +141,16 @@ clusters:
 		},
 	)
 
+	httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/storage/local-lvm",
+		func(req *http.Request) (*http.Response, error) {
+			return httpmock.NewJsonResponse(200, map[string]interface{}{
+				"data": map[string]interface{}{
+					"shared": 0,
+				},
+			})
+		},
+	)
+
 	httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/nodes/pve-1/storage/local-lvm/status",
 		func(req *http.Request) (*http.Response, error) {
 			return httpmock.NewJsonResponse(200, map[string]interface{}{

pkg/csi/utils.go

@@ -107,9 +107,10 @@ func waitForVolumeDetach(_ *pxapi.Client, _ *pxapi.VmRef, _ int) error {
 }
 
 func createVolume(cl *pxapi.Client, vol *volume.Volume, sizeGB int) error {
+	filename := strings.Split(vol.Disk(), "/")
 	diskParams := map[string]interface{}{
 		"vmid":     vmID,
-		"filename": vol.Disk(),
+		"filename": filename[len(filename)-1],
 		"size":     fmt.Sprintf("%dG", sizeGB),
 	}
 

pkg/volume/volume.go

@@ -46,7 +46,7 @@ func NewVolumeFromVolumeID(volume string) (*Volume, error) {
 }
 
 func parseVolumeID(vol string) (*Volume, error) {
-	parts := strings.Split(vol, "/")
+	parts := strings.SplitN(vol, "/", 4)
 	if len(parts) != 4 {
 		return nil, fmt.Errorf("VolumeID must be in the format of region/zone/storageName/diskName")
 	}

pkg/volume/volume_test.go

@@ -48,6 +48,16 @@ func TestNewVolumeFromVolumeID(t *testing.T) {
 	assert.Equal(t, "disk", v.Disk())
 }
 
+func TestNewVolumeFromVolumeIDWithFolder(t *testing.T) {
+	v, err := volume.NewVolumeFromVolumeID("region/zone/storage/folder/disk")
+	assert.Nil(t, err)
+	assert.NotNil(t, v)
+	assert.Equal(t, "region", v.Cluster())
+	assert.Equal(t, "zone", v.Node())
+	assert.Equal(t, "storage", v.Storage())
+	assert.Equal(t, "folder/disk", v.Disk())
+}
+
 func TestNewVolumeFromVolumeIDError(t *testing.T) {
 	_, err := volume.NewVolumeFromVolumeID("region/storage/disk")
 	assert.NotNil(t, err)