VAST Data CSI Plugin - v2.3.0

  (from b67d356)

.github/workflows/prepare_releaser_configuration.py

@@ -0,0 +1,42 @@
+"""
+This script generates the releaser configuration file for the following `Run chart-releaser` step.
+Releases are separated into two categories: beta and stable.
+Beta releases are created from branches with name pattern <version>-beta
+Stable releases are created from branches with a valid version number (e.g. `1.0.0`).
+"""
+import os
+import re
+import sys
+from pathlib import Path
+import fileinput
+
+ROOT = Path.cwd()
+BRANCH = os.environ["GITHUB_REF_NAME"]
+SHA = os.environ["GITHUB_SHA"][:7]
+VERSION = ROOT.joinpath("version.txt").read_text().strip().lstrip("v")
+CHART = ROOT / "charts" / "vastcsi" / "Chart.yaml"
+
+if __name__ == '__main__':
+    if not re.search('[0-9]+\.[0-9]+\.?[0-9]*', BRANCH):
+        sys.stderr.write(
+            f"Branch name must contain a valid version number. "
+            f"Got: {BRANCH}. Skipping release...\n"
+        )
+        sys.exit(0)
+    is_beta = "beta" in BRANCH
+
+    release_name_template = "helm-{{ .Version }}"
+    pages_branch = "gh-pages-beta" if is_beta else "gh-pages"
+    version = f"{VERSION}-beta.{SHA}" if is_beta else VERSION
+
+    # Create unique release name based on version and commit sha
+    for line in fileinput.input(CHART, inplace=True):
+        if line.startswith("version:"):
+            line = line.replace(line, f"version: {version}\n")
+        sys.stdout.write(line)
+
+    ROOT.joinpath("releaser-config.yml").open("w").write(
+        f"""
+            pages-branch: {pages_branch}
+            release-name-template: {release_name_template}
+        """)

.github/workflows/release.yml

@@ -1,9 +1,9 @@
 name: Release Charts
 
-on:
-  push:
-    branches:
-      - v2.2
+on: [push]
+
+permissions:
+  contents: write
 
 jobs:
   release:
@@ -14,12 +14,20 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Prepare releaser configuration
+        run: |
+          python .github/workflows/prepare_releaser_configuration.py
+
       - name: Configure Git
+        if: ${{ hashFiles('releaser-config.yml') != '' }}
         run: |
           git config user.name "$GITHUB_ACTOR"
           git config user.email "[email protected]"
 
       - name: Run chart-releaser
-        uses: helm/[email protected]
+        if: ${{ hashFiles('releaser-config.yml') != '' }}
+        uses: helm/[email protected]
+        with:
+          config: releaser-config.yml
         env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.gitlab-ci.yml

@@ -121,13 +121,13 @@ update_dockerhub [beta]:
   environment: beta
 
 
-update_github:
+update_github [prod]:  &update_github
   image:
     name: alpine/git
     entrypoint: [""]
   stage: deploy
   only:
-    - /v[\d]\.[\d]+/
+    - /^v[\d]\.[\d]+/
   script: |
     set -x
     VERSION=$(cat version.txt)
@@ -151,13 +151,17 @@ update_github:
     git tag -f $VERSION
     git push -f --tags github HEAD:$CI_COMMIT_REF_NAME
 
-  only:
-  - /^v\d.*/
   when: manual
   tags:
     - vast-dev-builder
 
 
+update_github [beta]:
+  <<: *update_github
+  only:
+    - /v[\d]\.[\d]+-beta.*/
+
+
 mark_stable:
   image: 
     name: amazon/aws-cli

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## Version 2.3.0
+* added CLONE_VOLUME support (VCSI-83)
+* clone volumes from snapshots in READ_WRITE mode (VCSI-103)
+
 ## Version 2.2.6
 * added `sslCertsSecretName` parameter, which points to a user-defined secret for the CSI driver to utilize for custom CA bundles. (VCSI-120)
 * removed kubernetes version check (VCSI-130)

charts/README.md

@@ -0,0 +1,49 @@
+# Install CSI driver with Helm 3
+
+## Prerequisites
+ - [install Helm](https://helm.sh/docs/intro/quickstart/#install-helm)
+
+
+### install production version of the driver:
+```console
+helm repo add vastcsi https://vast-data.github.io/vast-csi
+helm install csi-driver vastcsi/vast-csi -f values.yaml -n vast-csi --create-namespace
+```
+
+### install beta version of the driver:
+```console
+helm repo add vastcsi https://raw.githubusercontent.com/vast-data/vast-csi/gh_pages_beta
+helm install csi-driver vastcsi/vast-csi -f values.yaml -n vast-csi --create-namespace
+```
+
+> **NOTE:** Optionally modify values.yaml or set overrides via Helm command line 
+
+
+### install a specific version
+```console
+helm install csi-driver vastcsi/vast-csi -f values.yaml -n vast-csi --create-namespace --version 2.3.0
+```
+
+### Upgrade driver
+```console
+helm upgrade csi-driver vastcsi/vast-csi -f values.yaml -n vast-csi
+```
+
+### Upgrade helm repository
+```console
+helm repo update vastcsi
+```
+
+### Uninstall driver
+```console
+helm uninstall csi-driver  -n vast-csi
+```
+
+### search for all available chart versions
+```console
+helm search repo -l vastcsi
+```
+
+### troubleshooting
+ - Add `--wait -v=5 --debug` in `helm install` command to get detailed error
+ - Use `kubectl describe` to acquire more info

examples/csi-clone.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+    name: csi-pvc-clone
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: vastdata-filesystem
+  resources:
+    requests:
+      storage: 2Gi
+  dataSource:
+    kind: PersistentVolumeClaim
+    name: csi-pvc

examples/csi-restore.yaml

@@ -1,7 +1,27 @@
+# ReadOnly mode. Volume is pointed directly to snapshot.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: hpvc-restore
+  name: hpvc-restore-1
+spec:
+  storageClassName: csi-vast-sc
+  dataSource:
+    name: new-snapshot-demo
+    kind: VolumeSnapshot
+    apiGroup: snapshot.storage.k8s.io
+  accessModes:
+    - ReadOnlyMany
+  resources:
+    requests:
+      storage: 1Gi
+
+---
+
+# ReadWrite mode. Snapshot data is fully replicated to VAST view folder via intermediate GlobalSnapshotStream.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: hpvc-restore-2
 spec:
   storageClassName: csi-vast-sc
   dataSource:

packaging/Dockerfile

@@ -16,7 +16,7 @@ ARG NAME=csi.vastdata.com
 ARG VERSION
 ARG GIT_COMMIT
 ARG CI_PIPELINE_ID
-RUN echo "$NAME $VERSION $GIT_COMMIT" > version.info 
+RUN echo "$NAME $VERSION $GIT_COMMIT $CI_PIPELINE_ID" > version.info
 
 LABEL name=$NAME
 LABEL version=$VERSION.$GIT_COMMIT.$CI_PIPELINE_ID

packaging/build.sh

@@ -19,7 +19,7 @@ docker build \
     --cache-from vast-csi:latest \
     --build-arg=GIT_COMMIT=$CI_COMMIT_SHA \
     --build-arg=VERSION=$VERSION \
-    --build-arg=CI_PIPELINE_ID=$CI_PIPELINE_ID \
+    --build-arg=CI_PIPELINE_ID=${CI_PIPELINE_ID:-local} \
     --build-arg=BASE_IMAGE_NAME=$BASE_IMAGE_NAME \
     -f packaging/Dockerfile \
     .

tests/conftest.py

@@ -18,7 +18,7 @@
 with local.cwd(gettempdir()) as tempdir:
     # Temporary change working directory and create version.info file in order to allow reading
     # driver name, version and git commit by Config.
-    tempdir["version.info"].open("w").write("csi.vastdata.com v0.0.0 ####")
+    tempdir["version.info"].open("w").write("csi.vastdata.com v0.0.0 #### local")
     from vast_csi.server import Controller, Node, Config
     import vast_csi.csi_types as types
 

tests/test_controller.py

@@ -10,7 +10,7 @@ class TestControllerSuite:
 
     @pytest.mark.parametrize("fs_type, mount_flags, mode, err_message", [
         ("abc", "abc", types.AccessModeType.SINGLE_NODE_WRITER, "Unsupported file system type: abc"),
-        ("ext4", "", types.AccessModeType.SINGLE_NODE_READER_ONLY, "Unsupported access mode: 2 (use [1, 5])"),
+        ("ext4", "", types.AccessModeType.MULTI_NODE_SINGLE_WRITER, "Unsupported access mode: 4 (use [1, 2, 3, 5])"),
     ])
     def test_create_volume_invalid_capability(self, volume_capabilities, fs_type, mount_flags, mode, err_message):
         """Test invalid VolumeCapabilities must be validated"""

vast_csi/configuration.py

@@ -20,7 +20,7 @@ class Path(TypedEnv.Str):
         convert = staticmethod(local.path)
 
     vms_credentials_store = local.path("/opt/vms-auth")
-    plugin_name, plugin_version, git_commit = (
+    plugin_name, plugin_version, git_commit, ci_pipe = (
         open("version.info").read().strip().split()
     )
     plugin_name = TypedEnv.Str("X_CSI_PLUGIN_NAME", default=plugin_name)
@@ -45,7 +45,7 @@ class Path(TypedEnv.Str):
     worker_threads = TypedEnv.Int("X_CSI_WORKER_THREADS", default=10)
     dont_use_trash_api = TypedEnv.Bool("X_CSI_DONT_USE_TRASH_API", default=True)
 
-    _mode = TypedEnv.Str("CSI_MODE", default="controller_and_node")
+    _mode = TypedEnv.Str("X_CSI_MODE", default="controller_and_node")
     _endpoint = TypedEnv.Str("CSI_ENDPOINT", default="unix:///var/run/csi.sock")
     _mount_options = TypedEnv.Str("X_CSI_MOUNT_OPTIONS", default="")  # For example: "port=2049,nolock,vers=3"
     name_fmt = "csi:{namespace}:{name}:{id}"

vast_csi/server.py

@@ -46,7 +46,7 @@
 from . import csi_pb2_grpc
 from .csi_pb2_grpc import ControllerServicer, NodeServicer, IdentityServicer
 from . import csi_types as types
-from .volume_builder import EmptyVolumeBuilder, VolumeFromSnapshotBuilder, TestVolumeBuilder
+from .volume_builder import EmptyVolumeBuilder, VolumeFromSnapshotBuilder, VolumeFromVolumeBuilder, TestVolumeBuilder
 from .exceptions import (
     Abort,
     ApiError,
@@ -56,7 +56,6 @@
     SourceNotFound,
     OperationNotSupported
 )
-
 from .vms_session import VmsSession, TestVmsSession
 from .configuration import Config
 
@@ -80,8 +79,8 @@
 
 SUPPORTED_ACCESS = [
     types.AccessModeType.SINGLE_NODE_WRITER,
-    # types.AccessModeType.SINGLE_NODE_READER_ONLY,
-    # types.AccessModeType.MULTI_NODE_READER_ONLY,
+    types.AccessModeType.SINGLE_NODE_READER_ONLY,
+    types.AccessModeType.MULTI_NODE_READER_ONLY,
     # types.AccessModeType.MULTI_NODE_SINGLE_WRITER,
     types.AccessModeType.MULTI_NODE_MULTI_WRITER,
 ]
@@ -234,7 +233,7 @@ def Probe(self, request, context):
             return types.ProbeRespOK
         elif self.controller:
             try:
-                self.controller.vms_session.get_vip()
+                self.controller.vms_session.versions(status="success", log_result=False)
             except ApiError as exc:
                 raise Abort(FAILED_PRECONDITION, str(exc))
             return types.ProbeRespOK
@@ -258,8 +257,8 @@ class Controller(ControllerServicer, Instrumented):
         types.CtrlCapabilityType.EXPAND_VOLUME,
         types.CtrlCapabilityType.CREATE_DELETE_SNAPSHOT,
         types.CtrlCapabilityType.LIST_SNAPSHOTS,
+        types.CtrlCapabilityType.CLONE_VOLUME,
         # types.CtrlCapabilityType.GET_CAPACITY,
-        # types.CtrlCapabilityType.CLONE_VOLUME,
         # types.CtrlCapabilityType.PUBLISH_READONLY,
     ]
 
@@ -344,7 +343,11 @@ def CreateVolume(
             mount_options = ",".join(re.sub(r"[\[\]]", "", mount_options).replace(",", " ").split())
         except StopIteration:
             mount_options = ""
-
+        # check if list of provided access modes contains read-write mode
+        rw_access_modes = [types.AccessModeType.SINGLE_NODE_WRITER, types.AccessModeType.MULTI_NODE_MULTI_WRITER]
+        rw_access_mode = any(
+            cap.access_mode.mode in rw_access_modes for cap in volume_capabilities if cap.HasField("access_mode")
+        )
         # Take appropriate builder for volume, snapshot or test builder
         if CONF.mock_vast:
             root_export = volume_name_fmt = lb_strategy = view_policy = vip_pool_name = mount_options = qos_policy = ""
@@ -367,6 +370,9 @@ def CreateVolume(
             elif volume_content_source.snapshot.snapshot_id:
                 builder = VolumeFromSnapshotBuilder
 
+            elif volume_content_source.volume.volume_id:
+                builder = VolumeFromVolumeBuilder
+
             else:
                 raise ValueError(
                     "Invalid condition. Either volume_content_source"
@@ -379,6 +385,7 @@ def CreateVolume(
             controller=self,
             configuration=CONF,
             name=name,
+            rw_access_mode=rw_access_mode,
             capacity_range=capacity_range,
             pvc_name=parameters.get("csi.storage.k8s.io/pvc/name"),
             pvc_namespace=parameters.get("csi.storage.k8s.io/pvc/namespace"),
@@ -468,6 +475,7 @@ def _delete_data_from_storage(self, path, tenant_id):
                 os.rmdir(tmpdir)  # will fail if not empty directory
 
     def DeleteVolume(self, volume_id):
+        self.vms_session.ensure_snapshot_stream_deleted(f"strm-{volume_id}")
         if quota := self.vms_session.get_quota(volume_id):
             # this is a check we have to do until Vast provides access to orphaned snapshots (ORION-135599)
             might_use_trash_folder = not CONF.dont_use_trash_api