171 inject Secret Store CSI driver deployment_spec.tpl (#172)

* docs: Update README.adoc - added a row to features section * test: add failing test to check if csi block is present * test: check env section is added * chore: update deployment spec - added csi block in the volumes section - tpl needs a mix of funcitonality from volume and env types of secrets - added service account name to test * Update TestK8SServiceDeploymentCheckSecretStoreCSIBlock test * Move check for injecting secrets as env var from csi to another file * simplify values definition * add CSI block to the secrets{} docs * move readOnly to volume setting * small changes * remove comments about failing tests * update README --------- Co-authored-by: nadiia-caspar <[email protected]>
gruntwork-io · Aug 24, 2023 · 5675e6d · 5675e6d
1 parent 75c2eb8
commit 5675e6d
Show file tree

Hide file tree

Showing 7 changed files with 167 additions and 8 deletions.
diff --git a/README.adoc b/README.adoc
@@ -41,7 +41,8 @@ toc::[]
 * Deploy your application containers on to Kubernetes
 * Zero-downtime rolling deployments
 * Auto scaling and auto healing
-* Configuration management and Secrets management
+* Configuration management and Secrets management 
+** Secrets as Environment/Volumes/Secret Store CSI
 * Ingress and Service endpoints
 
 

diff --git a/charts/k8s-service/README.md b/charts/k8s-service/README.md
@@ -896,13 +896,31 @@ secrets:
         filePath: password
 ```
 
+**Mounting secrets with CSI**: In this example, we mount the `my-secret` `Secret` as the file `/etc/db`, and specify that the secret will sync with Secret Manager store (AWS, GCP, Vault) secret named `my-secret`. We also details the csi block were we define the driver and secreteProviderClass.
+
+```yaml
+secrets:
+  my-secret:
+    as: csi
+    mountPath: /etc/db
+    readOnly: true
+    csi:
+      driver: secrets-store.csi.k8s.io
+      secretProviderClass: secret-provider-class
+    items:
+      my-secret:
+        envVarName: SECRET_VAR
+```
+
 **NOTE**: The volumes are different between `secrets` and `configMaps`. This means that if you use the same `mountPath`
 for different secrets and config maps, you can end up with only one. It is undefined which `Secret` or `ConfigMap` ends
 up getting mounted. To be safe, use a different `mountPath` for each one.
 
 **NOTE**: If you want mount the volumes created with `secrets` or `configMaps` on your init or sidecar containers, you will 
 have to append `-volume` to the volume name in . In the example above, the resulting volume will be `my-secret-volume`.
 
+**Note** When installing the CSI driver on your cluster you have an option to activate syncing of secrets 
+
 ```yaml
 sideCarContainers:
   sidecar:

diff --git a/charts/k8s-service/templates/_deployment_spec.tpl b/charts/k8s-service/templates/_deployment_spec.tpl
@@ -43,6 +43,9 @@ We need this because certain sections are omitted if there are no volumes or env
     {{- $_ := set $hasInjectionTypes "hasVolume" true -}}
   {{- else if eq (index . "as") "environment" -}}
     {{- $_ := set $hasInjectionTypes "hasEnvVars" true -}}
+  {{- else if eq (index . "as") "csi" -}}
+    {{- $_ := set $hasInjectionTypes "hasEnvVars" true -}}
+    {{- $_ := set $hasInjectionTypes "hasVolume" true -}}
   {{- else if eq (index . "as") "envFrom" }}
     {{- $_ := set $hasInjectionTypes "hasEnvFrom" true -}}
   {{- else if eq (index . "as") "none" -}}
@@ -285,9 +288,9 @@ spec:
             {{- end }}
           {{- end }}
           {{- range $name, $value := .Values.secrets }}
-            {{- if eq $value.as "environment" }}
+            {{- if or (eq $value.as "environment") (eq $value.as "csi") }}
             {{- range $secretKey, $keyEnvVarConfig := $value.items }}
-            - name: {{ required "envVarName is required on secrets items when using environment" $keyEnvVarConfig.envVarName | quote }}
+            - name: {{ required "envVarName is required on secrets items when using environment or csi" $keyEnvVarConfig.envVarName | quote }}
               valueFrom:
                 secretKeyRef:
                   name: {{ $name }}
@@ -327,12 +330,13 @@ spec:
             {{- end }}
           {{- end }}
           {{- range $name, $value := .Values.secrets }}
-            {{- if eq $value.as "volume" }}
+            {{- if or (eq $value.as "volume") (eq $value.as "csi") }}
             - name: {{ $name }}-volume
               mountPath: {{ quote $value.mountPath }}
               {{- if $value.subPath }}
               subPath: {{ quote $value.subPath }}
               {{- end }}
+              readOnly: {{ $value.readOnly }}
             {{- end }}
           {{- end }}
           {{- range $name, $value := .Values.persistentVolumes }}
@@ -415,6 +419,15 @@ spec:
               {{- end }}
             {{- end }}
       {{- end }}
+      {{- if eq $value.as "csi" }}
+        - name: {{ $name }}-volume
+          csi: 
+            readOnly: {{ $value.readOnly }}
+            driver:  {{ $value.csi.driver }}
+            volumeAttributes:
+              secretProviderClass: {{ $value.csi.secretProviderClass }}
+
+      {{- end }}    
     {{- end }}
     {{- range $name, $value := .Values.persistentVolumes }}
         - name: {{ $name }}

diff --git a/charts/k8s-service/values.yaml b/charts/k8s-service/values.yaml
@@ -605,9 +605,9 @@ emptyDirs: {}
 # secrets is a map that specifies the Secret resources that should be exposed to the main application container. Each entry in
 # the map represents a Secret resource. The key refers to the name of the Secret that should be exposed, with the value
 # specifying how to expose the Secret. The value is also a map and has the following attributes:
-#   - as (enum[volume,environment,envFrom,none]) (required)
+#   - as (enum[volume,environment,envFrom,csi,none]) (required)
 #     : Secrets can be exposed to Pods as a volume mount, or as environment variables. This attribute is a string enum
-#       that is expected to be either "volume", "environment", or "envFrom", specifying that the Secret should be
+#       that is expected to be either "volume", "environment", "envFrom", or "csi" specifying that the Secret should be
 #       exposed as a mounted volume, via environment variables, or loaded in its entirety as environment variables
 #       respectively. This attribute can also be set to "none", which disables the `Secret` on the container.
 #   - mountPath (string)
@@ -632,14 +632,23 @@ emptyDirs: {}
 #                           exposed as environment variables. Expected to be the octal (e.g 777, 644). Defaults to 644.
 #   - envVarName (string) : The name of the environment variable where the value of the Secret keyed at the given key of
 #                           the item should be stored. Ignored when the Secret is exposed as a volume mount.
-#
+#   - csi (map)
+#     : For Secrets exposed as a volume using a CSI driver, specify the CSI driver details. This field should contain the
+#       following attributes:
+#       - driver (string)    : The name of the CSI driver.
+#       - readOnly (boolean) : Specify whether the volume should be mounted read-only.
+#       - secretProviderClass (string) : The name of the SecretProviderClass.
+#   - readOnly (boolean) : Specify whether the volume should be mounted read-only.
 # NOTE: These secrets are only automatically injected to the main application container. To add them to the side car
 # containers, use the official Kubernetes Pod syntax:
 # https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets
 #
 # The following example exposes the Secret `mysecret` as a volume mounted to `/etc/mysecret`, while it exposes the
 # Secret `myothersecret` as an environment variable. Additionally, it automatically mounts all of the keys
 # `anothersecret` as environment variables using the `envFrom` keyword.
+# The following example exposes the Secret `onemoresecret` as a volume mounted to `/mnt/secrets-store-volume`,
+# using the CSI driver `secrets-store.csi.k8s.io`, and configures an environment variable `SECRET_ONEMORESECRET`
+# with the corresponding value from the Secret.
 #
 # EXAMPLE:
 #
@@ -654,6 +663,16 @@ emptyDirs: {}
 #         envVarName: SECRET_FOO
 #   anothersecret:
 #     as: envFrom
+#   onemoresecret:
+#     as: csi
+#     mountPath: /mnt/secrets-store-volume
+#     readOnly: true
+#     csi:
+#       driver: secrets-store.csi.k8s.io
+#       secretProviderClass: mysecretproviderclass
+#     items:
+#       onemoresecret:
+#         envVarName: SECRET_VAR
 secrets: {}
 
 # containerResources specifies the amount of resources the application container will require. Only specify if you have

diff --git a/test/k8s_service_config_injection_template_test.go b/test/k8s_service_config_injection_template_test.go
@@ -446,6 +446,62 @@ func TestK8SServiceEnvironmentSecretAddsEnvVarsToPod(t *testing.T) {
 	assert.Equal(t, renderedEnvVar["DB_PORT"].ValueFrom.SecretKeyRef.Name, "dbsettings")
 }
 
+// Test that setting the `secrets` input value with environment include those csi vars
+// We test by injecting to secrets:
+// secrets:
+//
+//	dbsettings:
+//	  as: csi
+//	  items:
+//	    host:
+//	      envVarName: DB_HOST
+//	    port:
+//	      envVarName: DB_PORT
+func TestK8SServiceCSISecretAddsEnvVarsToPod(t *testing.T) {
+	t.Parallel()
+
+	deployment := renderK8SServiceDeploymentWithSetValues(
+		t,
+		map[string]string{			
+			"secrets.dbsettings.as":                    "csi",
+			"secrets.dbsettings.readOnly":              "true",
+
+			"secrets.dbsettings.csi.driver":              "secrets-store.csi.k8s.io",
+			"secrets.dbsettings.csi.secretProviderClass": "secret-provider-class",
+
+			"secrets.dbsettings.items.host.envVarName": "DB_HOST",
+			"secrets.dbsettings.items.port.envVarName": "DB_PORT",
+		},
+	)
+
+	// Verify that there is only one container and that the environments section is empty.
+	renderedPodContainers := deployment.Spec.Template.Spec.Containers
+	require.Equal(t, len(renderedPodContainers), 1)
+	appContainer := renderedPodContainers[0]
+	environments := appContainer.Env
+	assert.Equal(t, len(environments), 2)
+
+	// Read in the configured env vars for convenient mapping of env var name
+	renderedEnvVar := map[string]corev1.EnvVar{}
+	for _, env := range environments {
+		renderedEnvVar[env.Name] = env
+	}
+
+	// Verify the DB_HOST env var comes from secret host key of dbsettings
+	assert.Equal(t, renderedEnvVar["DB_HOST"].Value, "")
+	require.NotNil(t, renderedEnvVar["DB_HOST"].ValueFrom)
+	require.NotNil(t, renderedEnvVar["DB_HOST"].ValueFrom.SecretKeyRef)
+	assert.Equal(t, renderedEnvVar["DB_HOST"].ValueFrom.SecretKeyRef.Key, "host")
+	assert.Equal(t, renderedEnvVar["DB_HOST"].ValueFrom.SecretKeyRef.Name, "dbsettings")
+
+	// Verify the DB_PORT env var comes from secret port key of dbsettings
+	assert.Equal(t, renderedEnvVar["DB_PORT"].Value, "")
+	require.NotNil(t, renderedEnvVar["DB_PORT"].ValueFrom)
+	require.NotNil(t, renderedEnvVar["DB_PORT"].ValueFrom.SecretKeyRef)
+	assert.Equal(t, renderedEnvVar["DB_PORT"].ValueFrom.SecretKeyRef.Key, "port")
+	assert.Equal(t, renderedEnvVar["DB_PORT"].ValueFrom.SecretKeyRef.Name, "dbsettings")
+}
+
 // Test that setting the `secrets` input value with volume include the volume mount for the secret
 // We test by injecting to secrets:
 // secrets:

diff --git a/test/k8s_service_template_test.go b/test/k8s_service_template_test.go
@@ -924,7 +924,6 @@ func TestK8SServiceEnvFrom(t *testing.T) {
 		assert.Equal(t, len(deployment.Spec.Template.Spec.Containers[0].EnvFrom), 1)
 		assert.Equal(t, deployment.Spec.Template.Spec.Containers[0].EnvFrom[0].SecretRef.Name, "test-secret")
 	})
-
 }
 
 func TestK8SServiceMinPodsAvailableZeroMeansNoPDB(t *testing.T) {

diff --git a/test/k8s_service_volume_secret_store_csi_template_test.go b/test/k8s_service_volume_secret_store_csi_template_test.go
@@ -0,0 +1,53 @@
+//go:build all || tpl
+// +build all tpl
+
+// NOTE: We use build flags to differentiate between template tests and integration tests so that you can conveniently
+// run just the template tests. See the test README for more information.
+
+package test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestK8SServiceDeploymentCheckSecretStoreCSIBlock(t *testing.T) {
+	t.Parallel()
+
+	deployment := renderK8SServiceDeploymentWithSetValues(
+		t,
+		map[string]string{
+			"secrets.dbsettings.as":        "csi",
+			"secrets.dbsettings.mountPath": "/etc/db",
+			"secrets.dbsettings.readOnly":  "true",
+
+			"secrets.dbsettings.csi.driver":              "secrets-store.csi.k8s.io",
+			"secrets.dbsettings.csi.secretProviderClass": "secret-provider-class",
+
+			"secrets.dbsettings.items.host.envVarName": "DB_HOST",
+			"secrets.dbsettings.items.port.envVarName": "DB_PORT",
+		},
+	)
+
+	// Verify that there is only one container and only one volume
+	renderedPodContainers := deployment.Spec.Template.Spec.Containers
+	require.Equal(t, len(renderedPodContainers), 1)
+	renderedPodVolumes := deployment.Spec.Template.Spec.Volumes
+	require.Equal(t, len(renderedPodVolumes), 1)
+	podVolume := renderedPodVolumes[0]
+
+	// Check that the pod volume has a correct name
+	assert.Equal(t, podVolume.Name, "dbsettings-volume")
+
+	// Check that the pod volume has CSI block
+	assert.NotNil(t, podVolume.CSI)
+
+	// Check that the pod volume has correct CSI driver and attributes
+	assert.Equal(t, podVolume.CSI.Driver, "secrets-store.csi.k8s.io")
+	assert.NotNil(t, podVolume.CSI.VolumeAttributes)
+	assert.Equal(t, podVolume.CSI.VolumeAttributes, map[string]string{
+		"secretProviderClass": "secret-provider-class",
+	})
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -924,7 +924,6 @@ func TestK8SServiceEnvFrom(t *testing.T) { @@
     		assert.Equal(t, len(deployment.Spec.Template.Spec.Containers[0].EnvFrom), 1)
     		assert.Equal(t, deployment.Spec.Template.Spec.Containers[0].EnvFrom[0].SecretRef.Name, "test-secret")
     	})
     }
     func TestK8SServiceMinPodsAvailableZeroMeansNoPDB(t *testing.T) {
@@ Expand Down @@