Add k8s env froms (flyteorg#4969)

* Adding allowance for default configMap/secrets to pull env froms Signed-off-by: Neil Stout <[email protected]> * Updating Helm charts Signed-off-by: Neil Stout <[email protected]> * Fixing typos Signed-off-by: Neil Stout <[email protected]> * More typo Signed-off-by: Neil Stout <[email protected]> * Ran 'make helm' Signed-off-by: Neil Stout <[email protected]> * Updating docs from `make helm` Signed-off-by: Neil Stout <[email protected]> * `make helm` results Signed-off-by: Neil Stout <[email protected]> * Fixing issues with env vs envFrom Signed-off-by: Neil Stout <[email protected]> * Fixing merge issue Signed-off-by: Neil Stout <[email protected]> * Better naming for variable Signed-off-by: Neil Stout <[email protected]> * Renaming to be more accurate Signed-off-by: Neil Stout <[email protected]> * Fixing typo Signed-off-by: Neil Stout <[email protected]> * Forgot one change Signed-off-by: Neil Stout <[email protected]> * Fixing linting error Signed-off-by: Neil Stout <[email protected]> * Updating Helm charts Signed-off-by: Neil Stout <[email protected]> --------- Signed-off-by: Neil Stout <[email protected]>
MortalHappiness · Mar 18, 2024 · eb24107 · eb24107
1 parent 9818423
commit eb24107
Show file tree

Hide file tree

Showing 11 changed files with 39 additions and 17 deletions.
diff --git a/charts/flyte/README.md b/charts/flyte/README.md
diff --git a/charts/flyte/values.yaml b/charts/flyte/values.yaml
@@ -547,6 +547,8 @@ flyte:
             - FLYTE_AWS_ACCESS_KEY_ID: minio
             - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage
           default-memory: 200Mi
+          default-env-from-configmaps: []
+          default-env-from-secrets: []
 
     # -- Logger configuration
     logger:

diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml
@@ -646,6 +646,8 @@ data:
     plugins:
       k8s:
         default-cpus: 100m
+        default-env-from-configmaps: []
+        default-env-from-secrets: []
         default-env-vars:
         - FLYTE_AWS_ENDPOINT: http://minio.flyte:9000
         - FLYTE_AWS_ACCESS_KEY_ID: minio
@@ -7170,7 +7172,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "ecefcb1d6df38b32a9ec77d91de63cd40da935a3239b70a5bb0396435878c80"
+        configChecksum: "3500386bee9f0763c6f14460f607cea4e6e24be4224a806c0673db71e13e287"
       labels: 
         app.kubernetes.io/name: flytepropeller
         app.kubernetes.io/instance: flyte
@@ -7244,7 +7246,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.11.0
       annotations:
-        configChecksum: "ecefcb1d6df38b32a9ec77d91de63cd40da935a3239b70a5bb0396435878c80"
+        configChecksum: "3500386bee9f0763c6f14460f607cea4e6e24be4224a806c0673db71e13e287"
     spec:
       securityContext: 
         fsGroup: 65534

diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml
@@ -816,7 +816,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: VTg5RFBvY205cXRtcEVFbw==
+  haSharedSecret: U0tNSVV0aUwzZzYxNVZnVQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1412,7 +1412,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: b8339a7b42c1e04b994b517d4ddd749074787df4a0eaef9e1aa2181988aea2d7
+        checksum/secret: e0d4db2f1f52d2e4d16959b463e9470d657899ab22db5d8eb2fbc605cdcda0a2
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml
@@ -796,7 +796,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: ZTNEUmhqY1VsWDBXNFowTg==
+  haSharedSecret: bmNtdzJkQ0l2Rm5BS3VZSg==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1360,7 +1360,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: cedf489b22e27428631c5c365e58bfd51cf8465bf04d07a74462677278017b80
+        checksum/secret: b1ca10c95becb2e7214c9ece8101acc322ecde650f1e70bdb1bbb37323c948f6
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml
@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: bkV2U2JzN0o2TXNVcHJTMg==
+  haSharedSecret: S2hmRkxmVjdKZUpoTkR6OQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -934,7 +934,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 2f2e4b3f4abc8a670a75900bd0c7987fb3afa6b2a287fdf7292fbeb755c00c12
+        checksum/secret: 446da9acad72026ea6684e9f0cf3e47f8a458fa10d29d61ac3f8bc509afc7c11
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config.go
@@ -90,6 +90,10 @@ type K8sPluginConfig struct {
 	DefaultEnvVars map[string]string `json:"default-env-vars" pflag:"-,Additional environment variable that should be injected into every resource"`
 	// Provide additional environment variable pairs whose values resolve from the plugin's execution environment.
 	DefaultEnvVarsFromEnv map[string]string `json:"default-env-vars-from-env" pflag:"-,Additional environment variable that should be injected into every resource"`
+	// Provide additional environment variable parts from configMaps
+	DefaultEnvFromConfigMaps []string `json:"default-env-from-configmaps" pflag:"-,Additional environment variable sets that should be injected into each pod from these configMaps"`
+	// Provide additional environment variable parts from secrets
+	DefaultEnvFromSecrets []string `json:"default-env-from-secrets" pflag:"-,Additional environment variable sets that should be injected into each pod from these secret"`
 
 	// default cpu requests for a container
 	DefaultCPURequest resource.Quantity `json:"default-cpus" pflag:",Defines a default value for cpu for containers if not specified."`

diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper.go
@@ -291,7 +291,7 @@ func AddFlyteCustomizationsToContainer(ctx context.Context, parameters template.
 	}
 	container.Args = modifiedArgs
 
-	container.Env = DecorateEnvVars(ctx, container.Env, parameters.TaskExecMetadata.GetEnvironmentVariables(), parameters.TaskExecMetadata.GetTaskExecutionID())
+	container.Env, container.EnvFrom = DecorateEnvVars(ctx, container.Env, parameters.TaskExecMetadata.GetEnvironmentVariables(), parameters.TaskExecMetadata.GetTaskExecutionID())
 
 	// retrieve platformResources and overrideResources to use when aggregating container resources
 	platformResources := parameters.TaskExecMetadata.GetPlatformResources()

diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds.go
@@ -113,7 +113,7 @@ func GetExecutionEnvVars(id pluginsCore.TaskExecutionID) []v1.EnvVar {
 	return envVars
 }
 
-func DecorateEnvVars(ctx context.Context, envVars []v1.EnvVar, taskEnvironmentVariables map[string]string, id pluginsCore.TaskExecutionID) []v1.EnvVar {
+func DecorateEnvVars(ctx context.Context, envVars []v1.EnvVar, taskEnvironmentVariables map[string]string, id pluginsCore.TaskExecutionID) ([]v1.EnvVar, []v1.EnvFromSource) {
 	envVars = append(envVars, GetContextEnvVars(ctx)...)
 	envVars = append(envVars, GetExecutionEnvVars(id)...)
 
@@ -128,7 +128,21 @@ func DecorateEnvVars(ctx context.Context, envVars []v1.EnvVar, taskEnvironmentVa
 		envVars = append(envVars, v1.EnvVar{Name: k, Value: value})
 	}
 
-	return envVars
+	envFroms := []v1.EnvFromSource{}
+
+	for _, secretName := range config.GetK8sPluginConfig().DefaultEnvFromSecrets {
+		optional := true
+		secretRef := v1.SecretEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: secretName}, Optional: &optional}
+		envFroms = append(envFroms, v1.EnvFromSource{SecretRef: &secretRef})
+	}
+
+	for _, cmName := range config.GetK8sPluginConfig().DefaultEnvFromConfigMaps {
+		optional := true
+		cmRef := v1.ConfigMapEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: cmName}, Optional: &optional}
+		envFroms = append(envFroms, v1.EnvFromSource{ConfigMapRef: &cmRef})
+	}
+
+	return envVars, envFroms
 }
 
 func GetPodTolerations(interruptible bool, resourceRequirements ...v1.ResourceRequirements) []v1.Toleration {

diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go
@@ -283,7 +283,7 @@ func TestDecorateEnvVars(t *testing.T) {
 				DefaultEnvVars:        tt.additionEnvVar,
 				DefaultEnvVarsFromEnv: tt.additionEnvVarFromEnv,
 			}))
-			if got := DecorateEnvVars(ctx, tt.args.envVars, tt.executionEnvVar, tt.args.id); !reflect.DeepEqual(got, tt.want) {
+			if got, _ := DecorateEnvVars(ctx, tt.args.envVars, tt.executionEnvVar, tt.args.id); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("DecorateEnvVars() = %v, want %v", got, tt.want)
 			}
 		})

diff --git a/flyteplugins/go/tasks/plugins/array/awsbatch/transformer.go b/flyteplugins/go/tasks/plugins/array/awsbatch/transformer.go
@@ -136,7 +136,7 @@ func UpdateBatchInputForArray(_ context.Context, batchInput *batch.SubmitJobInpu
 
 func getEnvVarsForTask(ctx context.Context, execID pluginCore.TaskExecutionID, containerEnvVars []*core.KeyValuePair,
 	defaultEnvVars map[string]string) []v1.EnvVar {
-	envVars := flytek8s.DecorateEnvVars(ctx, flytek8s.ToK8sEnvVar(containerEnvVars), nil, execID)
+	envVars, _ := flytek8s.DecorateEnvVars(ctx, flytek8s.ToK8sEnvVar(containerEnvVars), nil, execID)
 	m := make(map[string]string, len(envVars))
 	for _, envVar := range envVars {
 		m[envVar.Name] = envVar.Value