Replace Storage To Ephemeral Storage in Helm Chart (#4782)

Signed-off-by: Future-Outlier <[email protected]>
flyteorg · Jan 26, 2024 · abef3f5 · abef3f5
1 parent 6a46e3c
commit abef3f5
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 49 deletions.
diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md
@@ -111,8 +111,8 @@ helm install gateway bitnami/contour -n flyte
 | configmap.schedulerConfig.scheduler.profilerPort | int | `10254` |  |
 | configmap.task_logs | object | `{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines |
 | configmap.task_logs.plugins.logs.cloudwatch-enabled | bool | `false` | One option is to enable cloudwatch logging for EKS, update the region and log group accordingly |
-| configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"500Mi","storage":"500Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}` | Task default resources configuration Refer to the full [structure](https://pkg.go.dev/github.com/lyft/[email protected]/pkg/runtime/interfaces#TaskResourceConfiguration). |
-| configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","memory":"500Mi","storage":"500Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}` | Task default resources parameters |
+| configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","ephemeralStorage":"500Mi","memory":"500Mi"},"limits":{"cpu":2,"ephemeralStorage":"20Mi","gpu":1,"memory":"1Gi"}}}` | Task default resources configuration Refer to the full [structure](https://pkg.go.dev/github.com/lyft/[email protected]/pkg/runtime/interfaces#TaskResourceConfiguration). |
+| configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","ephemeralStorage":"500Mi","memory":"500Mi"},"limits":{"cpu":2,"ephemeralStorage":"20Mi","gpu":1,"memory":"1Gi"}}` | Task default resources parameters |
 | daskoperator | object | `{"enabled":false}` | Optional: Dask Plugin using the Dask Operator |
 | daskoperator.enabled | bool | `false` | - enable or disable the dask operator deployment installation |
 | databricks | object | `{"enabled":false,"plugin_config":{"plugins":{"databricks":{"databricksInstance":"dbc-a53b7a3c-614c","entrypointFile":"dbfs:///FileStore/tables/entrypoint.py"}}}}` | Optional: Databricks Plugin allows us to run the spark job on the Databricks platform. |

diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml
@@ -89,7 +89,7 @@ flyteadmin:
         - limitranges
       # -- Specifies the verbs (actions) that this ClusterRole can perform on the specified resources
       verbs:
-        - '*'
+        - "*"
   # -- Annotations for Flyteadmin pods
   podAnnotations: {}
   # -- nodeSelector for Flyteadmin deployment
@@ -357,7 +357,6 @@ flyteconsole:
   # -- Sets priorityClassName for flyte console pod(s).
   priorityClassName: ""
 
-
 # It will enable the redoc route in ingress
 deployRedoc: false
 
@@ -489,37 +488,37 @@ db:
 configmap:
   clusters:
     labelClusterMap: {}
-#  labelClusterMap:
-#    team1:
-#      - id: testcluster
-#        weight: 1
-#    team2:
-#      - id: testcluster2
-#        weight: 0.5
-#      - id: testcluster3
-#        weight: 0.5
+    #  labelClusterMap:
+    #    team1:
+    #      - id: testcluster
+    #        weight: 1
+    #    team2:
+    #      - id: testcluster2
+    #        weight: 0.5
+    #      - id: testcluster3
+    #        weight: 0.5
     clusterConfigs: []
-#  clusterConfigs:
-#    - name: "testcluster"
-#      endpoint: "testcluster_endpoint"
-#      auth:
-#        type: "file_path"
-#        tokenPath: "/path/to/testcluster/token"
-#        certPath: "/path/to/testcluster/cert"
-#    - name: "testcluster2"
-#      endpoint: "testcluster2_endpoint"
-#      enabled: true
-#      auth:
-#        type: "file_path"
-#        tokenPath: "/path/to/testcluster2/token"
-#        certPath: "/path/to/testcluster2/cert"
-#    - name: "testcluster3"
-#      endpoint: "testcluster3_endpoint"
-#      enabled: true
-#      auth:
-#        type: "file_path"
-#        tokenPath: "/path/to/testcluster3/token"
-#        certPath: "/path/to/testcluster3/cert"
+  #  clusterConfigs:
+  #    - name: "testcluster"
+  #      endpoint: "testcluster_endpoint"
+  #      auth:
+  #        type: "file_path"
+  #        tokenPath: "/path/to/testcluster/token"
+  #        certPath: "/path/to/testcluster/cert"
+  #    - name: "testcluster2"
+  #      endpoint: "testcluster2_endpoint"
+  #      enabled: true
+  #      auth:
+  #        type: "file_path"
+  #        tokenPath: "/path/to/testcluster2/token"
+  #        certPath: "/path/to/testcluster2/cert"
+  #    - name: "testcluster3"
+  #      endpoint: "testcluster3_endpoint"
+  #      enabled: true
+  #      auth:
+  #        type: "file_path"
+  #        tokenPath: "/path/to/testcluster3/token"
+  #        certPath: "/path/to/testcluster3/cert"
 
   # -- Configuration for Flyte console UI
   console:
@@ -571,7 +570,7 @@ configmap:
       eventVersion: 2
       testing:
         host: http://flyteadmin
-      
+
     # -- Authentication configuration
     auth:
       authorizedUris:
@@ -621,11 +620,11 @@ configmap:
       defaults:
         cpu: 100m
         memory: 500Mi
-        storage: 500Mi
+        ephemeralStorage: 500Mi
       limits:
         cpu: 2
         memory: 1Gi
-        storage: 20Mi
+        ephemeralStorage: 20Mi
         gpu: 1
 
   # -- Admin Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/subworkflow/launchplan#AdminConfig)
@@ -922,4 +921,4 @@ databricks:
       databricks:
         entrypointFile: dbfs:///FileStore/tables/entrypoint.py
         # Databricks account
-        databricksInstance: dbc-a53b7a3c-614c
+        databricksInstance: dbc-a53b7a3c-614c
diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml
@@ -192,10 +192,12 @@ data:
     task_resources:
       defaults:
         cpu: 1000m
+        ephemeralStorage: 500Mi
         memory: 1000Mi
         storage: 1000Mi
       limits:
         cpu: 2
+        ephemeralStorage: 20Mi
         gpu: 1
         memory: 1Gi
         storage: 2000Mi
@@ -847,7 +849,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "2b5c85969f2bd85bb51a084f9fd72c20c3aca94be99e53cb4c4e9f78e77ebc5"
+        configChecksum: "85f2694a4138443026b87878dbbc5f1e9f52aa54eb87ef4c64117d1d91e1a7f"
       labels: 
         app.kubernetes.io/name: flyteadmin
         app.kubernetes.io/instance: flyte

diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml
@@ -173,10 +173,12 @@ data:
     task_resources:
       defaults:
         cpu: 1000m
+        ephemeralStorage: 500Mi
         memory: 1000Mi
         storage: 1000Mi
       limits:
         cpu: 2
+        ephemeralStorage: 20Mi
         gpu: 1
         memory: 1Gi
         storage: 2000Mi
@@ -553,7 +555,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f"
+        configChecksum: "61fa8a4eebe7e96a3e25b0b2c4baaf7d6af84924167f57e569632fdd282b442"
       labels: 
         app.kubernetes.io/name: flyteadmin
         app.kubernetes.io/instance: flyte
@@ -932,7 +934,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f"
+        configChecksum: "61fa8a4eebe7e96a3e25b0b2c4baaf7d6af84924167f57e569632fdd282b442"
       labels: 
         app.kubernetes.io/name: flytescheduler
         app.kubernetes.io/instance: flyte

diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml
@@ -204,10 +204,12 @@ data:
     task_resources:
       defaults:
         cpu: 1000m
+        ephemeralStorage: 500Mi
         memory: 1000Mi
         storage: 1000Mi
       limits:
         cpu: 2
+        ephemeralStorage: 20Mi
         gpu: 1
         memory: 1Gi
         storage: 2000Mi
@@ -878,7 +880,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f"
+        configChecksum: "61fa8a4eebe7e96a3e25b0b2c4baaf7d6af84924167f57e569632fdd282b442"
       labels: 
         app.kubernetes.io/name: flyteadmin
         app.kubernetes.io/instance: flyte
@@ -1257,7 +1259,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f"
+        configChecksum: "61fa8a4eebe7e96a3e25b0b2c4baaf7d6af84924167f57e569632fdd282b442"
       labels: 
         app.kubernetes.io/name: flytescheduler
         app.kubernetes.io/instance: flyte

diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml
@@ -178,10 +178,12 @@ data:
     task_resources:
       defaults:
         cpu: 500m
+        ephemeralStorage: 500Mi
         memory: 500Mi
         storage: 500Mi
       limits:
         cpu: 2
+        ephemeralStorage: 20Mi
         gpu: 1
         memory: 1Gi
         storage: 2000Mi
@@ -568,7 +570,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3"
+        configChecksum: "c7d43aa7ff4bf67124616d00a83d3c45926ea5ca36bdebdfac1cbcd0e465270"
       labels: 
         app.kubernetes.io/name: flyteadmin
         app.kubernetes.io/instance: flyte
@@ -947,7 +949,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3"
+        configChecksum: "c7d43aa7ff4bf67124616d00a83d3c45926ea5ca36bdebdfac1cbcd0e465270"
       labels: 
         app.kubernetes.io/name: flytescheduler
         app.kubernetes.io/instance: flyte

diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml
@@ -209,10 +209,12 @@ data:
     task_resources:
       defaults:
         cpu: 500m
+        ephemeralStorage: 500Mi
         memory: 500Mi
         storage: 500Mi
       limits:
         cpu: 2
+        ephemeralStorage: 20Mi
         gpu: 1
         memory: 1Gi
         storage: 2000Mi
@@ -901,7 +903,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3"
+        configChecksum: "c7d43aa7ff4bf67124616d00a83d3c45926ea5ca36bdebdfac1cbcd0e465270"
       labels: 
         app.kubernetes.io/name: flyteadmin
         app.kubernetes.io/instance: flyte
@@ -1280,7 +1282,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3"
+        configChecksum: "c7d43aa7ff4bf67124616d00a83d3c45926ea5ca36bdebdfac1cbcd0e465270"
       labels: 
         app.kubernetes.io/name: flytescheduler
         app.kubernetes.io/instance: flyte

diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml
@@ -334,10 +334,12 @@ data:
     task_resources:
       defaults:
         cpu: 100m
+        ephemeralStorage: 500Mi
         memory: 200Mi
         storage: 5Mi
       limits:
         cpu: 2
+        ephemeralStorage: 20Mi
         gpu: 1
         memory: 1Gi
         storage: 20Mi
@@ -6690,7 +6692,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "45f0232531c0d1494809cf83387a95b2fc802019ea095de7a24ccd4f8de86ec"
+        configChecksum: "82d6ffa2a2dd83eb11c491a95af43fdede659d6b5b400b6edcd88291a28c4f4"
       labels: 
         app.kubernetes.io/name: flyteadmin
         app.kubernetes.io/instance: flyte
@@ -7040,7 +7042,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "45f0232531c0d1494809cf83387a95b2fc802019ea095de7a24ccd4f8de86ec"
+        configChecksum: "82d6ffa2a2dd83eb11c491a95af43fdede659d6b5b400b6edcd88291a28c4f4"
       labels: 
         app.kubernetes.io/name: flytescheduler
         app.kubernetes.io/instance: flyte