Merge remote-tracking branch 'origin' into buf-experiment

Signed-off-by: Eduardo Apolinario <[email protected]>

.github/workflows/flyteidl-checks.yml

@@ -50,3 +50,5 @@ jobs:
     with:
       component: flyteidl
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}

charts/flyte-core/README.md

@@ -60,13 +60,15 @@ helm install gateway bitnami/contour -n flyte
 | cloud_events.eventsPublisher.eventTypes[0] | string | `"all"` |  |
 | cloud_events.eventsPublisher.topicName | string | `"arn:aws:sns:us-east-2:123456:123-my-topic"` |  |
 | cloud_events.type | string | `"aws"` |  |
-| cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"podAnnotations":{},"service_account_name":"flyteadmin","standaloneDeployment":false,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: project-quota\n  namespace: {{ namespace }}\nspec:\n  hard:\n    limits.cpu: {{ projectQuotaCpu }}\n    limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain |
+| cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"podAnnotations":{},"podEnv":{},"podLabels":{},"service_account_name":"flyteadmin","standaloneDeployment":false,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: project-quota\n  namespace: {{ namespace }}\nspec:\n  hard:\n    limits.cpu: {{ projectQuotaCpu }}\n    limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain |
 | cluster_resource_manager.config | object | `{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}}` | Configmap for ClusterResource parameters |
 | cluster_resource_manager.config.cluster_resources | object | `{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/[email protected]/pkg/runtime/interfaces#ClusterResourceConfig) to customize. |
 | cluster_resource_manager.config.cluster_resources.refreshInterval | string | `"5m"` | How frequently to run the sync process |
 | cluster_resource_manager.config.cluster_resources.standaloneDeployment | bool | `false` | Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints |
 | cluster_resource_manager.enabled | bool | `true` | Enables the Cluster resource manager component |
 | cluster_resource_manager.podAnnotations | object | `{}` | Annotations for ClusterResource pods |
+| cluster_resource_manager.podEnv | object | `{}` | Additional ClusterResource container environment variables |
+| cluster_resource_manager.podLabels | object | `{}` | Labels for ClusterResource pods |
 | cluster_resource_manager.service_account_name | string | `"flyteadmin"` | Service account name to run with |
 | cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: project-quota\n  namespace: {{ namespace }}\nspec:\n  hard:\n    limits.cpu: {{ projectQuotaCpu }}\n    limits.memory: {{ projectQuotaMemory }}\n"}]` | Resource templates that should be applied |
 | cluster_resource_manager.templates[0] | object | `{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"}` | Template for namespaces resources |
@@ -111,8 +113,8 @@ helm install gateway bitnami/contour -n flyte
 | configmap.schedulerConfig.scheduler.profilerPort | int | `10254` |  |
 | configmap.task_logs | object | `{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines |
 | configmap.task_logs.plugins.logs.cloudwatch-enabled | bool | `false` | One option is to enable cloudwatch logging for EKS, update the region and log group accordingly |
-| configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"500Mi","storage":"500Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}` | Task default resources configuration Refer to the full [structure](https://pkg.go.dev/github.com/lyft/[email protected]/pkg/runtime/interfaces#TaskResourceConfiguration). |
-| configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","memory":"500Mi","storage":"500Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}` | Task default resources parameters |
+| configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","ephemeralStorage":"500Mi","memory":"500Mi"},"limits":{"cpu":2,"ephemeralStorage":"20Mi","gpu":1,"memory":"1Gi"}}}` | Task default resources configuration Refer to the full [structure](https://pkg.go.dev/github.com/lyft/[email protected]/pkg/runtime/interfaces#TaskResourceConfiguration). |
+| configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","ephemeralStorage":"500Mi","memory":"500Mi"},"limits":{"cpu":2,"ephemeralStorage":"20Mi","gpu":1,"memory":"1Gi"}}` | Task default resources parameters |
 | daskoperator | object | `{"enabled":false}` | Optional: Dask Plugin using the Dask Operator |
 | daskoperator.enabled | bool | `false` | - enable or disable the dask operator deployment installation |
 | databricks | object | `{"enabled":false,"plugin_config":{"plugins":{"databricks":{"databricksInstance":"dbc-a53b7a3c-614c","entrypointFile":"dbfs:///FileStore/tables/entrypoint.py"}}}}` | Optional: Databricks Plugin allows us to run the spark job on the Databricks platform. |
@@ -128,6 +130,8 @@ helm install gateway bitnami/contour -n flyte
 | datacatalog.image.tag | string | `"v1.10.7-b2"` | Docker image tag |
 | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment |
 | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods |
+| datacatalog.podEnv | object | `{}` | Additional Datacatalog container environment variables |
+| datacatalog.podLabels | object | `{}` | Labels for Datacatalog pods |
 | datacatalog.priorityClassName | string | `""` | Sets priorityClassName for datacatalog pod(s). |
 | datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment |
 | datacatalog.resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Datacatalog deployment |
@@ -161,6 +165,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create |
 | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment |
 | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods |
+| flyteadmin.podLabels | object | `{}` | Labels for Flyteadmin pods |
 | flyteadmin.priorityClassName | string | `""` | Sets priorityClassName for flyteadmin pod(s). |
 | flyteadmin.replicaCount | int | `1` | Replicas count for Flyteadmin deployment |
 | flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment |
@@ -192,6 +197,8 @@ helm install gateway bitnami/contour -n flyte
 | flyteconsole.image.tag | string | `"v1.10.2"` |  |
 | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment |
 | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods |
+| flyteconsole.podEnv | object | `{}` | Additional Flyteconsole container environment variables |
+| flyteconsole.podLabels | object | `{}` | Labels for Flyteconsole pods |
 | flyteconsole.priorityClassName | string | `""` | Sets priorityClassName for flyte console pod(s). |
 | flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment |
 | flyteconsole.resources | object | `{"limits":{"cpu":"500m","memory":"250Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Flyteconsole deployment |
@@ -213,6 +220,8 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.manager | bool | `false` |  |
 | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment |
 | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods |
+| flytepropeller.podEnv | object | `{}` | Additional Flytepropeller container environment variables |
+| flytepropeller.podLabels | object | `{}` | Labels for Flytepropeller pods |
 | flytepropeller.priorityClassName | string | `""` | Sets priorityClassName for propeller pod(s). |
 | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment |
 | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"100Mi"}}` | Default resources requests and limits for Flytepropeller deployment |
@@ -239,6 +248,8 @@ helm install gateway bitnami/contour -n flyte
 | flytescheduler.image.tag | string | `"v1.10.7-b2"` | Docker image tag |
 | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment |
 | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods |
+| flytescheduler.podEnv | object | `{}` | Additional Flytescheduler container environment variables |
+| flytescheduler.podLabels | object | `{}` | Labels for Flytescheduler pods |
 | flytescheduler.priorityClassName | string | `""` | Sets priorityClassName for flyte scheduler pod(s). |
 | flytescheduler.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytescheduler deployment |
 | flytescheduler.runPrecheck | bool | `true` | Whether to inject an init container which waits on flyteadmin |

charts/flyte-core/templates/_helpers.tpl

@@ -28,6 +28,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flyteadmin.podLabels" -}}
+{{ include "flyteadmin.labels" . }}
+{{- with .Values.flyteadmin.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flytescheduler.name" -}}
 flytescheduler
 {{- end -}}
@@ -44,6 +51,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flytescheduler.podLabels" -}}
+{{ include "flytescheduler.labels" . }}
+{{- with .Values.flytescheduler.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flyteclusterresourcesync.name" -}}
 flyteclusterresourcesync
 {{- end -}}
@@ -59,6 +73,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flyteclusterresourcesync.podLabels" -}}
+{{ include "flyteclusterresourcesync.labels" . }}
+{{- with .Values.cluster_resource_manager.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "datacatalog.name" -}}
 datacatalog
 {{- end -}}
@@ -74,6 +95,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "datacatalog.podLabels" -}}
+{{ include "datacatalog.labels" . }}
+{{- with .Values.datacatalog.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flyteagent.name" -}}
 flyteagent
 {{- end -}}
@@ -89,6 +117,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flyteagent.podLabels" -}}
+{{ include "flyteagent.labels" . }}
+{{- with .Values.flyteagent.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flytepropeller.name" -}}
 flytepropeller
 {{- end -}}
@@ -104,6 +139,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flytepropeller.podLabels" -}}
+{{ include "flytepropeller.labels" . }}
+{{- with .Values.flytepropeller.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flytepropeller-manager.name" -}}
 flytepropeller-manager
 {{- end -}}
@@ -119,6 +161,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flytepropeller-manager.podLabels" -}}
+{{ include "flytepropeller-manager.labels" . }}
+{{- with .Values.flytepropeller.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flyte-pod-webhook.name" -}}
 flyte-pod-webhook
 {{- end -}}
@@ -139,6 +188,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flyteconsole.podLabels" -}}
+{{ include "flyteconsole.labels" . }}
+{{- with .Values.flyteconsole.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 # Optional blocks for secret mount
 
 {{- define "databaseSecret.volume" -}}

charts/flyte-core/templates/admin/deployment.yaml

@@ -16,7 +16,7 @@ spec:
         {{- with .Values.flyteadmin.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      labels: {{ include "flyteadmin.labels" . | nindent 8 }}
+      labels: {{ include "flyteadmin.podLabels" . | nindent 8 }}
     spec:
       securityContext:
         fsGroup: 65534

charts/flyte-core/templates/clusterresourcesync/deployment.yaml

@@ -16,7 +16,7 @@ spec:
         {{- with .Values.cluster_resource_manager.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      labels: {{ include "flyteclusterresourcesync.labels" . | nindent 8 }}
+      labels: {{ include "flyteclusterresourcesync.podLabels" . | nindent 8 }}
     spec:
       containers:
         - command:
@@ -25,6 +25,12 @@ spec:
             - {{ .Values.flyteadmin.configPath }}
             - clusterresource
             - run
+          {{- if .Values.cluster_resource_manager.podEnv }}
+          env:
+            {{- with .Values.cluster_resource_manager.podEnv }}
+            {{- toYaml . | nindent 10 }}
+            {{- end }}
+          {{- end }}
           image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
           imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}"
           name: sync-cluster-resources

charts/flyte-core/templates/console/deployment.yaml

@@ -16,7 +16,7 @@ spec:
         {{- with .Values.flyteconsole.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      labels: {{ include "flyteconsole.labels" . | nindent 8 }}
+      labels: {{ include "flyteconsole.podLabels" . | nindent 8 }}
     spec:
       securityContext:
         runAsUser: 1000
@@ -33,13 +33,18 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
-        {{- if .Values.flyteconsole.ga.enabled }}
         env:
+        {{- if .Values.flyteconsole.ga.enabled }}
         - name: ENABLE_GA
           value: "{{ .Values.flyteconsole.ga.enabled }}"
         - name: GA_TRACKING_ID
           value: "{{ .Values.flyteconsole.ga.tracking_id }}"
         {{- end }}
+        {{- if .Values.flyteconsole.podEnv -}}
+        {{- with .Values.flyteconsole.podEnv }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- end }}
         resources: {{ toYaml .Values.flyteconsole.resources | nindent 10 }}
         volumeMounts:
         - mountPath: /srv/flyte

charts/flyte-core/templates/datacatalog/deployment.yaml

@@ -16,7 +16,7 @@ spec:
         {{- with .Values.datacatalog.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      labels: {{ include "datacatalog.labels" . | nindent 8 }}
+      labels: {{ include "datacatalog.podLabels" . | nindent 8 }}
     spec:
       securityContext:
         fsGroup: 1001
@@ -47,6 +47,12 @@ spec:
         {{- with .Values.datacatalog.extraArgs }}
         {{- tpl (toYaml .) $ | nindent 8 }}
         {{- end }}
+        {{- if .Values.datacatalog.podEnv }}
+        env:
+          {{- with .Values.datacatalog.podEnv }}
+          {{- toYaml . | nindent 8 }}
+          {{- end }}
+        {{- end }}
         image: "{{ .Values.datacatalog.image.repository }}:{{ .Values.datacatalog.image.tag }}"
         imagePullPolicy: "{{ .Values.datacatalog.image.pullPolicy }}"
         name: datacatalog

charts/flyte-core/templates/flytescheduler/deployment.yaml

@@ -17,7 +17,7 @@ spec:
         {{- with .Values.flytescheduler.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      labels: {{ include "flytescheduler.labels" . | nindent 8 }}
+      labels: {{ include "flytescheduler.podLabels" . | nindent 8 }}
     spec:
       securityContext:
         fsGroup: 65534
@@ -48,6 +48,12 @@ spec:
         - run
         - --config
         - {{ .Values.flytescheduler.configPath }}
+        {{- if .Values.flytescheduler.podEnv }}
+        env:
+          {{- with .Values.flytescheduler.podEnv -}}
+          {{- toYaml . | nindent 8 }}
+          {{- end }}
+        {{- end }}
         image: "{{ .Values.flytescheduler.image.repository }}:{{ .Values.flytescheduler.image.tag }}"
         imagePullPolicy: "{{ .Values.flytescheduler.image.pullPolicy }}"
         name: flytescheduler