Merge remote-tracking branch 'origin/master' into jeev/gpu-selection

flyteorg · Oct 12, 2023 · 6526390 · 6526390
2 parents 93d15fd + 0ca2d22
commit 6526390
Show file tree

Hide file tree

Showing 67 changed files with 4,098 additions and 563 deletions.
diff --git a/.github/workflows/flyteidl-buf-publish.yml b/.github/workflows/flyteidl-buf-publish.yml
@@ -0,0 +1,18 @@
+name: Publish flyteidl Buf Package
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'flyteidl/**'
+jobs:
+  buf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: bufbuild/buf-setup-action@v1
+      - uses: bufbuild/buf-push-action@v1
+        with:
+          buf_token: ${{ secrets.BUF_TOKEN }}
+          input: 'flyteidl/protos'
diff --git a/README.md b/README.md
@@ -31,8 +31,8 @@ Flyte is an **open-source orchestrator** that facilitates building production-gr
     <img src="https://img.shields.io/badge/openssf%20best%20practices-passing-green?style=for-the-badge" alt="OpenSSF Best Practices" /></a>
   <a href="https://artifacthub.io/packages/search?repo=flyte">
     <img src="https://img.shields.io/endpoint?style=for-the-badge&url=https://artifacthub.io/badge/repository/flyte" alt="Flyte Helm Chart" /></a>
-  <a href="https://twitter.com/flyteorg">
-    <img src="https://img.shields.io/badge/Twitter-Follow-blue?style=for-the-badge&logo=twitter" alt="Flyte Twitter" /></a>
+  <a href="https://twitter.com/flyteorg"> 
+    <img src="https://img.shields.io/badge/X-000000.svg?style=for-the-badge&logo=X&logoColor=white" height=30px/></a>
   <a href="https://slack.flyte.org">
     <img src="https://img.shields.io/badge/Slack-Chat-pink?style=for-the-badge&logo=slack" alt="Flyte Slack" /></a>
 </p>

diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md
@@ -21,7 +21,11 @@ Chart for basic single Flyte executable deployment
 | clusterResourceTemplates.labels | object | `{}` |  |
 | commonAnnotations | object | `{}` |  |
 | commonLabels | object | `{}` |  |
-| configuration.agentService | object | `{}` |  |
+| configuration.agentService.defaultAgent.defaultTimeout | string | `"10s"` |  |
+| configuration.agentService.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` |  |
+| configuration.agentService.defaultAgent.insecure | bool | `true` |  |
+| configuration.agentService.defaultAgent.timeouts.GetTask | string | `"10s"` |  |
+| configuration.agentService.supportedTaskTypes[0] | string | `"default_task"` |  |
 | configuration.annotations | object | `{}` |  |
 | configuration.auth.authorizedUris | list | `[]` |  |
 | configuration.auth.clientSecretsExternalSecretRef | string | `""` |  |
@@ -103,9 +107,9 @@ Chart for basic single Flyte executable deployment
 | deployment.waitForDB.image.pullPolicy | string | `"IfNotPresent"` |  |
 | deployment.waitForDB.image.repository | string | `"postgres"` |  |
 | deployment.waitForDB.image.tag | string | `"15-alpine"` |  |
-| enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) |
-| enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) |
-| enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins |
+| enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) |
+| enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) |
+| enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array","agent-service"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins |
 | flyte-core-components.admin.disableClusterResourceManager | bool | `false` |  |
 | flyte-core-components.admin.disableScheduler | bool | `false` |  |
 | flyte-core-components.admin.disabled | bool | `false` |  |

diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
@@ -151,11 +151,16 @@ configuration:
       # tag CoPilot sidecar image tag
       tag: v1.9.4 # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
-  agentService: {}
+  agentService:
+    defaultAgent:
+      endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000"
+      insecure: true
+      timeouts:
+        GetTask: 10s
+      defaultTimeout: 10s
   #   Uncomment and modify to include configuration for Flyte Agent
-  #   defaultGrpcEndpoint: agent-service.agent-namespace:8000
-  #   supportedTaskTypes:
-  #     - custom_task_type
+    supportedTaskTypes:
+      - default_task
   # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte
   # If set, no Flyte configuration will be generated by this chart
   externalConfigMap: ""
@@ -382,8 +387,7 @@ enabled_plugins:
         - container
         - sidecar
         - k8s-array
-        # -- Uncomment to enable agent service
-        # - agent-service
+        - agent-service
       default-for-task-types:
         container: container
         sidecar: sidecar

diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml
@@ -108,6 +108,7 @@ data:
         - container
         - sidecar
         - k8s-array
+        - agent-service
     plugins:
       logs:
         kubernetes-enabled: false
@@ -122,6 +123,15 @@ data:
             kubernetes-enabled: false
             cloudwatch-enabled: false
             stackdriver-enabled: false
+      agent-service:
+        defaultAgent:
+          defaultTimeout: 10s
+          endpoint: dns:///flyteagent.flyte.svc.cluster.local:8000
+          insecure: true
+          timeouts:
+            GetTask: 10s
+        supportedTaskTypes:
+        - default_task
   002-database.yaml: |
     database:
       postgres:
@@ -357,7 +367,7 @@ spec:
         app.kubernetes.io/instance: flyte
         app.kubernetes.io/component: flyte-binary
       annotations:
-        checksum/configuration: 528ce4a42638a7810c99802dfd49525967db2a99dbc1019544b7799de2490b61
+        checksum/configuration: d220769393e7acbe0372fdccbf3d588797864ec934661f08912e88ec084cdfde
         checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929
         checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
     spec:

diff --git a/docker/sandbox-bundled/Makefile b/docker/sandbox-bundled/Makefile
@@ -19,6 +19,7 @@ flyte:
 manifests:
 	mkdir -p manifests
 	helm dependency update ../../charts/flyte-sandbox
+	helm dependency update ../../charts/flyteagent
 	kustomize build \
 		--enable-helm \
 		--load-restrictor=LoadRestrictionsNone \
@@ -27,6 +28,10 @@ manifests:
 		--enable-helm \
 		--load-restrictor=LoadRestrictionsNone \
 		kustomize/dev > manifests/dev.yaml
+	kustomize build \
+		--enable-helm \
+		--load-restrictor=LoadRestrictionsNone \
+		kustomize/complete-agent > manifests/complete-agent.yaml
 
 .PHONY: build
 build: flyte manifests

diff --git a/docker/sandbox-bundled/bootstrap/cmd/bootstrap/main.go b/docker/sandbox-bundled/bootstrap/cmd/bootstrap/main.go
@@ -17,13 +17,15 @@ const (
 	clusterResourceTemplatesConfigMapName = "flyte-sandbox-extra-cluster-resource-templates"
 	deploymentName                        = "flyte-sandbox"
 	devModeEnvVar                         = "FLYTE_DEV"
+	disableAgentModeEnvVar                = "DISABLE_AGENT"
 	dockerHost                            = "host.docker.internal"
 	namespace                             = "flyte"
 
 	// Template paths
-	devTemplatePath      = "/var/lib/rancher/k3s/server/manifests-staging/dev.yaml"
-	fullTemplatePath     = "/var/lib/rancher/k3s/server/manifests-staging/complete.yaml"
-	renderedManifestPath = "/var/lib/rancher/k3s/server/manifests/flyte.yaml"
+	devTemplatePath       = "/var/lib/rancher/k3s/server/manifests-staging/dev.yaml"
+	fullTemplatePath      = "/var/lib/rancher/k3s/server/manifests-staging/complete.yaml"
+	fullAgentTemplatePath = "/var/lib/rancher/k3s/server/manifests-staging/complete-agent.yaml"
+	renderedManifestPath  = "/var/lib/rancher/k3s/server/manifests/flyte.yaml"
 )
 
 func main() {
@@ -35,7 +37,11 @@ func main() {
 	} else {
 		// If we are not running in dev mode, look for user-specified configuration
 		// to load into the sandbox deployment
-		tmplPath = fullTemplatePath
+		tmplPath = fullAgentTemplatePath
+		if os.Getenv(disableAgentModeEnvVar) == "True" {
+			tmplPath = fullTemplatePath
+		}
+
 		cOpts := config.LoaderOpts{
 			ConfigurationConfigMapName:            configurationConfigMapName,
 			ClusterResourceTemplatesConfigMapName: clusterResourceTemplatesConfigMapName,

diff --git a/docker/sandbox-bundled/kustomize/complete-agent/kustomization.yaml b/docker/sandbox-bundled/kustomize/complete-agent/kustomization.yaml
@@ -0,0 +1,12 @@
+helmGlobals:
+  chartHome: ../../../../charts
+helmCharts:
+- name: flyte-sandbox
+  releaseName: flyte-sandbox
+  namespace: flyte
+- name: flyteagent
+  releaseName: flyteagent
+  namespace: flyte
+namespace: flyte
+resources:
+- ../namespace.yaml