From 033c743fc14cd083350658fa6082226c4cb6047c Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Mon, 2 Sep 2024 21:10:05 -0700 Subject: [PATCH] Update helm/docs per changes in supported task discovery (#5694) * Update helm/docs per changes in supported task discovery Signed-off-by: Jason Parraga * Cleanup flyte-binary Signed-off-by: Jason Parraga * fixes Signed-off-by: Jason Parraga --------- Signed-off-by: Jason Parraga Signed-off-by: pmahindrakar-oss --- charts/flyte-binary/README.md | 1 - charts/flyte-binary/values.yaml | 2 -- charts/flyte-core/README.md | 4 ++-- charts/flyte-core/values.yaml | 8 +++----- .../manifests/complete-agent.yaml | 8 +++----- docker/sandbox-bundled/manifests/complete.yaml | 4 ++-- docker/sandbox-bundled/manifests/dev.yaml | 4 ++-- docs/deployment/agents/airflow.rst | 11 +---------- docs/deployment/agents/bigquery.rst | 11 +---------- docs/deployment/agents/chatgpt.rst | 6 +----- docs/deployment/agents/databricks.rst | 18 +++--------------- docs/deployment/agents/mmcloud.rst | 2 -- docs/deployment/agents/openai_batch.rst | 12 ++---------- docs/deployment/agents/sagemaker_inference.rst | 14 ++------------ docs/deployment/agents/sensor.rst | 11 +---------- docs/deployment/agents/snowflake.rst | 11 +---------- .../generated/flytepropeller_config.rst | 3 --- docs/flyte_agents/developing_agents.md | 6 +----- 18 files changed, 25 insertions(+), 111 deletions(-) diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md index cc39b38cdc3..43e0a714371 100644 --- a/charts/flyte-binary/README.md +++ b/charts/flyte-binary/README.md @@ -25,7 +25,6 @@ Chart for basic single Flyte executable deployment | configuration.agentService.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | | | configuration.agentService.defaultAgent.insecure | bool | `true` | | | configuration.agentService.defaultAgent.timeouts.GetTask | string | `"10s"` | | -| configuration.agentService.supportedTaskTypes[0] | string | `"default_task"` | | | configuration.annotations | object | `{}` | | | configuration.auth.authorizedUris | list | `[]` | | | configuration.auth.clientSecretsExternalSecretRef | string | `""` | | diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml index 0f7261d86e7..3f87556c3ed 100644 --- a/charts/flyte-binary/values.yaml +++ b/charts/flyte-binary/values.yaml @@ -168,8 +168,6 @@ configuration: timeouts: GetTask: 10s defaultTimeout: 10s - supportedTaskTypes: - - default_task # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte # If set, no Flyte configuration will be generated by this chart externalConfigMap: "" diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 673ba7b6ef5..ba11205264f 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -195,11 +195,11 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out | | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | | flyteagent.enabled | bool | `false` | | -| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":["sensor"]}` | Agent service configuration for propeller. | +| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":[]}` | Agent service configuration for propeller. | | flyteagent.plugin_config.plugins.agent-service.defaultAgent | object | `{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true}` | The default agent service to use for plugin tasks. | | flyteagent.plugin_config.plugins.agent-service.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | The agent service endpoint propeller should connect to. | | flyteagent.plugin_config.plugins.agent-service.defaultAgent.insecure | bool | `true` | Whether the connection from propeller to the agent service should use TLS. | -| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `["sensor"]` | The task types supported by the default agent. | +| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `[]` | The task types supported by the default agent. As of #5460 these are discovered automatically and don't need to be configured. | | flyteagent.podLabels | object | `{}` | Labels for flyteagent pods | | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyteconsole.enabled | bool | `true` | | diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 93c0d9b3894..7442222e9bc 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -287,11 +287,9 @@ flyteagent: endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000" # -- Whether the connection from propeller to the agent service should use TLS. insecure: true - # -- The task types supported by the default agent. - supportedTaskTypes: - - sensor - # -- Uncomment to enable task type that uses Flyte Agent - # - bigquery_query_job_task + # -- The task types supported by the default agent. As of #5460 these are discovered automatically and don't + # need to be configured. + supportedTaskTypes: [] # -- Labels for flyteagent pods podLabels: {} diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index 78a678ae34f..1279b64c601 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -485,8 +485,6 @@ data: insecure: true timeouts: GetTask: 10s - supportedTaskTypes: - - default_task 002-database.yaml: | database: postgres: @@ -818,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: cWlOc1c1bnl5ZGI3YTlzSw== + haSharedSecret: WVl4Y1pFNm1JTkxOMjJpZQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1249,7 +1247,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 6e8a4cc6177037f26cee65d09c37c010437ea3f0989a2a2dfef380fed9f468c2 + checksum/configuration: f746817691b502fb50f04992e148847f89c0a32d7df822dbda7f1f5fdf84f420 checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1415,7 +1413,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 7f8247a0b84f43018fdf11a598132b8a67ed9fde6573ffce801b725a6f955012 + checksum/secret: f0211dde23276e14b7254edd85e3afd32044562e409b56d79b27202e23dc224c labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index 5d46b89edf6..e9e3007dc9b 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -798,7 +798,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: UUxqaW5SeGlBbFNlQzVoag== + haSharedSecret: Q3BDekMwb0JHM1pxaXAycg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1362,7 +1362,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: bea0c8f293b54e309a353e0e8563e709ad817d372d2b1dce1114188693aa3f12 + checksum/secret: 65e52b61d9d5af16c4217a1ea9f625d8d18e3251ede28f012cc14ef665412685 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 917645af33f..d0983c6f0d6 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: ZmdJNWs5RUg4cWNVTVBzRw== + haSharedSecret: RU9mMGpZWGNCcnp4cEJ3bg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: a896f2c43dff6c05c154b51e4c9ec21c9e2f03ecaf4c1fed045d84523219cf63 + checksum/secret: 90ba33230f60ed1ee81e3088a4c88b0c9408c36ad5d40270cf05f503149f25a8 labels: app: docker-registry release: flyte-sandbox diff --git a/docs/deployment/agents/airflow.rst b/docs/deployment/agents/airflow.rst index ad6a6dab360..174967e20c4 100644 --- a/docs/deployment/agents/airflow.rst +++ b/docs/deployment/agents/airflow.rst @@ -21,7 +21,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -35,11 +35,6 @@ Specify agent configuration - container_array: k8s-array - airflow: agent-service - plugins: - agent-service: - supportedTaskTypes: - - airflow - .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following configuration to it. @@ -63,10 +58,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array airflow: agent-service - plugins: - agent-service: - supportedTaskTypes: - - airflow Upgrade the Flyte Helm release diff --git a/docs/deployment/agents/bigquery.rst b/docs/deployment/agents/bigquery.rst index d3e4ee490e3..0d30d0d3b35 100644 --- a/docs/deployment/agents/bigquery.rst +++ b/docs/deployment/agents/bigquery.rst @@ -29,7 +29,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -42,11 +42,6 @@ Specify agent configuration - container: container - container_array: k8s-array - bigquery_query_job_task: agent-service - - plugins: - agent-service: - supportedTaskTypes: - - bigquery_query_job_task .. group-tab:: Flyte core @@ -71,10 +66,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array bigquery_query_job_task: agent-service - plugins: - agent-service: - supportedTaskTypes: - - bigquery_query_job_task Ensure that the propeller has the correct service account for BigQuery. diff --git a/docs/deployment/agents/chatgpt.rst b/docs/deployment/agents/chatgpt.rst index afc569222f4..cb0b44fa390 100644 --- a/docs/deployment/agents/chatgpt.rst +++ b/docs/deployment/agents/chatgpt.rst @@ -20,7 +20,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -36,8 +36,6 @@ Specify agent configuration plugins: agent-service: - supportedTaskTypes: - - chatgpt # Configuring the timeout is optional. # Tasks like using ChatGPT with a large model might require a longer time, # so we have the option to adjust the timeout setting here. @@ -70,8 +68,6 @@ Specify agent configuration chatgpt: agent-service plugins: agent-service: - supportedTaskTypes: - - chatgpt # Configuring the timeout is optional. # Tasks like using ChatGPT with a large model might require a longer time, # so we have the option to adjust the timeout setting here. diff --git a/docs/deployment/agents/databricks.rst b/docs/deployment/agents/databricks.rst index b21fab3c571..0458fb36678 100644 --- a/docs/deployment/agents/databricks.rst +++ b/docs/deployment/agents/databricks.rst @@ -139,7 +139,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,12,16 + :emphasize-lines: 7,12 tasks: task-plugins: @@ -153,17 +153,13 @@ Specify agent configuration - sidecar - k8s-array - agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark .. group-tab:: Helm chart Edit the relevant YAML file to specify the plugin. .. code-block:: yaml - :emphasize-lines: 7,11,15 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -176,17 +172,13 @@ Specify agent configuration - container: container - container_array: k8s-array - spark: agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following config to it: .. code-block:: yaml - :emphasize-lines: 9,14-17 + :emphasize-lines: 9 enabled_plugins: tasks: @@ -201,10 +193,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array spark: agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark Add the Databricks access token ------------------------------- diff --git a/docs/deployment/agents/mmcloud.rst b/docs/deployment/agents/mmcloud.rst index 422162af27f..9ccb101aa2f 100644 --- a/docs/deployment/agents/mmcloud.rst +++ b/docs/deployment/agents/mmcloud.rst @@ -71,8 +71,6 @@ Enable the MMCloud agent by adding the following config to the relevant YAML fil mmcloud-agent: endpoint: insecure: true - supportedTaskTypes: - - mmcloud_task agentForTaskTypes: - mmcloud_task: mmcloud-agent diff --git a/docs/deployment/agents/openai_batch.rst b/docs/deployment/agents/openai_batch.rst index 7aff9d262e8..2cfa70471aa 100644 --- a/docs/deployment/agents/openai_batch.rst +++ b/docs/deployment/agents/openai_batch.rst @@ -19,7 +19,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,15 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -32,17 +32,13 @@ Specify agent configuration - container: container - container_array: k8s-array - openai-batch: agent-service - plugins: - agent-service: - supportedTaskTypes: - - openai-batch .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following configuration to it: .. code-block:: yaml - :emphasize-lines: 9,14,18 + :emphasize-lines: 9,14 configmap: enabled_plugins: @@ -58,10 +54,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array openai-batch: agent-service - plugins: - agent-service: - supportedTaskTypes: - - openai-batch Add the OpenAI API token ------------------------ diff --git a/docs/deployment/agents/sagemaker_inference.rst b/docs/deployment/agents/sagemaker_inference.rst index 5ceb248c2d9..3f03e08f55c 100644 --- a/docs/deployment/agents/sagemaker_inference.rst +++ b/docs/deployment/agents/sagemaker_inference.rst @@ -19,7 +19,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11-12,16-17 + :emphasize-lines: 7,11-12 tasks: task-plugins: @@ -33,18 +33,13 @@ Specify agent configuration - container_array: k8s-array - boto: agent-service - sagemaker-endpoint: agent-service - plugins: - agent-service: - supportedTaskTypes: - - boto - - sagemaker-endpoint .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following configuration to it: .. code-block:: yaml - :emphasize-lines: 9,14-15,19-20 + :emphasize-lines: 9,14-15 configmap: enabled_plugins: @@ -61,11 +56,6 @@ Specify agent configuration container_array: k8s-array boto: agent-service sagemaker-endpoint: agent-service - plugins: - agent-service: - supportedTaskTypes: - - boto - - sagemaker-endpoint AWS credentials --------------- diff --git a/docs/deployment/agents/sensor.rst b/docs/deployment/agents/sensor.rst index 958e5d896ac..312e34bcd1a 100644 --- a/docs/deployment/agents/sensor.rst +++ b/docs/deployment/agents/sensor.rst @@ -56,7 +56,7 @@ Enable the sensor agent by adding the following config to the relevant YAML file kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -69,11 +69,6 @@ Enable the sensor agent by adding the following config to the relevant YAML file - container: container - container_array: k8s-array - sensor: agent-service - - plugins: - agent-service: - supportedTaskTypes: - - sensor .. group-tab:: Flyte core @@ -98,10 +93,6 @@ Enable the sensor agent by adding the following config to the relevant YAML file sidecar: sidecar container_array: k8s-array sensor: agent-service - plugins: - agent-service: - supportedTaskTypes: - - sensor Upgrade the deployment diff --git a/docs/deployment/agents/snowflake.rst b/docs/deployment/agents/snowflake.rst index d6ee74125bd..dad62a3795c 100644 --- a/docs/deployment/agents/snowflake.rst +++ b/docs/deployment/agents/snowflake.rst @@ -35,7 +35,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -49,11 +49,6 @@ Specify agent configuration - container_array: k8s-array - snowflake: agent-service - plugins: - agent-service: - supportedTaskTypes: - - snowflake - .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following configuration to it. @@ -77,10 +72,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array snowflake: agent-service - plugins: - agent-service: - supportedTaskTypes: - - snowflake Ensure that the propeller has the correct service account for Snowflake. diff --git a/docs/deployment/configuration/generated/flytepropeller_config.rst b/docs/deployment/configuration/generated/flytepropeller_config.rst index be6f7ee7f05..554ebec849d 100644 --- a/docs/deployment/configuration/generated/flytepropeller_config.rst +++ b/docs/deployment/configuration/generated/flytepropeller_config.rst @@ -994,9 +994,6 @@ agent-service (`agent.Config`_) Value: 50 ProjectScopeResourceConstraint: Value: 100 - supportedTaskTypes: - - task_type_1 - - task_type_2 webApi: caching: maxSystemFailures: 5 diff --git a/docs/flyte_agents/developing_agents.md b/docs/flyte_agents/developing_agents.md index ee989b812f8..fd310088efd 100644 --- a/docs/flyte_agents/developing_agents.md +++ b/docs/flyte_agents/developing_agents.md @@ -197,10 +197,6 @@ you can route particular task requests to designated agent services by adjusting ```yaml plugins: agent-service: - supportedTaskTypes: - - bigquery_query_job_task - - default_task - - custom_task # By default, all requests will be sent to the default agent. defaultAgent: endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000" @@ -224,4 +220,4 @@ you can route particular task requests to designated agent services by adjusting agentForTaskTypes: # It will override the default agent for custom_task, which means propeller will send the request to this agent. - custom_task: custom_agent -``` \ No newline at end of file +```