Merge branch 'master' into terryhung/fix-the-config-error

Makefile

@@ -6,10 +6,10 @@ define PIP_COMPILE
 pip-compile $(1) --upgrade --verbose --resolver=backtracking --annotation-style=line
 endef
 
-GIT_VERSION := $(shell git describe --always --tags)
+GIT_VERSION := $(shell git describe --tags --long --match "v*" --first-parent)
 GIT_HASH := $(shell git rev-parse --short HEAD)
 TIMESTAMP := $(shell date '+%Y-%m-%d')
-PACKAGE ?=github.com/flyteorg/flytestdlib
+PACKAGE ?=github.com/flyteorg/flyte/flytestdlib
 LD_FLAGS="-s -w -X $(PACKAGE)/version.Version=$(GIT_VERSION) -X $(PACKAGE)/version.Build=$(GIT_HASH) -X $(PACKAGE)/version.BuildTime=$(TIMESTAMP)"
 TMP_BUILD_DIR := .tmp_build
 

charts/flyte-core/README.md

@@ -55,10 +55,25 @@ helm install gateway bitnami/contour -n flyte
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| cloud_events.aws.region | string | `"us-east-2"` |  |
+| cloud_events.aws | object | `{"region":"us-east-2"}` | Configuration for sending cloud events to AWS SNS |
 | cloud_events.enable | bool | `false` |  |
 | cloud_events.eventsPublisher.eventTypes[0] | string | `"all"` |  |
 | cloud_events.eventsPublisher.topicName | string | `"arn:aws:sns:us-east-2:123456:123-my-topic"` |  |
+| cloud_events.gcp | object | `{"region":"us-east1"}` | Configuration for sending cloud events to GCP Pub Sub |
+| cloud_events.kafka | object | `{"brokers":["mybroker:443"],"saslConfig":{"enabled":false,"handshake":true,"mechanism":"PLAIN","password":"","passwordPath":"","user":"kafka"},"tlsConfig":{"certPath":"/etc/ssl/certs/kafka-client.crt","enabled":false,"keyPath":"/etc/ssl/certs/kafka-client.key"},"version":"3.7.0"}` | Configuration for sending cloud events to Kafka |
+| cloud_events.kafka.brokers | list | `["mybroker:443"]` | The kafka brokers to talk to |
+| cloud_events.kafka.saslConfig | object | `{"enabled":false,"handshake":true,"mechanism":"PLAIN","password":"","passwordPath":"","user":"kafka"}` | SASL based authentication |
+| cloud_events.kafka.saslConfig.enabled | bool | `false` | Whether to use SASL authentication |
+| cloud_events.kafka.saslConfig.handshake | bool | `true` | Whether the send the SASL handsahke first |
+| cloud_events.kafka.saslConfig.mechanism | string | `"PLAIN"` | Which SASL mechanism to use. Defaults to PLAIN |
+| cloud_events.kafka.saslConfig.password | string | `""` | The password for the kafka user |
+| cloud_events.kafka.saslConfig.passwordPath | string | `""` | Optional mount path of file containing the kafka password. |
+| cloud_events.kafka.saslConfig.user | string | `"kafka"` | The kafka user |
+| cloud_events.kafka.tlsConfig | object | `{"certPath":"/etc/ssl/certs/kafka-client.crt","enabled":false,"keyPath":"/etc/ssl/certs/kafka-client.key"}` | Certificate based authentication |
+| cloud_events.kafka.tlsConfig.certPath | string | `"/etc/ssl/certs/kafka-client.crt"` | Path to the client certificate |
+| cloud_events.kafka.tlsConfig.enabled | bool | `false` | Whether to use certificate based authentication or TLS |
+| cloud_events.kafka.tlsConfig.keyPath | string | `"/etc/ssl/certs/kafka-client.key"` | Path to the client private key |
+| cloud_events.kafka.version | string | `"3.7.0"` | The version of Kafka |
 | cloud_events.type | string | `"aws"` |  |
 | cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"nodeSelector":{},"podAnnotations":{},"podEnv":{},"podLabels":{},"prometheus":{"enabled":false,"path":"/metrics","port":10254},"resources":{},"service_account_name":"flyteadmin","standaloneDeployment":false,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: project-quota\n  namespace: {{ namespace }}\nspec:\n  hard:\n    limits.cpu: {{ projectQuotaCpu }}\n    limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain |
 | cluster_resource_manager.config | object | `{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}}` | Configmap for ClusterResource parameters |

charts/flyte-core/values.yaml

@@ -946,15 +946,48 @@ external_events:
 # an SNS topic (or gcp equivalent)
 cloud_events:
   enable: false
-  type: aws
-  aws:
-    region: us-east-2
   eventsPublisher:
     # Make sure this is not a fifo queue. Admin does not yet support
     # writing to fifo sns topics.
     topicName: "arn:aws:sns:us-east-2:123456:123-my-topic"
     eventTypes:
       - all  # Or workflow, node, task. Or "*"
+  type: aws
+  # -- Configuration for sending cloud events to AWS SNS
+  aws:
+    region: us-east-2
+  # -- Configuration for sending cloud events to GCP Pub Sub
+  gcp:
+    region: us-east1
+  # -- Configuration for sending cloud events to Kafka
+  kafka:
+    # -- The version of Kafka
+    version: "3.7.0"
+    # -- The kafka brokers to talk to
+    brokers:
+      - mybroker:443
+    # -- SASL based authentication
+    saslConfig:
+      # -- Whether to use SASL authentication
+      enabled: false
+      # -- The kafka user
+      user: kafka
+      # -- The password for the kafka user
+      password: ""
+      # -- Optional mount path of file containing the kafka password.
+      passwordPath: ""
+      # -- Whether the send the SASL handsahke first
+      handshake: true
+      # -- Which SASL mechanism to use. Defaults to PLAIN
+      mechanism: PLAIN
+    # -- Certificate based authentication
+    tlsConfig:
+      # -- Whether to use certificate based authentication or TLS
+      enabled: false
+      # -- Path to the client certificate
+      certPath: /etc/ssl/certs/kafka-client.crt
+      # -- Path to the client private key
+      keyPath: /etc/ssl/certs/kafka-client.key
 
 # -- Configuration for the Cluster resource manager component. This is an optional component, that enables automatic
 # cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain

docker/sandbox-bundled/manifests/complete-agent.yaml

@@ -444,6 +444,9 @@ data:
         disabled: false
         seedProjects:
         - flytesnacks
+        seedProjectsWithDetails:
+        - description: Default project setup.
+          name: flytesnacks
       dataCatalog:
         disabled: false
       propeller:
@@ -816,7 +819,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: M0MwZjRDRlRMRVg5eFlNWA==
+  haSharedSecret: SlI1TDFkTXBMaThuc0hlSQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1247,7 +1250,7 @@ spec:
     metadata:
       annotations:
         checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
-        checksum/configuration: dc6e26fec37cad413a92bf06f2840ea1e497284312275ff06e22b152dee1566b
+        checksum/configuration: a823eaadac5f3a4358c8acf628ebeb3719f88312af520d2c253de2579dff262d
         checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
       labels:
         app.kubernetes.io/component: flyte-binary
@@ -1413,7 +1416,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 49b88f7ed6b4bec4cdb0305c1d990514d9b75690607d7ae75d5862da9a3b2a29
+        checksum/secret: ffc8aa05a602edd8f9b1d7ef35aa1cc5e383bceb9b91307eef99e86f53e13d4e
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/complete.yaml

@@ -433,6 +433,9 @@ data:
         disabled: false
         seedProjects:
         - flytesnacks
+        seedProjectsWithDetails:
+        - description: Default project setup.
+          name: flytesnacks
       dataCatalog:
         disabled: false
       propeller:
@@ -798,7 +801,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: ekx6Z2kxS3FBYjV5dExlMw==
+  haSharedSecret: YjdMdE9yejJzZ2xXSDFBRQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1196,7 +1199,7 @@ spec:
     metadata:
       annotations:
         checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
-        checksum/configuration: a6f3ea502338c626b7824453ce7dc8b6fcd441d68865c075e2e74d797bc607fa
+        checksum/configuration: c2649df6bcb523f120c73b0fdeec5d9516f555eab12e4eae78b04dea2cf2abae
         checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
       labels:
         app.kubernetes.io/component: flyte-binary
@@ -1362,7 +1365,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 9b64bfe991cd6ce4394fa9c2651b0bbfe4834024ece293b3ac9688111d6fe5d3
+        checksum/secret: 956ac1b58c049a630c94605eedaba7ba9de3fc01233701ef403ab4bf24fe2a7a
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/dev.yaml

@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: MW90empzaUNBd2FlV09QSw==
+  haSharedSecret: YUpzb25xNTM1eml3Rmpueg==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -934,7 +934,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: ba78cd87d2f6685980b95bd20913088b3a07fa48e9a414693277e3df134710ad
+        checksum/secret: 2720f13bd64051a7acb512e59e426b9f6c5f6c3c7d1d9a3a423e2df4cf9bab46
       labels:
         app: docker-registry
         release: flyte-sandbox

docs/deployment/configuration/auth_setup.rst

@@ -254,15 +254,15 @@ Apply OIDC Configuration
                httpPort: 8088
                grpc:
                  port: 8089
-             security:
-               secure: false
-               useAuth: true
-               allowCors: true
-               allowedOrigins:
-          # Accepting all domains for Sandbox installation
-                 - "*"
-               allowedHeaders:
-                 - "Content-Type"
+               security:
+                 secure: false
+                 useAuth: true
+                 allowCors: true
+                 allowedOrigins:
+            # Accepting all domains for Sandbox installation
+                   - "*"
+                 allowedHeaders:
+                   - "Content-Type"
              auth:
                appAuth:
                  thirdPartyConfig:

docs/deployment/configuration/general.rst

@@ -66,7 +66,7 @@ Notice how in this example we are defining a new PodTemplate inline, which allow
 `V1PodSpec <https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1PodSpec.md>`__ and also define
 the name of the primary container, labels, and annotations.
 
-The term compile-time here refers to the fact that the pod template definition is part of the `TaskSpec <https://docs.flyte.org/en/latest/protos/docs/admin/admin.html#ref-flyteidl-admin-taskclosure>`__.
+The term compile-time here refers to the fact that the pod template definition is part of the `TaskSpec <https://docs.flyte.org/en/latest/api/flyteidl/docs/admin/admin.html#ref-flyteidl-admin-taskclosure>`__.
 
 ********************
 Runtime PodTemplates
@@ -88,7 +88,7 @@ initializes a K8s informer internally to track system PodTemplate updates
 `aware <https://docs.flyte.org/en/latest/deployment/cluster_config/flytepropeller_config.html#config-k8spluginconfig>`__
 of the latest PodTemplate definitions in the K8s environment. You can find this
 setting in `FlytePropeller <https://github.com/flyteorg/flyte/blob/e3e4978838f3caee0d156348ca966b7f940e3d45/deployment/eks/flyte_generated.yaml#L8239-L8244>`__
-config map, which is not set by default. 
+config map, which is not set by default.
 
 An example configuration is:
 
@@ -101,14 +101,14 @@ An example configuration is:
           image: "cr.flyte.org/flyteorg/flytecopilot:v0.0.15"
           start-timeout: "30s"
         default-pod-template-name: <your_template_name>
- 
+
 Create a PodTemplate resource
 =============================
 
-Flyte recognizes PodTemplate definitions with the ``default-pod-template-name`` at two granularities. 
+Flyte recognizes PodTemplate definitions with the ``default-pod-template-name`` at two granularities.
 
 1. A system-wide configuration can be created in the same namespace that
-   FlytePropeller is running in (typically `flyte`). 
+   FlytePropeller is running in (typically `flyte`).
 2. PodTemplates can be applied from the same namespace that the Pod will be
    created in. FlytePropeller always favors the PodTemplate with the more
    specific namespace. For example, a Pod created in the ``flytesnacks-development``
@@ -196,7 +196,7 @@ where you start the Pod.
 An example PodTemplate is shown:
 
 .. code-block:: yaml
-    
+
     apiVersion: v1
     kind: PodTemplate
     metadata:
@@ -220,7 +220,7 @@ In addition, the K8s plugin configuration in FlytePropeller defines the default
 Pod Labels, Annotations, and enables the host networking.
 
 .. code-block:: yaml
-    
+
     plugins:
        k8s:
         default-labels:
@@ -233,7 +233,7 @@ Pod Labels, Annotations, and enables the host networking.
 To construct a Pod, FlytePropeller initializes a Pod definition using the default
 PodTemplate. This definition is applied to the K8s plugin configuration values,
 and any task-specific configuration is overlaid. During the process, when lists
-are merged, values are appended and when maps are merged, the values are overridden. 
+are merged, values are appended and when maps are merged, the values are overridden.
 The resultant Pod using the above default PodTemplate and K8s Plugin configuration is shown:
 
 .. code-block:: yaml

docs/deployment/configuration/performance.rst

@@ -270,7 +270,7 @@ The hash shard Strategy, denoted by ``type: Hash`` in the configuration below, u
             type: Hash     # use the "hash" shard strategy
             shard-count: 4 # the total number of shards
  
-The project and domain shard strategies, denoted by ``type: project`` and ``type: domain`` respectively, use the Flyte workflow project and domain metadata to shard Flyte workflows. These shard strategies are configured using a ``per-shard-mapping`` option, which is a list of IDs. Each element in the ``per-shard-mapping`` list defines a new shard, and the ID list assigns responsibility for the specified IDs to that shard. A shard configured as a single wildcard ID (i.e. ``*``) is responsible for all IDs that are not covered by other shards. Only a single shard may be configured with a wildcard ID and, on that shard, there must be only one ID, namely the wildcard.
+The project and domain shard strategies, denoted by ``type: Project`` and ``type: Domain`` respectively, use the Flyte workflow project and domain metadata to shard Flyte workflows. These shard strategies are configured using a ``per-shard-mapping`` option, which is a list of IDs. Each element in the ``per-shard-mapping`` list defines a new shard, and the ID list assigns responsibility for the specified IDs to that shard. A shard configured as a single wildcard ID (i.e. ``*``) is responsible for all IDs that are not covered by other shards. Only a single shard may be configured with a wildcard ID and, on that shard, there must be only one ID, namely the wildcard.
 
 .. code-block:: yaml
 
@@ -281,7 +281,7 @@ The project and domain shard strategies, denoted by ``type: project`` and ``type
           # pod and scanning configuration redacted
           # ...
           shard:
-            type: project       # use the "project" shard strategy
+            type: Project       # use the "Project" shard strategy
             per-shard-mapping:  # a list of per shard mappings - one shard is created for each element
               - ids:            # the list of ids to be managed by the first shard
                 - flytesnacks
@@ -298,7 +298,7 @@ The project and domain shard strategies, denoted by ``type: project`` and ``type
           # pod and scanning configuration redacted
           # ...
           shard:
-            type: domain        # use the "domain" shard strategy
+            type: Domain        # use the "Domain" shard strategy
             per-shard-mapping:  # a list of per shard mappings - one shard is created for each element
               - ids:            # the list of ids to be managed by the first shard
                 - production

docs/user_guide/basics/tasks.md

@@ -51,9 +51,9 @@ We create a task that computes the slope of a regression line:
 ```
 
 :::{note}
-Flytekit will assign a default name to the output variable like `out0`.
+Flytekit will assign a default name to the output variable like `o0`.
 In case of multiple outputs, each output will be numbered in the order
-starting with 0, e.g., -> `out0, out1, out2, ...`.
+starting with 0, e.g., `o0`, `o1`, `o2`, etc.
 :::
 
 You can execute a Flyte task just like any regular Python function: