diff --git a/, b/, new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aea9e0fabb..831beda79d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,20 +27,35 @@ jobs: docs: runs-on: ubuntu-latest steps: - - name: Fetch the code + - name: Fetch flyte code + uses: actions/checkout@v4 + with: + path: "${{ github.workspace }}/flyte" + - name: Fetch flytekit code uses: actions/checkout@v4 + with: + repository: flyteorg/flytekit + path: "${{ github.workspace }}/flytekit" - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: 3.9 - shell: bash -el {0} + working-directory: ${{ github.workspace }}/flyte run: | conda install -c conda-forge conda-lock conda-lock install -n monodocs-env monodocs-environment.lock.yaml - shell: bash -el {0} + working-directory: ${{ github.workspace }}/flyte run: | conda activate monodocs-env + export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0" pip install -e ./flyteidl + - shell: bash -el {0} + working-directory: ${{ github.workspace }}/flytekit + run: | + conda activate monodocs-env + pip install -e . conda info conda list conda config --show-sources @@ -49,6 +64,7 @@ jobs: - name: Setup Graphviz uses: ts-graphviz/setup-graphviz@v1 - name: Build the documentation + working-directory: ${{ github.workspace }}/flyte shell: bash -el {0} run: | conda activate monodocs-env diff --git a/CHANGELOG/CHANGELOG-v1.11.0-b1.md b/CHANGELOG/CHANGELOG-v1.11.0-b1.md new file mode 100644 index 0000000000..be0c9414a2 --- /dev/null +++ b/CHANGELOG/CHANGELOG-v1.11.0-b1.md @@ -0,0 +1,3 @@ +# Flyte v1.11.0-b1 + +Second beta release for 1.11.0. \ No newline at end of file diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md index 99aa1c40b1..78c301f2b3 100644 --- a/charts/flyte-binary/README.md +++ b/charts/flyte-binary/README.md @@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment | configuration.auth.oidc.clientId | string | `""` | | | configuration.auth.oidc.clientSecret | string | `""` | | | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` | | -| configuration.co-pilot.image.tag | string | `"v1.11.0-b0"` | | +| configuration.co-pilot.image.tag | string | `"v1.11.0-b1"` | | | configuration.database.dbname | string | `"flyte"` | | | configuration.database.host | string | `"127.0.0.1"` | | | configuration.database.options | string | `"sslmode=disable"` | | diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml index 0da15a1855..61bb73e72d 100644 --- a/charts/flyte-binary/values.yaml +++ b/charts/flyte-binary/values.yaml @@ -159,7 +159,7 @@ configuration: # repository CoPilot sidecar image repository repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE # tag CoPilot sidecar image tag - tag: v1.11.0-b0 # FLYTECOPILOT_TAG + tag: v1.11.0-b1 # FLYTECOPILOT_TAG # agentService Flyte Agent configuration agentService: defaultAgent: diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 31fd4791b9..5f0e3dcbb5 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -95,8 +95,8 @@ helm install gateway bitnami/contour -n flyte | configmap.clusters.clusterConfigs | list | `[]` | | | configmap.clusters.labelClusterMap | object | `{}` | | | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). | | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | @@ -130,7 +130,7 @@ helm install gateway bitnami/contour -n flyte | datacatalog.extraArgs | object | `{}` | Appends extra command line arguments to the main command | | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| datacatalog.image.tag | string | `"v1.11.0-b0"` | Docker image tag | +| datacatalog.image.tag | string | `"v1.11.0-b1"` | Docker image tag | | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.podEnv | object | `{}` | Additional Datacatalog container environment variables | @@ -165,7 +165,7 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command | | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | | | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyteadmin.image.tag | string | `"v1.11.0-b0"` | | +| flyteadmin.image.tag | string | `"v1.11.0-b1"` | | | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create | | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | @@ -202,7 +202,7 @@ helm install gateway bitnami/contour -n flyte | flyteconsole.ga.tracking_id | string | `"G-0QW4DJWJ20"` | | | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | | flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyteconsole.image.tag | string | `"v1.10.3"` | | +| flyteconsole.image.tag | string | `"v1.11.0"` | | | flyteconsole.imagePullSecrets | list | `[]` | ImagePullSecrets to assign to the Flyteconsole deployment | | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | @@ -226,7 +226,7 @@ helm install gateway bitnami/contour -n flyte | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command | | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | | | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flytepropeller.image.tag | string | `"v1.11.0-b0"` | | +| flytepropeller.image.tag | string | `"v1.11.0-b1"` | | | flytepropeller.manager | bool | `false` | | | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | @@ -256,7 +256,7 @@ helm install gateway bitnami/contour -n flyte | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | -| flytescheduler.image.tag | string | `"v1.11.0-b0"` | Docker image tag | +| flytescheduler.image.tag | string | `"v1.11.0-b1"` | Docker image tag | | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | | flytescheduler.podEnv | object | `{}` | Additional Flytescheduler container environment variables | @@ -290,6 +290,9 @@ helm install gateway bitnami/contour -n flyte | storage.s3.secretKey | string | `""` | AWS IAM user secret access key to use for S3 bucket auth, only used if authType is set to accesskey | | storage.type | string | `"sandbox"` | Sets the storage type. Supported values are sandbox, s3, gcs and custom. | | webhook.enabled | bool | `true` | enable or disable secrets webhook | +| webhook.resources.requests.cpu | string | `"200m"` | | +| webhook.resources.requests.ephemeral-storage | string | `"500Mi"` | | +| webhook.resources.requests.memory | string | `"500Mi"` | | | webhook.securityContext | object | `{"fsGroup":65534,"fsGroupChangePolicy":"Always","runAsNonRoot":true,"runAsUser":1001,"seLinuxOptions":{"type":"spc_t"}}` | Sets securityContext for webhook pod(s). | | webhook.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"}` | Service settings for the webhook | | webhook.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for the webhook | diff --git a/charts/flyte-core/templates/propeller/webhook.yaml b/charts/flyte-core/templates/propeller/webhook.yaml index 941ed1add1..e34241d905 100644 --- a/charts/flyte-core/templates/propeller/webhook.yaml +++ b/charts/flyte-core/templates/propeller/webhook.yaml @@ -103,6 +103,9 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + {{- with .Values.webhook.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - name: config-volume mountPath: /etc/flyte/config diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index d5c21c5131..32fb5c5827 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -16,7 +16,7 @@ flyteadmin: image: # -- Docker image for Flyteadmin deployment repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE - tag: v1.11.0-b0 # FLYTEADMIN_TAG + tag: v1.11.0-b1 # FLYTEADMIN_TAG pullPolicy: IfNotPresent # -- Additional flyteadmin container environment variables # @@ -142,7 +142,7 @@ flytescheduler: # -- Docker image for Flytescheduler deployment repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE # -- Docker image tag - tag: v1.11.0-b0 # FLYTESCHEDULER_TAG + tag: v1.11.0-b1 # FLYTESCHEDULER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytescheduler deployment @@ -208,7 +208,7 @@ datacatalog: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE # -- Docker image tag - tag: v1.11.0-b0 # DATACATALOG_TAG + tag: v1.11.0-b1 # DATACATALOG_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment @@ -296,7 +296,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE - tag: v1.11.0-b0 # FLYTEPROPELLER_TAG + tag: v1.11.0-b1 # FLYTEPROPELLER_TAG pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -379,7 +379,7 @@ flyteconsole: image: # -- Docker image for Flyteconsole deployment repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE - tag: v1.10.3 # FLYTECONSOLE_TAG + tag: v1.11.0 # FLYTECONSOLE_TAG pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment resources: @@ -466,6 +466,12 @@ webhook: seLinuxOptions: type: spc_t + resources: + requests: + cpu: 200m + ephemeral-storage: 500Mi + memory: 500Mi + # ------------------------------------------------ # # COMMON SETTINGS @@ -725,7 +731,7 @@ configmap: # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0 # FLYTECOPILOT_IMAGE + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 # FLYTECOPILOT_IMAGE start-timeout: 30s # -- Core propeller configuration diff --git a/charts/flyte/README.md b/charts/flyte/README.md index 0bce53a417..f2ac67bdc2 100644 --- a/charts/flyte/README.md +++ b/charts/flyte/README.md @@ -71,7 +71,7 @@ helm upgrade -f values-sandbox.yaml flyte . | contour.tolerations | list | `[]` | tolerations for Contour deployment | | daskoperator | object | `{"enabled":false}` | Optional: Dask Plugin using the Dask Operator | | daskoperator.enabled | bool | `false` | - enable or disable the dask operator deployment installation | -| flyte | object | `{"cluster_resource_manager":{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]},"common":{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"host":"","separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":true}},"configmap":{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}},"datacatalog":{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v1.11.0-b0"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"db":{"admin":{"database":{"dbname":"flyteadmin","host":"postgres","port":5432,"username":"postgres"}},"datacatalog":{"database":{"dbname":"datacatalog","host":"postgres","port":5432,"username":"postgres"}}},"deployRedoc":true,"flyteadmin":{"additionalVolumeMounts":[],"additionalVolumes":[],"affinity":{},"configPath":"/etc/flyte/config/*.yaml","env":[],"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteadmin","tag":"v1.11.0-b0"},"initialProjects":["flytesnacks","flytetester","flyteexamples"],"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flyteconsole":{"affinity":{},"ga":{"enabled":true,"tracking_id":"G-0QW4DJWJ20"},"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteconsole","tag":"v1.10.3"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"275Mi"},"requests":{"cpu":"10m","memory":"250Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]},"flytepropeller":{"affinity":{},"cacheSizeMbs":0,"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytepropeller","tag":"v1.11.0-b0"},"manager":false,"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flytescheduler":{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v1.11.0-b0"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"storage":{"bucketName":"my-s3-bucket","custom":{},"gcs":null,"s3":{"region":"us-east-1"},"type":"sandbox"},"webhook":{"enabled":true,"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]}},"workflow_notifications":{"config":{},"enabled":false},"workflow_scheduler":{"enabled":true,"type":"native"}}` | ------------------------------------------------------------------- Core System settings This section consists of Core components of Flyte and their deployment settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and Flyteconsole | +| flyte | object | `{"cluster_resource_manager":{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]},"common":{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"host":"","separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":true}},"configmap":{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}},"datacatalog":{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v1.11.0-b1"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"db":{"admin":{"database":{"dbname":"flyteadmin","host":"postgres","port":5432,"username":"postgres"}},"datacatalog":{"database":{"dbname":"datacatalog","host":"postgres","port":5432,"username":"postgres"}}},"deployRedoc":true,"flyteadmin":{"additionalVolumeMounts":[],"additionalVolumes":[],"affinity":{},"configPath":"/etc/flyte/config/*.yaml","env":[],"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteadmin","tag":"v1.11.0-b1"},"initialProjects":["flytesnacks","flytetester","flyteexamples"],"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flyteconsole":{"affinity":{},"ga":{"enabled":true,"tracking_id":"G-0QW4DJWJ20"},"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteconsole","tag":"v1.11.0"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"275Mi"},"requests":{"cpu":"10m","memory":"250Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]},"flytepropeller":{"affinity":{},"cacheSizeMbs":0,"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytepropeller","tag":"v1.11.0-b1"},"manager":false,"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flytescheduler":{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v1.11.0-b1"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"storage":{"bucketName":"my-s3-bucket","custom":{},"gcs":null,"s3":{"region":"us-east-1"},"type":"sandbox"},"webhook":{"enabled":true,"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]}},"workflow_notifications":{"config":{},"enabled":false},"workflow_scheduler":{"enabled":true,"type":"native"}}` | ------------------------------------------------------------------- Core System settings This section consists of Core components of Flyte and their deployment settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and Flyteconsole | | flyte.cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | | flyte.cluster_resource_manager.config.cluster_resources | object | `{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. | | flyte.cluster_resource_manager.config.cluster_resources.standaloneDeployment | bool | `false` | Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints | @@ -91,15 +91,15 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.common.ingress.separateGrpcIngressAnnotations | object | `{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"}` | - Extra Ingress annotations applied only to the GRPC ingress. Only makes sense if `separateGrpcIngress` is enabled. | | flyte.common.ingress.tls | object | `{"enabled":false}` | - TLS Settings | | flyte.common.ingress.webpackHMR | bool | `true` | - Enable or disable HMR route to flyteconsole. This is useful only for frontend development. | -| flyte.configmap | object | `{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ----------------------------------------------------------------- CONFIGMAPS SETTINGS | +| flyte.configmap | object | `{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ----------------------------------------------------------------- CONFIGMAPS SETTINGS | | flyte.configmap.adminServer | object | `{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | | flyte.configmap.adminServer.auth | object | `{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}}` | Authentication configuration | | flyte.configmap.adminServer.server.security.secure | bool | `false` | Controls whether to serve requests over SSL/TLS. | | flyte.configmap.adminServer.server.security.useAuth | bool | `false` | Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. | | flyte.configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) | | flyte.configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| flyte.configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| flyte.configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| flyte.configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| flyte.configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | flyte.configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | flyte.configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | flyte.configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | @@ -120,7 +120,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | | flyte.datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| flyte.datacatalog.image.tag | string | `"v1.11.0-b0"` | Docker image tag | +| flyte.datacatalog.image.tag | string | `"v1.11.0-b1"` | Docker image tag | | flyte.datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | flyte.datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | flyte.datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | @@ -136,7 +136,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flyteadmin.env | list | `[]` | Additional flyteadmin container environment variables e.g. SendGrid's API key - name: SENDGRID_API_KEY value: "" e.g. secret environment variable (you can combine it with .additionalVolumes): - name: SENDGRID_API_KEY valueFrom: secretKeyRef: name: sendgrid-secret key: api_key | | flyte.flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyte.flyteadmin.image.tag | string | `"v1.11.0-b0"` | Docker image tag | +| flyte.flyteadmin.image.tag | string | `"v1.11.0-b1"` | Docker image tag | | flyte.flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create | | flyte.flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyte.flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | @@ -151,7 +151,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyte.flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyte.flyteconsole.image.tag | string | `"v1.10.3"` | Docker image tag | +| flyte.flyteconsole.image.tag | string | `"v1.11.0"` | Docker image tag | | flyte.flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyte.flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | | flyte.flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment | @@ -162,7 +162,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flytepropeller.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyte.flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flyte.flytepropeller.image.tag | string | `"v1.11.0-b0"` | Docker image tag | +| flyte.flytepropeller.image.tag | string | `"v1.11.0-b1"` | Docker image tag | | flyte.flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flyte.flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | | flyte.flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | @@ -176,7 +176,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyte.flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | -| flyte.flytescheduler.image.tag | string | `"v1.11.0-b0"` | Docker image tag | +| flyte.flytescheduler.image.tag | string | `"v1.11.0-b1"` | Docker image tag | | flyte.flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | | flyte.flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | | flyte.flytescheduler.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytescheduler deployment | diff --git a/charts/flyte/values.yaml b/charts/flyte/values.yaml index aecfce362f..50a1fc5402 100755 --- a/charts/flyte/values.yaml +++ b/charts/flyte/values.yaml @@ -16,7 +16,7 @@ flyte: # -- Docker image for Flyteadmin deployment repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE # -- Docker image tag - tag: v1.11.0-b0 # FLYTEADMIN_TAG + tag: v1.11.0-b1 # FLYTEADMIN_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Additional flyteadmin container environment variables @@ -84,7 +84,7 @@ flyte: # -- Docker image for Flytescheduler deployment repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE # -- Docker image tag - tag: v1.11.0-b0 # FLYTESCHEDULER_TAG + tag: v1.11.0-b1 # FLYTESCHEDULER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytescheduler deployment @@ -129,7 +129,7 @@ flyte: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE # -- Docker image tag - tag: v1.11.0-b0 # DATACATALOG_TAG + tag: v1.11.0-b1 # DATACATALOG_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment @@ -178,7 +178,7 @@ flyte: # -- Docker image for Flytepropeller deployment repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE # -- Docker image tag - tag: v1.11.0-b0 # FLYTEPROPELLER_TAG + tag: v1.11.0-b1 # FLYTEPROPELLER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment @@ -223,7 +223,7 @@ flyte: # -- Docker image for Flyteconsole deployment repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE # -- Docker image tag - tag: v1.10.3 # FLYTECONSOLE_TAG + tag: v1.11.0 # FLYTECONSOLE_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment @@ -471,7 +471,7 @@ flyte: # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0 # FLYTECOPILOT_IMAGE + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 # FLYTECOPILOT_IMAGE start-timeout: 30s # -- Core propeller configuration diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml index c1865eeb87..83c0d249c5 100644 --- a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml +++ b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml @@ -429,7 +429,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -868,7 +868,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -889,7 +889,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -907,7 +907,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -924,7 +924,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -951,7 +951,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1056,7 +1056,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1112,7 +1112,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1186,7 +1186,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1204,7 +1204,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1267,7 +1267,7 @@ spec: template: metadata: annotations: - configChecksum: "30e5fce341e4344cb6253ef4321f37c1e0895b9b55a927f94dfbc303d65c15b" + configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1293,7 +1293,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1347,9 +1347,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b0 + app.kubernetes.io/version: v1.11.0-b1 annotations: - configChecksum: "30e5fce341e4344cb6253ef4321f37c1e0895b9b55a927f94dfbc303d65c15b" + configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" spec: securityContext: fsGroup: 65534 @@ -1361,7 +1361,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1388,7 +1388,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1411,6 +1411,11 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 200m + ephemeral-storage: 500Mi + memory: 500Mi volumeMounts: - name: config-volume mountPath: /etc/flyte/config diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index b4b8f63584..b4ee31f106 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -8640,7 +8640,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0 + image: cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -8663,7 +8663,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0 + image: cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8724,7 +8724,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -8751,7 +8751,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1 imagePullPolicy: IfNotPresent name: generate-secrets volumeMounts: @@ -8799,7 +8799,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8846,7 +8846,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8863,7 +8863,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8877,7 +8877,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8897,7 +8897,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 imagePullPolicy: IfNotPresent name: generate-secrets volumeMounts: @@ -8951,7 +8951,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: cr.flyte.org/flyteorg/flyteconsole:v1.10.3 + image: cr.flyte.org/flyteorg/flyteconsole:v1.11.0 name: flyteconsole ports: - containerPort: 8080 @@ -9002,7 +9002,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -9270,7 +9270,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml index 1ae984cf69..b60694f947 100644 --- a/deployment/eks/flyte_helm_controlplane_generated.yaml +++ b/deployment/eks/flyte_helm_controlplane_generated.yaml @@ -574,7 +574,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -595,7 +595,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -613,7 +613,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -630,7 +630,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -657,7 +657,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -762,7 +762,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -818,7 +818,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -892,7 +892,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -910,7 +910,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -993,7 +993,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1013,7 +1013,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml index 510ef5c3c8..90d327cb9e 100644 --- a/deployment/eks/flyte_helm_dataplane_generated.yaml +++ b/deployment/eks/flyte_helm_dataplane_generated.yaml @@ -94,7 +94,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -427,7 +427,7 @@ spec: template: metadata: annotations: - configChecksum: "30e5fce341e4344cb6253ef4321f37c1e0895b9b55a927f94dfbc303d65c15b" + configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -453,7 +453,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -507,9 +507,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b0 + app.kubernetes.io/version: v1.11.0-b1 annotations: - configChecksum: "30e5fce341e4344cb6253ef4321f37c1e0895b9b55a927f94dfbc303d65c15b" + configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" spec: securityContext: fsGroup: 65534 @@ -521,7 +521,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -548,7 +548,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -571,6 +571,11 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 200m + ephemeral-storage: 500Mi + memory: 500Mi volumeMounts: - name: config-volume mountPath: /etc/flyte/config diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index 4cd67923ba..8f34f4f236 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -460,7 +460,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -899,7 +899,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -920,7 +920,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -938,7 +938,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -955,7 +955,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -982,7 +982,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1087,7 +1087,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1143,7 +1143,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1217,7 +1217,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1235,7 +1235,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1318,7 +1318,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1338,7 +1338,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -1397,7 +1397,7 @@ spec: template: metadata: annotations: - configChecksum: "30e5fce341e4344cb6253ef4321f37c1e0895b9b55a927f94dfbc303d65c15b" + configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1423,7 +1423,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1477,9 +1477,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b0 + app.kubernetes.io/version: v1.11.0-b1 annotations: - configChecksum: "30e5fce341e4344cb6253ef4321f37c1e0895b9b55a927f94dfbc303d65c15b" + configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" spec: securityContext: fsGroup: 65534 @@ -1491,7 +1491,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1518,7 +1518,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1541,6 +1541,11 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 200m + ephemeral-storage: 500Mi + memory: 500Mi volumeMounts: - name: config-volume mountPath: /etc/flyte/config diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml index 0f1ebf1381..7e0b58b832 100644 --- a/deployment/gcp/flyte_helm_controlplane_generated.yaml +++ b/deployment/gcp/flyte_helm_controlplane_generated.yaml @@ -589,7 +589,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -610,7 +610,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -628,7 +628,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -645,7 +645,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -672,7 +672,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -777,7 +777,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -833,7 +833,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -907,7 +907,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -925,7 +925,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1008,7 +1008,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1028,7 +1028,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml index 59a0fca4f6..d19a4d48c4 100644 --- a/deployment/gcp/flyte_helm_dataplane_generated.yaml +++ b/deployment/gcp/flyte_helm_dataplane_generated.yaml @@ -94,7 +94,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -435,7 +435,7 @@ spec: template: metadata: annotations: - configChecksum: "bfe89fce66aa8eee9543c676ab07345b9c05c4ec7859daefd51da6bf414f0f4" + configChecksum: "9c6856d58b3168a752486076eccbc6783bbb94d62b47139c6cab6e1fe12174c" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -460,7 +460,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -514,9 +514,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b0 + app.kubernetes.io/version: v1.11.0-b1 annotations: - configChecksum: "bfe89fce66aa8eee9543c676ab07345b9c05c4ec7859daefd51da6bf414f0f4" + configChecksum: "9c6856d58b3168a752486076eccbc6783bbb94d62b47139c6cab6e1fe12174c" spec: securityContext: fsGroup: 65534 @@ -528,7 +528,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -555,7 +555,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -578,6 +578,11 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 200m + ephemeral-storage: 500Mi + memory: 500Mi volumeMounts: - name: config-volume mountPath: /etc/flyte/config diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index f220536479..5b8757459c 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -473,7 +473,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -922,7 +922,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -943,7 +943,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -961,7 +961,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -978,7 +978,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -1005,7 +1005,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1110,7 +1110,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1166,7 +1166,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1240,7 +1240,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1258,7 +1258,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1341,7 +1341,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1361,7 +1361,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -1420,7 +1420,7 @@ spec: template: metadata: annotations: - configChecksum: "bfe89fce66aa8eee9543c676ab07345b9c05c4ec7859daefd51da6bf414f0f4" + configChecksum: "9c6856d58b3168a752486076eccbc6783bbb94d62b47139c6cab6e1fe12174c" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1445,7 +1445,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1499,9 +1499,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b0 + app.kubernetes.io/version: v1.11.0-b1 annotations: - configChecksum: "bfe89fce66aa8eee9543c676ab07345b9c05c4ec7859daefd51da6bf414f0f4" + configChecksum: "9c6856d58b3168a752486076eccbc6783bbb94d62b47139c6cab6e1fe12174c" spec: securityContext: fsGroup: 65534 @@ -1513,7 +1513,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1540,7 +1540,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1563,6 +1563,11 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 200m + ephemeral-storage: 500Mi + memory: 500Mi volumeMounts: - name: config-volume mountPath: /etc/flyte/config diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml index 2d93910a5c..1aeec5ee6d 100644 --- a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml +++ b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml @@ -116,7 +116,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1" k8s-array: logs: config: @@ -358,7 +358,7 @@ spec: app.kubernetes.io/instance: flyte app.kubernetes.io/component: flyte-binary annotations: - checksum/configuration: 882c31ec18bdac7aa4f1a9057f9e549b1307b60b5d76839dfb6bc526958bee57 + checksum/configuration: 9c864736e7d08baebc6de026def33e542b35097dd158017c89f85d7672401b92 checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929 checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae spec: diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index bfd40edd37..c398c5fbf8 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -585,7 +585,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -6705,7 +6705,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -6725,7 +6725,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -6742,7 +6742,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -6758,7 +6758,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -6785,7 +6785,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -6880,7 +6880,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -6933,7 +6933,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -7005,7 +7005,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -7022,7 +7022,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -7095,7 +7095,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -7114,7 +7114,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -7170,7 +7170,7 @@ spec: template: metadata: annotations: - configChecksum: "8d992b3c2174350d363ddbf3b1ac0d7f8017a546ec794a9551a4f2b1f4e6ea7" + configChecksum: "fa64b9672600e5eeed8ca407828a7490b17dadfe16c273e2aacfb52d9b5b085" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -7195,7 +7195,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -7242,9 +7242,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b0 + app.kubernetes.io/version: v1.11.0-b1 annotations: - configChecksum: "8d992b3c2174350d363ddbf3b1ac0d7f8017a546ec794a9551a4f2b1f4e6ea7" + configChecksum: "fa64b9672600e5eeed8ca407828a7490b17dadfe16c273e2aacfb52d9b5b085" spec: securityContext: fsGroup: 65534 @@ -7256,7 +7256,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -7283,7 +7283,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -7306,6 +7306,11 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 200m + ephemeral-storage: 500Mi + memory: 500Mi volumeMounts: - name: config-volume mountPath: /etc/flyte/config diff --git a/docker/sandbox-bundled/Makefile b/docker/sandbox-bundled/Makefile index d2ec89de8b..1c080db16a 100644 --- a/docker/sandbox-bundled/Makefile +++ b/docker/sandbox-bundled/Makefile @@ -3,6 +3,7 @@ mkdir -p images/tar/$(1) docker buildx build \ --build-arg FLYTECONSOLE_VERSION=$(FLYTECONSOLE_VERSION) \ + --builder flyte-sandbox \ --platform linux/$(1) \ --tag flyte-binary:sandbox \ --output type=docker,dest=images/tar/$(1)/flyte-binary.tar \ @@ -10,9 +11,17 @@ docker buildx build \ endef +.PHONY: create_builder +create_builder: + [ -n "$(shell docker buildx ls | awk '/^flyte-sandbox / {print $$1}')" ] || \ + docker buildx create --name flyte-sandbox \ + --driver docker-container --driver-opt image=moby/buildkit:master \ + --buildkitd-flags '--allow-insecure-entitlement security.insecure' \ + --platform linux/arm64,linux/amd64 + .PHONY: flyte flyte: FLYTECONSOLE_VERSION := latest -flyte: +flyte: create_builder $(foreach arch,amd64 arm64,$(call FLYTE_BINARY_BUILD,$(arch))) .PHONY: dep_update @@ -39,11 +48,6 @@ manifests: dep_update .PHONY: build build: flyte dep_update manifests - [ -n "$(shell docker buildx ls | awk '/^flyte-sandbox / {print $$1}')" ] || \ - docker buildx create --name flyte-sandbox \ - --driver docker-container --driver-opt image=moby/buildkit:master \ - --buildkitd-flags '--allow-insecure-entitlement security.insecure' \ - --platform linux/arm64,linux/amd64 docker buildx build --builder flyte-sandbox --allow security.insecure --load \ --tag flyte-sandbox:latest . diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index f3b70f9af4..3dc2da8d69 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -468,7 +468,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1" k8s-array: logs: config: @@ -816,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: aXMxM0ZiZFBUQTVKQzZzag== + haSharedSecret: MDZkcjZpSWtxQ0l1U3BkRg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1246,7 +1246,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 9ab632fe7ac69bcf63d6965a44986a05e23798beda4a3175d1601e61057a9832 + checksum/configuration: 286b381aca83e796ac69572dba6d2071241be79d4f00f3c2af55415a95157efd checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1412,7 +1412,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 5efa2a8b90cb09695695fe2117018010cba089397a6ec9b91191e1ba1cdb7ee0 + checksum/secret: 0e1cd1e2d9516f5d078a3175e0dca1176c488a2a1b26a520ffb5e8567b028371 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index a6a0cb641e..9f41c7583a 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -457,7 +457,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b0" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1" k8s-array: logs: config: @@ -796,7 +796,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: YjFvdVhNdW52SHpKQ2FJSw== + haSharedSecret: VmVKR1JscnRLS0NlcGR0aA== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1194,7 +1194,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 11cd65708fd872839c6e561e84c30e045567486f06757f4549c69cc22aea5697 + checksum/configuration: 1362bc266b00c161aac0a24f7db312b9d6b9e68d7d8d8859ec8e171b9dc3e2bd checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1360,7 +1360,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 2963c5428fd79c68e963854f7c4154d3f2dec35e6a52c93d0e7807205aa7c43c + checksum/secret: a0053cd8b344fa27669b64ed7abc8d0579ecce282a81be07cd7d4c07734ac269 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 69624160e0..6a45fee772 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: MTg1TFE5S0JzU2NJNjM0NQ== + haSharedSecret: VFFoSGVGVXNlZG5HamZ0Vw== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 1711d3d34542992ce0d26da8299f55a0c534553eae7ba7dfb6b1151b87f52e98 + checksum/secret: ea52a66e2d5ced849a5c3de4e1877b5bb4f945e9240457b48bb5cdffa3c25b97 labels: app: docker-registry release: flyte-sandbox diff --git a/docs/community/troubleshoot.rst b/docs/community/troubleshoot.rst index 1228b5f5a0..41bc6624c3 100644 --- a/docs/community/troubleshoot.rst +++ b/docs/community/troubleshoot.rst @@ -14,7 +14,7 @@ Before getting started, collect the following information from the underlying in .. prompt:: bash $ - $ kubectl describe pod -n + kubectl describe pod -n Where will typically correspond to the node execution string that you can find in the UI. @@ -23,7 +23,7 @@ Where will typically correspond to the node execution string that you .. prompt:: bash $ - $ kubectl logs pods -n + kubectl logs pods -n Where will typically correspond to the Flyte -, e.g. flytesnacks-development. @@ -70,7 +70,7 @@ This issue is more common on MacOS devices. Make sure that your Docker daemon ha .. prompt:: bash $ - $ flytectl demo start --env HTTP_PROXY= + flytectl demo start --env HTTP_PROXY= - If you're building a custom Docker image, make sure to use a tag other than ``latest``. Otherwise, the Kubernetes default pull policy will be changed from ``IfNotPresent`` to ``Always``, forcing an image pull with every Pod deployment. @@ -85,14 +85,14 @@ Issues running workloads .. prompt:: bash $ - $ export FLYTECTL_CONFIG=~/.flyte/config-sandbox.yaml + export FLYTECTL_CONFIG=~/.flyte/config-sandbox.yaml ``ModuleNotFoundError`` ^^^^^^^^^^^^^^^^^^^^^^^ - If you're using a custom container image and using Docker, make sure your ``Dockerfile`` is located at the same level of the ``flyte`` directory and that there is an empty ``__init__.py`` file in your project's folder : -.. prompt:: bash $ +.. prompt:: myflyteapp ├── Dockerfile @@ -111,11 +111,11 @@ Issues running workloads .. prompt:: bash $ - $ kubectl describe sa -n + kubectl describe sa -n Example output: -.. prompt:: bash $ +.. prompt:: Name: Namespace: flyte @@ -130,7 +130,7 @@ Example output: .. prompt:: bash $ - $ kubectl annotate serviceaccount -n eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/ + kubectl annotate serviceaccount -n eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/ - Refer to this community-maintained `guides `_ for further information about Flyte deployment on EKS diff --git a/docs/conf.py b/docs/conf.py index b4666d94a4..6d20ecf4b9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,7 +35,7 @@ # The short X.Y version version = "" # The full version, including alpha/beta/rc tags -release = "1.11.0-b0" +release = "1.11.0-b1" # -- General configuration --------------------------------------------------- diff --git a/docs/deployment/agents/bigquery.rst b/docs/deployment/agents/bigquery.rst index d706ac7c37..d3e4ee490e 100644 --- a/docs/deployment/agents/bigquery.rst +++ b/docs/deployment/agents/bigquery.rst @@ -103,4 +103,4 @@ Upgrade the Flyte Helm release and ```` with the name of your namespace (e.g., ``flyte``). -For BigQuery agent on the Flyte cluster, see `BigQuery agent `_. +For BigQuery agent on the Flyte cluster, see `BigQuery agent `_. diff --git a/docs/deployment/agents/chatgpt.rst b/docs/deployment/agents/chatgpt.rst new file mode 100644 index 0000000000..c3e5a490ea --- /dev/null +++ b/docs/deployment/agents/chatgpt.rst @@ -0,0 +1,138 @@ +.. _deployment-agent-setup-chatgpt: + +ChatGPT agent +================= + +This guide provides an overview of how to set up the ChatGPT agent in your Flyte deployment. +Please note that you have to set up the OpenAI API key in the agent server to to run ChatGPT tasks. + +Specify agent configuration +---------------------------- + +.. tabs:: + + .. group-tab:: Flyte binary + + Edit the relevant YAML file to specify the agent. + + .. code-block:: bash + + kubectl edit configmap flyte-sandbox-config -n flyte + + .. code-block:: yaml + :emphasize-lines: 7,11,16 + + tasks: + task-plugins: + enabled-plugins: + - container + - sidecar + - k8s-array + - agent-service + default-for-task-types: + - container: container + - container_array: k8s-array + - chatgpt: agent-service + + plugins: + agent-service: + supportedTaskTypes: + - chatgpt + + .. group-tab:: Flyte core + + Create a file named ``values-override.yaml`` and add the following configuration to it: + + .. code-block:: yaml + + configmap: + enabled_plugins: + # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) + tasks: + # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) + task-plugins: + # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend + enabled-plugins: + - container + - sidecar + - k8s-array + - agent-service + default-for-task-types: + container: container + sidecar: sidecar + container_array: k8s-array + chatgpt: agent-service + plugins: + agent-service: + supportedTaskTypes: + - chatgpt + +Add the OpenAI API token +------------------------------- + +1. Install flyteagent pod using helm: + +.. code-block:: + + helm repo add flyteorg https://flyteorg.github.io/flyte + helm install flyteagent flyteorg/flyteagent --namespace flyte + +2. Get the base64 value of your OpenAI API token: + +.. code-block:: + + echo -n "" | base64 + +3. Edit the flyteagent secret: + + .. code-block:: bash + + kubectl edit secret flyteagent -n flyte + + .. code-block:: yaml + :emphasize-lines: 3 + + apiVersion: v1 + data: + flyte_openai_access_token: + kind: Secret + metadata: + annotations: + meta.helm.sh/release-name: flyteagent + meta.helm.sh/release-namespace: flyte + creationTimestamp: "2023-10-04T04:09:03Z" + labels: + app.kubernetes.io/managed-by: Helm + name: flyteagent + namespace: flyte + resourceVersion: "753" + uid: 5ac1e1b6-2a4c-4e26-9001-d4ba72c39e54 + type: Opaque + + +Upgrade the Flyte Helm release +------------------------------ + +.. tabs:: + + .. group-tab:: Flyte binary + + .. code-block:: bash + + helm upgrade flyteorg/flyte-binary -n --values + + Replace ```` with the name of your release (e.g., ``flyte-backend``), + ```` with the name of your namespace (e.g., ``flyte``), + and ```` with the name of your YAML file. + + .. group-tab:: Flyte core + + .. code-block:: bash + + helm upgrade flyte/flyte-core -n --values values-override.yaml + + Replace ```` with the name of your release (e.g., ``flyte``) + + and ```` with the name of your namespace (e.g., ``flyte``). + +For ChatGPT agent on the Flyte cluster, see `ChatGPT agent `_. diff --git a/docs/deployment/agents/databricks.rst b/docs/deployment/agents/databricks.rst index 3dbf7731c5..b21fab3c57 100644 --- a/docs/deployment/agents/databricks.rst +++ b/docs/deployment/agents/databricks.rst @@ -291,4 +291,4 @@ Wait for the upgrade to complete. You can check the status of the deployment pod kubectl get pods -n flyte -For Databricks agent on the Flyte cluster, see `Databricks agent `_. +For Databricks agent on the Flyte cluster, see `Databricks agent `_. diff --git a/docs/deployment/agents/index.md b/docs/deployment/agents/index.md index 0e114c8d06..7c7f3db92d 100644 --- a/docs/deployment/agents/index.md +++ b/docs/deployment/agents/index.md @@ -17,6 +17,8 @@ If you are using a managed deployment of Flyte, you will need to contact your de * - {ref}`Airflow Agent ` - Configuring your Flyte deployment for the Airflow agent +* - {ref}`ChatGPT Agent ` + - Configuring your Flyte deployment for the ChatGPT agent. * - {ref}`Databricks Agent ` - Configuring your Flyte deployment for the Databricks agent. * - {ref}`Google BigQuery Agent ` @@ -33,6 +35,7 @@ If you are using a managed deployment of Flyte, you will need to contact your de :hidden: airflow +chatgpt databricks bigquery mmcloud diff --git a/docs/deployment/agents/mmcloud.rst b/docs/deployment/agents/mmcloud.rst index ac08f4fcdf..422162af27 100644 --- a/docs/deployment/agents/mmcloud.rst +++ b/docs/deployment/agents/mmcloud.rst @@ -118,4 +118,4 @@ Wait for the upgrade to complete. You can check the status of the deployment pod kubectl get pods -n flyte -For MMCloud agent on the Flyte cluster, see `MMCloud agent `_. +For MMCloud agent on the Flyte cluster, see `MMCloud agent `_. diff --git a/docs/deployment/agents/snowflake.rst b/docs/deployment/agents/snowflake.rst index f4d82c0eb2..fe1c8482ae 100644 --- a/docs/deployment/agents/snowflake.rst +++ b/docs/deployment/agents/snowflake.rst @@ -100,4 +100,4 @@ Upgrade the Flyte Helm release and ```` with the name of your namespace (e.g., ``flyte``). -For Snowflake agent on the Flyte cluster, see `Snowflake agent `_. +For Snowflake agent on the Flyte cluster, see `Snowflake agent `_. diff --git a/docs/deployment/deployment/cloud_simple.rst b/docs/deployment/deployment/cloud_simple.rst index c71708f1f4..94833a0db4 100644 --- a/docs/deployment/deployment/cloud_simple.rst +++ b/docs/deployment/deployment/cloud_simple.rst @@ -112,8 +112,8 @@ hello world example: .. prompt:: bash $ git clone https://github.com/flyteorg/flytesnacks - cd flytesnacks/cookbook - pyflyte run --remote core/flyte_basics/hello_world.py my_wf + cd flytesnacks/examples/basics + pyflyte run --remote basics/hello_world.py hello_world_wf *********************************** Flyte in on-premises infrastructure diff --git a/docs/deployment/deployment/multicluster.rst b/docs/deployment/deployment/multicluster.rst index e8c6b84f13..52e0378955 100644 --- a/docs/deployment/deployment/multicluster.rst +++ b/docs/deployment/deployment/multicluster.rst @@ -398,11 +398,14 @@ label has to be 1. 11. Verify that all Pods in the ``flyte`` namespace are ``Running``: +.. prompt:: bash $ + + kubectl get pods -n flyte + Example output: -.. prompt:: bash $ +.. prompt:: - kubectl get pods -n flyte NAME READY STATUS RESTARTS AGE datacatalog-86f6b9bf64-bp2cj 1/1 Running 0 23h datacatalog-86f6b9bf64-fjzcp 1/1 Running 0 23h @@ -658,4 +661,4 @@ The process can be repeated for additional clusters. 15. A successful execution should be visible on the UI, confirming it ran in the new cluster: - .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/common/multicluster-execution.png \ No newline at end of file + .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/common/multicluster-execution.png diff --git a/docs/flyte_agents/developing_agents.md b/docs/flyte_agents/developing_agents.md index ba114be6c7..a4a4adc7c7 100644 --- a/docs/flyte_agents/developing_agents.md +++ b/docs/flyte_agents/developing_agents.md @@ -15,7 +15,7 @@ If you need to create a new type of task, we recommend creating a new agent to r ```{note} -We strongly encourage you to contribute your agent to the Flyte community. To do so, follow the steps in "[Contributing to Flyte](https://docs.flyte.org/en/latest/community/contribute.html)", and reach out to us on [Slack](https://docs.flyte.org/en/latest/community/contribute.html#) if you have any questions. +We strongly encourage you to contribute your agent to the Flyte community. To do so, follow the steps in "[Contributing to Flyte](https://docs.flyte.org/en/latest/community/contribute.html)" to add your agent to [Flytekit](https://github.com/flyteorg/flytekit/tree/master/plugins) and [create an example](https://docs.flyte.org/en/latest/flytesnacks/contribute.html) of your agent for the [Integrations](https://docs.flyte.org/en/latest/flytesnacks/integrations.html) documentation. If you have any questions, reach out to us on [Slack](https://docs.flyte.org/en/latest/community/contribute.html#). ``` @@ -31,7 +31,7 @@ While agents can be written in any programming language, we currently only suppo ## Async agent interface specification -To create a new async agent, extend the `AsyncAgentBase` and implement `create`, `get`, and `delete` methods. These methods must be idempotent. +To create a new async agent, extend the [`AsyncAgentBase`](https://github.com/flyteorg/flytekit/blob/master/flytekit/extend/backend/base_agent.py#L127) class and implement `create`, `get`, and `delete` methods. These methods must be idempotent. - `create`: This method is used to initiate a new job. Users have the flexibility to use gRPC, REST, or an SDK to create a job. - `get`: This method retrieves the job resource (jobID or output literal) associated with the task, such as a BigQuery job ID or Databricks task ID. @@ -78,7 +78,7 @@ For an example implementation, see the [BigQuery agent](https://github.com/flyte ## Sync agent interface specification -To create a new sync agent, extend the `SyncAgentBase` class and implement a `do` method. This method must be idempotent. +To create a new sync agent, extend the [`SyncAgentBase`](https://github.com/flyteorg/flytekit/blob/master/flytekit/extend/backend/base_agent.py#L107) class and implement a `do` method. This method must be idempotent. - `do`: This method is used to execute the synchronous task, and the worker in Flyte will be blocked until the method returns. diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index 191143dd89..7b9045cbe3 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -4197,20 +4197,6 @@ ], "default": "SINGLE" }, - "ComparisonExpressionOperator": { - "type": "string", - "enum": [ - "EQ", - "NEQ", - "GT", - "GTE", - "LT", - "LTE" - ], - "default": "EQ", - "description": "- GT: Greater Than\n - LT: Less Than", - "title": "Binary Operator for each expression" - }, "ConjunctionExpressionLogicalOperator": { "type": "string", "enum": [ @@ -6481,8 +6467,8 @@ "bind_to_time_partition": { "type": "boolean" }, - "transform": { - "type": "string", + "time_transform": { + "$ref": "#/definitions/coreTimeTransform", "title": "This is only relevant in the time partition case" } }, @@ -6750,7 +6736,7 @@ "type": "object", "properties": { "operator": { - "$ref": "#/definitions/ComparisonExpressionOperator" + "$ref": "#/definitions/coreComparisonExpressionOperator" }, "left_value": { "$ref": "#/definitions/coreOperand" @@ -6761,6 +6747,20 @@ }, "description": "Defines a 2-level tree where the root is a comparison operator and Operands are primitives or known variables.\nEach expression results in a boolean result." }, + "coreComparisonExpressionOperator": { + "type": "string", + "enum": [ + "EQ", + "NEQ", + "GT", + "GTE", + "LT", + "LTE" + ], + "default": "EQ", + "description": "- GT: Greater Than\n - LT: Less Than", + "title": "Binary Operator for each expression" + }, "coreCompiledLaunchPlan": { "type": "object", "properties": { @@ -7038,6 +7038,18 @@ }, "description": "GateNode refers to the condition that is required for the gate to successfully complete." }, + "coreGranularity": { + "type": "string", + "enum": [ + "UNSET", + "MINUTE", + "HOUR", + "DAY", + "MONTH" + ], + "default": "UNSET", + "title": "- DAY: default" + }, "coreIOStrategy": { "type": "object", "properties": { @@ -7205,6 +7217,9 @@ }, "input_binding": { "$ref": "#/definitions/coreInputBindingData" + }, + "runtime_binding": { + "$ref": "#/definitions/coreRuntimeBinding" } } }, @@ -7688,6 +7703,9 @@ }, "description": "Retry strategy associated with an executable unit." }, + "coreRuntimeBinding": { + "type": "object" + }, "coreRuntimeMetadata": { "type": "object", "properties": { @@ -8145,6 +8163,20 @@ "properties": { "value": { "$ref": "#/definitions/coreLabelValue" + }, + "granularity": { + "$ref": "#/definitions/coreGranularity" + } + } + }, + "coreTimeTransform": { + "type": "object", + "properties": { + "transform": { + "type": "string" + }, + "op": { + "$ref": "#/definitions/flyteidlcoreOperator" } } }, @@ -8838,6 +8870,14 @@ }, "description": "A generic key value pair." }, + "flyteidlcoreOperator": { + "type": "string", + "enum": [ + "MINUS", + "PLUS" + ], + "default": "MINUS" + }, "flyteidlcoreSchema": { "type": "object", "properties": { diff --git a/flyteidl/gen/pb-es/flyteidl/core/artifact_id_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/artifact_id_pb.ts index 42d063161d..be92b47b70 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/artifact_id_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/artifact_id_pb.ts @@ -6,6 +6,66 @@ import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf"; import { Message, proto3, Timestamp } from "@bufbuild/protobuf"; +/** + * @generated from enum flyteidl.core.Granularity + */ +export enum Granularity { + /** + * @generated from enum value: UNSET = 0; + */ + UNSET = 0, + + /** + * @generated from enum value: MINUTE = 1; + */ + MINUTE = 1, + + /** + * @generated from enum value: HOUR = 2; + */ + HOUR = 2, + + /** + * default + * + * @generated from enum value: DAY = 3; + */ + DAY = 3, + + /** + * @generated from enum value: MONTH = 4; + */ + MONTH = 4, +} +// Retrieve enum metadata with: proto3.getEnumType(Granularity) +proto3.util.setEnumType(Granularity, "flyteidl.core.Granularity", [ + { no: 0, name: "UNSET" }, + { no: 1, name: "MINUTE" }, + { no: 2, name: "HOUR" }, + { no: 3, name: "DAY" }, + { no: 4, name: "MONTH" }, +]); + +/** + * @generated from enum flyteidl.core.Operator + */ +export enum Operator { + /** + * @generated from enum value: MINUS = 0; + */ + MINUS = 0, + + /** + * @generated from enum value: PLUS = 1; + */ + PLUS = 1, +} +// Retrieve enum metadata with: proto3.getEnumType(Operator) +proto3.util.setEnumType(Operator, "flyteidl.core.Operator", [ + { no: 0, name: "MINUS" }, + { no: 1, name: "PLUS" }, +]); + /** * @generated from message flyteidl.core.ArtifactKey */ @@ -76,13 +136,13 @@ export class ArtifactBindingData extends Message { */ partitionData: { /** - * @generated from field: string partition_key = 2; + * @generated from field: string partition_key = 5; */ value: string; case: "partitionKey"; } | { /** - * @generated from field: bool bind_to_time_partition = 3; + * @generated from field: bool bind_to_time_partition = 6; */ value: boolean; case: "bindToTimePartition"; @@ -91,9 +151,9 @@ export class ArtifactBindingData extends Message { /** * This is only relevant in the time partition case * - * @generated from field: string transform = 4; + * @generated from field: flyteidl.core.TimeTransform time_transform = 7; */ - transform = ""; + timeTransform?: TimeTransform; constructor(data?: PartialMessage) { super(); @@ -103,9 +163,9 @@ export class ArtifactBindingData extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "flyteidl.core.ArtifactBindingData"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 2, name: "partition_key", kind: "scalar", T: 9 /* ScalarType.STRING */, oneof: "partition_data" }, - { no: 3, name: "bind_to_time_partition", kind: "scalar", T: 8 /* ScalarType.BOOL */, oneof: "partition_data" }, - { no: 4, name: "transform", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 5, name: "partition_key", kind: "scalar", T: 9 /* ScalarType.STRING */, oneof: "partition_data" }, + { no: 6, name: "bind_to_time_partition", kind: "scalar", T: 8 /* ScalarType.BOOL */, oneof: "partition_data" }, + { no: 7, name: "time_transform", kind: "message", T: TimeTransform }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ArtifactBindingData { @@ -125,6 +185,49 @@ export class ArtifactBindingData extends Message { } } +/** + * @generated from message flyteidl.core.TimeTransform + */ +export class TimeTransform extends Message { + /** + * @generated from field: string transform = 1; + */ + transform = ""; + + /** + * @generated from field: flyteidl.core.Operator op = 2; + */ + op = Operator.MINUS; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "flyteidl.core.TimeTransform"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "transform", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "op", kind: "enum", T: proto3.getEnumType(Operator) }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): TimeTransform { + return new TimeTransform().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): TimeTransform { + return new TimeTransform().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): TimeTransform { + return new TimeTransform().fromJsonString(jsonString, options); + } + + static equals(a: TimeTransform | PlainMessage | undefined, b: TimeTransform | PlainMessage | undefined): boolean { + return proto3.util.equals(TimeTransform, a, b); + } +} + /** * @generated from message flyteidl.core.InputBindingData */ @@ -162,6 +265,37 @@ export class InputBindingData extends Message { } } +/** + * @generated from message flyteidl.core.RuntimeBinding + */ +export class RuntimeBinding extends Message { + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "flyteidl.core.RuntimeBinding"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): RuntimeBinding { + return new RuntimeBinding().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): RuntimeBinding { + return new RuntimeBinding().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): RuntimeBinding { + return new RuntimeBinding().fromJsonString(jsonString, options); + } + + static equals(a: RuntimeBinding | PlainMessage | undefined, b: RuntimeBinding | PlainMessage | undefined): boolean { + return proto3.util.equals(RuntimeBinding, a, b); + } +} + /** * @generated from message flyteidl.core.LabelValue */ @@ -197,6 +331,12 @@ export class LabelValue extends Message { */ value: InputBindingData; case: "inputBinding"; + } | { + /** + * @generated from field: flyteidl.core.RuntimeBinding runtime_binding = 5; + */ + value: RuntimeBinding; + case: "runtimeBinding"; } | { case: undefined; value?: undefined } = { case: undefined }; constructor(data?: PartialMessage) { @@ -211,6 +351,7 @@ export class LabelValue extends Message { { no: 2, name: "time_value", kind: "message", T: Timestamp, oneof: "value" }, { no: 3, name: "triggered_binding", kind: "message", T: ArtifactBindingData, oneof: "value" }, { no: 4, name: "input_binding", kind: "message", T: InputBindingData, oneof: "value" }, + { no: 5, name: "runtime_binding", kind: "message", T: RuntimeBinding, oneof: "value" }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): LabelValue { @@ -276,6 +417,11 @@ export class TimePartition extends Message { */ value?: LabelValue; + /** + * @generated from field: flyteidl.core.Granularity granularity = 2; + */ + granularity = Granularity.UNSET; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -285,6 +431,7 @@ export class TimePartition extends Message { static readonly typeName = "flyteidl.core.TimePartition"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "value", kind: "message", T: LabelValue }, + { no: 2, name: "granularity", kind: "enum", T: proto3.getEnumType(Granularity) }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TimePartition { diff --git a/flyteidl/gen/pb-go/flyteidl/core/artifact_id.pb.go b/flyteidl/gen/pb-go/flyteidl/core/artifact_id.pb.go index 7d692916a7..9fd82504dd 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/artifact_id.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/artifact_id.pb.go @@ -21,6 +21,107 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type Granularity int32 + +const ( + Granularity_UNSET Granularity = 0 + Granularity_MINUTE Granularity = 1 + Granularity_HOUR Granularity = 2 + Granularity_DAY Granularity = 3 // default + Granularity_MONTH Granularity = 4 +) + +// Enum value maps for Granularity. +var ( + Granularity_name = map[int32]string{ + 0: "UNSET", + 1: "MINUTE", + 2: "HOUR", + 3: "DAY", + 4: "MONTH", + } + Granularity_value = map[string]int32{ + "UNSET": 0, + "MINUTE": 1, + "HOUR": 2, + "DAY": 3, + "MONTH": 4, + } +) + +func (x Granularity) Enum() *Granularity { + p := new(Granularity) + *p = x + return p +} + +func (x Granularity) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Granularity) Descriptor() protoreflect.EnumDescriptor { + return file_flyteidl_core_artifact_id_proto_enumTypes[0].Descriptor() +} + +func (Granularity) Type() protoreflect.EnumType { + return &file_flyteidl_core_artifact_id_proto_enumTypes[0] +} + +func (x Granularity) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Granularity.Descriptor instead. +func (Granularity) EnumDescriptor() ([]byte, []int) { + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{0} +} + +type Operator int32 + +const ( + Operator_MINUS Operator = 0 + Operator_PLUS Operator = 1 +) + +// Enum value maps for Operator. +var ( + Operator_name = map[int32]string{ + 0: "MINUS", + 1: "PLUS", + } + Operator_value = map[string]int32{ + "MINUS": 0, + "PLUS": 1, + } +) + +func (x Operator) Enum() *Operator { + p := new(Operator) + *p = x + return p +} + +func (x Operator) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Operator) Descriptor() protoreflect.EnumDescriptor { + return file_flyteidl_core_artifact_id_proto_enumTypes[1].Descriptor() +} + +func (Operator) Type() protoreflect.EnumType { + return &file_flyteidl_core_artifact_id_proto_enumTypes[1] +} + +func (x Operator) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Operator.Descriptor instead. +func (Operator) EnumDescriptor() ([]byte, []int) { + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{1} +} + type ArtifactKey struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -107,7 +208,7 @@ type ArtifactBindingData struct { // *ArtifactBindingData_BindToTimePartition PartitionData isArtifactBindingData_PartitionData `protobuf_oneof:"partition_data"` // This is only relevant in the time partition case - Transform string `protobuf:"bytes,4,opt,name=transform,proto3" json:"transform,omitempty"` + TimeTransform *TimeTransform `protobuf:"bytes,7,opt,name=time_transform,json=timeTransform,proto3" json:"time_transform,omitempty"` } func (x *ArtifactBindingData) Reset() { @@ -163,11 +264,11 @@ func (x *ArtifactBindingData) GetBindToTimePartition() bool { return false } -func (x *ArtifactBindingData) GetTransform() string { +func (x *ArtifactBindingData) GetTimeTransform() *TimeTransform { if x != nil { - return x.Transform + return x.TimeTransform } - return "" + return nil } type isArtifactBindingData_PartitionData interface { @@ -175,17 +276,72 @@ type isArtifactBindingData_PartitionData interface { } type ArtifactBindingData_PartitionKey struct { - PartitionKey string `protobuf:"bytes,2,opt,name=partition_key,json=partitionKey,proto3,oneof"` + PartitionKey string `protobuf:"bytes,5,opt,name=partition_key,json=partitionKey,proto3,oneof"` } type ArtifactBindingData_BindToTimePartition struct { - BindToTimePartition bool `protobuf:"varint,3,opt,name=bind_to_time_partition,json=bindToTimePartition,proto3,oneof"` + BindToTimePartition bool `protobuf:"varint,6,opt,name=bind_to_time_partition,json=bindToTimePartition,proto3,oneof"` } func (*ArtifactBindingData_PartitionKey) isArtifactBindingData_PartitionData() {} func (*ArtifactBindingData_BindToTimePartition) isArtifactBindingData_PartitionData() {} +type TimeTransform struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Transform string `protobuf:"bytes,1,opt,name=transform,proto3" json:"transform,omitempty"` + Op Operator `protobuf:"varint,2,opt,name=op,proto3,enum=flyteidl.core.Operator" json:"op,omitempty"` +} + +func (x *TimeTransform) Reset() { + *x = TimeTransform{} + if protoimpl.UnsafeEnabled { + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TimeTransform) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TimeTransform) ProtoMessage() {} + +func (x *TimeTransform) ProtoReflect() protoreflect.Message { + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TimeTransform.ProtoReflect.Descriptor instead. +func (*TimeTransform) Descriptor() ([]byte, []int) { + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{2} +} + +func (x *TimeTransform) GetTransform() string { + if x != nil { + return x.Transform + } + return "" +} + +func (x *TimeTransform) GetOp() Operator { + if x != nil { + return x.Op + } + return Operator_MINUS +} + type InputBindingData struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -197,7 +353,7 @@ type InputBindingData struct { func (x *InputBindingData) Reset() { *x = InputBindingData{} if protoimpl.UnsafeEnabled { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[2] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -210,7 +366,7 @@ func (x *InputBindingData) String() string { func (*InputBindingData) ProtoMessage() {} func (x *InputBindingData) ProtoReflect() protoreflect.Message { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[2] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -223,7 +379,7 @@ func (x *InputBindingData) ProtoReflect() protoreflect.Message { // Deprecated: Use InputBindingData.ProtoReflect.Descriptor instead. func (*InputBindingData) Descriptor() ([]byte, []int) { - return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{2} + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{3} } func (x *InputBindingData) GetVar() string { @@ -233,6 +389,44 @@ func (x *InputBindingData) GetVar() string { return "" } +type RuntimeBinding struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RuntimeBinding) Reset() { + *x = RuntimeBinding{} + if protoimpl.UnsafeEnabled { + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RuntimeBinding) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RuntimeBinding) ProtoMessage() {} + +func (x *RuntimeBinding) ProtoReflect() protoreflect.Message { + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RuntimeBinding.ProtoReflect.Descriptor instead. +func (*RuntimeBinding) Descriptor() ([]byte, []int) { + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{4} +} + type LabelValue struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -244,13 +438,14 @@ type LabelValue struct { // *LabelValue_TimeValue // *LabelValue_TriggeredBinding // *LabelValue_InputBinding + // *LabelValue_RuntimeBinding Value isLabelValue_Value `protobuf_oneof:"value"` } func (x *LabelValue) Reset() { *x = LabelValue{} if protoimpl.UnsafeEnabled { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[3] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -263,7 +458,7 @@ func (x *LabelValue) String() string { func (*LabelValue) ProtoMessage() {} func (x *LabelValue) ProtoReflect() protoreflect.Message { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[3] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -276,7 +471,7 @@ func (x *LabelValue) ProtoReflect() protoreflect.Message { // Deprecated: Use LabelValue.ProtoReflect.Descriptor instead. func (*LabelValue) Descriptor() ([]byte, []int) { - return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{3} + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{5} } func (m *LabelValue) GetValue() isLabelValue_Value { @@ -314,6 +509,13 @@ func (x *LabelValue) GetInputBinding() *InputBindingData { return nil } +func (x *LabelValue) GetRuntimeBinding() *RuntimeBinding { + if x, ok := x.GetValue().(*LabelValue_RuntimeBinding); ok { + return x.RuntimeBinding + } + return nil +} + type isLabelValue_Value interface { isLabelValue_Value() } @@ -336,6 +538,10 @@ type LabelValue_InputBinding struct { InputBinding *InputBindingData `protobuf:"bytes,4,opt,name=input_binding,json=inputBinding,proto3,oneof"` } +type LabelValue_RuntimeBinding struct { + RuntimeBinding *RuntimeBinding `protobuf:"bytes,5,opt,name=runtime_binding,json=runtimeBinding,proto3,oneof"` +} + func (*LabelValue_StaticValue) isLabelValue_Value() {} func (*LabelValue_TimeValue) isLabelValue_Value() {} @@ -344,6 +550,8 @@ func (*LabelValue_TriggeredBinding) isLabelValue_Value() {} func (*LabelValue_InputBinding) isLabelValue_Value() {} +func (*LabelValue_RuntimeBinding) isLabelValue_Value() {} + type Partitions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -355,7 +563,7 @@ type Partitions struct { func (x *Partitions) Reset() { *x = Partitions{} if protoimpl.UnsafeEnabled { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[4] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -368,7 +576,7 @@ func (x *Partitions) String() string { func (*Partitions) ProtoMessage() {} func (x *Partitions) ProtoReflect() protoreflect.Message { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[4] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -381,7 +589,7 @@ func (x *Partitions) ProtoReflect() protoreflect.Message { // Deprecated: Use Partitions.ProtoReflect.Descriptor instead. func (*Partitions) Descriptor() ([]byte, []int) { - return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{4} + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{6} } func (x *Partitions) GetValue() map[string]*LabelValue { @@ -396,13 +604,14 @@ type TimePartition struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Value *LabelValue `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` + Value *LabelValue `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` + Granularity Granularity `protobuf:"varint,2,opt,name=granularity,proto3,enum=flyteidl.core.Granularity" json:"granularity,omitempty"` } func (x *TimePartition) Reset() { *x = TimePartition{} if protoimpl.UnsafeEnabled { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[5] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -415,7 +624,7 @@ func (x *TimePartition) String() string { func (*TimePartition) ProtoMessage() {} func (x *TimePartition) ProtoReflect() protoreflect.Message { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[5] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -428,7 +637,7 @@ func (x *TimePartition) ProtoReflect() protoreflect.Message { // Deprecated: Use TimePartition.ProtoReflect.Descriptor instead. func (*TimePartition) Descriptor() ([]byte, []int) { - return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{5} + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{7} } func (x *TimePartition) GetValue() *LabelValue { @@ -438,6 +647,13 @@ func (x *TimePartition) GetValue() *LabelValue { return nil } +func (x *TimePartition) GetGranularity() Granularity { + if x != nil { + return x.Granularity + } + return Granularity_UNSET +} + type ArtifactID struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -455,7 +671,7 @@ type ArtifactID struct { func (x *ArtifactID) Reset() { *x = ArtifactID{} if protoimpl.UnsafeEnabled { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[6] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -468,7 +684,7 @@ func (x *ArtifactID) String() string { func (*ArtifactID) ProtoMessage() {} func (x *ArtifactID) ProtoReflect() protoreflect.Message { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[6] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -481,7 +697,7 @@ func (x *ArtifactID) ProtoReflect() protoreflect.Message { // Deprecated: Use ArtifactID.ProtoReflect.Descriptor instead. func (*ArtifactID) Descriptor() ([]byte, []int) { - return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{6} + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{8} } func (x *ArtifactID) GetArtifactKey() *ArtifactKey { @@ -524,7 +740,7 @@ type ArtifactTag struct { func (x *ArtifactTag) Reset() { *x = ArtifactTag{} if protoimpl.UnsafeEnabled { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[7] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -537,7 +753,7 @@ func (x *ArtifactTag) String() string { func (*ArtifactTag) ProtoMessage() {} func (x *ArtifactTag) ProtoReflect() protoreflect.Message { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[7] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -550,7 +766,7 @@ func (x *ArtifactTag) ProtoReflect() protoreflect.Message { // Deprecated: Use ArtifactTag.ProtoReflect.Descriptor instead. func (*ArtifactTag) Descriptor() ([]byte, []int) { - return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{7} + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{9} } func (x *ArtifactTag) GetArtifactKey() *ArtifactKey { @@ -590,7 +806,7 @@ type ArtifactQuery struct { func (x *ArtifactQuery) Reset() { *x = ArtifactQuery{} if protoimpl.UnsafeEnabled { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[8] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -603,7 +819,7 @@ func (x *ArtifactQuery) String() string { func (*ArtifactQuery) ProtoMessage() {} func (x *ArtifactQuery) ProtoReflect() protoreflect.Message { - mi := &file_flyteidl_core_artifact_id_proto_msgTypes[8] + mi := &file_flyteidl_core_artifact_id_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -616,7 +832,7 @@ func (x *ArtifactQuery) ProtoReflect() protoreflect.Message { // Deprecated: Use ArtifactQuery.ProtoReflect.Descriptor instead. func (*ArtifactQuery) Descriptor() ([]byte, []int) { - return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{8} + return file_flyteidl_core_artifact_id_proto_rawDescGZIP(), []int{10} } func (m *ArtifactQuery) GetIdentifier() isArtifactQuery_Identifier { @@ -700,101 +916,125 @@ var file_flyteidl_core_artifact_id_proto_rawDesc = []byte{ 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x22, 0xa3, 0x01, 0x0a, 0x13, 0x41, 0x72, 0x74, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x22, 0xd0, 0x01, 0x0a, 0x13, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x12, 0x25, 0x0a, 0x0d, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, - 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x35, 0x0a, 0x16, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x13, 0x62, 0x69, 0x6e, 0x64, 0x54, - 0x6f, 0x54, 0x69, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1c, - 0x0a, 0x09, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x42, 0x10, 0x0a, 0x0e, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x22, 0x24, - 0x0a, 0x10, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, - 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x76, 0x61, 0x72, 0x22, 0x92, 0x02, 0x0a, 0x0a, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x61, 0x74, 0x69, 0x63, 0x5f, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x61, - 0x74, 0x69, 0x63, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3b, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, - 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x51, 0x0a, 0x11, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, - 0x65, 0x64, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x22, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, - 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x10, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x65, - 0x64, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x46, 0x0a, 0x0d, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1f, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x49, 0x6e, 0x70, 0x75, 0x74, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, - 0x48, 0x00, 0x52, 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, - 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x9d, 0x01, 0x0a, 0x0a, 0x50, 0x61, - 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, - 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x53, 0x0a, 0x0a, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x40, 0x0a, 0x0d, 0x54, 0x69, 0x6d, - 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2f, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xe5, 0x01, 0x0a, 0x0a, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x12, 0x3d, 0x0a, 0x0c, 0x61, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x0b, 0x61, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, - 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x43, - 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, - 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x22, 0x7d, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, - 0x61, 0x67, 0x12, 0x3d, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, - 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x4b, 0x65, 0x79, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4b, 0x65, - 0x79, 0x12, 0x2f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x22, 0xf0, 0x01, 0x0a, 0x0d, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x51, - 0x75, 0x65, 0x72, 0x79, 0x12, 0x3c, 0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x49, 0x44, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x49, 0x64, 0x12, 0x3f, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, - 0x61, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, + 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x13, 0x62, 0x69, 0x6e, 0x64, 0x54, + 0x6f, 0x54, 0x69, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x43, + 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x6f, 0x72, 0x6d, 0x42, 0x10, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x64, 0x61, 0x74, 0x61, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x05, 0x22, 0x56, 0x0a, 0x0d, 0x54, + 0x69, 0x6d, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x1c, 0x0a, 0x09, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x27, 0x0a, 0x02, 0x6f, 0x70, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x52, + 0x02, 0x6f, 0x70, 0x22, 0x24, 0x0a, 0x10, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x42, 0x69, 0x6e, 0x64, + 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x76, 0x61, 0x72, 0x22, 0x10, 0x0a, 0x0e, 0x52, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x22, 0xdc, 0x02, 0x0a, 0x0a, + 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, + 0x61, 0x74, 0x69, 0x63, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x69, 0x63, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x3b, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, + 0x00, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x51, 0x0a, 0x11, + 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, + 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x10, 0x74, + 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x65, 0x64, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, + 0x46, 0x0a, 0x0d, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x42, 0x69, 0x6e, 0x64, + 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x72, 0x75, 0x6e, 0x74, 0x69, + 0x6d, 0x65, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x48, + 0x00, 0x52, 0x0e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, + 0x67, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x9d, 0x01, 0x0a, 0x0a, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, + 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x53, 0x0a, 0x0a, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7e, 0x0a, 0x0d, 0x54, 0x69, + 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2f, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, + 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3c, 0x0a, 0x0b, + 0x67, 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x47, 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x52, 0x0b, 0x67, + 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x22, 0xe5, 0x01, 0x0a, 0x0a, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x12, 0x3d, 0x0a, 0x0c, 0x61, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x0b, 0x61, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x43, 0x0a, + 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x7d, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x61, + 0x67, 0x12, 0x3d, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x4b, 0x65, 0x79, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4b, 0x65, 0x79, + 0x12, 0x2f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x22, 0xf0, 0x01, 0x0a, 0x0d, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x12, 0x3c, 0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x54, 0x61, 0x67, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x54, 0x61, 0x67, 0x12, 0x12, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x48, 0x00, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, 0x3e, 0x0a, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, - 0x6e, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, - 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x07, - 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x42, 0x0c, 0x0a, 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, - 0x69, 0x66, 0x69, 0x65, 0x72, 0x42, 0xb5, 0x01, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6c, - 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3a, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, - 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, - 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, - 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x43, 0x58, - 0xaa, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x43, 0x6f, 0x72, 0x65, - 0xca, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, - 0xe2, 0x02, 0x19, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, - 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x46, - 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x49, 0x44, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, + 0x64, 0x12, 0x3f, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x61, + 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x54, 0x61, 0x67, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, + 0x61, 0x67, 0x12, 0x12, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, 0x3e, 0x0a, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, + 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x07, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x42, 0x0c, 0x0a, 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x66, 0x69, 0x65, 0x72, 0x2a, 0x42, 0x0a, 0x0b, 0x47, 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, 0x72, + 0x69, 0x74, 0x79, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x4e, 0x53, 0x45, 0x54, 0x10, 0x00, 0x12, 0x0a, + 0x0a, 0x06, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x4f, + 0x55, 0x52, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x41, 0x59, 0x10, 0x03, 0x12, 0x09, 0x0a, + 0x05, 0x4d, 0x4f, 0x4e, 0x54, 0x48, 0x10, 0x04, 0x2a, 0x1f, 0x0a, 0x08, 0x4f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x6f, 0x72, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x49, 0x4e, 0x55, 0x53, 0x10, 0x00, 0x12, + 0x08, 0x0a, 0x04, 0x50, 0x4c, 0x55, 0x53, 0x10, 0x01, 0x42, 0xb5, 0x01, 0x0a, 0x11, 0x63, 0x6f, + 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, + 0x0f, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x50, 0x01, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, + 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, 0x66, 0x6c, + 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, 0x67, 0x6f, + 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0xa2, 0x02, + 0x03, 0x46, 0x43, 0x58, 0xaa, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, + 0x43, 0x6f, 0x72, 0x65, 0xca, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, + 0x43, 0x6f, 0x72, 0x65, 0xe2, 0x02, 0x19, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, + 0x43, 0x6f, 0x72, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0xea, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x72, + 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -809,40 +1049,49 @@ func file_flyteidl_core_artifact_id_proto_rawDescGZIP() []byte { return file_flyteidl_core_artifact_id_proto_rawDescData } -var file_flyteidl_core_artifact_id_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_flyteidl_core_artifact_id_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_flyteidl_core_artifact_id_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_flyteidl_core_artifact_id_proto_goTypes = []interface{}{ - (*ArtifactKey)(nil), // 0: flyteidl.core.ArtifactKey - (*ArtifactBindingData)(nil), // 1: flyteidl.core.ArtifactBindingData - (*InputBindingData)(nil), // 2: flyteidl.core.InputBindingData - (*LabelValue)(nil), // 3: flyteidl.core.LabelValue - (*Partitions)(nil), // 4: flyteidl.core.Partitions - (*TimePartition)(nil), // 5: flyteidl.core.TimePartition - (*ArtifactID)(nil), // 6: flyteidl.core.ArtifactID - (*ArtifactTag)(nil), // 7: flyteidl.core.ArtifactTag - (*ArtifactQuery)(nil), // 8: flyteidl.core.ArtifactQuery - nil, // 9: flyteidl.core.Partitions.ValueEntry - (*timestamppb.Timestamp)(nil), // 10: google.protobuf.Timestamp + (Granularity)(0), // 0: flyteidl.core.Granularity + (Operator)(0), // 1: flyteidl.core.Operator + (*ArtifactKey)(nil), // 2: flyteidl.core.ArtifactKey + (*ArtifactBindingData)(nil), // 3: flyteidl.core.ArtifactBindingData + (*TimeTransform)(nil), // 4: flyteidl.core.TimeTransform + (*InputBindingData)(nil), // 5: flyteidl.core.InputBindingData + (*RuntimeBinding)(nil), // 6: flyteidl.core.RuntimeBinding + (*LabelValue)(nil), // 7: flyteidl.core.LabelValue + (*Partitions)(nil), // 8: flyteidl.core.Partitions + (*TimePartition)(nil), // 9: flyteidl.core.TimePartition + (*ArtifactID)(nil), // 10: flyteidl.core.ArtifactID + (*ArtifactTag)(nil), // 11: flyteidl.core.ArtifactTag + (*ArtifactQuery)(nil), // 12: flyteidl.core.ArtifactQuery + nil, // 13: flyteidl.core.Partitions.ValueEntry + (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp } var file_flyteidl_core_artifact_id_proto_depIdxs = []int32{ - 10, // 0: flyteidl.core.LabelValue.time_value:type_name -> google.protobuf.Timestamp - 1, // 1: flyteidl.core.LabelValue.triggered_binding:type_name -> flyteidl.core.ArtifactBindingData - 2, // 2: flyteidl.core.LabelValue.input_binding:type_name -> flyteidl.core.InputBindingData - 9, // 3: flyteidl.core.Partitions.value:type_name -> flyteidl.core.Partitions.ValueEntry - 3, // 4: flyteidl.core.TimePartition.value:type_name -> flyteidl.core.LabelValue - 0, // 5: flyteidl.core.ArtifactID.artifact_key:type_name -> flyteidl.core.ArtifactKey - 4, // 6: flyteidl.core.ArtifactID.partitions:type_name -> flyteidl.core.Partitions - 5, // 7: flyteidl.core.ArtifactID.time_partition:type_name -> flyteidl.core.TimePartition - 0, // 8: flyteidl.core.ArtifactTag.artifact_key:type_name -> flyteidl.core.ArtifactKey - 3, // 9: flyteidl.core.ArtifactTag.value:type_name -> flyteidl.core.LabelValue - 6, // 10: flyteidl.core.ArtifactQuery.artifact_id:type_name -> flyteidl.core.ArtifactID - 7, // 11: flyteidl.core.ArtifactQuery.artifact_tag:type_name -> flyteidl.core.ArtifactTag - 1, // 12: flyteidl.core.ArtifactQuery.binding:type_name -> flyteidl.core.ArtifactBindingData - 3, // 13: flyteidl.core.Partitions.ValueEntry.value:type_name -> flyteidl.core.LabelValue - 14, // [14:14] is the sub-list for method output_type - 14, // [14:14] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 4, // 0: flyteidl.core.ArtifactBindingData.time_transform:type_name -> flyteidl.core.TimeTransform + 1, // 1: flyteidl.core.TimeTransform.op:type_name -> flyteidl.core.Operator + 14, // 2: flyteidl.core.LabelValue.time_value:type_name -> google.protobuf.Timestamp + 3, // 3: flyteidl.core.LabelValue.triggered_binding:type_name -> flyteidl.core.ArtifactBindingData + 5, // 4: flyteidl.core.LabelValue.input_binding:type_name -> flyteidl.core.InputBindingData + 6, // 5: flyteidl.core.LabelValue.runtime_binding:type_name -> flyteidl.core.RuntimeBinding + 13, // 6: flyteidl.core.Partitions.value:type_name -> flyteidl.core.Partitions.ValueEntry + 7, // 7: flyteidl.core.TimePartition.value:type_name -> flyteidl.core.LabelValue + 0, // 8: flyteidl.core.TimePartition.granularity:type_name -> flyteidl.core.Granularity + 2, // 9: flyteidl.core.ArtifactID.artifact_key:type_name -> flyteidl.core.ArtifactKey + 8, // 10: flyteidl.core.ArtifactID.partitions:type_name -> flyteidl.core.Partitions + 9, // 11: flyteidl.core.ArtifactID.time_partition:type_name -> flyteidl.core.TimePartition + 2, // 12: flyteidl.core.ArtifactTag.artifact_key:type_name -> flyteidl.core.ArtifactKey + 7, // 13: flyteidl.core.ArtifactTag.value:type_name -> flyteidl.core.LabelValue + 10, // 14: flyteidl.core.ArtifactQuery.artifact_id:type_name -> flyteidl.core.ArtifactID + 11, // 15: flyteidl.core.ArtifactQuery.artifact_tag:type_name -> flyteidl.core.ArtifactTag + 3, // 16: flyteidl.core.ArtifactQuery.binding:type_name -> flyteidl.core.ArtifactBindingData + 7, // 17: flyteidl.core.Partitions.ValueEntry.value:type_name -> flyteidl.core.LabelValue + 18, // [18:18] is the sub-list for method output_type + 18, // [18:18] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_flyteidl_core_artifact_id_proto_init() } @@ -877,7 +1126,7 @@ func file_flyteidl_core_artifact_id_proto_init() { } } file_flyteidl_core_artifact_id_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InputBindingData); i { + switch v := v.(*TimeTransform); i { case 0: return &v.state case 1: @@ -889,7 +1138,7 @@ func file_flyteidl_core_artifact_id_proto_init() { } } file_flyteidl_core_artifact_id_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LabelValue); i { + switch v := v.(*InputBindingData); i { case 0: return &v.state case 1: @@ -901,7 +1150,7 @@ func file_flyteidl_core_artifact_id_proto_init() { } } file_flyteidl_core_artifact_id_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Partitions); i { + switch v := v.(*RuntimeBinding); i { case 0: return &v.state case 1: @@ -913,7 +1162,7 @@ func file_flyteidl_core_artifact_id_proto_init() { } } file_flyteidl_core_artifact_id_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TimePartition); i { + switch v := v.(*LabelValue); i { case 0: return &v.state case 1: @@ -925,7 +1174,7 @@ func file_flyteidl_core_artifact_id_proto_init() { } } file_flyteidl_core_artifact_id_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactID); i { + switch v := v.(*Partitions); i { case 0: return &v.state case 1: @@ -937,7 +1186,7 @@ func file_flyteidl_core_artifact_id_proto_init() { } } file_flyteidl_core_artifact_id_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactTag); i { + switch v := v.(*TimePartition); i { case 0: return &v.state case 1: @@ -949,6 +1198,30 @@ func file_flyteidl_core_artifact_id_proto_init() { } } file_flyteidl_core_artifact_id_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ArtifactID); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_flyteidl_core_artifact_id_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ArtifactTag); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_flyteidl_core_artifact_id_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ArtifactQuery); i { case 0: return &v.state @@ -965,13 +1238,14 @@ func file_flyteidl_core_artifact_id_proto_init() { (*ArtifactBindingData_PartitionKey)(nil), (*ArtifactBindingData_BindToTimePartition)(nil), } - file_flyteidl_core_artifact_id_proto_msgTypes[3].OneofWrappers = []interface{}{ + file_flyteidl_core_artifact_id_proto_msgTypes[5].OneofWrappers = []interface{}{ (*LabelValue_StaticValue)(nil), (*LabelValue_TimeValue)(nil), (*LabelValue_TriggeredBinding)(nil), (*LabelValue_InputBinding)(nil), + (*LabelValue_RuntimeBinding)(nil), } - file_flyteidl_core_artifact_id_proto_msgTypes[8].OneofWrappers = []interface{}{ + file_flyteidl_core_artifact_id_proto_msgTypes[10].OneofWrappers = []interface{}{ (*ArtifactQuery_ArtifactId)(nil), (*ArtifactQuery_ArtifactTag)(nil), (*ArtifactQuery_Uri)(nil), @@ -982,13 +1256,14 @@ func file_flyteidl_core_artifact_id_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_flyteidl_core_artifact_id_proto_rawDesc, - NumEnums: 0, - NumMessages: 10, + NumEnums: 2, + NumMessages: 12, NumExtensions: 0, NumServices: 0, }, GoTypes: file_flyteidl_core_artifact_id_proto_goTypes, DependencyIndexes: file_flyteidl_core_artifact_id_proto_depIdxs, + EnumInfos: file_flyteidl_core_artifact_id_proto_enumTypes, MessageInfos: file_flyteidl_core_artifact_id_proto_msgTypes, }.Build() File_flyteidl_core_artifact_id_proto = out.File diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index 191143dd89..7b9045cbe3 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -4197,20 +4197,6 @@ ], "default": "SINGLE" }, - "ComparisonExpressionOperator": { - "type": "string", - "enum": [ - "EQ", - "NEQ", - "GT", - "GTE", - "LT", - "LTE" - ], - "default": "EQ", - "description": "- GT: Greater Than\n - LT: Less Than", - "title": "Binary Operator for each expression" - }, "ConjunctionExpressionLogicalOperator": { "type": "string", "enum": [ @@ -6481,8 +6467,8 @@ "bind_to_time_partition": { "type": "boolean" }, - "transform": { - "type": "string", + "time_transform": { + "$ref": "#/definitions/coreTimeTransform", "title": "This is only relevant in the time partition case" } }, @@ -6750,7 +6736,7 @@ "type": "object", "properties": { "operator": { - "$ref": "#/definitions/ComparisonExpressionOperator" + "$ref": "#/definitions/coreComparisonExpressionOperator" }, "left_value": { "$ref": "#/definitions/coreOperand" @@ -6761,6 +6747,20 @@ }, "description": "Defines a 2-level tree where the root is a comparison operator and Operands are primitives or known variables.\nEach expression results in a boolean result." }, + "coreComparisonExpressionOperator": { + "type": "string", + "enum": [ + "EQ", + "NEQ", + "GT", + "GTE", + "LT", + "LTE" + ], + "default": "EQ", + "description": "- GT: Greater Than\n - LT: Less Than", + "title": "Binary Operator for each expression" + }, "coreCompiledLaunchPlan": { "type": "object", "properties": { @@ -7038,6 +7038,18 @@ }, "description": "GateNode refers to the condition that is required for the gate to successfully complete." }, + "coreGranularity": { + "type": "string", + "enum": [ + "UNSET", + "MINUTE", + "HOUR", + "DAY", + "MONTH" + ], + "default": "UNSET", + "title": "- DAY: default" + }, "coreIOStrategy": { "type": "object", "properties": { @@ -7205,6 +7217,9 @@ }, "input_binding": { "$ref": "#/definitions/coreInputBindingData" + }, + "runtime_binding": { + "$ref": "#/definitions/coreRuntimeBinding" } } }, @@ -7688,6 +7703,9 @@ }, "description": "Retry strategy associated with an executable unit." }, + "coreRuntimeBinding": { + "type": "object" + }, "coreRuntimeMetadata": { "type": "object", "properties": { @@ -8145,6 +8163,20 @@ "properties": { "value": { "$ref": "#/definitions/coreLabelValue" + }, + "granularity": { + "$ref": "#/definitions/coreGranularity" + } + } + }, + "coreTimeTransform": { + "type": "object", + "properties": { + "transform": { + "type": "string" + }, + "op": { + "$ref": "#/definitions/flyteidlcoreOperator" } } }, @@ -8838,6 +8870,14 @@ }, "description": "A generic key value pair." }, + "flyteidlcoreOperator": { + "type": "string", + "enum": [ + "MINUS", + "PLUS" + ], + "default": "MINUS" + }, "flyteidlcoreSchema": { "type": "object", "properties": { diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index ff115cad21..2680d36bda 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -843,8 +843,8 @@ "bind_to_time_partition": { "type": "boolean" }, - "transform": { - "type": "string", + "time_transform": { + "$ref": "#/definitions/coreTimeTransform", "title": "This is only relevant in the time partition case" } }, @@ -1093,6 +1093,18 @@ }, "description": "Metadata associated with the GPU accelerator to allocate to a task. Contains\ninformation about device type, and for multi-instance GPUs, the partition size to\nuse." }, + "coreGranularity": { + "type": "string", + "enum": [ + "UNSET", + "MINUTE", + "HOUR", + "DAY", + "MONTH" + ], + "default": "UNSET", + "title": "- DAY: default" + }, "coreIOStrategy": { "type": "object", "properties": { @@ -1222,6 +1234,9 @@ }, "input_binding": { "$ref": "#/definitions/coreInputBindingData" + }, + "runtime_binding": { + "$ref": "#/definitions/coreRuntimeBinding" } } }, @@ -1472,6 +1487,9 @@ }, "description": "Retry strategy associated with an executable unit." }, + "coreRuntimeBinding": { + "type": "object" + }, "coreRuntimeMetadata": { "type": "object", "properties": { @@ -1847,6 +1865,20 @@ "properties": { "value": { "$ref": "#/definitions/coreLabelValue" + }, + "granularity": { + "$ref": "#/definitions/coreGranularity" + } + } + }, + "coreTimeTransform": { + "type": "object", + "properties": { + "transform": { + "type": "string" + }, + "op": { + "$ref": "#/definitions/flyteidlcoreOperator" } } }, @@ -2071,6 +2103,14 @@ }, "description": "A generic key value pair." }, + "flyteidlcoreOperator": { + "type": "string", + "enum": [ + "MINUS", + "PLUS" + ], + "default": "MINUS" + }, "flyteidlcoreSchema": { "type": "object", "properties": { diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json index eda2434bcd..cebc48b75b 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json @@ -173,8 +173,8 @@ "bind_to_time_partition": { "type": "boolean" }, - "transform": { - "type": "string", + "time_transform": { + "$ref": "#/definitions/coreTimeTransform", "title": "This is only relevant in the time partition case" } }, @@ -409,6 +409,18 @@ }, "description": "Metadata associated with the GPU accelerator to allocate to a task. Contains\ninformation about device type, and for multi-instance GPUs, the partition size to\nuse." }, + "coreGranularity": { + "type": "string", + "enum": [ + "UNSET", + "MINUTE", + "HOUR", + "DAY", + "MONTH" + ], + "default": "UNSET", + "title": "- DAY: default" + }, "coreIOStrategy": { "type": "object", "properties": { @@ -538,6 +550,9 @@ }, "input_binding": { "$ref": "#/definitions/coreInputBindingData" + }, + "runtime_binding": { + "$ref": "#/definitions/coreRuntimeBinding" } } }, @@ -776,6 +791,9 @@ }, "description": "Retry strategy associated with an executable unit." }, + "coreRuntimeBinding": { + "type": "object" + }, "coreRuntimeMetadata": { "type": "object", "properties": { @@ -1084,6 +1102,20 @@ "properties": { "value": { "$ref": "#/definitions/coreLabelValue" + }, + "granularity": { + "$ref": "#/definitions/coreGranularity" + } + } + }, + "coreTimeTransform": { + "type": "object", + "properties": { + "transform": { + "type": "string" + }, + "op": { + "$ref": "#/definitions/flyteidlcoreOperator" } } }, @@ -1217,6 +1249,14 @@ }, "description": "A generic key value pair." }, + "flyteidlcoreOperator": { + "type": "string", + "enum": [ + "MINUS", + "PLUS" + ], + "default": "MINUS" + }, "flyteidlcoreSchema": { "type": "object", "properties": { diff --git a/flyteidl/gen/pb-js/flyteidl.d.ts b/flyteidl/gen/pb-js/flyteidl.d.ts index c876340a61..5cc6b36eeb 100644 --- a/flyteidl/gen/pb-js/flyteidl.d.ts +++ b/flyteidl/gen/pb-js/flyteidl.d.ts @@ -84,8 +84,8 @@ export namespace flyteidl { /** ArtifactBindingData bindToTimePartition */ bindToTimePartition?: (boolean|null); - /** ArtifactBindingData transform */ - transform?: (string|null); + /** ArtifactBindingData timeTransform */ + timeTransform?: (flyteidl.core.ITimeTransform|null); } /** Represents an ArtifactBindingData. */ @@ -103,8 +103,8 @@ export namespace flyteidl { /** ArtifactBindingData bindToTimePartition. */ public bindToTimePartition: boolean; - /** ArtifactBindingData transform. */ - public transform: string; + /** ArtifactBindingData timeTransform. */ + public timeTransform?: (flyteidl.core.ITimeTransform|null); /** ArtifactBindingData partitionData. */ public partitionData?: ("partitionKey"|"bindToTimePartition"); @@ -142,6 +142,79 @@ export namespace flyteidl { public static verify(message: { [k: string]: any }): (string|null); } + /** Granularity enum. */ + enum Granularity { + UNSET = 0, + MINUTE = 1, + HOUR = 2, + DAY = 3, + MONTH = 4 + } + + /** Operator enum. */ + enum Operator { + MINUS = 0, + PLUS = 1 + } + + /** Properties of a TimeTransform. */ + interface ITimeTransform { + + /** TimeTransform transform */ + transform?: (string|null); + + /** TimeTransform op */ + op?: (flyteidl.core.Operator|null); + } + + /** Represents a TimeTransform. */ + class TimeTransform implements ITimeTransform { + + /** + * Constructs a new TimeTransform. + * @param [properties] Properties to set + */ + constructor(properties?: flyteidl.core.ITimeTransform); + + /** TimeTransform transform. */ + public transform: string; + + /** TimeTransform op. */ + public op: flyteidl.core.Operator; + + /** + * Creates a new TimeTransform instance using the specified properties. + * @param [properties] Properties to set + * @returns TimeTransform instance + */ + public static create(properties?: flyteidl.core.ITimeTransform): flyteidl.core.TimeTransform; + + /** + * Encodes the specified TimeTransform message. Does not implicitly {@link flyteidl.core.TimeTransform.verify|verify} messages. + * @param message TimeTransform message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: flyteidl.core.ITimeTransform, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Decodes a TimeTransform message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns TimeTransform + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.core.TimeTransform; + + /** + * Verifies a TimeTransform message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): (string|null); + } + /** Properties of an InputBindingData. */ interface IInputBindingData { @@ -194,6 +267,52 @@ export namespace flyteidl { public static verify(message: { [k: string]: any }): (string|null); } + /** Properties of a RuntimeBinding. */ + interface IRuntimeBinding { + } + + /** Represents a RuntimeBinding. */ + class RuntimeBinding implements IRuntimeBinding { + + /** + * Constructs a new RuntimeBinding. + * @param [properties] Properties to set + */ + constructor(properties?: flyteidl.core.IRuntimeBinding); + + /** + * Creates a new RuntimeBinding instance using the specified properties. + * @param [properties] Properties to set + * @returns RuntimeBinding instance + */ + public static create(properties?: flyteidl.core.IRuntimeBinding): flyteidl.core.RuntimeBinding; + + /** + * Encodes the specified RuntimeBinding message. Does not implicitly {@link flyteidl.core.RuntimeBinding.verify|verify} messages. + * @param message RuntimeBinding message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: flyteidl.core.IRuntimeBinding, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Decodes a RuntimeBinding message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns RuntimeBinding + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.core.RuntimeBinding; + + /** + * Verifies a RuntimeBinding message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): (string|null); + } + /** Properties of a LabelValue. */ interface ILabelValue { @@ -208,6 +327,9 @@ export namespace flyteidl { /** LabelValue inputBinding */ inputBinding?: (flyteidl.core.IInputBindingData|null); + + /** LabelValue runtimeBinding */ + runtimeBinding?: (flyteidl.core.IRuntimeBinding|null); } /** Represents a LabelValue. */ @@ -231,8 +353,11 @@ export namespace flyteidl { /** LabelValue inputBinding. */ public inputBinding?: (flyteidl.core.IInputBindingData|null); + /** LabelValue runtimeBinding. */ + public runtimeBinding?: (flyteidl.core.IRuntimeBinding|null); + /** LabelValue value. */ - public value?: ("staticValue"|"timeValue"|"triggeredBinding"|"inputBinding"); + public value?: ("staticValue"|"timeValue"|"triggeredBinding"|"inputBinding"|"runtimeBinding"); /** * Creates a new LabelValue instance using the specified properties. @@ -324,6 +449,9 @@ export namespace flyteidl { /** TimePartition value */ value?: (flyteidl.core.ILabelValue|null); + + /** TimePartition granularity */ + granularity?: (flyteidl.core.Granularity|null); } /** Represents a TimePartition. */ @@ -338,6 +466,9 @@ export namespace flyteidl { /** TimePartition value. */ public value?: (flyteidl.core.ILabelValue|null); + /** TimePartition granularity. */ + public granularity: flyteidl.core.Granularity; + /** * Creates a new TimePartition instance using the specified properties. * @param [properties] Properties to set diff --git a/flyteidl/gen/pb-js/flyteidl.js b/flyteidl/gen/pb-js/flyteidl.js index 161d05475b..3400b971b3 100644 --- a/flyteidl/gen/pb-js/flyteidl.js +++ b/flyteidl/gen/pb-js/flyteidl.js @@ -203,7 +203,7 @@ * @interface IArtifactBindingData * @property {string|null} [partitionKey] ArtifactBindingData partitionKey * @property {boolean|null} [bindToTimePartition] ArtifactBindingData bindToTimePartition - * @property {string|null} [transform] ArtifactBindingData transform + * @property {flyteidl.core.ITimeTransform|null} [timeTransform] ArtifactBindingData timeTransform */ /** @@ -238,12 +238,12 @@ ArtifactBindingData.prototype.bindToTimePartition = false; /** - * ArtifactBindingData transform. - * @member {string} transform + * ArtifactBindingData timeTransform. + * @member {flyteidl.core.ITimeTransform|null|undefined} timeTransform * @memberof flyteidl.core.ArtifactBindingData * @instance */ - ArtifactBindingData.prototype.transform = ""; + ArtifactBindingData.prototype.timeTransform = null; // OneOf field names bound to virtual getters and setters var $oneOfFields; @@ -284,11 +284,11 @@ if (!writer) writer = $Writer.create(); if (message.partitionKey != null && message.hasOwnProperty("partitionKey")) - writer.uint32(/* id 2, wireType 2 =*/18).string(message.partitionKey); + writer.uint32(/* id 5, wireType 2 =*/42).string(message.partitionKey); if (message.bindToTimePartition != null && message.hasOwnProperty("bindToTimePartition")) - writer.uint32(/* id 3, wireType 0 =*/24).bool(message.bindToTimePartition); - if (message.transform != null && message.hasOwnProperty("transform")) - writer.uint32(/* id 4, wireType 2 =*/34).string(message.transform); + writer.uint32(/* id 6, wireType 0 =*/48).bool(message.bindToTimePartition); + if (message.timeTransform != null && message.hasOwnProperty("timeTransform")) + $root.flyteidl.core.TimeTransform.encode(message.timeTransform, writer.uint32(/* id 7, wireType 2 =*/58).fork()).ldelim(); return writer; }; @@ -310,14 +310,14 @@ while (reader.pos < end) { var tag = reader.uint32(); switch (tag >>> 3) { - case 2: + case 5: message.partitionKey = reader.string(); break; - case 3: + case 6: message.bindToTimePartition = reader.bool(); break; - case 4: - message.transform = reader.string(); + case 7: + message.timeTransform = $root.flyteidl.core.TimeTransform.decode(reader, reader.uint32()); break; default: reader.skipType(tag & 7); @@ -351,13 +351,181 @@ if (typeof message.bindToTimePartition !== "boolean") return "bindToTimePartition: boolean expected"; } + if (message.timeTransform != null && message.hasOwnProperty("timeTransform")) { + var error = $root.flyteidl.core.TimeTransform.verify(message.timeTransform); + if (error) + return "timeTransform." + error; + } + return null; + }; + + return ArtifactBindingData; + })(); + + /** + * Granularity enum. + * @name flyteidl.core.Granularity + * @enum {string} + * @property {number} UNSET=0 UNSET value + * @property {number} MINUTE=1 MINUTE value + * @property {number} HOUR=2 HOUR value + * @property {number} DAY=3 DAY value + * @property {number} MONTH=4 MONTH value + */ + core.Granularity = (function() { + var valuesById = {}, values = Object.create(valuesById); + values[valuesById[0] = "UNSET"] = 0; + values[valuesById[1] = "MINUTE"] = 1; + values[valuesById[2] = "HOUR"] = 2; + values[valuesById[3] = "DAY"] = 3; + values[valuesById[4] = "MONTH"] = 4; + return values; + })(); + + /** + * Operator enum. + * @name flyteidl.core.Operator + * @enum {string} + * @property {number} MINUS=0 MINUS value + * @property {number} PLUS=1 PLUS value + */ + core.Operator = (function() { + var valuesById = {}, values = Object.create(valuesById); + values[valuesById[0] = "MINUS"] = 0; + values[valuesById[1] = "PLUS"] = 1; + return values; + })(); + + core.TimeTransform = (function() { + + /** + * Properties of a TimeTransform. + * @memberof flyteidl.core + * @interface ITimeTransform + * @property {string|null} [transform] TimeTransform transform + * @property {flyteidl.core.Operator|null} [op] TimeTransform op + */ + + /** + * Constructs a new TimeTransform. + * @memberof flyteidl.core + * @classdesc Represents a TimeTransform. + * @implements ITimeTransform + * @constructor + * @param {flyteidl.core.ITimeTransform=} [properties] Properties to set + */ + function TimeTransform(properties) { + if (properties) + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * TimeTransform transform. + * @member {string} transform + * @memberof flyteidl.core.TimeTransform + * @instance + */ + TimeTransform.prototype.transform = ""; + + /** + * TimeTransform op. + * @member {flyteidl.core.Operator} op + * @memberof flyteidl.core.TimeTransform + * @instance + */ + TimeTransform.prototype.op = 0; + + /** + * Creates a new TimeTransform instance using the specified properties. + * @function create + * @memberof flyteidl.core.TimeTransform + * @static + * @param {flyteidl.core.ITimeTransform=} [properties] Properties to set + * @returns {flyteidl.core.TimeTransform} TimeTransform instance + */ + TimeTransform.create = function create(properties) { + return new TimeTransform(properties); + }; + + /** + * Encodes the specified TimeTransform message. Does not implicitly {@link flyteidl.core.TimeTransform.verify|verify} messages. + * @function encode + * @memberof flyteidl.core.TimeTransform + * @static + * @param {flyteidl.core.ITimeTransform} message TimeTransform message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + TimeTransform.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + if (message.transform != null && message.hasOwnProperty("transform")) + writer.uint32(/* id 1, wireType 2 =*/10).string(message.transform); + if (message.op != null && message.hasOwnProperty("op")) + writer.uint32(/* id 2, wireType 0 =*/16).int32(message.op); + return writer; + }; + + /** + * Decodes a TimeTransform message from the specified reader or buffer. + * @function decode + * @memberof flyteidl.core.TimeTransform + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {flyteidl.core.TimeTransform} TimeTransform + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + TimeTransform.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.core.TimeTransform(); + while (reader.pos < end) { + var tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.transform = reader.string(); + break; + case 2: + message.op = reader.int32(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Verifies a TimeTransform message. + * @function verify + * @memberof flyteidl.core.TimeTransform + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + TimeTransform.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; if (message.transform != null && message.hasOwnProperty("transform")) if (!$util.isString(message.transform)) return "transform: string expected"; + if (message.op != null && message.hasOwnProperty("op")) + switch (message.op) { + default: + return "op: enum value expected"; + case 0: + case 1: + break; + } return null; }; - return ArtifactBindingData; + return TimeTransform; })(); core.InputBindingData = (function() { @@ -470,6 +638,99 @@ return InputBindingData; })(); + core.RuntimeBinding = (function() { + + /** + * Properties of a RuntimeBinding. + * @memberof flyteidl.core + * @interface IRuntimeBinding + */ + + /** + * Constructs a new RuntimeBinding. + * @memberof flyteidl.core + * @classdesc Represents a RuntimeBinding. + * @implements IRuntimeBinding + * @constructor + * @param {flyteidl.core.IRuntimeBinding=} [properties] Properties to set + */ + function RuntimeBinding(properties) { + if (properties) + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * Creates a new RuntimeBinding instance using the specified properties. + * @function create + * @memberof flyteidl.core.RuntimeBinding + * @static + * @param {flyteidl.core.IRuntimeBinding=} [properties] Properties to set + * @returns {flyteidl.core.RuntimeBinding} RuntimeBinding instance + */ + RuntimeBinding.create = function create(properties) { + return new RuntimeBinding(properties); + }; + + /** + * Encodes the specified RuntimeBinding message. Does not implicitly {@link flyteidl.core.RuntimeBinding.verify|verify} messages. + * @function encode + * @memberof flyteidl.core.RuntimeBinding + * @static + * @param {flyteidl.core.IRuntimeBinding} message RuntimeBinding message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + RuntimeBinding.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + return writer; + }; + + /** + * Decodes a RuntimeBinding message from the specified reader or buffer. + * @function decode + * @memberof flyteidl.core.RuntimeBinding + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {flyteidl.core.RuntimeBinding} RuntimeBinding + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + RuntimeBinding.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.core.RuntimeBinding(); + while (reader.pos < end) { + var tag = reader.uint32(); + switch (tag >>> 3) { + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Verifies a RuntimeBinding message. + * @function verify + * @memberof flyteidl.core.RuntimeBinding + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + RuntimeBinding.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; + return null; + }; + + return RuntimeBinding; + })(); + core.LabelValue = (function() { /** @@ -480,6 +741,7 @@ * @property {google.protobuf.ITimestamp|null} [timeValue] LabelValue timeValue * @property {flyteidl.core.IArtifactBindingData|null} [triggeredBinding] LabelValue triggeredBinding * @property {flyteidl.core.IInputBindingData|null} [inputBinding] LabelValue inputBinding + * @property {flyteidl.core.IRuntimeBinding|null} [runtimeBinding] LabelValue runtimeBinding */ /** @@ -529,17 +791,25 @@ */ LabelValue.prototype.inputBinding = null; + /** + * LabelValue runtimeBinding. + * @member {flyteidl.core.IRuntimeBinding|null|undefined} runtimeBinding + * @memberof flyteidl.core.LabelValue + * @instance + */ + LabelValue.prototype.runtimeBinding = null; + // OneOf field names bound to virtual getters and setters var $oneOfFields; /** * LabelValue value. - * @member {"staticValue"|"timeValue"|"triggeredBinding"|"inputBinding"|undefined} value + * @member {"staticValue"|"timeValue"|"triggeredBinding"|"inputBinding"|"runtimeBinding"|undefined} value * @memberof flyteidl.core.LabelValue * @instance */ Object.defineProperty(LabelValue.prototype, "value", { - get: $util.oneOfGetter($oneOfFields = ["staticValue", "timeValue", "triggeredBinding", "inputBinding"]), + get: $util.oneOfGetter($oneOfFields = ["staticValue", "timeValue", "triggeredBinding", "inputBinding", "runtimeBinding"]), set: $util.oneOfSetter($oneOfFields) }); @@ -575,6 +845,8 @@ $root.flyteidl.core.ArtifactBindingData.encode(message.triggeredBinding, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim(); if (message.inputBinding != null && message.hasOwnProperty("inputBinding")) $root.flyteidl.core.InputBindingData.encode(message.inputBinding, writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim(); + if (message.runtimeBinding != null && message.hasOwnProperty("runtimeBinding")) + $root.flyteidl.core.RuntimeBinding.encode(message.runtimeBinding, writer.uint32(/* id 5, wireType 2 =*/42).fork()).ldelim(); return writer; }; @@ -608,6 +880,9 @@ case 4: message.inputBinding = $root.flyteidl.core.InputBindingData.decode(reader, reader.uint32()); break; + case 5: + message.runtimeBinding = $root.flyteidl.core.RuntimeBinding.decode(reader, reader.uint32()); + break; default: reader.skipType(tag & 7); break; @@ -663,6 +938,16 @@ return "inputBinding." + error; } } + if (message.runtimeBinding != null && message.hasOwnProperty("runtimeBinding")) { + if (properties.value === 1) + return "value: multiple values"; + properties.value = 1; + { + var error = $root.flyteidl.core.RuntimeBinding.verify(message.runtimeBinding); + if (error) + return "runtimeBinding." + error; + } + } return null; }; @@ -802,6 +1087,7 @@ * @memberof flyteidl.core * @interface ITimePartition * @property {flyteidl.core.ILabelValue|null} [value] TimePartition value + * @property {flyteidl.core.Granularity|null} [granularity] TimePartition granularity */ /** @@ -827,6 +1113,14 @@ */ TimePartition.prototype.value = null; + /** + * TimePartition granularity. + * @member {flyteidl.core.Granularity} granularity + * @memberof flyteidl.core.TimePartition + * @instance + */ + TimePartition.prototype.granularity = 0; + /** * Creates a new TimePartition instance using the specified properties. * @function create @@ -853,6 +1147,8 @@ writer = $Writer.create(); if (message.value != null && message.hasOwnProperty("value")) $root.flyteidl.core.LabelValue.encode(message.value, writer.uint32(/* id 1, wireType 2 =*/10).fork()).ldelim(); + if (message.granularity != null && message.hasOwnProperty("granularity")) + writer.uint32(/* id 2, wireType 0 =*/16).int32(message.granularity); return writer; }; @@ -877,6 +1173,9 @@ case 1: message.value = $root.flyteidl.core.LabelValue.decode(reader, reader.uint32()); break; + case 2: + message.granularity = reader.int32(); + break; default: reader.skipType(tag & 7); break; @@ -901,6 +1200,17 @@ if (error) return "value." + error; } + if (message.granularity != null && message.hasOwnProperty("granularity")) + switch (message.granularity) { + default: + return "granularity: enum value expected"; + case 0: + case 1: + case 2: + case 3: + case 4: + break; + } return null; }; diff --git a/flyteidl/gen/pb_python/flyteidl/core/artifact_id_pb2.py b/flyteidl/gen/pb_python/flyteidl/core/artifact_id_pb2.py index cffe1484c7..a0cde0f864 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/artifact_id_pb2.py +++ b/flyteidl/gen/pb_python/flyteidl/core/artifact_id_pb2.py @@ -15,7 +15,7 @@ from flyteidl.core import identifier_pb2 as flyteidl_dot_core_dot_identifier__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66lyteidl/core/artifact_id.proto\x12\rflyteidl.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x66lyteidl/core/identifier.proto\"e\n\x0b\x41rtifactKey\x12\x18\n\x07project\x18\x01 \x01(\tR\x07project\x12\x16\n\x06\x64omain\x18\x02 \x01(\tR\x06\x64omain\x12\x12\n\x04name\x18\x03 \x01(\tR\x04name\x12\x10\n\x03org\x18\x04 \x01(\tR\x03org\"\xa3\x01\n\x13\x41rtifactBindingData\x12%\n\rpartition_key\x18\x02 \x01(\tH\x00R\x0cpartitionKey\x12\x35\n\x16\x62ind_to_time_partition\x18\x03 \x01(\x08H\x00R\x13\x62indToTimePartition\x12\x1c\n\ttransform\x18\x04 \x01(\tR\ttransformB\x10\n\x0epartition_data\"$\n\x10InputBindingData\x12\x10\n\x03var\x18\x01 \x01(\tR\x03var\"\x92\x02\n\nLabelValue\x12#\n\x0cstatic_value\x18\x01 \x01(\tH\x00R\x0bstaticValue\x12;\n\ntime_value\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\ttimeValue\x12Q\n\x11triggered_binding\x18\x03 \x01(\x0b\x32\".flyteidl.core.ArtifactBindingDataH\x00R\x10triggeredBinding\x12\x46\n\rinput_binding\x18\x04 \x01(\x0b\x32\x1f.flyteidl.core.InputBindingDataH\x00R\x0cinputBindingB\x07\n\x05value\"\x9d\x01\n\nPartitions\x12:\n\x05value\x18\x01 \x03(\x0b\x32$.flyteidl.core.Partitions.ValueEntryR\x05value\x1aS\n\nValueEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.flyteidl.core.LabelValueR\x05value:\x02\x38\x01\"@\n\rTimePartition\x12/\n\x05value\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.LabelValueR\x05value\"\xe5\x01\n\nArtifactID\x12=\n\x0c\x61rtifact_key\x18\x01 \x01(\x0b\x32\x1a.flyteidl.core.ArtifactKeyR\x0b\x61rtifactKey\x12\x18\n\x07version\x18\x02 \x01(\tR\x07version\x12\x39\n\npartitions\x18\x03 \x01(\x0b\x32\x19.flyteidl.core.PartitionsR\npartitions\x12\x43\n\x0etime_partition\x18\x04 \x01(\x0b\x32\x1c.flyteidl.core.TimePartitionR\rtimePartition\"}\n\x0b\x41rtifactTag\x12=\n\x0c\x61rtifact_key\x18\x01 \x01(\x0b\x32\x1a.flyteidl.core.ArtifactKeyR\x0b\x61rtifactKey\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.flyteidl.core.LabelValueR\x05value\"\xf0\x01\n\rArtifactQuery\x12<\n\x0b\x61rtifact_id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.ArtifactIDH\x00R\nartifactId\x12?\n\x0c\x61rtifact_tag\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.ArtifactTagH\x00R\x0b\x61rtifactTag\x12\x12\n\x03uri\x18\x03 \x01(\tH\x00R\x03uri\x12>\n\x07\x62inding\x18\x04 \x01(\x0b\x32\".flyteidl.core.ArtifactBindingDataH\x00R\x07\x62indingB\x0c\n\nidentifierB\xb5\x01\n\x11\x63om.flyteidl.coreB\x0f\x41rtifactIdProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66lyteidl/core/artifact_id.proto\x12\rflyteidl.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x66lyteidl/core/identifier.proto\"e\n\x0b\x41rtifactKey\x12\x18\n\x07project\x18\x01 \x01(\tR\x07project\x12\x16\n\x06\x64omain\x18\x02 \x01(\tR\x06\x64omain\x12\x12\n\x04name\x18\x03 \x01(\tR\x04name\x12\x10\n\x03org\x18\x04 \x01(\tR\x03org\"\xd0\x01\n\x13\x41rtifactBindingData\x12%\n\rpartition_key\x18\x05 \x01(\tH\x00R\x0cpartitionKey\x12\x35\n\x16\x62ind_to_time_partition\x18\x06 \x01(\x08H\x00R\x13\x62indToTimePartition\x12\x43\n\x0etime_transform\x18\x07 \x01(\x0b\x32\x1c.flyteidl.core.TimeTransformR\rtimeTransformB\x10\n\x0epartition_dataJ\x04\x08\x01\x10\x05\"V\n\rTimeTransform\x12\x1c\n\ttransform\x18\x01 \x01(\tR\ttransform\x12\'\n\x02op\x18\x02 \x01(\x0e\x32\x17.flyteidl.core.OperatorR\x02op\"$\n\x10InputBindingData\x12\x10\n\x03var\x18\x01 \x01(\tR\x03var\"\x10\n\x0eRuntimeBinding\"\xdc\x02\n\nLabelValue\x12#\n\x0cstatic_value\x18\x01 \x01(\tH\x00R\x0bstaticValue\x12;\n\ntime_value\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\ttimeValue\x12Q\n\x11triggered_binding\x18\x03 \x01(\x0b\x32\".flyteidl.core.ArtifactBindingDataH\x00R\x10triggeredBinding\x12\x46\n\rinput_binding\x18\x04 \x01(\x0b\x32\x1f.flyteidl.core.InputBindingDataH\x00R\x0cinputBinding\x12H\n\x0fruntime_binding\x18\x05 \x01(\x0b\x32\x1d.flyteidl.core.RuntimeBindingH\x00R\x0eruntimeBindingB\x07\n\x05value\"\x9d\x01\n\nPartitions\x12:\n\x05value\x18\x01 \x03(\x0b\x32$.flyteidl.core.Partitions.ValueEntryR\x05value\x1aS\n\nValueEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.flyteidl.core.LabelValueR\x05value:\x02\x38\x01\"~\n\rTimePartition\x12/\n\x05value\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.LabelValueR\x05value\x12<\n\x0bgranularity\x18\x02 \x01(\x0e\x32\x1a.flyteidl.core.GranularityR\x0bgranularity\"\xe5\x01\n\nArtifactID\x12=\n\x0c\x61rtifact_key\x18\x01 \x01(\x0b\x32\x1a.flyteidl.core.ArtifactKeyR\x0b\x61rtifactKey\x12\x18\n\x07version\x18\x02 \x01(\tR\x07version\x12\x39\n\npartitions\x18\x03 \x01(\x0b\x32\x19.flyteidl.core.PartitionsR\npartitions\x12\x43\n\x0etime_partition\x18\x04 \x01(\x0b\x32\x1c.flyteidl.core.TimePartitionR\rtimePartition\"}\n\x0b\x41rtifactTag\x12=\n\x0c\x61rtifact_key\x18\x01 \x01(\x0b\x32\x1a.flyteidl.core.ArtifactKeyR\x0b\x61rtifactKey\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.flyteidl.core.LabelValueR\x05value\"\xf0\x01\n\rArtifactQuery\x12<\n\x0b\x61rtifact_id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.ArtifactIDH\x00R\nartifactId\x12?\n\x0c\x61rtifact_tag\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.ArtifactTagH\x00R\x0b\x61rtifactTag\x12\x12\n\x03uri\x18\x03 \x01(\tH\x00R\x03uri\x12>\n\x07\x62inding\x18\x04 \x01(\x0b\x32\".flyteidl.core.ArtifactBindingDataH\x00R\x07\x62indingB\x0c\n\nidentifier*B\n\x0bGranularity\x12\t\n\x05UNSET\x10\x00\x12\n\n\x06MINUTE\x10\x01\x12\x08\n\x04HOUR\x10\x02\x12\x07\n\x03\x44\x41Y\x10\x03\x12\t\n\x05MONTH\x10\x04*\x1f\n\x08Operator\x12\t\n\x05MINUS\x10\x00\x12\x08\n\x04PLUS\x10\x01\x42\xb5\x01\n\x11\x63om.flyteidl.coreB\x0f\x41rtifactIdProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -26,24 +26,32 @@ DESCRIPTOR._serialized_options = b'\n\021com.flyteidl.coreB\017ArtifactIdProtoP\001Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\242\002\003FCX\252\002\rFlyteidl.Core\312\002\rFlyteidl\\Core\342\002\031Flyteidl\\Core\\GPBMetadata\352\002\016Flyteidl::Core' _PARTITIONS_VALUEENTRY._options = None _PARTITIONS_VALUEENTRY._serialized_options = b'8\001' + _globals['_GRANULARITY']._serialized_start=1814 + _globals['_GRANULARITY']._serialized_end=1880 + _globals['_OPERATOR']._serialized_start=1882 + _globals['_OPERATOR']._serialized_end=1913 _globals['_ARTIFACTKEY']._serialized_start=115 _globals['_ARTIFACTKEY']._serialized_end=216 _globals['_ARTIFACTBINDINGDATA']._serialized_start=219 - _globals['_ARTIFACTBINDINGDATA']._serialized_end=382 - _globals['_INPUTBINDINGDATA']._serialized_start=384 - _globals['_INPUTBINDINGDATA']._serialized_end=420 - _globals['_LABELVALUE']._serialized_start=423 - _globals['_LABELVALUE']._serialized_end=697 - _globals['_PARTITIONS']._serialized_start=700 - _globals['_PARTITIONS']._serialized_end=857 - _globals['_PARTITIONS_VALUEENTRY']._serialized_start=774 - _globals['_PARTITIONS_VALUEENTRY']._serialized_end=857 - _globals['_TIMEPARTITION']._serialized_start=859 - _globals['_TIMEPARTITION']._serialized_end=923 - _globals['_ARTIFACTID']._serialized_start=926 - _globals['_ARTIFACTID']._serialized_end=1155 - _globals['_ARTIFACTTAG']._serialized_start=1157 - _globals['_ARTIFACTTAG']._serialized_end=1282 - _globals['_ARTIFACTQUERY']._serialized_start=1285 - _globals['_ARTIFACTQUERY']._serialized_end=1525 + _globals['_ARTIFACTBINDINGDATA']._serialized_end=427 + _globals['_TIMETRANSFORM']._serialized_start=429 + _globals['_TIMETRANSFORM']._serialized_end=515 + _globals['_INPUTBINDINGDATA']._serialized_start=517 + _globals['_INPUTBINDINGDATA']._serialized_end=553 + _globals['_RUNTIMEBINDING']._serialized_start=555 + _globals['_RUNTIMEBINDING']._serialized_end=571 + _globals['_LABELVALUE']._serialized_start=574 + _globals['_LABELVALUE']._serialized_end=922 + _globals['_PARTITIONS']._serialized_start=925 + _globals['_PARTITIONS']._serialized_end=1082 + _globals['_PARTITIONS_VALUEENTRY']._serialized_start=999 + _globals['_PARTITIONS_VALUEENTRY']._serialized_end=1082 + _globals['_TIMEPARTITION']._serialized_start=1084 + _globals['_TIMEPARTITION']._serialized_end=1210 + _globals['_ARTIFACTID']._serialized_start=1213 + _globals['_ARTIFACTID']._serialized_end=1442 + _globals['_ARTIFACTTAG']._serialized_start=1444 + _globals['_ARTIFACTTAG']._serialized_end=1569 + _globals['_ARTIFACTQUERY']._serialized_start=1572 + _globals['_ARTIFACTQUERY']._serialized_end=1812 # @@protoc_insertion_point(module_scope) diff --git a/flyteidl/gen/pb_python/flyteidl/core/artifact_id_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/core/artifact_id_pb2.pyi index e1e688c8e4..c8e5a61a4a 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/artifact_id_pb2.pyi +++ b/flyteidl/gen/pb_python/flyteidl/core/artifact_id_pb2.pyi @@ -1,12 +1,33 @@ from google.protobuf import timestamp_pb2 as _timestamp_pb2 from flyteidl.core import identifier_pb2 as _identifier_pb2 from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor +class Granularity(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] + UNSET: _ClassVar[Granularity] + MINUTE: _ClassVar[Granularity] + HOUR: _ClassVar[Granularity] + DAY: _ClassVar[Granularity] + MONTH: _ClassVar[Granularity] + +class Operator(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] + MINUS: _ClassVar[Operator] + PLUS: _ClassVar[Operator] +UNSET: Granularity +MINUTE: Granularity +HOUR: Granularity +DAY: Granularity +MONTH: Granularity +MINUS: Operator +PLUS: Operator + class ArtifactKey(_message.Message): __slots__ = ["project", "domain", "name", "org"] PROJECT_FIELD_NUMBER: _ClassVar[int] @@ -20,14 +41,22 @@ class ArtifactKey(_message.Message): def __init__(self, project: _Optional[str] = ..., domain: _Optional[str] = ..., name: _Optional[str] = ..., org: _Optional[str] = ...) -> None: ... class ArtifactBindingData(_message.Message): - __slots__ = ["partition_key", "bind_to_time_partition", "transform"] + __slots__ = ["partition_key", "bind_to_time_partition", "time_transform"] PARTITION_KEY_FIELD_NUMBER: _ClassVar[int] BIND_TO_TIME_PARTITION_FIELD_NUMBER: _ClassVar[int] - TRANSFORM_FIELD_NUMBER: _ClassVar[int] + TIME_TRANSFORM_FIELD_NUMBER: _ClassVar[int] partition_key: str bind_to_time_partition: bool + time_transform: TimeTransform + def __init__(self, partition_key: _Optional[str] = ..., bind_to_time_partition: bool = ..., time_transform: _Optional[_Union[TimeTransform, _Mapping]] = ...) -> None: ... + +class TimeTransform(_message.Message): + __slots__ = ["transform", "op"] + TRANSFORM_FIELD_NUMBER: _ClassVar[int] + OP_FIELD_NUMBER: _ClassVar[int] transform: str - def __init__(self, partition_key: _Optional[str] = ..., bind_to_time_partition: bool = ..., transform: _Optional[str] = ...) -> None: ... + op: Operator + def __init__(self, transform: _Optional[str] = ..., op: _Optional[_Union[Operator, str]] = ...) -> None: ... class InputBindingData(_message.Message): __slots__ = ["var"] @@ -35,17 +64,23 @@ class InputBindingData(_message.Message): var: str def __init__(self, var: _Optional[str] = ...) -> None: ... +class RuntimeBinding(_message.Message): + __slots__ = [] + def __init__(self) -> None: ... + class LabelValue(_message.Message): - __slots__ = ["static_value", "time_value", "triggered_binding", "input_binding"] + __slots__ = ["static_value", "time_value", "triggered_binding", "input_binding", "runtime_binding"] STATIC_VALUE_FIELD_NUMBER: _ClassVar[int] TIME_VALUE_FIELD_NUMBER: _ClassVar[int] TRIGGERED_BINDING_FIELD_NUMBER: _ClassVar[int] INPUT_BINDING_FIELD_NUMBER: _ClassVar[int] + RUNTIME_BINDING_FIELD_NUMBER: _ClassVar[int] static_value: str time_value: _timestamp_pb2.Timestamp triggered_binding: ArtifactBindingData input_binding: InputBindingData - def __init__(self, static_value: _Optional[str] = ..., time_value: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., triggered_binding: _Optional[_Union[ArtifactBindingData, _Mapping]] = ..., input_binding: _Optional[_Union[InputBindingData, _Mapping]] = ...) -> None: ... + runtime_binding: RuntimeBinding + def __init__(self, static_value: _Optional[str] = ..., time_value: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., triggered_binding: _Optional[_Union[ArtifactBindingData, _Mapping]] = ..., input_binding: _Optional[_Union[InputBindingData, _Mapping]] = ..., runtime_binding: _Optional[_Union[RuntimeBinding, _Mapping]] = ...) -> None: ... class Partitions(_message.Message): __slots__ = ["value"] @@ -61,10 +96,12 @@ class Partitions(_message.Message): def __init__(self, value: _Optional[_Mapping[str, LabelValue]] = ...) -> None: ... class TimePartition(_message.Message): - __slots__ = ["value"] + __slots__ = ["value", "granularity"] VALUE_FIELD_NUMBER: _ClassVar[int] + GRANULARITY_FIELD_NUMBER: _ClassVar[int] value: LabelValue - def __init__(self, value: _Optional[_Union[LabelValue, _Mapping]] = ...) -> None: ... + granularity: Granularity + def __init__(self, value: _Optional[_Union[LabelValue, _Mapping]] = ..., granularity: _Optional[_Union[Granularity, str]] = ...) -> None: ... class ArtifactID(_message.Message): __slots__ = ["artifact_key", "version", "partitions", "time_partition"] diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index 07973186b4..a36add78f2 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -731,10 +731,10 @@ pub struct ArtifactKey { #[derive(Clone, PartialEq, ::prost::Message)] pub struct ArtifactBindingData { /// This is only relevant in the time partition case - #[prost(string, tag="4")] - pub transform: ::prost::alloc::string::String, + #[prost(message, optional, tag="7")] + pub time_transform: ::core::option::Option, /// These two fields are only relevant in the partition value case - #[prost(oneof="artifact_binding_data::PartitionData", tags="2, 3")] + #[prost(oneof="artifact_binding_data::PartitionData", tags="5, 6")] pub partition_data: ::core::option::Option, } /// Nested message and enum types in `ArtifactBindingData`. @@ -743,22 +743,34 @@ pub mod artifact_binding_data { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum PartitionData { - #[prost(string, tag="2")] + #[prost(string, tag="5")] PartitionKey(::prost::alloc::string::String), - #[prost(bool, tag="3")] + #[prost(bool, tag="6")] BindToTimePartition(bool), } } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct TimeTransform { + #[prost(string, tag="1")] + pub transform: ::prost::alloc::string::String, + #[prost(enumeration="Operator", tag="2")] + pub op: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct InputBindingData { #[prost(string, tag="1")] pub var: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct RuntimeBinding { +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct LabelValue { - #[prost(oneof="label_value::Value", tags="1, 2, 3, 4")] + #[prost(oneof="label_value::Value", tags="1, 2, 3, 4, 5")] pub value: ::core::option::Option, } /// Nested message and enum types in `LabelValue`. @@ -776,6 +788,8 @@ pub mod label_value { TriggeredBinding(super::ArtifactBindingData), #[prost(message, tag="4")] InputBinding(super::InputBindingData), + #[prost(message, tag="5")] + RuntimeBinding(super::RuntimeBinding), } } #[allow(clippy::derive_partial_eq_without_eq)] @@ -789,6 +803,8 @@ pub struct Partitions { pub struct TimePartition { #[prost(message, optional, tag="1")] pub value: ::core::option::Option, + #[prost(enumeration="Granularity", tag="2")] + pub granularity: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -841,6 +857,68 @@ pub mod artifact_query { Binding(super::ArtifactBindingData), } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Granularity { + Unset = 0, + Minute = 1, + Hour = 2, + /// default + Day = 3, + Month = 4, +} +impl Granularity { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Granularity::Unset => "UNSET", + Granularity::Minute => "MINUTE", + Granularity::Hour => "HOUR", + Granularity::Day => "DAY", + Granularity::Month => "MONTH", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNSET" => Some(Self::Unset), + "MINUTE" => Some(Self::Minute), + "HOUR" => Some(Self::Hour), + "DAY" => Some(Self::Day), + "MONTH" => Some(Self::Month), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Operator { + Minus = 0, + Plus = 1, +} +impl Operator { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Operator::Minus => "MINUS", + Operator::Plus => "PLUS", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "MINUS" => Some(Self::Minus), + "PLUS" => Some(Self::Plus), + _ => None, + } + } +} /// Defines a strongly typed variable. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/flyteidl/protos/flyteidl/core/artifact_id.proto b/flyteidl/protos/flyteidl/core/artifact_id.proto index fe887e336a..022bc20cff 100644 --- a/flyteidl/protos/flyteidl/core/artifact_id.proto +++ b/flyteidl/protos/flyteidl/core/artifact_id.proto @@ -18,20 +18,41 @@ message ArtifactKey { // Only valid for triggers message ArtifactBindingData { + reserved 1 to 4; // These two fields are only relevant in the partition value case oneof partition_data { - string partition_key = 2; - bool bind_to_time_partition = 3; + string partition_key = 5; + bool bind_to_time_partition = 6; } // This is only relevant in the time partition case - string transform = 4; + TimeTransform time_transform = 7; +} + +enum Granularity { + UNSET = 0; + MINUTE = 1; + HOUR = 2; + DAY = 3; // default + MONTH = 4; +} + +enum Operator { + MINUS = 0; + PLUS = 1; +} + +message TimeTransform { + string transform = 1; + Operator op = 2; } message InputBindingData { string var = 1; } +message RuntimeBinding {} + message LabelValue { oneof value { // The string static value is for use in the Partitions object @@ -41,6 +62,7 @@ message LabelValue { google.protobuf.Timestamp time_value = 2; ArtifactBindingData triggered_binding = 3; InputBindingData input_binding = 4; + RuntimeBinding runtime_binding = 5; } } @@ -50,6 +72,7 @@ message Partitions { message TimePartition { LabelValue value = 1; + Granularity granularity = 2; } message ArtifactID { diff --git a/flyteplugins/go/tasks/logs/logging_utils.go b/flyteplugins/go/tasks/logs/logging_utils.go index 04d23f4ec8..45d12624de 100644 --- a/flyteplugins/go/tasks/logs/logging_utils.go +++ b/flyteplugins/go/tasks/logs/logging_utils.go @@ -52,6 +52,7 @@ func GetLogsForContainerInPod(ctx context.Context, logPlugin tasklog.Plugin, tas TaskExecutionID: taskExecID, ExtraTemplateVars: extraLogTemplateVars, TaskTemplate: taskTemplate, + HostName: pod.Spec.Hostname, }, ) diff --git a/flyteplugins/go/tasks/logs/logging_utils_test.go b/flyteplugins/go/tasks/logs/logging_utils_test.go index 946d069d50..e8c716e341 100644 --- a/flyteplugins/go/tasks/logs/logging_utils_test.go +++ b/flyteplugins/go/tasks/logs/logging_utils_test.go @@ -210,6 +210,40 @@ func TestGetLogsForContainerInPod_All(t *testing.T) { assert.Len(t, logs, 2) } +func TestGetLogsForContainerInPod_HostName(t *testing.T) { + logPlugin, err := InitializeLogPlugins(&LogConfig{ + IsKubernetesEnabled: true, + KubernetesURL: "k8s.com", + IsCloudwatchEnabled: true, + CloudwatchRegion: "us-east-1", + CloudwatchLogGroup: "/kubernetes/flyte-production", + }) + assert.NoError(t, err) + + pod := &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "ContainerName", + }, + }, + Hostname: "my-hostname", + }, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + ContainerID: "ContainerID", + }, + }, + }, + } + pod.Name = podName + + logs, err := GetLogsForContainerInPod(context.TODO(), logPlugin, dummyTaskExecID(), pod, 0, " Suffix", nil, nil) + assert.Nil(t, err) + assert.Len(t, logs, 2) +} + func TestGetLogsForContainerInPod_Stackdriver(t *testing.T) { logPlugin, err := InitializeLogPlugins(&LogConfig{ IsStackDriverEnabled: true, @@ -268,7 +302,7 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Stackdriver Logs my-Suffix", }, - }) + }, "") }) t.Run("StackDriver", func(t *testing.T) { @@ -281,11 +315,11 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Stackdriver Logs my-Suffix", }, - }) + }, "") }) } -func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog) { +func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string) { logPlugin, err := InitializeLogPlugins(config) assert.NoError(tb, err) @@ -300,6 +334,7 @@ func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.Ta Name: "ContainerName", }, }, + Hostname: hostname, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -347,7 +382,27 @@ func TestGetLogsForContainerInPod_Templates(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Internal my-Suffix", }, - }) + }, "") +} + +func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) { + assertTestSucceeded(t, &LogConfig{ + Templates: []tasklog.TemplateLogPlugin{ + { + DisplayName: "StackDriver", + TemplateURIs: []string{ + "{{ .hostname }}/{{ .namespace }}/{{ .podName }}/{{ .containerName }}/{{ .containerId }}", + }, + MessageFormat: core.TaskLog_JSON, + }, + }, + }, nil, []*core.TaskLog{ + { + Uri: "my-hostname/my-namespace/my-pod/ContainerName/ContainerID", + MessageFormat: core.TaskLog_JSON, + Name: "StackDriver my-Suffix", + }, + }, "my-hostname") } func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) { @@ -507,7 +562,7 @@ func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs) + assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "") }) } } diff --git a/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go b/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go index bf6ef791ac..99a3ccdf7a 100644 --- a/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go +++ b/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go @@ -2,6 +2,7 @@ package webapi import ( "context" + pluginErrors "github.com/flyteorg/flyte/flyteplugins/go/tasks/errors" "time" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core" @@ -15,7 +16,7 @@ func launch(ctx context.Context, p webapi.AsyncPlugin, tCtx core.TaskExecutionCo rMeta, r, err := p.Create(ctx, tCtx) if err != nil { logger.Errorf(ctx, "Failed to create resource. Error: %v", err) - return nil, core.PhaseInfo{}, err + return state, core.PhaseInfoRetryableFailure(pluginErrors.TaskFailedWithError, err.Error(), nil), nil } // If the plugin also returned the created resource, check to see if it's already in a terminal state. diff --git a/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher_test.go b/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher_test.go index 85ba42d0c6..7836cc591d 100644 --- a/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher_test.go @@ -79,8 +79,9 @@ func Test_launch(t *testing.T) { plgn := newPluginWithProperties(webapi.PluginConfig{}) plgn.OnCreate(ctx, tCtx).Return("", nil, fmt.Errorf("error creating")) - _, _, err := launch(ctx, plgn, tCtx, c, &s) - assert.Error(t, err) + _, phase, err := launch(ctx, plgn, tCtx, c, &s) + assert.Nil(t, err) + assert.Equal(t, core.PhaseRetryableFailure, phase.Phase()) }) t.Run("Failed to cache", func(t *testing.T) { diff --git a/flyteplugins/go/tasks/plugins/webapi/agent/integration_test.go b/flyteplugins/go/tasks/plugins/webapi/agent/integration_test.go index a4ddc5e303..689527ee3b 100644 --- a/flyteplugins/go/tasks/plugins/webapi/agent/integration_test.go +++ b/flyteplugins/go/tasks/plugins/webapi/agent/integration_test.go @@ -137,8 +137,8 @@ func TestEndToEnd(t *testing.T) { tCtx.OnInputReader().Return(inputReader) trns, err := plugin.Handle(context.Background(), tCtx) - assert.Error(t, err) - assert.Equal(t, trns.Info().Phase(), core.PhaseUndefined) + assert.Nil(t, err) + assert.Equal(t, trns.Info().Phase(), core.PhaseRetryableFailure) err = plugin.Abort(context.Background(), tCtx) assert.Nil(t, err) }) @@ -155,8 +155,8 @@ func TestEndToEnd(t *testing.T) { assert.NoError(t, err) trns, err := plugin.Handle(context.Background(), tCtx) - assert.Error(t, err) - assert.Equal(t, trns.Info().Phase(), core.PhaseUndefined) + assert.Nil(t, err) + assert.Equal(t, trns.Info().Phase(), core.PhaseRetryableFailure) }) t.Run("failed to read inputs", func(t *testing.T) { @@ -176,8 +176,8 @@ func TestEndToEnd(t *testing.T) { assert.NoError(t, err) trns, err := plugin.Handle(context.Background(), tCtx) - assert.Error(t, err) - assert.Equal(t, trns.Info().Phase(), core.PhaseUndefined) + assert.Nil(t, err) + assert.Equal(t, trns.Info().Phase(), core.PhaseRetryableFailure) }) } diff --git a/flyteplugins/go/tasks/plugins/webapi/databricks/integration_test.go b/flyteplugins/go/tasks/plugins/webapi/databricks/integration_test.go index 651892f672..d18f4ba79e 100644 --- a/flyteplugins/go/tasks/plugins/webapi/databricks/integration_test.go +++ b/flyteplugins/go/tasks/plugins/webapi/databricks/integration_test.go @@ -109,8 +109,8 @@ func newFakeDatabricksServer() *httptest.Server { runID := "065168461" jobID := "019e7546" return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if request.URL.Path == fmt.Sprintf("%v/submit", databricksAPI) && request.Method == post { - writer.WriteHeader(202) + if request.URL.Path == fmt.Sprintf("%v/submit", databricksAPI) && request.Method == http.MethodPost { + writer.WriteHeader(http.StatusOK) bytes := []byte(fmt.Sprintf(`{ "run_id": "%v" }`, runID)) @@ -118,8 +118,8 @@ func newFakeDatabricksServer() *httptest.Server { return } - if request.URL.Path == fmt.Sprintf("%v/get", databricksAPI) && request.Method == get { - writer.WriteHeader(200) + if request.URL.Path == fmt.Sprintf("%v/get", databricksAPI) && request.Method == http.MethodGet { + writer.WriteHeader(http.StatusOK) bytes := []byte(fmt.Sprintf(`{ "job_id": "%v", "state": {"state_message": "execution in progress.", "life_cycle_state": "TERMINATED", "result_state": "SUCCESS"} @@ -128,12 +128,12 @@ func newFakeDatabricksServer() *httptest.Server { return } - if request.URL.Path == fmt.Sprintf("%v/cancel", databricksAPI) && request.Method == post { - writer.WriteHeader(200) + if request.URL.Path == fmt.Sprintf("%v/cancel", databricksAPI) && request.Method == http.MethodPost { + writer.WriteHeader(http.StatusOK) return } - writer.WriteHeader(500) + writer.WriteHeader(http.StatusInternalServerError) })) } diff --git a/flyteplugins/go/tasks/plugins/webapi/databricks/plugin.go b/flyteplugins/go/tasks/plugins/webapi/databricks/plugin.go index 3e9b37ea93..5ebe1d0075 100644 --- a/flyteplugins/go/tasks/plugins/webapi/databricks/plugin.go +++ b/flyteplugins/go/tasks/plugins/webapi/databricks/plugin.go @@ -6,6 +6,7 @@ import ( "encoding/gob" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "time" @@ -26,20 +27,20 @@ import ( ) const ( - ErrSystem errors.ErrorCode = "System" - post string = "POST" - get string = "GET" - databricksAPI string = "/api/2.1/jobs/runs" - newCluster string = "new_cluster" - dockerImage string = "docker_image" - sparkConfig string = "spark_conf" - sparkPythonTask string = "spark_python_task" - pythonFile string = "python_file" - parameters string = "parameters" - url string = "url" + create string = "create" + get string = "get" + cancel string = "cancel" + databricksAPI string = "/api/2.1/jobs/runs" + newCluster string = "new_cluster" + dockerImage string = "docker_image" + sparkConfig string = "spark_conf" + sparkPythonTask string = "spark_python_task" + pythonFile string = "python_file" + parameters string = "parameters" + url string = "url" ) -// for mocking/testing purposes, and we'll override this method +// HTTPClient for mocking/testing purposes, and we'll override this method type HTTPClient interface { Do(req *http.Request) (*http.Response, error) } @@ -127,60 +128,41 @@ func (p Plugin) Create(ctx context.Context, taskCtx webapi.TaskExecutionContextR } databricksJob[sparkPythonTask] = map[string]interface{}{pythonFile: p.cfg.EntrypointFile, parameters: modifiedArgs} - req, err := buildRequest(post, databricksJob, p.cfg.databricksEndpoint, - p.cfg.DatabricksInstance, token, "", false) + data, err := p.sendRequest(create, databricksJob, token, "") if err != nil { return nil, nil, err } - resp, err := p.client.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - data, err := buildResponse(resp) - if err != nil { - return nil, nil, err - } - if data["run_id"] == "" { - return nil, nil, pluginErrors.Wrapf(pluginErrors.RuntimeFailure, err, - "Unable to fetch statementHandle from http response") + if _, ok := data["run_id"]; !ok { + return nil, nil, errors.Errorf("CorruptedPluginState", "can't get the run_id") } runID := fmt.Sprintf("%.0f", data["run_id"]) - return ResourceMetaWrapper{runID, p.cfg.DatabricksInstance, token}, - ResourceWrapper{StatusCode: resp.StatusCode}, nil + return ResourceMetaWrapper{runID, p.cfg.DatabricksInstance, token}, nil, nil } func (p Plugin) Get(ctx context.Context, taskCtx webapi.GetContext) (latest webapi.Resource, err error) { exec := taskCtx.ResourceMeta().(ResourceMetaWrapper) - req, err := buildRequest(get, nil, p.cfg.databricksEndpoint, - p.cfg.DatabricksInstance, exec.Token, exec.RunID, false) - if err != nil { - logger.Errorf(ctx, "Failed to build databricks job request [%v]", err) - return nil, err - } - resp, err := p.client.Do(req) - logger.Debugf(ctx, "Get databricks job response", "resp", resp) + res, err := p.sendRequest(get, nil, exec.Token, exec.RunID) if err != nil { - logger.Errorf(ctx, "Failed to get databricks job status [%v]", resp) return nil, err } - defer resp.Body.Close() - data, err := buildResponse(resp) - if err != nil { - return nil, err - } - if data == nil || data["state"] == nil { + if _, ok := res["state"]; !ok { return nil, errors.Errorf("CorruptedPluginState", "can't get the job state") } - jobState := data["state"].(map[string]interface{}) + jobState := res["state"].(map[string]interface{}) + jobID := fmt.Sprintf("%.0f", res["job_id"]) message := fmt.Sprintf("%s", jobState["state_message"]) - jobID := fmt.Sprintf("%.0f", data["job_id"]) lifeCycleState := fmt.Sprintf("%s", jobState["life_cycle_state"]) - resultState := fmt.Sprintf("%s", jobState["result_state"]) + var resultState string + if _, ok := jobState["result_state"]; !ok { + // The result_state is not available until the job is finished. + // https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runresultstate + resultState = "" + } else { + resultState = fmt.Sprintf("%s", jobState["result_state"]) + } return ResourceWrapper{ - StatusCode: resp.StatusCode, JobID: jobID, LifeCycleState: lifeCycleState, ResultState: resultState, @@ -193,63 +175,111 @@ func (p Plugin) Delete(ctx context.Context, taskCtx webapi.DeleteContext) error return nil } exec := taskCtx.ResourceMeta().(ResourceMetaWrapper) - req, err := buildRequest(post, nil, p.cfg.databricksEndpoint, - p.cfg.DatabricksInstance, exec.Token, exec.RunID, true) + _, err := p.sendRequest(cancel, nil, exec.Token, exec.RunID) if err != nil { return err } + logger.Info(ctx, "Deleted Databricks job execution.") + + return nil +} + +func (p Plugin) sendRequest(method string, databricksJob map[string]interface{}, token string, runID string) (map[string]interface{}, error) { + var databricksURL string + // for mocking/testing purposes + if p.cfg.databricksEndpoint == "" { + databricksURL = fmt.Sprintf("https://%v%v", p.cfg.DatabricksInstance, databricksAPI) + } else { + databricksURL = fmt.Sprintf("%v%v", p.cfg.databricksEndpoint, databricksAPI) + } + + // build the request spec + var body io.Reader + var httpMethod string + switch method { + case create: + databricksURL += "/submit" + mJSON, err := json.Marshal(databricksJob) + if err != nil { + return nil, fmt.Errorf("failed to marshal the job request: %v", err) + } + body = bytes.NewBuffer(mJSON) + httpMethod = http.MethodPost + case get: + databricksURL += "/get?run_id=" + runID + httpMethod = http.MethodGet + case cancel: + databricksURL += "/cancel" + body = bytes.NewBuffer([]byte(fmt.Sprintf("{ \"run_id\": %v }", runID))) + httpMethod = http.MethodPost + } + + req, err := http.NewRequest(httpMethod, databricksURL, body) + if err != nil { + return nil, err + } + req.Header.Add("Authorization", "Bearer "+token) + req.Header.Add("Content-Type", "application/json") + + // Send the request resp, err := p.client.Do(req) if err != nil { - return err + return nil, fmt.Errorf("failed to send request to Databricks platform with err: [%v]", err) } defer resp.Body.Close() - logger.Infof(ctx, "Deleted query execution [%v]", resp) - return nil + // Parse the response body + responseBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var data map[string]interface{} + err = json.Unmarshal(responseBody, &data) + if err != nil { + return nil, fmt.Errorf("failed to parse response with err: [%v]", err) + } + if resp.StatusCode != http.StatusOK { + message := "" + if v, ok := data["message"]; ok { + message = v.(string) + } + return nil, fmt.Errorf("failed to %v Databricks job with error [%v]", method, message) + } + return data, nil } func (p Plugin) Status(ctx context.Context, taskCtx webapi.StatusContext) (phase core.PhaseInfo, err error) { exec := taskCtx.ResourceMeta().(ResourceMetaWrapper) resource := taskCtx.Resource().(ResourceWrapper) message := resource.Message - statusCode := resource.StatusCode jobID := resource.JobID lifeCycleState := resource.LifeCycleState resultState := resource.ResultState - if statusCode == 0 { - return core.PhaseInfoUndefined, errors.Errorf(ErrSystem, "No Status field set.") - } - taskInfo := createTaskInfo(exec.RunID, jobID, exec.DatabricksInstance) - switch statusCode { - // Job response format. https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunsSubmit - case http.StatusAccepted: - return core.PhaseInfoRunning(pluginsCore.DefaultPhaseVersion, taskInfo), nil - case http.StatusOK: - if lifeCycleState == "TERMINATED" || lifeCycleState == "TERMINATING" || lifeCycleState == "INTERNAL_ERROR" { - if resultState == "SUCCESS" { - if err := writeOutput(ctx, taskCtx); err != nil { - pluginsCore.PhaseInfoFailure(string(rune(statusCode)), "failed to write output", taskInfo) - } - return pluginsCore.PhaseInfoSuccess(taskInfo), nil + switch lifeCycleState { + // Job response format. https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runlifecyclestate + case "PENDING": + return core.PhaseInfoInitializing(time.Now(), core.DefaultPhaseVersion, message, taskInfo), nil + case "RUNNING": + fallthrough + case "TERMINATING": + return core.PhaseInfoRunning(core.DefaultPhaseVersion, taskInfo), nil + case "TERMINATED": + if resultState == "SUCCESS" { + // Result state details. https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runresultstate + if err := writeOutput(ctx, taskCtx); err != nil { + return core.PhaseInfoFailure(string(rune(http.StatusInternalServerError)), "failed to write output", taskInfo), nil } - return pluginsCore.PhaseInfoFailure(string(rune(statusCode)), message, taskInfo), nil - } - - if lifeCycleState == "PENDING" { - return core.PhaseInfoInitializing(time.Now(), core.DefaultPhaseVersion, message, taskInfo), nil + return core.PhaseInfoSuccess(taskInfo), nil } - - return core.PhaseInfoRunning(pluginsCore.DefaultPhaseVersion, taskInfo), nil - case http.StatusBadRequest: - fallthrough - case http.StatusInternalServerError: - fallthrough - case http.StatusUnauthorized: - return pluginsCore.PhaseInfoFailure(string(rune(statusCode)), message, taskInfo), nil + return core.PhaseInfoFailure(pluginErrors.TaskFailedWithError, message, taskInfo), nil + case "SKIPPED": + return core.PhaseInfoFailure(string(rune(http.StatusConflict)), message, taskInfo), nil + case "INTERNAL_ERROR": + return core.PhaseInfoFailure(string(rune(http.StatusInternalServerError)), message, taskInfo), nil } - return core.PhaseInfoUndefined, pluginErrors.Errorf(pluginsCore.SystemErrorCode, "unknown execution phase [%v].", statusCode) + return core.PhaseInfoUndefined, pluginErrors.Errorf(pluginsCore.SystemErrorCode, "unknown execution phase [%v].", lifeCycleState) } func writeOutput(ctx context.Context, taskCtx webapi.StatusContext) error { @@ -266,66 +296,6 @@ func writeOutput(ctx context.Context, taskCtx webapi.StatusContext) error { return taskCtx.OutputWriter().Put(ctx, outputReader) } -func buildRequest( - method string, - databricksJob map[string]interface{}, - databricksEndpoint string, - databricksInstance string, - token string, - runID string, - isCancel bool, -) (*http.Request, error) { - var databricksURL string - // for mocking/testing purposes - if databricksEndpoint == "" { - databricksURL = fmt.Sprintf("https://%v%v", databricksInstance, databricksAPI) - } else { - databricksURL = fmt.Sprintf("%v%v", databricksEndpoint, databricksAPI) - } - - var data []byte - var req *http.Request - var err error - if isCancel { - databricksURL += "/cancel" - data = []byte(fmt.Sprintf("{ \"run_id\": %v }", runID)) - } else if method == post { - databricksURL += "/submit" - mJSON, err := json.Marshal(databricksJob) - if err != nil { - return nil, err - } - data = []byte(string(mJSON)) - } else { - databricksURL += "/get?run_id=" + runID - } - - if data == nil { - req, err = http.NewRequest(method, databricksURL, nil) - } else { - req, err = http.NewRequest(method, databricksURL, bytes.NewBuffer(data)) - } - if err != nil { - return nil, err - } - req.Header.Add("Authorization", "Bearer "+token) - req.Header.Add("Content-Type", "application/json") - return req, nil -} - -func buildResponse(response *http.Response) (map[string]interface{}, error) { - responseBody, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err - } - var data map[string]interface{} - err = json.Unmarshal(responseBody, &data) - if err != nil { - return nil, err - } - return data, nil -} - func createTaskInfo(runID, jobID, databricksInstance string) *core.TaskInfo { timeNow := time.Now() diff --git a/flyteplugins/go/tasks/plugins/webapi/databricks/plugin_test.go b/flyteplugins/go/tasks/plugins/webapi/databricks/plugin_test.go index fda3ab61b0..228914af93 100644 --- a/flyteplugins/go/tasks/plugins/webapi/databricks/plugin_test.go +++ b/flyteplugins/go/tasks/plugins/webapi/databricks/plugin_test.go @@ -2,10 +2,8 @@ package databricks import ( "context" - "encoding/json" - "io/ioutil" + "errors" "net/http" - "strings" "testing" "time" @@ -13,21 +11,22 @@ import ( pluginsCore "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core" pluginCoreMocks "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core/mocks" + "github.com/flyteorg/flyte/flytestdlib/ioutils" "github.com/flyteorg/flyte/flytestdlib/promutils" ) type MockClient struct { + MockDo func(req *http.Request) (*http.Response, error) +} + +func (m MockClient) Do(req *http.Request) (*http.Response, error) { + return m.MockDo(req) } var ( - MockDo func(req *http.Request) (*http.Response, error) testInstance = "test-account.cloud.databricks.com" ) -func (m *MockClient) Do(req *http.Request) (*http.Response, error) { - return MockDo(req) -} - func TestPlugin(t *testing.T) { fakeSetupContext := pluginCoreMocks.SetupContext{} fakeSetupContext.OnMetricsScope().Return(promutils.NewScope("test")) @@ -35,7 +34,9 @@ func TestPlugin(t *testing.T) { plugin := Plugin{ metricScope: fakeSetupContext.MetricsScope(), cfg: GetConfig(), - client: &MockClient{}, + client: &MockClient{func(req *http.Request) (*http.Response, error) { + return nil, nil + }}, } t.Run("get config", func(t *testing.T) { cfg := defaultConfig @@ -53,61 +54,101 @@ func TestPlugin(t *testing.T) { }) } -func TestCreateTaskInfo(t *testing.T) { - t.Run("create task info", func(t *testing.T) { - taskInfo := createTaskInfo("run-id", "job-id", testInstance) +func TestSendRequest(t *testing.T) { + fakeSetupContext := pluginCoreMocks.SetupContext{} + fakeSetupContext.OnMetricsScope().Return(promutils.NewScope("test1")) + databricksJob := map[string]interface{}{"sparkConfig": map[string]interface{}{"sparkVersion": "7.3.x-scala2.12"}} + token := "token" - assert.Equal(t, 1, len(taskInfo.Logs)) - assert.Equal(t, taskInfo.Logs[0].Uri, "https://test-account.cloud.databricks.com/#job/job-id/run/run-id") - assert.Equal(t, taskInfo.Logs[0].Name, "Databricks Console") + plugin := Plugin{ + metricScope: fakeSetupContext.MetricsScope(), + cfg: GetConfig(), + client: &MockClient{MockDo: func(req *http.Request) (*http.Response, error) { + assert.Equal(t, req.Method, http.MethodPost) + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutils.NewBytesReadCloser([]byte(`{"id":"someID","data":"someData"}`)), + }, nil + }}, + } + + t.Run("create a Databricks job", func(t *testing.T) { + data, err := plugin.sendRequest(create, databricksJob, token, "") + assert.NotNil(t, data) + assert.Equal(t, "someID", data["id"]) + assert.Equal(t, "someData", data["data"]) + assert.Nil(t, err) }) -} -func TestBuildRequest(t *testing.T) { - token := "test-token" - runID := "019e70eb" - databricksEndpoint := "" - databricksURL := "https://" + testInstance + "/api/2.1/jobs/runs" - t.Run("build http request for submitting a databricks job", func(t *testing.T) { - req, err := buildRequest(post, nil, databricksEndpoint, testInstance, token, runID, false) - header := http.Header{} - header.Add("Authorization", "Bearer "+token) - header.Add("Content-Type", "application/json") + t.Run("failed to create a Databricks job", func(t *testing.T) { + plugin.client = &MockClient{MockDo: func(req *http.Request) (*http.Response, error) { + assert.Equal(t, req.Method, http.MethodPost) + return &http.Response{ + StatusCode: http.StatusBadRequest, + Body: ioutils.NewBytesReadCloser([]byte(`{"message":"failed"}`)), + }, nil + }} + data, err := plugin.sendRequest(create, databricksJob, token, "") + assert.Nil(t, data) + assert.Equal(t, err.Error(), "failed to create Databricks job with error [failed]") + }) - assert.NoError(t, err) - assert.Equal(t, header, req.Header) - assert.Equal(t, databricksURL+"/submit", req.URL.String()) - assert.Equal(t, post, req.Method) + t.Run("failed to send request to Databricks", func(t *testing.T) { + plugin.client = &MockClient{MockDo: func(req *http.Request) (*http.Response, error) { + assert.Equal(t, req.Method, http.MethodPost) + return nil, errors.New("failed to send request") + }} + data, err := plugin.sendRequest(create, databricksJob, token, "") + assert.Nil(t, data) + assert.Equal(t, err.Error(), "failed to send request to Databricks platform with err: [failed to send request]") }) - t.Run("Get a databricks spark job status", func(t *testing.T) { - req, err := buildRequest(get, nil, databricksEndpoint, testInstance, token, runID, false) - assert.NoError(t, err) - assert.Equal(t, databricksURL+"/get?run_id="+runID, req.URL.String()) - assert.Equal(t, get, req.Method) + t.Run("failed to send request to Databricks", func(t *testing.T) { + plugin.client = &MockClient{MockDo: func(req *http.Request) (*http.Response, error) { + assert.Equal(t, req.Method, http.MethodPost) + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutils.NewBytesReadCloser([]byte(`123`)), + }, nil + }} + data, err := plugin.sendRequest(create, databricksJob, token, "") + assert.Nil(t, data) + assert.Equal(t, err.Error(), "failed to parse response with err: [json: cannot unmarshal number into Go value of type map[string]interface {}]") }) - t.Run("Cancel a spark job", func(t *testing.T) { - req, err := buildRequest(post, nil, databricksEndpoint, testInstance, token, runID, true) - assert.NoError(t, err) - assert.Equal(t, databricksURL+"/cancel", req.URL.String()) - assert.Equal(t, post, req.Method) + t.Run("get a Databricks job", func(t *testing.T) { + plugin.client = &MockClient{MockDo: func(req *http.Request) (*http.Response, error) { + assert.Equal(t, req.Method, http.MethodGet) + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutils.NewBytesReadCloser([]byte(`{"message":"ok"}`)), + }, nil + }} + data, err := plugin.sendRequest(get, databricksJob, token, "") + assert.NotNil(t, data) + assert.Nil(t, err) + }) + + t.Run("cancel a Databricks job", func(t *testing.T) { + plugin.client = &MockClient{MockDo: func(req *http.Request) (*http.Response, error) { + assert.Equal(t, req.Method, http.MethodPost) + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutils.NewBytesReadCloser([]byte(`{"message":"ok"}`)), + }, nil + }} + data, err := plugin.sendRequest(cancel, databricksJob, token, "") + assert.NotNil(t, data) + assert.Nil(t, err) }) } -func TestBuildResponse(t *testing.T) { - t.Run("build http response", func(t *testing.T) { - bodyStr := `{"job_id":"019c06a4-0000", "message":"Statement executed successfully."}` - responseBody := ioutil.NopCloser(strings.NewReader(bodyStr)) - response := &http.Response{Body: responseBody} - actualData, err := buildResponse(response) - assert.NoError(t, err) +func TestCreateTaskInfo(t *testing.T) { + t.Run("create task info", func(t *testing.T) { + taskInfo := createTaskInfo("run-id", "job-id", testInstance) - bodyByte, err := ioutil.ReadAll(strings.NewReader(bodyStr)) - assert.NoError(t, err) - var expectedData map[string]interface{} - err = json.Unmarshal(bodyByte, &expectedData) - assert.NoError(t, err) - assert.Equal(t, expectedData, actualData) + assert.Equal(t, 1, len(taskInfo.Logs)) + assert.Equal(t, taskInfo.Logs[0].Uri, "https://test-account.cloud.databricks.com/#job/job-id/run/run-id") + assert.Equal(t, taskInfo.Logs[0].Name, "Databricks Console") }) } diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml index d50b42c64f..b29d932a80 100644 --- a/kustomize/overlays/eks/kustomization.yaml +++ b/kustomize/overlays/eks/kustomization.yaml @@ -21,23 +21,23 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v1.11.0-b0 # FLYTEADMIN_TAG override the tag + newTag: v1.11.0-b1 # FLYTEADMIN_TAG override the tag newName: cr.flyte.org/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v1.10.3 # FLYTECONSOLE_TAG the tag + newTag: v1.11.0 # FLYTECONSOLE_TAG the tag newName: cr.flyte.org/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v1.11.0-b0 # DATACATALOG_TAG override the tag + newTag: v1.11.0-b1 # DATACATALOG_TAG override the tag newName: cr.flyte.org/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v1.11.0-b0 # FLYTEPROPELLER_TAG override the tag + newTag: v1.11.0-b1 # FLYTEPROPELLER_TAG override the tag newName: cr.flyte.org/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v1.11.0-b0 # FLYTEPROPELLER_TAG override the tag + newTag: v1.11.0-b1 # FLYTEPROPELLER_TAG override the tag newName: cr.flyte.org/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres diff --git a/monodocs-environment.yaml b/monodocs-environment.yaml index 140552aa2d..78f85c19cf 100644 --- a/monodocs-environment.yaml +++ b/monodocs-environment.yaml @@ -56,6 +56,7 @@ dependencies: - mlflow==2.7.0 # mlflow - snowflake-connector-python # snowflake - vaex-core # vaex + - openai # chatgpt - pip: - readthedocs-sphinx-ext diff --git a/script/generate_helm.sh b/script/generate_helm.sh index 43caee8203..1c836b9002 100755 --- a/script/generate_helm.sh +++ b/script/generate_helm.sh @@ -7,7 +7,7 @@ echo "Generating Helm" HELM_SKIP_INSTALL=${HELM_SKIP_INSTALL:-false} if [ "${HELM_SKIP_INSTALL}" != "true" ]; then - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash fi helm version @@ -15,7 +15,7 @@ helm version # All the values files to be built DEPLOYMENT_CORE=${1:-eks gcp} -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" HELM_CAPABILITIES="-a rbac.authorization.k8s.io/v1 -a networking.k8s.io/v1/Ingress -a apiextensions.k8s.io/v1/CustomResourceDefinition" helm dep update ${DIR}/../charts/flyte-deps/ @@ -24,49 +24,48 @@ helm dep update ${DIR}/../charts/flyte-binary/ helm dep update ${DIR}/../charts/flyte-sandbox/ helm dep update ${DIR}/../charts/flyte/ -helm template flyte -n flyte ${DIR}/../charts/flyte/ -f ${DIR}/../charts/flyte/values.yaml ${HELM_CAPABILITIES} --debug > ${DIR}/../deployment/sandbox/flyte_helm_generated.yaml +helm template flyte -n flyte ${DIR}/../charts/flyte/ -f ${DIR}/../charts/flyte/values.yaml ${HELM_CAPABILITIES} --debug >${DIR}/../deployment/sandbox/flyte_helm_generated.yaml for deployment in ${DEPLOYMENT_CORE}; do - helm template flyte -n flyte ${DIR}/../charts/flyte-core/ -f ${DIR}/../charts/flyte-core/values.yaml -f ${DIR}/../charts/flyte-core/values-${deployment}.yaml ${HELM_CAPABILITIES} > ${DIR}/../deployment/${deployment}/flyte_helm_generated.yaml - helm template flyte -n flyte ${DIR}/../charts/flyte-core/ -f ${DIR}/../charts/flyte-core/values.yaml -f ${DIR}/../charts/flyte-core/values-${deployment}.yaml -f ${DIR}/../charts/flyte-core/values-controlplane.yaml ${HELM_CAPABILITIES} > ${DIR}/../deployment/${deployment}/flyte_helm_controlplane_generated.yaml - helm template flyte -n flyte ${DIR}/../charts/flyte-core/ -f ${DIR}/../charts/flyte-core/values.yaml -f ${DIR}/../charts/flyte-core/values-${deployment}.yaml -f ${DIR}/../charts/flyte-core/values-dataplane.yaml ${HELM_CAPABILITIES} > ${DIR}/../deployment/${deployment}/flyte_helm_dataplane_generated.yaml + helm template flyte -n flyte ${DIR}/../charts/flyte-core/ -f ${DIR}/../charts/flyte-core/values.yaml -f ${DIR}/../charts/flyte-core/values-${deployment}.yaml ${HELM_CAPABILITIES} >${DIR}/../deployment/${deployment}/flyte_helm_generated.yaml + helm template flyte -n flyte ${DIR}/../charts/flyte-core/ -f ${DIR}/../charts/flyte-core/values.yaml -f ${DIR}/../charts/flyte-core/values-${deployment}.yaml -f ${DIR}/../charts/flyte-core/values-controlplane.yaml ${HELM_CAPABILITIES} >${DIR}/../deployment/${deployment}/flyte_helm_controlplane_generated.yaml + helm template flyte -n flyte ${DIR}/../charts/flyte-core/ -f ${DIR}/../charts/flyte-core/values.yaml -f ${DIR}/../charts/flyte-core/values-${deployment}.yaml -f ${DIR}/../charts/flyte-core/values-dataplane.yaml ${HELM_CAPABILITIES} >${DIR}/../deployment/${deployment}/flyte_helm_dataplane_generated.yaml done # Generate manifest AWS Scheduler -helm template flyte -n flyte ${DIR}/../charts/flyte-core/ -f ${DIR}/../charts/flyte-core/values.yaml -f ${DIR}/../charts/flyte-core/values-eks.yaml -f ${DIR}/../charts/flyte-core/values-eks-override.yaml ${HELM_CAPABILITIES} --debug > ${DIR}/../deployment/eks/flyte_aws_scheduler_helm_generated.yaml +helm template flyte -n flyte ${DIR}/../charts/flyte-core/ -f ${DIR}/../charts/flyte-core/values.yaml -f ${DIR}/../charts/flyte-core/values-eks.yaml -f ${DIR}/../charts/flyte-core/values-eks-override.yaml ${HELM_CAPABILITIES} --debug >${DIR}/../deployment/eks/flyte_aws_scheduler_helm_generated.yaml # Generate manifest deps chart -helm template flyte -n flyte ${DIR}/../charts/flyte-deps/ ${HELM_CAPABILITIES} --debug > ${DIR}/../deployment/sandbox/flyte_sandbox_deps_helm_generated.yaml +helm template flyte -n flyte ${DIR}/../charts/flyte-deps/ ${HELM_CAPABILITIES} --debug >${DIR}/../deployment/sandbox/flyte_sandbox_deps_helm_generated.yaml # Generate manifest single binary chart -helm template flyte -n flyte ${DIR}/../charts/flyte-binary/ ${HELM_CAPABILITIES} --debug > ${DIR}/../deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml +helm template flyte -n flyte ${DIR}/../charts/flyte-binary/ ${HELM_CAPABILITIES} --debug >${DIR}/../deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml # Generate manifest flyte agent -helm template flyte -n flyte ${DIR}/../charts/flyteagent/ ${HELM_CAPABILITIES} --debug > ${DIR}/../deployment/agent/flyte_agent_helm_generated.yaml - +helm template flyte -n flyte ${DIR}/../charts/flyteagent/ ${HELM_CAPABILITIES} --debug >${DIR}/../deployment/agent/flyte_agent_helm_generated.yaml echo "Generating helm docs" -if command -v helm-docs &> /dev/null -then - rm $(which helm-docs) +if command -v helm-docs &>/dev/null; then + rm $(which helm-docs) fi -GO111MODULE=on go install github.com/norwoodj/helm-docs/cmd/helm-docs@latest +# TODO: (https://github.com/flyteorg/flyte/issues/4994) Unpin when moving past go 1.21 +GO111MODULE=on go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.12.0 ${GOPATH:-~/go}/bin/helm-docs -c ${DIR}/../charts/ # This section is used by GitHub workflow to ensure that the generation step was run if [ -n "$DELTA_CHECK" ]; then - DIRTY=$(git status --porcelain) - if [ -n "$DIRTY" ]; then - echo "FAILED: helm code updated without committing generated code." - echo "Ensure make helm has run and all changes are committed." - DIFF=$(git diff) - echo "diff detected: $DIFF" - DIFF=$(git diff --name-only) - echo "files different: $DIFF" - exit 1 - else - echo "SUCCESS: Generated code is up to date." - fi + DIRTY=$(git status --porcelain) + if [ -n "$DIRTY" ]; then + echo "FAILED: helm code updated without committing generated code." + echo "Ensure make helm has run and all changes are committed." + DIFF=$(git diff) + echo "diff detected: $DIFF" + DIFF=$(git diff --name-only) + echo "files different: $DIFF" + exit 1 + else + echo "SUCCESS: Generated code is up to date." + fi fi