Merge branch 'master' of https://github.com/Future-Outlier/flyte into…

… agent-metadata-proto-service

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,51 +1,51 @@
-<!--
-Thank you for sending the PR! 
-Please fill the applicable details below
-Happy contributing!
--->
-
 ## Tracking issue
+_https://github.com/flyteorg/flyte/issues/<number>_
 
 <!-- If your PR fixes an open issue, use `Closes #999` to link your PR with the issue. #999 stands for the issue number you are fixing -->
 
 <!-- Remove this section if not applicable -->
 
 <!-- Example: Closes #31 -->
 
-## Docs link
+## Why are the changes needed?
 
-<!-- Add documentation link built by CI jobs here, and specify the changed place -->
+<!--
+Please clarify why the changes are needed. For instance,
+1. If you propose a new API, clarify the use case for a new API.
+2. If you fix a bug, you can clarify why it is a bug.
+-->
 
-## Describe your changes
+## What changes were proposed in this pull request?
 
-<!-- List all the proposed changes in your PR -->
+<!--
+Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
+If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
+1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
+2. If there is design documentation, please add the link.
+-->
+
+## How was this patch tested?
 
-<!-- Mark all the applicable boxes. To mark the box as done follow the following conventions -->
 <!--
-[x] - Correct; marked as done
-[X] - Correct; marked as done
-[ ] - Not correct; marked as **not** done
+If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
+If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
+If tests were not added, please describe why they were not added and/or why it was difficult to add.
 -->
 
+### Setup process
+
+### Screenshots
+
 ## Check all the applicable boxes <!-- Follow the above conventions to check the box -->
 
 - [ ] I updated the documentation accordingly.
 - [ ] All new and existing tests passed.
 - [ ] All commits are signed-off.
 
-## Setup Process
-
-<!-- Describe how you set up this PR's environment to help maintainers reproduce your results more easily -->
-
-## Screenshots
-
-<!-- Add all the screenshots which support your changes -->
-
-## Note to reviewers
-
-<!-- Add notes to reviewers if applicable -->
-
 ## Related PRs
 
 <!-- Add related pull requests for reviewers to check -->
 
+## Docs link
+
+<!-- Add documentation link built by CI jobs here, and specify the changed place -->

.github/workflows/create_release.yml

@@ -105,7 +105,11 @@ jobs:
       - name: Get Latest Version of component
         id: set_version
         run: |
-          echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
+          if [ ${{ matrix.component }} = "flytecopilot" ]; then
+             echo ::set-output name=version::$(yq eval '.configmap.copilot.plugins.k8s.co-pilot.image' charts/flyte-core/values.yaml | cut -d ":" -f 2 )
+          else
+             echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
+          fi
         shell: bash
 
       - name: Login to GitHub Container Registry

.github/workflows/generate_flyte_manifest.yml

@@ -11,8 +11,6 @@ jobs:
   update-flyte-releases:
     name: Update Flyte components
     runs-on: ubuntu-latest
-    needs:
-      - generate-tags
     steps:
       - uses: actions/checkout@v2
         with:

.goreleaser.yml

@@ -11,4 +11,13 @@ release:
   name_template: "Flyte v{{.Version}} milestone release"
   disable: false
   extra_files:
-    - glob: ./release/*
+    - glob: ./release/*
+git:
+  ignore_tags:
+    - "datacatalog/*"
+    - "flyteadmin/*"
+    - "flytecopilot/*"
+    - "flyteidl/*"
+    - "flyteplugins/*"
+    - "flytepropeller/*"
+    - "flytestdlib/*"

CHANGELOG/CHANGELOG-v1.10.6-b0.md

@@ -0,0 +1,3 @@
+# Flyte v1.10.6-b0 Release
+
+Beta release.

Makefile

@@ -100,3 +100,14 @@ build_native_flyte:
 	docker build \
 	--build-arg FLYTECONSOLE_VERSION=$(FLYTECONSOLE_VERSION) \
 	--tag flyte-binary:native .
+
+.PHONY: go-tidy
+go-tidy:
+	go mod tidy
+	make -C datacatalog go-tidy
+	make -C flyteadmin go-tidy
+	make -C flyteidl go-tidy
+	make -C flytepropeller go-tidy
+	make -C flyteplugins go-tidy
+	make -C flytestdlib go-tidy
+	make -C flytecopilot go-tidy

charts/flyte-binary/README.md

@@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment
 | configuration.auth.oidc.clientId | string | `""` |  |
 | configuration.auth.oidc.clientSecret | string | `""` |  |
 | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` |  |
-| configuration.co-pilot.image.tag | string | `"v1.9.37"` |  |
+| configuration.co-pilot.image.tag | string | `"v1.10.6-b0"` |  |
 | configuration.database.dbname | string | `"flyte"` |  |
 | configuration.database.host | string | `"127.0.0.1"` |  |
 | configuration.database.options | string | `"sslmode=disable"` |  |
@@ -66,6 +66,10 @@ Chart for basic single Flyte executable deployment
 | configuration.logging.plugins.stackdriver.templateUri | string | `""` |  |
 | configuration.storage.metadataContainer | string | `"my-organization-flyte-container"` |  |
 | configuration.storage.provider | string | `"s3"` |  |
+| configuration.storage.providerConfig.azure.account | string | `"storage-account-name"` |  |
+| configuration.storage.providerConfig.azure.configDomainSuffix | string | `""` |  |
+| configuration.storage.providerConfig.azure.configUploadConcurrency | int | `4` |  |
+| configuration.storage.providerConfig.azure.key | string | `""` |  |
 | configuration.storage.providerConfig.gcs.project | string | `"my-organization-gcp-project"` |  |
 | configuration.storage.providerConfig.s3.accessKey | string | `""` |  |
 | configuration.storage.providerConfig.s3.authType | string | `"iam"` |  |

charts/flyte-binary/templates/_helpers.tpl

@@ -99,6 +99,8 @@ Get the Flyte user data prefix.
 {{- printf "s3://%s/data" $userDataContainer -}}
 {{- else if eq "gcs" .Values.configuration.storage.provider -}}
 {{- printf "gs://%s/data" $userDataContainer -}}
+{{- else if eq "azure" .Values.configuration.storage.provider -}}
+{{- printf "abfs://%s/data" $userDataContainer -}}
 {{- end -}}
 {{- end -}}
 

charts/flyte-binary/templates/configmap.yaml

@@ -110,6 +110,21 @@ data:
           {{- printf "Invalid value for S3 storage provider authentication type. Expected one of (iam, accesskey), but got: %s" .authType | fail }}
           {{- end }}
         {{- end }}
+        {{- else if eq "azure" .provider }}
+        {{- with .providerConfig.azure }}
+        kind: azure
+        config:
+          account: {{ .account }}
+          {{- if .key }}
+          key: {{ .key }}
+          {{- end }}
+          {{- if .configDomainSuffix }}
+          configDomainSuffix: {{ .configDomainSuffix }}
+          {{- end }}
+          {{- if .configUploadConcurrency }}
+          configUploadConcurrency: {{ .configUploadConcurrency }}
+          {{- end }}
+        {{- end }}
         {{- else if eq "gcs" .provider }}
         kind: google
         config:

charts/flyte-binary/values.yaml

@@ -81,6 +81,16 @@ configuration:
       gcs:
         # project Google Cloud project in which bucket resides
         project: "my-organization-gcp-project"
+      # azure Provider configuration for Azure object store
+      azure:
+        # configDomainSuffix Domain name suffix
+        configDomainSuffix: ""
+        # configUploadConcurrency Upload Concurrency (default 4)
+        configUploadConcurrency: 4
+        # account Storage Account name
+        account: "storage-account-name"
+        # key Storage Account key if used
+        key: ""
   # logging Specify configuration for logs emitted by Flyte
   logging:
     # level Set the log level
@@ -149,7 +159,7 @@ configuration:
       # repository CoPilot sidecar image repository
       repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE
       # tag CoPilot sidecar image tag
-      tag: v1.9.37 # FLYTECOPILOT_TAG
+      tag: v1.10.6-b0 # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
   agentService:
     defaultAgent:

charts/flyte-core/README.md

@@ -90,8 +90,8 @@ helm install gateway bitnami/contour -n flyte
 | configmap.clusters.clusterConfigs | list | `[]` |  |
 | configmap.clusters.labelClusterMap | object | `{}` |  |
 | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI |
-| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.9.37","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
-| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.9.37","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
+| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.10.6-b0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
+| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.10.6-b0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
 | configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration |
 | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). |
 | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). |
@@ -125,7 +125,7 @@ helm install gateway bitnami/contour -n flyte
 | datacatalog.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment |
-| datacatalog.image.tag | string | `"v1.9.37"` | Docker image tag |
+| datacatalog.image.tag | string | `"v1.10.6-b0"` | Docker image tag |
 | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment |
 | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods |
 | datacatalog.priorityClassName | string | `""` | Sets priorityClassName for datacatalog pod(s). |
@@ -157,7 +157,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command |
 | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment |
-| flyteadmin.image.tag | string | `"v1.9.37"` |  |
+| flyteadmin.image.tag | string | `"v1.10.6-b0"` |  |
 | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create |
 | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment |
 | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods |
@@ -209,7 +209,7 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment |
-| flytepropeller.image.tag | string | `"v1.9.37"` |  |
+| flytepropeller.image.tag | string | `"v1.10.6-b0"` |  |
 | flytepropeller.manager | bool | `false` |  |
 | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment |
 | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods |
@@ -236,7 +236,7 @@ helm install gateway bitnami/contour -n flyte
 | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files |
 | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment |
-| flytescheduler.image.tag | string | `"v1.9.37"` | Docker image tag |
+| flytescheduler.image.tag | string | `"v1.10.6-b0"` | Docker image tag |
 | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment |
 | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods |
 | flytescheduler.priorityClassName | string | `""` | Sets priorityClassName for flyte scheduler pod(s). |

charts/flyte-core/values.yaml

@@ -16,7 +16,7 @@ flyteadmin:
   image:
     # -- Docker image for Flyteadmin deployment
     repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE
-    tag: v1.9.37 # FLYTEADMIN_TAG
+    tag: v1.10.6-b0 # FLYTEADMIN_TAG
     pullPolicy: IfNotPresent
   # -- Additional flyteadmin container environment variables
   #
@@ -132,7 +132,7 @@ flytescheduler:
     # -- Docker image for Flytescheduler deployment
     repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE
     # -- Docker image tag
-    tag: v1.9.37 # FLYTESCHEDULER_TAG
+    tag: v1.10.6-b0 # FLYTESCHEDULER_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytescheduler deployment
@@ -186,7 +186,7 @@ datacatalog:
     # -- Docker image for Datacatalog deployment
     repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE
     # -- Docker image tag
-    tag: v1.9.37 # DATACATALOG_TAG
+    tag: v1.10.6-b0 # DATACATALOG_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Datacatalog deployment
@@ -254,7 +254,7 @@ flytepropeller:
   image:
     # -- Docker image for Flytepropeller deployment
     repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE
-    tag: v1.9.37 # FLYTEPROPELLER_TAG
+    tag: v1.10.6-b0 # FLYTEPROPELLER_TAG
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytepropeller deployment
   resources:
@@ -654,7 +654,7 @@ configmap:
         # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig)
         co-pilot:
           name: flyte-copilot-
-          image: cr.flyte.org/flyteorg/flytecopilot:v1.9.37 # FLYTECOPILOT_IMAGE
+          image: cr.flyte.org/flyteorg/flytecopilot:v1.10.6-b0 # FLYTECOPILOT_IMAGE
           start-timeout: 30s
 
   # -- Core propeller configuration