Merge remote-tracking branch 'origin' into buf-experiment

Signed-off-by: Eduardo Apolinario <[email protected]>

.github/workflows/flyteidl-release.yml

@@ -19,19 +19,13 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Set version in python package
-        run: |
-          # from refs/tags/v1.2.3 get 1.2.3
-          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
-          VERSION=$VERSION make update_pyversion
-        shell: bash
+          pip install build twine
       - name: Build and publish
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build
           twine upload dist/*
   deploy-to-npm:
     runs-on: ubuntu-latest

charts/flyte-core/templates/admin/deployment.yaml

@@ -38,6 +38,12 @@ spec:
           volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }}
           - mountPath: /etc/flyte/config
             name: base-config-volume
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- if .Values.flyteadmin.initialProjects }}
         - command:
           - flyteadmin
@@ -54,6 +60,12 @@ spec:
           volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }}
           - mountPath: /etc/flyte/config
             name: base-config-volume
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- end }}
         {{- if and (.Values.cluster_resource_manager.enabled) (not .Values.cluster_resource_manager.standalone_deploy) }}
         - command:
@@ -77,6 +89,12 @@ spec:
           {{ tpl (toYaml .) $ | nindent 10 }}
           {{- end }}
           {{- end }}
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- end }}
         - name: generate-secrets
           image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
@@ -96,6 +114,9 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
       containers:
       - command:
         - flyteadmin

charts/flyte-core/templates/console/deployment.yaml

@@ -37,7 +37,9 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
+        {{- if or .Values.flyteconsole.ga.enabled .Values.flyteconsole.podEnv }}
         env:
+        {{- end }}
         {{- if .Values.flyteconsole.ga.enabled }}
         - name: ENABLE_GA
           value: "{{ .Values.flyteconsole.ga.enabled }}"

charts/flyte-core/templates/datacatalog/deployment.yaml

@@ -38,6 +38,12 @@ spec:
         volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }}
         - mountPath: /etc/datacatalog/config
           name: config-volume
+        {{- if .Values.datacatalog.podEnv }}
+        env:
+          {{- with .Values.datacatalog.podEnv }}
+          {{- toYaml . | nindent 8 }}
+          {{- end }}
+        {{- end }}
       containers:
       - command:
         - datacatalog

charts/flyte-core/templates/flytescheduler/deployment.yaml

@@ -33,6 +33,12 @@ spec:
         - precheck
         - --config
         - {{ .Values.flytescheduler.configPath }}
+        {{- if .Values.flytescheduler.podEnv }}
+        env:
+          {{- with .Values.flytescheduler.podEnv -}}
+          {{- toYaml . | nindent 8 }}
+          {{- end }}
+        {{- end }}
         image: "{{ .Values.flytescheduler.image.repository }}:{{ .Values.flytescheduler.image.tag }}"
         imagePullPolicy: "{{ .Values.flytescheduler.image.pullPolicy }}"
         name: flytescheduler-check

charts/flyte-core/templates/propeller/webhook.yaml

@@ -61,6 +61,11 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
+        {{- if .Values.flytepropeller.podEnv -}}
+        {{- with .Values.flytepropeller.podEnv -}}
+        {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
         volumeMounts:
           - name: config-volume
             mountPath: /etc/flyte/config
@@ -89,6 +94,8 @@ spec:
         {{- toYaml . | nindent 12 }}
         {{- end }}
         {{- end }}
+          ports:
+          - containerPort: 9443
           volumeMounts:
             - name: config-volume
               mountPath: /etc/flyte/config

deployment/eks/flyte_aws_scheduler_helm_generated.yaml

@@ -1096,7 +1096,6 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
-        env:
         resources: 
           limits:
             cpu: 250m
@@ -1360,6 +1359,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          ports:
+          - containerPort: 9443
           volumeMounts:
             - name: config-volume
               mountPath: /etc/flyte/config

deployment/eks/flyte_helm_controlplane_generated.yaml

@@ -802,7 +802,6 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
-        env:
         resources: 
           limits:
             cpu: 250m

deployment/eks/flyte_helm_dataplane_generated.yaml

@@ -558,6 +558,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          ports:
+          - containerPort: 9443
           volumeMounts:
             - name: config-volume
               mountPath: /etc/flyte/config

deployment/eks/flyte_helm_generated.yaml

@@ -1127,7 +1127,6 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
-        env:
         resources: 
           limits:
             cpu: 250m
@@ -1479,6 +1478,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          ports:
+          - containerPort: 9443
           volumeMounts:
             - name: config-volume
               mountPath: /etc/flyte/config

deployment/gcp/flyte_helm_controlplane_generated.yaml

@@ -817,7 +817,6 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
-        env:
         resources: 
           limits:
             cpu: 250m

deployment/gcp/flyte_helm_dataplane_generated.yaml

@@ -565,6 +565,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          ports:
+          - containerPort: 9443
           volumeMounts:
             - name: config-volume
               mountPath: /etc/flyte/config

deployment/gcp/flyte_helm_generated.yaml

@@ -1150,7 +1150,6 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
-        env:
         resources: 
           limits:
             cpu: 250m
@@ -1501,6 +1500,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          ports:
+          - containerPort: 9443
           volumeMounts:
             - name: config-volume
               mountPath: /etc/flyte/config

deployment/sandbox/flyte_helm_generated.yaml

@@ -7249,6 +7249,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          ports:
+          - containerPort: 9443
           volumeMounts:
             - name: config-volume
               mountPath: /etc/flyte/config

docs/community/contribute.rst

@@ -403,11 +403,11 @@ that integrates all Flyte components into a single binary.
    # We will now create a simple template that allows the automatic creation of required namespaces for projects.
    # For example, with Flyte's default project "flytesnacks", the controller will auto-create the following namespaces:
    # flytesnacks-staging, flytesnacks-development, and flytesnacks-production.
-   mkdir $HOME/.flyte/cluster-resource-templates/
+   mkdir $HOME/.flyte/sandbox/cluster-resource-templates/
    echo "apiVersion: v1
    kind: Namespace
    metadata:
-     name: '{{ namespace }}'" > $HOME/.flyte/cluster-resource-templates/namespace.yaml
+     name: '{{ namespace }}'" > $HOME/.flyte/sandbox/cluster-resource-templates/namespace.yaml
 
    # Step5: Running the single binary.
    # The POD_NAMESPACE environment variable is necessary for the webhook to function correctly. 

docs/introduction/core_use_cases/analytics.md → docs/core_use_cases/analytics.md

@@ -13,7 +13,7 @@ Flyte is ideal for data cleaning, statistical summarization, and plotting
 because with `flytekit` you can leverage the rich Python ecosystem of data
 processing and visualization tools.
 
-## Cleaning Data
+## Cleaning data
 
 In this example, we are going to analyze some covid vaccination data:
 
@@ -41,7 +41,7 @@ As you can see, we're using `pandas` for data processing, and in the task
 below we use `plotly` to create a choropleth map of the percent of a country's
 population that has received at least one COVID-19 vaccination.
 
-## Rendering Plots
+## Rendering plots
 
 We can use {ref}`Flyte Decks <decks>` for rendering a static HTML report
 of the map. In this case, we normalize the `people_vaccinated` by the
@@ -158,7 +158,7 @@ logger.removeFilter(deck_filter)
 HTML(filename=cp_deck(deck_filter.deck_files["plot"]))
 ```
 
-## Custom Flyte Deck Renderers
+## Custom Flyte deck renderers
 
 You can also create your own {ref}`custom Flyte Deck renderers <getting_started_customer_renderers>`
 to visualize data with any plotting/visualization library of your choice, as

docs/introduction/core_use_cases/data_engineering.md → docs/core_use_cases/data_engineering.md

@@ -7,7 +7,7 @@ kernelspec:
 
 (getting_started_data_engineering)=
 
-# Data Engineering
+# Data engineering
 
 Flyte is well-suited for data engineering use cases, where you can interleave
 SQL queries with data processing logic implemented in Python with whichever
@@ -100,7 +100,7 @@ def load(df: pd.DataFrame) -> CSVFile:
     return CSVFile(path=csv_file)
 ```
 
-## ETL Workflow
+## ETL workflow
 
 Putting all the pieces together, we create an `etl_workflow` that produces a
 dataset based on the parameters you give it.
@@ -130,7 +130,7 @@ csv_file = etl_workflow(limit=5)
 pd.read_csv(csv_file)
 ```
 
-## Workflows as Reusable Components
+## Workflows as reusable components
 
 Because Flyte tasks and workflows are simply functions, we can embed
 `etl_workflow` as part of a larger workflow, where it's used to create a

docs/introduction/core_use_cases/index.md → docs/core_use_cases/index.md

@@ -1,8 +1,8 @@
 (getting_started_core_use_cases)=
 
-# Core Use Cases
+# Core use cases
 
-This section of the **Getting Started** documentation will take you through the
+This section of the documentation will take you through the
 core use cases for which Flyte is designed. Within the context of these guides,
 we're going to assume that the discipline of data science can be broken down into
 at least three specializations: data engineering, machine learning (or
@@ -16,9 +16,9 @@ practice.
 :header-rows: 0
 :widths: 10 30
 
-* - {doc}`🛠 Data Engineering <data_engineering>`
+* - {doc}`🛠 Data engineering <data_engineering>`
   - Create an ETL workflow for processing data with SQLAlchemy and Pandas.
-* - {doc}`🤖 Machine Learning <machine_learning>`
+* - {doc}`🤖 Machine learning <machine_learning>`
   - Train a classifier with Scikit-Learn and Pandas.
 * - {doc}`📈 Analytics <analytics>`
   - Develop a data cleaning and plotting pipeline with Plotly and Pandas.

docs/introduction/core_use_cases/machine_learning.md → docs/core_use_cases/machine_learning.md

@@ -7,13 +7,13 @@ kernelspec:
 
 (getting_started_machine_learning)=
 
-# Machine Learning
+# Machine learning
 
 Flyte can handle a full spectrum of machine learning workloads, from
 training small models to gpu-accelerated deep learning and hyperparameter
 optimization.
 
-## Getting the Data
+## Getting the data
 
 In this simple example, we train a binary classification model on the
 [wine dataset](https://scikit-learn.org/stable/datasets/toy_dataset.html#wine-dataset)
@@ -34,7 +34,7 @@ def get_data() -> pd.DataFrame:
     return load_wine(as_frame=True).frame
 ```
 
-## Define a Training Workflow
+## Define a training workflow
 
 Then, we define `process_data` and `train_model` tasks along with a
 `training_workflow` to put all the pieces together for a model-training
@@ -76,7 +76,7 @@ type-safe launch forms on the Flyte UI. Learn more in the
 {ref}`Extending Flyte <customizing_flyte_types>` guide.
 ```
 
-## Computing Predictions
+## Computing predictions
 
 Executing this workflow locally, we can call the `model.predict` method to make
 sure we can use our newly trained model to make predictions based on some
@@ -88,7 +88,7 @@ X, _ = load_wine(as_frame=True, return_X_y=True)
 model.predict(X.sample(10, random_state=41))
 ```
 
-## Extending your ML Workloads
+## Extending your ML workloads
 
 There are many ways to extend your workloads:
 
@@ -119,7 +119,7 @@ There are many ways to extend your workloads:
     supports ScikitLearn, TensorFlow, and PyTorch.
 ```
 
-```{admonition} Learn More
+```{admonition} Learn more
 :class: important
 
 See the {ref}`Tutorials <tutorials>` for more machine learning examples.