Merge remote-tracking branch 'origin' into flyte-issues-4870

Signed-off-by: Eduardo Apolinario <[email protected]>

.gitattributes

@@ -1 +1,5 @@
 docs/**/*html linguist-generated=true
+flyteidl/gen/** linguist-generated=true
+flyteidl/protos/**/*.rst linguist-generated=true
+flyteidl/clients/go/assets/admin.swagger.json linguist-generated=true
+

.github/workflows/generate_flyte_manifest.yml

@@ -23,7 +23,6 @@ jobs:
           VERSION: ${{ github.event.inputs.next-version }}
         run: |
           make release_automation
-          make kustomize
           make helm
       - name: Create Pull Request
         id: cpr
@@ -39,17 +38,12 @@ jobs:
           title: 'Update Flyte components'
           body: |
             Updated flyte deployment
-            - Updated GCP Flyte kustomize generated manifest file
-            - Updated EKS Flyte kustomize generated manifest file
-            - Updated Sandbox Flyte  kustomize generated manifest file
-            - Updated TEST Flyte kustomize generated manifest file
             - Updated GCP Flyte helm generated manifest file
             - Updated EKS Flyte helm generated manifest file
             - Updated Sandbox Flyte helm generated manifest file
             - Updated TEST Flyte helm generated manifest file
             - Auto-generated by [flyte-bot]
           labels: |
-            kustomize
             helm
           team-reviewers: |
             flyte-maintainers

.github/workflows/sandbox.yml

@@ -62,60 +62,3 @@ jobs:
           tags: ${{ steps.dind-names.outputs.tags }}
           build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}"
           file: docker/sandbox/Dockerfile
-
-  trigger-sandbox-lite-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-      - name: Setup Golang caches
-        uses: actions/cache@v3
-        with:
-          path: |
-            /root/.cache/go-build
-            /root/go/pkg/mod
-          key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-golang-
-      - name: Set flyte version to release
-        id: set_version
-        run: |
-          if [ ${{ github.event_name}} = "release" ]; then
-            echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }})
-          else
-            echo ::set-output name=flyte_version::latest
-          fi
-      - name: Prepare DIND Image Names
-        id: dind-names
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ghcr.io/${{ github.repository_owner }}/flyte-sandbox-lite
-          tags: |
-            ${{ steps.set_version.outputs.flyte_version }}
-            type=sha,format=long
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to GitHub Container Registry
-        if: ${{ github.event_name == 'release' }}
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
-          password: "${{ secrets.FLYTE_BOT_PAT }}"
-      - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          platforms: linux/arm64, linux/amd64
-          push: ${{ github.event_name == 'release' }}
-          target: dind
-          tags: ${{ steps.dind-names.outputs.tags }}
-          build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}"
-          file: Dockerfile.sandbox-lite

.github/workflows/single-binary.yml

@@ -156,7 +156,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - uses: unionai/[email protected]
       - name: Setup sandbox
         run: |
@@ -181,17 +181,17 @@ jobs:
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks
-          # TODO: Enable this once refactored version produces a release tag
-          # ref: ${{ env.FLYTESNACKS_VERSION }}
+          ref: ${{ env.FLYTESNACKS_VERSION }}
       - name: Register specific tests
         run: |
+          flytekit_version=$(pip show flytekit | grep -i version | awk '{ print $2 }')
           while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
-                  --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
+                  --image cr.flyte.org/flyteorg/flytekit:py3.12-${flytekit_version} \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
                   flytesnacks/$line;
           done < flytesnacks/flyte_tests.txt

.github/workflows/tests.yml

@@ -27,20 +27,35 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-      - name: Fetch the code
+      - name: Fetch flyte code
+        uses: actions/checkout@v4
+        with:
+          path: "${{ github.workspace }}/flyte"
+      - name: Fetch flytekit code
         uses: actions/checkout@v4
+        with:
+          repository: flyteorg/flytekit
+          path: "${{ github.workspace }}/flytekit"
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           python-version: 3.9
       - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
         run: |
           conda install -c conda-forge conda-lock
           conda-lock install -n monodocs-env monodocs-environment.lock.yaml
       - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
         run: |
           conda activate monodocs-env
+          export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0"
           pip install -e ./flyteidl
+      - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flytekit
+        run: |
+          conda activate monodocs-env
+          pip install -e .
           conda info
           conda list
           conda config --show-sources
@@ -49,21 +64,12 @@ jobs:
       - name: Setup Graphviz
         uses: ts-graphviz/setup-graphviz@v1
       - name: Build the documentation
+        working-directory: ${{ github.workspace }}/flyte
         shell: bash -el {0}
         run: |
           conda activate monodocs-env
           make docs
 
-  generate_kustomize:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: "1.21"
-      - name: Kustomize and diff
-        run: DELTA_CHECK=true make kustomize
-
   generate_helm:
     runs-on: ubuntu-latest
     steps:

.github/workflows/validate-helm-charts.yaml

@@ -0,0 +1,89 @@
+name: Validate helm charts & manifests
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "charts/**"
+      - "deployment/**"
+      - "docker/sandbox-bundled/manifests/**"
+
+jobs:
+  lint-and-test-charts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          sparse-checkout: charts
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          check-latest: true
+
+      - uses: unionai/[email protected]
+        name: Setup flytectl
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2
+
+      - name: Detect charts changed (list-changed)
+        id: charts-changed
+        run: |
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run chart-testing (lint)
+        if: steps.charts-changed.outputs.changed == 'true'
+        run: |
+          ct lint \
+            --target-branch ${{ github.event.repository.default_branch }} \
+            --validate-maintainers=false \
+            --check-version-increment=false \
+            --chart-repos spark-operator=https://kubeflow.github.io/spark-operator \
+            --chart-repos dask=https://helm.dask.org \
+            --chart-repos bitnami=https://charts.bitnami.com/bitnami \
+            --chart-repos twuni=https://helm.twun.io \
+            --chart-repos kubernetes-dashboard=https://kubernetes.github.io/dashboard
+
+  validate-manifests:
+    needs:
+      - lint-and-test-charts
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        k8s_versions: [ "1.29.2", "1.28.7", "1.27.11" ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            deployment
+            docker/sandbox-bundled/manifests
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Install kubeconform
+        run: |
+          curl -L -o kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.4/kubeconform-linux-amd64.tar.gz
+          tar -zvxf kubeconform.tar.gz
+          chmod +x kubeconform
+          sudo mv kubeconform /usr/local/bin/kubeconform
+
+      - name: Validate manifests
+        run: |
+          kubeconform -strict -summary -skip CustomResourceDefinition -ignore-filename-pattern "deployment/stats/prometheus/*" -kubernetes-version ${{ matrix.k8s_versions }} ./deployment ./docker/sandbox-bundled/manifests

.gitignore

@@ -20,8 +20,6 @@ __pycache__/
 /charts/flyteagent/Chart.lock
 /charts/flyte-deps/Chart.lock
 /charts/flyte-core/Chart.lock
-/docker/sandbox-lite/storage/charts
-/docker/sandbox-lite/storage/Chart.lock
 **/*.bak-
 .python-version
 dist/

CHANGELOG/CHANGELOG-v1.11.0-b0.md

@@ -0,0 +1,3 @@
+# Flyte v1.11.0-b0
+
+Beta release to test new idl

CHANGELOG/CHANGELOG-v1.11.0-b1.md

@@ -0,0 +1,3 @@
+# Flyte v1.11.0-b1
+
+Second beta release for 1.11.0.

CHANGELOG/CHANGELOG-v1.11.0.md

@@ -0,0 +1,32 @@
+# Flyte v1.11.0 Release Notes
+
+We're excited to announce the release of Flyte v1.11.0! This version brings a host of improvements, bug fixes, and new features designed to enhance your experience with Flyte. From operational enhancements to documentation updates, this release aims to make Flyte more robust, user-friendly, and feature-rich.
+
+## Highlights
+
+- **Agents hit General Availability (GA):** Agents, now in General Availability, are long-running, stateless services that facilitate asynchronous job launches on platforms like Databricks or Snowflake and enable external service calls. They are versatile, supporting implementations in any language through a protobuf interface, enhancing Flyte's flexibility and operational efficiency.
+- **Improved Caching:** Support for loading cached sublists with multiple data types has been introduced, eliminating issues related to cache retrieval across varied data formats.
+- **Tracing and Observability:** The introduction of opentelemetry BlobstoreClientTracer in flyteadmin enhances observability, allowing for better monitoring and troubleshooting.
+- **Security Enhancements:** Added securityContext configuration to Flyte-core charts, strengthening the security posture of Flyte deployments.
+- **Documentation Overhaul:** Continuous improvements and updates have been made to the documentation, fixing broken links and updating content for better clarity and usability.
+- **Operational Improvements:** This release introduces enhancements such as adding a service account for V1 Ray Jobs, caching console assets in a single binary, and conditional mounting of secrets to improve the operational efficiency of Flyte. Additionally, we are removing `kustomize` from our deployment process to simplify the configuration and management of Flyte instances, making it easier for users to maintain and streamline their deployment workflows.
+
+
+## Bug Fixes
+
+- **Fixed Literal in Launchplan:** Added fixed_literal to the launchplan template, addressing issues with hardcoded values in workflows.
+- **Corrected Metadata and Resources:** Fixes have been applied to correct IsParent metadata in ArrayNode eventing and to address invalid "resources" scope issues in deployment configurations.
+- **Enhanced Stability and Performance:** Numerous bug fixes have been implemented to address stability and performance issues, including fixes for data catalog errors, yaml comment errors in pod template examples, and more.
+
+## Documentation and Guides
+
+- **Comprehensive Guides:** New guides and documentation updates have been added, including a ChatGPT Agent Setup guide and an Airflow migration guide. Improvements in documentation for developing agents have been integrated into the broader enhancements for this release.
+- **Updated Troubleshooting and Configuration Docs:** New troubleshooting guides for spark task execution and updates to deployment configuration documents enhance the knowledge base for Flyte users.
+
+## Contributors
+
+We extend our deepest gratitude to all the contributors who made this release possible. Special shoutouts to @neilisaur, @lowc1012, @MortalHappiness, @novahow, and @pryce-turner for making their first contributions!
+
+**For a full list of changes, enhancements, and bug fixes, visit our [changelog](https://github.com/flyteorg/flyte/compare/v1.10.7...v1.11.0).**
+
+Thank you for your continued support of Flyte. We look forward to hearing your feedback on this release!

CHANGELOG/CHANGELOG-v1.11.1-b0.md

@@ -0,0 +1,2 @@
+# Flyte v1.11.1-b0
+