Merge branch 'flyteorg:master' into master

.gitattributes

@@ -1 +1,5 @@
 docs/**/*html linguist-generated=true
+flyteidl/gen/** linguist-generated=true
+flyteidl/protos/**/*.rst linguist-generated=true
+flyteidl/clients/go/assets/admin.swagger.json linguist-generated=true
+

.github/workflows/flyteidl-release.yml

@@ -33,6 +33,7 @@ jobs:
       run:
         working-directory: flyteidl
     steps:
+      - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:
           node-version: "12.x"

.github/workflows/generate_flyte_manifest.yml

@@ -23,7 +23,6 @@ jobs:
           VERSION: ${{ github.event.inputs.next-version }}
         run: |
           make release_automation
-          make kustomize
           make helm
       - name: Create Pull Request
         id: cpr
@@ -39,17 +38,12 @@ jobs:
           title: 'Update Flyte components'
           body: |
             Updated flyte deployment
-            - Updated GCP Flyte kustomize generated manifest file
-            - Updated EKS Flyte kustomize generated manifest file
-            - Updated Sandbox Flyte  kustomize generated manifest file
-            - Updated TEST Flyte kustomize generated manifest file
             - Updated GCP Flyte helm generated manifest file
             - Updated EKS Flyte helm generated manifest file
             - Updated Sandbox Flyte helm generated manifest file
             - Updated TEST Flyte helm generated manifest file
             - Auto-generated by [flyte-bot]
           labels: |
-            kustomize
             helm
           team-reviewers: |
             flyte-maintainers

.github/workflows/sandbox.yml

@@ -62,60 +62,3 @@ jobs:
           tags: ${{ steps.dind-names.outputs.tags }}
           build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}"
           file: docker/sandbox/Dockerfile
-
-  trigger-sandbox-lite-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-      - name: Setup Golang caches
-        uses: actions/cache@v3
-        with:
-          path: |
-            /root/.cache/go-build
-            /root/go/pkg/mod
-          key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-golang-
-      - name: Set flyte version to release
-        id: set_version
-        run: |
-          if [ ${{ github.event_name}} = "release" ]; then
-            echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }})
-          else
-            echo ::set-output name=flyte_version::latest
-          fi
-      - name: Prepare DIND Image Names
-        id: dind-names
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ghcr.io/${{ github.repository_owner }}/flyte-sandbox-lite
-          tags: |
-            ${{ steps.set_version.outputs.flyte_version }}
-            type=sha,format=long
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to GitHub Container Registry
-        if: ${{ github.event_name == 'release' }}
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
-          password: "${{ secrets.FLYTE_BOT_PAT }}"
-      - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          platforms: linux/arm64, linux/amd64
-          push: ${{ github.event_name == 'release' }}
-          target: dind
-          tags: ${{ steps.dind-names.outputs.tags }}
-          build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}"
-          file: Dockerfile.sandbox-lite

.github/workflows/single-binary.yml

@@ -156,7 +156,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - uses: unionai/[email protected]
       - name: Setup sandbox
         run: |
@@ -181,17 +181,17 @@ jobs:
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks
-          # TODO: Enable this once refactored version produces a release tag
-          # ref: ${{ env.FLYTESNACKS_VERSION }}
+          ref: ${{ env.FLYTESNACKS_VERSION }}
       - name: Register specific tests
         run: |
+          flytekit_version=$(pip show flytekit | grep -i version | awk '{ print $2 }')
           while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
-                  --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
+                  --image cr.flyte.org/flyteorg/flytekit:py3.12-${flytekit_version} \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
                   flytesnacks/$line;
           done < flytesnacks/flyte_tests.txt

.github/workflows/tests.yml

@@ -27,20 +27,35 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-      - name: Fetch the code
+      - name: Fetch flyte code
+        uses: actions/checkout@v4
+        with:
+          path: "${{ github.workspace }}/flyte"
+      - name: Fetch flytekit code
         uses: actions/checkout@v4
+        with:
+          repository: flyteorg/flytekit
+          path: "${{ github.workspace }}/flytekit"
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           python-version: 3.9
       - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
         run: |
           conda install -c conda-forge conda-lock
           conda-lock install -n monodocs-env monodocs-environment.lock.yaml
       - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
         run: |
           conda activate monodocs-env
+          export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0"
           pip install -e ./flyteidl
+      - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flytekit
+        run: |
+          conda activate monodocs-env
+          pip install -e .
           conda info
           conda list
           conda config --show-sources
@@ -49,21 +64,12 @@ jobs:
       - name: Setup Graphviz
         uses: ts-graphviz/setup-graphviz@v1
       - name: Build the documentation
+        working-directory: ${{ github.workspace }}/flyte
         shell: bash -el {0}
         run: |
           conda activate monodocs-env
           make docs
 
-  generate_kustomize:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: "1.21"
-      - name: Kustomize and diff
-        run: DELTA_CHECK=true make kustomize
-
   generate_helm:
     runs-on: ubuntu-latest
     steps:

.github/workflows/validate-helm-charts.yaml

@@ -0,0 +1,89 @@
+name: Validate helm charts & manifests
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "charts/**"
+      - "deployment/**"
+      - "docker/sandbox-bundled/manifests/**"
+
+jobs:
+  lint-and-test-charts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          sparse-checkout: charts
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          check-latest: true
+
+      - uses: unionai/[email protected]
+        name: Setup flytectl
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2
+
+      - name: Detect charts changed (list-changed)
+        id: charts-changed
+        run: |
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run chart-testing (lint)
+        if: steps.charts-changed.outputs.changed == 'true'
+        run: |
+          ct lint \
+            --target-branch ${{ github.event.repository.default_branch }} \
+            --validate-maintainers=false \
+            --check-version-increment=false \
+            --chart-repos spark-operator=https://kubeflow.github.io/spark-operator \
+            --chart-repos dask=https://helm.dask.org \
+            --chart-repos bitnami=https://charts.bitnami.com/bitnami \
+            --chart-repos twuni=https://helm.twun.io \
+            --chart-repos kubernetes-dashboard=https://kubernetes.github.io/dashboard
+
+  validate-manifests:
+    needs:
+      - lint-and-test-charts
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        k8s_versions: [ "1.29.2", "1.28.7", "1.27.11" ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            deployment
+            docker/sandbox-bundled/manifests
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Install kubeconform
+        run: |
+          curl -L -o kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.4/kubeconform-linux-amd64.tar.gz
+          tar -zvxf kubeconform.tar.gz
+          chmod +x kubeconform
+          sudo mv kubeconform /usr/local/bin/kubeconform
+
+      - name: Validate manifests
+        run: |
+          kubeconform -strict -summary -skip CustomResourceDefinition -ignore-filename-pattern "deployment/stats/prometheus/*" -kubernetes-version ${{ matrix.k8s_versions }} ./deployment ./docker/sandbox-bundled/manifests

.gitignore

@@ -20,8 +20,6 @@ __pycache__/
 /charts/flyteagent/Chart.lock
 /charts/flyte-deps/Chart.lock
 /charts/flyte-core/Chart.lock
-/docker/sandbox-lite/storage/charts
-/docker/sandbox-lite/storage/Chart.lock
 **/*.bak-
 .python-version
 dist/

CHANGELOG/CHANGELOG-v0.11.0.md

@@ -4,7 +4,7 @@
 * New to flyte? https://start.flyte.org takes you through first run experience. (Thanks to @jeevb)
 * [Grafana templates](https://docs.flyte.org/en/latest/howto/monitoring/index.html) for monitoring Flyte System and User Workflows.
 * [Extend Flyte](https://docs.flyte.org/en/latest/plugins/index.html) docs.
-* [FlyteIdl Docs](https://docs.flyte.org/projects/flyteidl/en/latest/) are published! You can learn about the core language that makes it all work.
+* [FlyteIdl Docs](https://docs.flyte.org/en/latest/reference_flyteidl.html) are published! You can learn about the core language that makes it all work.
 * [Additional knob](https://github.com/flyteorg/flytepropeller/pull/219/files#diff-91657d6448dfbf87f4cecf126ad02bd668ea233edcf74e860ef4f54bdd4cb552R78) for fine tuning flyte propeller performance that speeds up executions drastically.
 * OidC support for Google Idp (And other OidC compliant Idps)
 * Various stabilization bugs.

CHANGELOG/CHANGELOG-v0.14.0.md

@@ -11,7 +11,7 @@
 - performance improvement for dynamic workflows
 
 ## Flyteconsole
- - Bug fixes 
+ - Bug fixes
  - More updates coming soon
 
 ## Flytekit
@@ -27,6 +27,6 @@
      - More use case driven examples in flytesnacks
 
 ## flytectl
- - flytectl is ready for BETA. check it out - https://docs.flyte.org/projects/flytectl/en/latest/
+ - flytectl is ready for BETA. check it out - https://docs.flyte.org/en/latest/flytectl/overview.html
 
 Please see the [flytekit release](https://github.com/flyteorg/flytekit/releases/tag/v0.18.0) for the full list and more details.