Merge branch 'flyteorg:master' into podsecurity

flyteorg · Jul 11, 2024 · 92bd3a0 · 92bd3a0
2 parents ce03fb7 + 81afb76
commit 92bd3a0
Show file tree

Hide file tree

Showing 179 changed files with 2,764 additions and 1,746 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -1,7 +1,7 @@
 name: Components Checks
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:

diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -2,7 +2,7 @@
 name: Codespell
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
@@ -88,6 +88,7 @@ jobs:
           [
               datacatalog,
               flyteadmin,
+              flyteagent,
               flyteconsole,
               flytecopilot,
               flytepropeller,
@@ -107,6 +108,8 @@ jobs:
         run: |
           if [ ${{ matrix.component }} = "flytecopilot" ]; then
              echo ::set-output name=version::$(yq eval '.configmap.copilot.plugins.k8s.co-pilot.image' charts/flyte-core/values.yaml | cut -d ":" -f 2 )
+          elif [ ${{ matrix.component }} = "flyteagent" ]; then
+             echo ::set-output name=version::$(yq eval '.image.tag' charts/flyteagent/values.yaml)
           else
              echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
           fi

diff --git a/.github/workflows/flytectl-install.yml b/.github/workflows/flytectl-install.yml
@@ -1,7 +1,7 @@
 name: Flytectl-specific checks
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:

diff --git a/.github/workflows/flyteidl-checks.yml b/.github/workflows/flyteidl-checks.yml
@@ -1,7 +1,7 @@
 name: Flyteidl Verification Tests
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:

diff --git a/.github/workflows/flyteidl-release.yml b/.github/workflows/flyteidl-release.yml
@@ -1,12 +1,12 @@
 name: Upload flyteidl to PyPI and npm
 
 on:
-  release:
-    types: [published]
+  push:
+    tags:
+      - flyteidl/v*.*.*
 
 jobs:
   deploy-to-pypi:
-    if: "!startsWith(github.event.release.tag_name, 'flytectl/')"
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -29,7 +29,6 @@ jobs:
           python -m build
           twine upload dist/*
   deploy-to-npm:
-    if: "!startsWith(github.event.release.tag_name, 'flytectl/')"
     runs-on: ubuntu-latest
     defaults:
       run:

diff --git a/.github/workflows/helm-charts.yaml b/.github/workflows/helm-charts.yaml
@@ -1,7 +1,7 @@
 name: Package & Push Flyte Helm Charts
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:

diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml
@@ -1,7 +1,7 @@
 name: Build & Push Sandbox Docker Image
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:

diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml
@@ -1,7 +1,7 @@
 name: Build & Push Flyte Single Binary Images
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -1,7 +1,7 @@
 name: tests
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:

diff --git a/CHANGELOG/CHANGELOG-v1.13.0-rc0.md b/CHANGELOG/CHANGELOG-v1.13.0-rc0.md
@@ -0,0 +1,40 @@
+# Flyte v1.13.0-rc0 Release Notes
+
+## Major Features and Improvements
+
+- **Key-value execution tags**: Enhanced execution tagging capabilities. (#5453)
+- **Watch agent metadata service**: Improved monitoring of agent metadata. (#5017)
+- **Distributed job refactoring**: Refactored distributed job using common ReplicaSpec. (#5355)
+- **Execution cluster label inheritance**: Executions now inherit cluster labels from source executions. (#5431)
+- **Domain API**: Added new API to retrieve domain information. (#5443)
+- **Execution environment versioning**: Added version to ExecutionEnv proto message. (#5506)
+- **OTLP and sampling in otelutils**: Enhanced observability with OpenTelemetry Protocol (OTLP) support. (#5504)
+
+## Notable Changes
+
+- Removed mmcloud plugin. (#5468)
+- Updated k3s version to 1.29.0. (#5475)
+- Improved auth flow to support custom base URLs in deployments. (#5192)
+- Added flyteconsole URL to FlyteWorkflow CRD. (#5449)
+- Enhanced 'flytectl compile' to consider launch plans within workflows. (#5463)
+- Introduced control over task phases for log link display. (#4726)
+- Improved Helm chart configurations for Prometheus, Webhook HPA, and Flyteconsole probes. (#5508)
+
+## Bug Fixes and Optimizations
+
+- Fixed flaky auto_refresh_test. (#5438)
+- Corrected NULL to empty string in SQL insert for migrations. (#5482)
+- Resolved issues with broken mermaid diagrams in documentation. (#5498)
+- Fixed Ray plugin to use default service account if not set in task metadata. (#5499)
+
+## Documentation and Usability
+
+- Updated community page. (#5496)
+- Improved documentation on logging link lifetime configuration. (#5503)
+- Replaced 'uctl' with 'flytectl' in OSS docs for consistency. (#5501)
+
+## New Contributors
+
+- @Dlougach, @flixr, @trevormcguire, @va6996, @mhotan, and @eltociear made their first contributions to the project.
+
+For a complete list of changes, please refer to the [full changelog](https://github.com/flyteorg/flyte/compare/flytectl/v0.8.24...v1.13.0-rc0).
diff --git a/CHANGELOG/CHANGELOG-v1.13.0.md b/CHANGELOG/CHANGELOG-v1.13.0.md
@@ -0,0 +1,55 @@
+# Flyte 1.13.0 Release Notes
+
+Flyte 1.13.0 brings a host of new features, optimizations, and fixes, enhancing the platform's functionality and user experience. This release also welcomes several new contributors to the Flyte community. Below are the highlights of this release.
+
+## 🚀 New Features & Improvements
+
+1. **Features**
+   - [Watch agent metadata service](https://github.com/flyteorg/flyte/pull/5017)
+   - [Allow controlling in which task phases log links are shown](https://github.com/flyteorg/flyte/pull/4726)
+   - [Add executionClusterLabel](https://github.com/flyteorg/flyte/pull/5394)
+
+2. **Helm Charts & Manifests**
+   - [Grafana dashboard updates](https://github.com/flyteorg/flyte/pull/5255) for better monitoring and observability.
+   - [Core helm chart updates for propeller configuration of agent service](https://github.com/flyteorg/flyte/pull/5402).
+   - [Helm chart updates related to Prometheus, Webhook HPA, and Flyteconsole probes](https://github.com/flyteorg/flyte/pull/5508).
+
+3. **Documentation Improvements**
+   - Added examples and guides, including [a guide for setting up the OpenAI batch agent backend](https://github.com/flyteorg/flyte/pull/5291) and [updated Optimization Performance docs](https://github.com/flyteorg/flyte/pull/5278).
+   - [Fix doc link to testing agent on local cluster](https://github.com/flyteorg/flyte/pull/5398) and [replace Azure AD OIDC URL](https://github.com/flyteorg/flyte/pull/4075).
+   - [Clarify networking configurations between data plane propeller and control plane data catalog in multi-cluster deployments](https://github.com/flyteorg/flyte/pull/5345).
+
+4. **Performance & Bug Fixes**
+   - [Fix cache and token management issues](https://github.com/flyteorg/flyte/pull/5388), including addressing auto-refresh cache race conditions and improving token refresh mechanisms.
+   - [Fix SQL insert operations](https://github.com/flyteorg/flyte/pull/5482) to handle NULL values appropriately.
+   - [Keep EnvFrom from pod template](https://github.com/flyteorg/flyte/pull/5423)
+   - [Miscellaneous fixes](https://github.com/flyteorg/flyte/pull/5416) for pod templates, Kubeflow webhook errors, and logging configurations.
+
+5. **Flytectl**
+   - [Add prefetch functionality for paginator](https://github.com/flyteorg/flyte/pull/5310)
+   - [Fix `upgrade` and `version` commands](https://github.com/flyteorg/flyte/pull/5470)
+
+
+## 🔧 Housekeeping & Deprecations
+
+1. **Miscellaneous**
+   - [Add OTLP and sampling to otelutils](https://github.com/flyteorg/flyte/pull/5504)
+   - [Refactor distributed job using common ReplicaSpec](https://github.com/flyteorg/flyte/pull/5355)
+   - [Fix typos using codespell CI job](https://github.com/flyteorg/flyte/pull/5418).
+   - [Remove obsolete Flyte config files](https://github.com/flyteorg/flyte/pull/5495).
+   - [Upgraded various dependencies](https://github.com/flyteorg/flyte/pull/5313), including Golang, Docker, and others.
+   - [Move to upstream mockery](https://github.com/flyteorg/flyte/pull/4937), which simplifies the migration to mockery-v2 
+   - [Key-value execution tags](https://github.com/flyteorg/flyte/pull/5453)
+
+## 🆕 New Contributors
+
+- **@zychen5186** for [adding prefetch functionality for paginator](https://github.com/flyteorg/flyte/pull/5310).
+- **@EraYaN** for [replacing Azure AD OIDC URL](https://github.com/flyteorg/flyte/pull/4075).
+- **@Dlougach** for [making BaseURL insensitive to trailing slashes for metadata endpoint redirect](https://github.com/flyteorg/flyte/pull/5458).
+- **@flixr** for [honoring redoc.enabled=false in charts](https://github.com/flyteorg/flyte/pull/5452).
+- **@trevormcguire** for [including group in apiVersion in plugin_collector](https://github.com/flyteorg/flyte/pull/5457).
+- **@va6996** for [inheriting execution cluster label from source execution](https://github.com/flyteorg/flyte/pull/5431).
+- **@mhotan** for [Helm chart updates related to Prometheus, Webhook HPA, and Flyteconsole probes](https://github.com/flyteorg/flyte/pull/5508).
+- **@eltociear** for [updating token_source.go](https://github.com/flyteorg/flyte/pull/5396).
+
+**[Full Changelog](https://github.com/flyteorg/flyte/compare/v1.12.0...v1.13.0)**
diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md
@@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment
 | configuration.auth.oidc.clientId | string | `""` |  |
 | configuration.auth.oidc.clientSecret | string | `""` |  |
 | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` |  |
-| configuration.co-pilot.image.tag | string | `"v1.12.1-rc0"` |  |
+| configuration.co-pilot.image.tag | string | `"v1.13.0"` |  |
 | configuration.database.dbname | string | `"flyte"` |  |
 | configuration.database.host | string | `"127.0.0.1"` |  |
 | configuration.database.options | string | `"sslmode=disable"` |  |

diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
@@ -159,7 +159,7 @@ configuration:
       # repository CoPilot sidecar image repository
       repository: cr.flyte.org/flyteorg/flytecopilot  # FLYTECOPILOT_IMAGE
       # tag CoPilot sidecar image tag
-      tag: v1.12.1-rc0  # FLYTECOPILOT_TAG
+      tag: v1.13.0  # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
   agentService:
     defaultAgent: