diff --git a/, b/, deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.gitattributes b/.gitattributes index e4b260b693..cbaca9f35c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,5 @@ docs/**/*html linguist-generated=true +flyteidl/gen/** linguist-generated=true +flyteidl/protos/**/*.rst linguist-generated=true +flyteidl/clients/go/assets/admin.swagger.json linguist-generated=true + diff --git a/.github/workflows/lite-image-manual.yml b/.github/workflows/lite-image-manual.yml deleted file mode 100644 index 8b5831b0e2..0000000000 --- a/.github/workflows/lite-image-manual.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Manually push - -on: - workflow_dispatch: - inputs: - flyte_version: - description: 'flyte version' - required: true - default: 'latest' - type: string - -jobs: - sandbox-lite-build-dind: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set flyte version to release - id: set_version - run: | - echo ::set-output name=flyte_version::$(echo ${{ github.event.inputs.flyte_version }}) - - name: Prepare DIND Image Names - id: dind-names - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/${{ github.repository_owner }}/flyte-sandbox-lite - tags: | - ${{ steps.set_version.outputs.flyte_version }} - type=sha,format=long, prefix=dind- - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-single-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-single-buildx - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - - name: Build and push DIND Image - uses: docker/build-push-action@v2 - with: - context: . - platforms: linux/arm64, linux/amd64 - push: true - target: dind - tags: ${{ steps.dind-names.outputs.tags }} - build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" - file: Dockerfile.sandbox-lite - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index dc78d4b559..5b9a6056a4 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -62,60 +62,3 @@ jobs: tags: ${{ steps.dind-names.outputs.tags }} build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" file: docker/sandbox/Dockerfile - - trigger-sandbox-lite-build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Setup Golang caches - uses: actions/cache@v3 - with: - path: | - /root/.cache/go-build - /root/go/pkg/mod - key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }} - restore-keys: | - ${{ runner.os }}-golang- - - name: Set flyte version to release - id: set_version - run: | - if [ ${{ github.event_name}} = "release" ]; then - echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }}) - else - echo ::set-output name=flyte_version::latest - fi - - name: Prepare DIND Image Names - id: dind-names - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/${{ github.repository_owner }}/flyte-sandbox-lite - tags: | - ${{ steps.set_version.outputs.flyte_version }} - type=sha,format=long - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Login to GitHub Container Registry - if: ${{ github.event_name == 'release' }} - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - - name: Build and push DIND Image - uses: docker/build-push-action@v2 - with: - context: . - platforms: linux/arm64, linux/amd64 - push: ${{ github.event_name == 'release' }} - target: dind - tags: ${{ steps.dind-names.outputs.tags }} - build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" - file: Dockerfile.sandbox-lite diff --git a/.gitignore b/.gitignore index a8e78b52d6..301bf266a8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,8 +20,6 @@ __pycache__/ /charts/flyteagent/Chart.lock /charts/flyte-deps/Chart.lock /charts/flyte-core/Chart.lock -/docker/sandbox-lite/storage/charts -/docker/sandbox-lite/storage/Chart.lock **/*.bak- .python-version dist/ diff --git a/Dockerfile.sandbox-lite b/Dockerfile.sandbox-lite deleted file mode 100644 index 095c83b6e1..0000000000 --- a/Dockerfile.sandbox-lite +++ /dev/null @@ -1,115 +0,0 @@ -# syntax=docker/dockerfile:1.3 - - -ARG FLYTE_VERSION="latest" -FROM ghcr.io/flyteorg/flyteconsole-release:${FLYTE_VERSION} AS flyteconsole - -FROM --platform=${BUILDPLATFORM} golang:1.19.0-alpine3.16 AS go_builder - -ARG TARGETARCH -ENV GOARCH=${TARGETARCH} -ENV GOOS=linux - -# Install dependencies -RUN apk add --no-cache build-base - -COPY go.mod go.sum /app/flyte/ -WORKDIR /app/flyte - -COPY datacatalog datacatalog -COPY flyteadmin flyteadmin -COPY flytecopilot flytecopilot -COPY flyteidl flyteidl -COPY flyteplugins flyteplugins -COPY flytepropeller flytepropeller -COPY flytestdlib flytestdlib - -RUN go mod download - -COPY --from=flyteconsole /app/ cmd/single/dist - -COPY cmd/ /app/flyte/cmd/ -RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod go build -tags console -v -o /flyte cmd/main.go - -FROM alpine:3.15 AS base - -# Install dependencies -RUN apk add --no-cache openssl - -# Make directory to store artifacts -RUN mkdir -p /flyteorg/bin /flyteorg/share - -# Install k3s -ARG K3S_VERSION="v1.21.1%2Bk3s1" -ARG TARGETARCH - -RUN case $TARGETARCH in \ - amd64) export SUFFIX=;; \ - arm64) export SUFFIX=-arm64;; \ - aarch64) export SUFFIX=-arm64;; \ - # TODO: Check if we need to add case fail - esac; \ - wget -q -O /flyteorg/bin/k3s https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s${SUFFIX} \ - && chmod +x /flyteorg/bin/k3s - -# Install Helm -ARG HELM_VERSION="v3.6.3" - -RUN wget -q -O /flyteorg/bin/get_helm.sh https://raw.githubusercontent.com/helm/helm/${HELM_VERSION}/scripts/get-helm-3 && \ - chmod 700 /flyteorg/bin/get_helm.sh && \ - sh /flyteorg/bin/get_helm.sh --version ${HELM_VERSION} && \ - mv /usr/local/bin/helm /flyteorg/bin/helm && \ - rm /flyteorg/bin/get_helm.sh - -# Install flytectl -RUN wget -q -O - https://raw.githubusercontent.com/flyteorg/flytectl/master/install.sh | BINDIR=/flyteorg/bin sh -s - -# Install buildkit-cli-for-kubectl -COPY --from=go_builder /flyte /flyteorg/bin/ - -# Copy flyte chart -COPY charts/flyte-deps/ /flyteorg/share/flyte-deps - -# Copy scripts -COPY docker/sandbox/kubectl docker/sandbox/cgroup-v2-hack.sh /flyteorg/bin/ - -# Copy Flyte config -COPY flyte.yaml /flyteorg/share/flyte.yaml - -FROM docker:20.10.14-dind-alpine3.15 AS dind - -# Install dependencies -RUN apk add --no-cache bash git make tini curl jq - -# Copy artifacts from base -COPY --from=base /flyteorg/ /flyteorg/ - -# Copy entrypoints -COPY docker/sandbox-lite/flyte-entrypoint-dind.sh /flyteorg/bin/flyte-entrypoint.sh - -# Copy cluster resource templates -COPY docker/sandbox-lite/templates/ /etc/flyte/clusterresource/templates/ - -ENV FLYTE_VERSION "${FLYTE_VERSION}" - -ARG FLYTE_TEST="release" -ENV FLYTE_TEST "${FLYTE_TEST}" -ENV FLYTE_DEV "False" - -RUN addgroup -S docker - -# Update PATH variable -ENV PATH "/flyteorg/bin:${PATH}" -ENV POD_NAMESPACE "flyte" - -# Declare volumes for k3s -VOLUME /var/lib/kubelet -VOLUME /var/lib/rancher/k3s -VOLUME /var/lib/cni -VOLUME /var/log - -# Expose Flyte ports -# 30080 for console, 30081 for gRPC, 30082 for k8s dashboard, 30084 for minio api, 30088 for minio console -EXPOSE 30080 30081 30082 30084 30088 30089 - -ENTRYPOINT ["tini", "flyte-entrypoint.sh"] diff --git a/charts/flyte-core/templates/_helpers.tpl b/charts/flyte-core/templates/_helpers.tpl index 2c3b059841..98a42655e3 100755 --- a/charts/flyte-core/templates/_helpers.tpl +++ b/charts/flyte-core/templates/_helpers.tpl @@ -247,7 +247,7 @@ storage: region: us-east-1 signedUrl: stowConfigOverride: - endpoint: http://localhost:30084 + endpoint: http://minio.{{ .Release.Namespace }}.svc.cluster.local:9000 {{- else if eq .Values.storage.type "custom" }} {{- with .Values.storage.custom -}} {{ tpl (toYaml .) $ | nindent 2 }} diff --git a/charts/flyte-core/templates/common/ingress.yaml b/charts/flyte-core/templates/common/ingress.yaml index 69f7c4ae07..d812f2abe3 100644 --- a/charts/flyte-core/templates/common/ingress.yaml +++ b/charts/flyte-core/templates/common/ingress.yaml @@ -7,154 +7,90 @@ - path: /flyteidl.service.SignalService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.SignalService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.DataProxyService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.DataProxyService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.AuthMetadataService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.AuthMetadataService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.IdentityService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.IdentityService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /grpc.health.v1.Health pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /grpc.health.v1.Health/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} {{- end }} {{- if .Values.common.ingress.enabled }} -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} apiVersion: networking.k8s.io/v1 -{{- else }} -apiVersion: networking.k8s.io/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ template "flyte.name" . }} @@ -171,288 +107,173 @@ spec: - path: /* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: ssl-redirect port: name: use-annotation -{{- else }} - serviceName: ssl-redirect - servicePort: use-annotation -{{- end }} {{- end }} # This is useful only for frontend development {{- if .Values.common.ingress.webpackHMR }} - path: /__webpack_hmr pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteconsole port: number: 80 -{{- else }} - serviceName: flyteconsole - servicePort: 80 -{{- end }} {{- end }} # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml - path: /console pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteconsole port: number: 80 -{{- else }} - serviceName: flyteconsole - servicePort: 80 -{{- end }} - path: /console/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteconsole port: number: 80 -{{- else }} - serviceName: flyteconsole - servicePort: 80 -{{- end }} - path: /api pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /api/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /healthcheck pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /v1/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} {{- if .Values.deployRedoc }} # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: redoc port: number: 87 -{{- else }} - serviceName: redoc - servicePort: 87 -{{- end }} # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: redoc port: number: 87 -{{- else }} - serviceName: redoc - servicePort: 87 -{{- end }} {{- end }} - path: /.well-known pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /.well-known/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /login pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /login/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /logout pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /logout/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /callback pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /callback/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /me pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /config pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /config/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /oauth2 pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /oauth2/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} {{- if not .Values.common.ingress.separateGrpcIngress }} {{- include "grpcRoutes" . | nindent 10 -}} {{- end }} @@ -469,11 +290,7 @@ spec: # Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only # enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). --- -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} apiVersion: networking.k8s.io/v1 -{{- else }} -apiVersion: networking.k8s.io/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ template "flyte.name" . }}-grpc diff --git a/charts/flyteagent/README.md b/charts/flyteagent/README.md index f889c095bd..d862716673 100644 --- a/charts/flyteagent/README.md +++ b/charts/flyteagent/README.md @@ -20,7 +20,7 @@ A Helm chart for Flyte agent | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | image.repository | string | `"ghcr.io/flyteorg/flyteagent"` | Docker image for flyteagent deployment | -| image.tag | string | `"1.10.7"` | Docker image tag | +| image.tag | string | `"1.10.8b4"` | Docker image tag | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | nodeSelector for flyteagent deployment | | podAnnotations | object | `{}` | Annotations for flyteagent pods | @@ -30,6 +30,7 @@ A Helm chart for Flyte agent | ports.containerPort | int | `8000` | | | ports.name | string | `"agent-grpc"` | | | priorityClassName | string | `""` | Sets priorityClassName for datacatalog pod(s). | +| readinessProbe | object | `{"grpc":{"port":8000},"initialDelaySeconds":1,"periodSeconds":3}` | https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out | | replicaCount | int | `1` | Replicas count for flyteagent deployment | | resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"200Mi"},"requests":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"200Mi"}}` | Default resources requests and limits for flyteagent deployment | | securityContext | object | `{"allowPrivilegeEscalation":false}` | Security context for container | diff --git a/charts/flyteagent/templates/agent/deployment.yaml b/charts/flyteagent/templates/agent/deployment.yaml index 2c6a903f8f..caad9ca44a 100644 --- a/charts/flyteagent/templates/agent/deployment.yaml +++ b/charts/flyteagent/templates/agent/deployment.yaml @@ -39,6 +39,10 @@ spec: ports: - containerPort: {{ .Values.ports.containerPort }} name: {{ .Values.ports.name }} + readinessProbe: + {{- with .Values.readinessProbe -}} + {{ tpl (toYaml .) $ | nindent 10 }} + {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} resources: {{- toYaml .Values.resources | nindent 10 }} diff --git a/charts/flyteagent/values.yaml b/charts/flyteagent/values.yaml index aee84dc2b2..266ec9c0eb 100755 --- a/charts/flyteagent/values.yaml +++ b/charts/flyteagent/values.yaml @@ -23,7 +23,7 @@ image: # -- Docker image for flyteagent deployment repository: ghcr.io/flyteorg/flyteagent # -- Docker image tag - tag: 1.10.7 # FLYTEAGENT_TAG + tag: 1.10.8b4 # FLYTEAGENT_TAG # -- Docker image pull policy pullPolicy: IfNotPresent ports: @@ -56,6 +56,13 @@ serviceAccount: imagePullSecrets: [] # -- Security context for pod podSecurityContext: {} +# -- Readiness probe for flyteagent. Use readinessProbe: {} if agent doesn't implement grpc-health-checking service. +# -- https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out +readinessProbe: + grpc: + port: 8000 + initialDelaySeconds: 1 + periodSeconds: 3 # -- Security context for container securityContext: allowPrivilegeEscalation: false diff --git a/deployment/agent/flyte_agent_helm_generated.yaml b/deployment/agent/flyte_agent_helm_generated.yaml index 46762b4cff..4e78b991ac 100644 --- a/deployment/agent/flyte_agent_helm_generated.yaml +++ b/deployment/agent/flyte_agent_helm_generated.yaml @@ -78,7 +78,7 @@ spec: - pyflyte - serve - agent - image: "ghcr.io/flyteorg/flyteagent:1.10.7" + image: "ghcr.io/flyteorg/flyteagent:1.10.8b4" imagePullPolicy: "IfNotPresent" name: flyteagent volumeMounts: @@ -87,6 +87,11 @@ spec: ports: - containerPort: 8000 name: agent-grpc + readinessProbe: + grpc: + port: 8000 + initialDelaySeconds: 1 + periodSeconds: 3 securityContext: allowPrivilegeEscalation: false resources: diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index c398c5fbf8..1959cc5bd8 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -326,7 +326,7 @@ data: region: us-east-1 signedUrl: stowConfigOverride: - endpoint: http://localhost:30084 + endpoint: http://minio.flyte.svc.cluster.local:9000 enable-multicontainer: false limits: maxDownloadMBs: 10 @@ -512,7 +512,7 @@ data: region: us-east-1 signedUrl: stowConfigOverride: - endpoint: http://localhost:30084 + endpoint: http://minio.flyte.svc.cluster.local:9000 enable-multicontainer: false limits: maxDownloadMBs: 10 @@ -674,7 +674,7 @@ data: region: us-east-1 signedUrl: stowConfigOverride: - endpoint: http://localhost:30084 + endpoint: http://minio.flyte.svc.cluster.local:9000 enable-multicontainer: false limits: maxDownloadMBs: 10 @@ -6684,7 +6684,7 @@ spec: template: metadata: annotations: - configChecksum: "45f0232531c0d1494809cf83387a95b2fc802019ea095de7a24ccd4f8de86ec" + configChecksum: "8702cc3d688d44938e6974b267f70fb01ce7fec4780de86b8f2e65f9446c711" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -6984,7 +6984,7 @@ spec: template: metadata: annotations: - configChecksum: "8feeaa7f7ec6506426db0d3e3cda6bc3ac0049a7eeba49d6cce62b3e1c7c424" + configChecksum: "ccdd0d27618b8053a8ae11046fd2b84b9a397144dd81c7113f398cddf001397" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -7075,7 +7075,7 @@ spec: template: metadata: annotations: - configChecksum: "45f0232531c0d1494809cf83387a95b2fc802019ea095de7a24ccd4f8de86ec" + configChecksum: "8702cc3d688d44938e6974b267f70fb01ce7fec4780de86b8f2e65f9446c711" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte @@ -7170,7 +7170,7 @@ spec: template: metadata: annotations: - configChecksum: "fa64b9672600e5eeed8ca407828a7490b17dadfe16c273e2aacfb52d9b5b085" + configChecksum: "a82c947246fac99ad98c86bc01cdd99569b2ea11f25d3e1361cd40d145bed87" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -7244,7 +7244,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.11.0-b1 annotations: - configChecksum: "fa64b9672600e5eeed8ca407828a7490b17dadfe16c273e2aacfb52d9b5b085" + configChecksum: "a82c947246fac99ad98c86bc01cdd99569b2ea11f25d3e1361cd40d145bed87" spec: securityContext: fsGroup: 65534 diff --git a/docker/sandbox-bundled/README.md b/docker/sandbox-bundled/README.md deleted file mode 100644 index 17ca62b002..0000000000 --- a/docker/sandbox-bundled/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Flyte Deployment Sandbox - -First make images -``` -ytong@Yees-MBP:~/go/src/github.com/flyteorg/flyte/docker/sandbox-ultra [flyte-sandbox] (cicd-sandbox-lite) $ make images -``` - -then build the k3s image. -``` -ytong@Yees-MBP:~/go/src/github.com/flyteorg/flyte/docker/sandbox-ultra [] (cicd-sandbox-lite) $ docker buildx build --file images/dockerfiles/k3s/Dockerfile --platform linux/arm64,linux/amd64 --push --tag ghcr.io/flyteorg/flyte-sandbox-lite:ultra7 . -``` diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index bc96ffac8f..60577c5716 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -816,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: YXFGVndTaXBySjFxWEIxNA== + haSharedSecret: UVdVTnB4cXBMVXMyRjhGUw== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1412,7 +1412,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: f5132fa475eb2386265cbda64d4e6d56922b771a5f7f5e6e8e4536a712663e45 + checksum/secret: 7dd7be244652f0c2d4d5651db7e9d879fb74fb3a4407a80c48cba3dcd7b74e53 labels: app: docker-registry release: flyte-sandbox @@ -1755,12 +1755,17 @@ spec: value: minio - name: FLYTE_AWS_SECRET_ACCESS_KEY value: miniostorage - image: ghcr.io/flyteorg/flyteagent:1.10.7 + image: ghcr.io/flyteorg/flyteagent:1.10.8b4 imagePullPolicy: IfNotPresent name: flyteagent ports: - containerPort: 8000 name: agent-grpc + readinessProbe: + grpc: + port: 8000 + initialDelaySeconds: 1 + periodSeconds: 3 resources: limits: cpu: 500m diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index 36aca312f3..f148ebf1c8 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -796,7 +796,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: QVo4T0pQZmZLcFZmNjIyNQ== + haSharedSecret: cnBScDd2Y3Y0a2JlcHlzVQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1360,7 +1360,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: f5e2b9fbf28a7d7815a094f72366cac395bd0aacd7890faa07e80ec700770fc5 + checksum/secret: 459fe43b90e7be7fce10bc94020ba94037c1a21501b61d4e7121c84251f1b198 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 80e06fc6a2..84e16dcaba 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: dktYanZDU0tsa0hTWlZmWA== + haSharedSecret: NlJkVzYyZ1MzZmZIRE1nOA== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 7956909ef2268e1aad9b4b132d938c8b1a5775321bacfaf61aea7b73cdbcc85e + checksum/secret: 2e417b7dd337346d064d1cd4edf6b5624d335fce92ccfe22acd20485be450ec9 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-lite/flyte-entrypoint-dind.sh b/docker/sandbox-lite/flyte-entrypoint-dind.sh deleted file mode 100755 index 0585624970..0000000000 --- a/docker/sandbox-lite/flyte-entrypoint-dind.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/sh - -set -euo pipefail - -# Apply cgroup v2 hack -cgroup-v2-hack.sh - -trap 'pkill -P $$' EXIT -FLYTE_TIMEOUT=${FLYTE_TIMEOUT:-600} - -monitor() { - while : ; do - for pid in $@ ; do - kill -0 $pid &> /dev/null || exit 1 - done - - sleep 1 - done -} - -# Start docker daemon -echo "Starting Docker daemon..." -file="/var/run/docker.pid" -if [ -f "$file" ] ; then - rm "$file" -fi -dockerd &> /var/log/dockerd.log & -DOCKERD_PID=$! -timeout "$FLYTE_TIMEOUT" sh -c "until docker info &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for dockerd to start"; exit 1 ) -echo "Done." - -# Start k3s -echo "Starting k3s cluster..." -KUBERNETES_API_PORT=${KUBERNETES_API_PORT:-6443} -k3s server --docker --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log & -K3S_PID=$! -timeout "$FLYTE_TIMEOUT" sh -c "until k3s kubectl get node $HOSTNAME &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to start"; exit 1 ) -k3s kubectl wait node $HOSTNAME --for condition=Ready --timeout ${FLYTE_TIMEOUT}s &> /dev/null || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to be ready"; exit 1 ) -echo "Done." - -# Deploy flyte -echo "Deploying Flyte..." -charts="/flyteorg/share/flyte-deps" -version="" - -if [[ $FLYTE_TEST = "release" ]] -then - helm repo add flyteorg https://flyteorg.github.io/flyte - helm fetch flyteorg/flyte-deps --version=$FLYTE_VERSION - version="--version $FLYTE_VERSION" - charts="flyteorg/flyte-deps" -fi - -if [[ $FLYTE_TEST = "local" ]] -then - helm dep update $charts -fi -helm upgrade -n flyte --create-namespace flyte $charts --kubeconfig /etc/rancher/k3s/k3s.yaml --install --wait - -timeout "$FLYTE_TIMEOUT" sh -c "until k3s kubectl get namespace flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Flyte namespace to be created"; exit 1 ) -timeout "$FLYTE_TIMEOUT" sh -c "until k3s kubectl rollout status deployment minio -n flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the minio rollout to be created"; exit 1 ) -timeout "$FLYTE_TIMEOUT" sh -c "until k3s kubectl rollout status deployment postgres -n flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the postgres rollout to be created"; exit 1 ) - -k3s kubectl wait --for=condition=available deployment/minio deployment/postgres -n flyte --timeout=5m || ( echo >&2 "Timed out while waiting for the Flyte deployment to start"; exit 1 ) -# Create directory to store certificate -mkdir -p /tmp/k8s-webhook-server/serving-certs - -# With flytectl sandbox --source flag, we mount the root volume to user source dir that will create helm & k8s cache specific directory. -# In Linux, These file belongs to root user that is different then current user -# In this case during fast serialization, Pyflyte will through error because of permission denied -rm -rf /root/.cache /root/.kube /root/.config - -if [[ $FLYTE_DEV = "True" ]] -then - # Namespaces must be manually created since cluster resource manager is disabled by default - k3s kubectl create ns flytesnacks-development - # Monitor running processes. Exit when the first process exits. - monitor ${DOCKERD_PID} ${K3S_PID} -else - flyte start --config /flyteorg/share/flyte.yaml & - FLYTE_PID=$! - # Monitor running processes. Exit when the first process exits. - monitor ${DOCKERD_PID} ${K3S_PID} ${FLYTE_PID} -fi diff --git a/docker/sandbox-lite/templates/namespace.yaml b/docker/sandbox-lite/templates/namespace.yaml deleted file mode 100644 index 90b53893ac..0000000000 --- a/docker/sandbox-lite/templates/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ namespace }} -spec: - finalizers: - - kubernetes \ No newline at end of file diff --git a/docker/sandbox-lite/templates/project_resource_quota.yaml b/docker/sandbox-lite/templates/project_resource_quota.yaml deleted file mode 100644 index 12e7711503..0000000000 --- a/docker/sandbox-lite/templates/project_resource_quota.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ResourceQuota -metadata: - name: project-quota - namespace: {{ namespace }} -spec: - hard: - limits.cpu: {{ projectQuotaCpu }} - limits.memory: {{ projectQuotaMemory }} \ No newline at end of file diff --git a/docs/deployment/configuration/general.rst b/docs/deployment/configuration/general.rst index b5278365d2..0b97f8b8ff 100644 --- a/docs/deployment/configuration/general.rst +++ b/docs/deployment/configuration/general.rst @@ -202,10 +202,10 @@ An example PodTemplate is shown: template: metadata: labels: - - foo + foo: from-pod-template annotations: - - foo: initial-value - - bar: initial-value + foo: initial-value + bar: initial-value spec: containers: - name: default @@ -221,10 +221,10 @@ Pod Labels, Annotations, and enables the host networking. plugins: k8s: default-labels: - - bar + bar: from-default-label default-annotations: - - foo: overridden-value - - baz: non-overridden-value + foo: overridden-value + baz: non-overridden-value enable-host-networking-pod: true To construct a Pod, FlytePropeller initializes a Pod definition using the default @@ -241,19 +241,19 @@ The resultant Pod using the above default PodTemplate and K8s Plugin configurati name: example-pod namespace: flytesnacks-development labels: - - foo // maintained initial value - - bar // value appended by k8s plugin configuration + foo: from-pod-template # maintained initial value + bar: from-default-label # value appended by k8s plugin configuration annotations: - - foo: overridden-value // value overridden by k8s plugin configuration - - bar: initial-value // maintained initial value - - baz: non-overridden-value // value added by k8s plugin configuration + foo: overridden-value # value overridden by k8s plugin configuration + bar: initial-value # maintained initial value + baz: non-overridden-value # value added by k8s plugin configuration spec: containers: - name: ax9kd5xb4p8r45bpdv7v-n0-0 image: ghcr.io/flyteorg/flytecookbook:core-bfee7e549ad749bfb55922e130f4330a0ebc25b0 terminationMessagePath: "/dev/foo" - // remaining container configuration omitted - hostNetwork: true // overridden by the k8s plugin configuration + # remaining container configuration omitted + hostNetwork: true # overridden by the k8s plugin configuration The last step in constructing a Pod is to apply any task-specific configuration. These options follow the same rules as merging the default PodTemplate and K8s @@ -280,8 +280,8 @@ of the task. For example: template: metadata: annotations: - - annotation_1: initial-value - - bar: initial-value + annotation_1: initial-value + bar: initial-value spec: containers: - name: default @@ -328,12 +328,12 @@ The resultant Pod is as follows: name: example-pod namespace: flytesnacks-development labels: - - label_1: value-1 # from Compile-time value - - label_2: value-2 # from Compile-time value + label_1: value-1 # from Compile-time value + label_2: value-2 # from Compile-time value annotations: - - annotation_1: value-1 # value overridden by Compile-time PodTemplate - - annotation_2: value-2 # from Compile-time PodTemplate - - bar: initial-value # from Runtime PodTemplate + annotation_1: value-1 # value overridden by Compile-time PodTemplate + annotation_2: value-2 # from Compile-time PodTemplate + bar: initial-value # from Runtime PodTemplate spec: containers: - name: default @@ -343,7 +343,7 @@ The resultant Pod is as follows: image: a.b.c/image:v1 command: cmd args: [] - // remaining container configuration omitted + # remaining container configuration omitted Notice how options follow the same merging rules, i.e. lists append and maps override. @@ -398,12 +398,12 @@ And a Runtime PodTemplate: template: metadata: labels: - - label_1: value-runtime - - label_2: value-runtime - - label_3: value-runtime + label_1: value-runtime + label_2: value-runtime + label_3: value-runtime annotations: - - foo: value-runtime - - bar: value-runtime + foo: value-runtime + bar: value-runtime spec: containers: - name: default @@ -418,10 +418,10 @@ And the following K8s Plugin Configuration: plugins: k8s: default-labels: - - label_1: value-plugin + label_1: value-plugin default-annotations: - - annotation_1: value-plugin - - baz: value-plugin + annotation_1: value-plugin + baz: value-plugin The resultant pod for that task is as follows: @@ -433,14 +433,14 @@ The resultant pod for that task is as follows: name: example-pod namespace: flytesnacks-development labels: - - label_1: value-plugin - - label_2: value-compile + label_1: value-plugin + label_2: value-compile annotations: - - annotation_1: value-plugin - - annotation_2: value-compile - - foo: value-runtime - - bar: value-runtime - - baz: value-plugin + annotation_1: value-plugin + annotation_2: value-compile + foo: value-runtime + bar: value-runtime + baz: value-plugin spec: containers: - name: default @@ -450,4 +450,4 @@ The resultant pod for that task is as follows: image: a.b.c/image:v1 command: cmd args: [] - // remaining container configuration omitted + # remaining container configuration omitted diff --git a/flyteadmin/pkg/manager/impl/validation/task_validator.go b/flyteadmin/pkg/manager/impl/validation/task_validator.go index c50e3311b3..02f2dcded0 100644 --- a/flyteadmin/pkg/manager/impl/validation/task_validator.go +++ b/flyteadmin/pkg/manager/impl/validation/task_validator.go @@ -166,6 +166,12 @@ func isWholeNumber(quantity resource.Quantity) bool { func resourceListToQuantity(resources corev1.ResourceList) map[core.Resources_ResourceName]resource.Quantity { var requestedToQuantity = make(map[core.Resources_ResourceName]resource.Quantity) for name, quantity := range resources { + // The name to refer to ephemeral storage defined in k8s (https://github.com/kubernetes/api/blob/05aa4bceed70af2652698a28fb144ee22b2dd2ba/core/v1/types.go#L5988) + // is different from the name defined in Flyte's proto (https://github.com/flyteorg/flyte/blob/fd42f65660069d9c164cda2de579d3a89cac5b0f/flyteidl/protos/flyteidl/core/tasks.proto#L25). + // This is a workaround to handle the conversion. + if name == corev1.ResourceEphemeralStorage { + name = corev1.ResourceName(core.Resources_EPHEMERAL_STORAGE.String()) + } resourceName := core.Resources_ResourceName(core.Resources_ResourceName_value[strings.ToUpper(name.String())]) requestedToQuantity[resourceName] = quantity } diff --git a/flyteadmin/pkg/manager/impl/validation/task_validator_test.go b/flyteadmin/pkg/manager/impl/validation/task_validator_test.go index c82d2e5f58..108a651517 100644 --- a/flyteadmin/pkg/manager/impl/validation/task_validator_test.go +++ b/flyteadmin/pkg/manager/impl/validation/task_validator_test.go @@ -278,6 +278,11 @@ func TestResourceListToQuantity(t *testing.T) { gpuQuantity := gpuResources[core.Resources_CPU] val = gpuQuantity.Value() assert.Equal(t, val, int64(2)) + + ephemeralStorageResources := resourceListToQuantity(corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("500Mi")}) + ephemeralStorageQuantity := ephemeralStorageResources[core.Resources_EPHEMERAL_STORAGE] + val = ephemeralStorageQuantity.Value() + assert.Equal(t, val, int64(524288000)) } func TestRequestedResourcesToQuantity(t *testing.T) { diff --git a/flyteidl/.gitattributes b/flyteidl/.gitattributes deleted file mode 100644 index a2236d5f13..0000000000 --- a/flyteidl/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -gen/** linguist-generated=true -protos/**/*.rst linguist-generated=true diff --git a/flyteidl/README.md b/flyteidl/README.md index a1be04d742..67685f45b7 100644 --- a/flyteidl/README.md +++ b/flyteidl/README.md @@ -7,7 +7,7 @@ This is one of the core repositories of Flyte. It contains the Specification of * [flyte.org](https://flyte.org) * [Flyte Docs](http://docs.flyte.org) -* [Flyteidl API reference documentation](https://docs.flyte.org/projects/flyteidl/en/stable/index.html) +* [Flyteidl API reference documentation](https://docs.flyte.org/en/latest/reference_flyteidl.html) ## Contributing to Flyteidl diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index 7b9045cbe3..64a6af5899 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -7908,7 +7908,7 @@ "properties": { "statement": { "type": "string", - "title": "The actual query to run, the query can have templated parameters.\nWe use Flyte's Golang templating format for Query templating.\nRefer to the templating documentation.\nhttps://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/hive.html#sphx-glr-auto-integrations-external-services-hive-hive-py\nFor example,\ninsert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet\nselect *\nfrom my_table\nwhere ds = '{{ .Inputs.ds }}'" + "title": "The actual query to run, the query can have templated parameters.\nWe use Flyte's Golang templating format for Query templating.\nFor example,\ninsert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet\nselect *\nfrom my_table\nwhere ds = '{{ .Inputs.ds }}'" }, "dialect": { "$ref": "#/definitions/SqlDialect" diff --git a/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts index afd4e5f98b..5cc011314c 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts @@ -1180,8 +1180,6 @@ export class Sql extends Message { /** * The actual query to run, the query can have templated parameters. * We use Flyte's Golang templating format for Query templating. - * Refer to the templating documentation. - * https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/hive.html#sphx-glr-auto-integrations-external-services-hive-hive-py * For example, * insert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet * select * diff --git a/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go b/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go index 92a40b8e01..122ddce559 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go @@ -1514,8 +1514,6 @@ type Sql struct { // The actual query to run, the query can have templated parameters. // We use Flyte's Golang templating format for Query templating. - // Refer to the templating documentation. - // https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/hive.html#sphx-glr-auto-integrations-external-services-hive-hive-py // For example, // insert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet // select * diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index 7b9045cbe3..64a6af5899 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -7908,7 +7908,7 @@ "properties": { "statement": { "type": "string", - "title": "The actual query to run, the query can have templated parameters.\nWe use Flyte's Golang templating format for Query templating.\nRefer to the templating documentation.\nhttps://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/hive.html#sphx-glr-auto-integrations-external-services-hive-hive-py\nFor example,\ninsert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet\nselect *\nfrom my_table\nwhere ds = '{{ .Inputs.ds }}'" + "title": "The actual query to run, the query can have templated parameters.\nWe use Flyte's Golang templating format for Query templating.\nFor example,\ninsert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet\nselect *\nfrom my_table\nwhere ds = '{{ .Inputs.ds }}'" }, "dialect": { "$ref": "#/definitions/SqlDialect" diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index 2680d36bda..9a6175929f 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -1624,7 +1624,7 @@ "properties": { "statement": { "type": "string", - "title": "The actual query to run, the query can have templated parameters.\nWe use Flyte's Golang templating format for Query templating.\nRefer to the templating documentation.\nhttps://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/hive.html#sphx-glr-auto-integrations-external-services-hive-hive-py\nFor example,\ninsert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet\nselect *\nfrom my_table\nwhere ds = '{{ .Inputs.ds }}'" + "title": "The actual query to run, the query can have templated parameters.\nWe use Flyte's Golang templating format for Query templating.\nFor example,\ninsert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet\nselect *\nfrom my_table\nwhere ds = '{{ .Inputs.ds }}'" }, "dialect": { "$ref": "#/definitions/SqlDialect" diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json index cebc48b75b..f488a49c00 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json @@ -928,7 +928,7 @@ "properties": { "statement": { "type": "string", - "title": "The actual query to run, the query can have templated parameters.\nWe use Flyte's Golang templating format for Query templating.\nRefer to the templating documentation.\nhttps://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/hive.html#sphx-glr-auto-integrations-external-services-hive-hive-py\nFor example,\ninsert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet\nselect *\nfrom my_table\nwhere ds = '{{ .Inputs.ds }}'" + "title": "The actual query to run, the query can have templated parameters.\nWe use Flyte's Golang templating format for Query templating.\nFor example,\ninsert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet\nselect *\nfrom my_table\nwhere ds = '{{ .Inputs.ds }}'" }, "dialect": { "$ref": "#/definitions/SqlDialect" diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index a36add78f2..280140c75d 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -1729,8 +1729,6 @@ pub struct K8sObjectMetadata { pub struct Sql { /// The actual query to run, the query can have templated parameters. /// We use Flyte's Golang templating format for Query templating. - /// Refer to the templating documentation. - /// /// For example, /// insert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet /// select * diff --git a/flyteidl/index.rst b/flyteidl/index.rst index f06f0e0e50..bbe9037b82 100644 --- a/flyteidl/index.rst +++ b/flyteidl/index.rst @@ -17,12 +17,12 @@ This specification is used to generate client stubs for `Flytekit - |book-reader| User Guide - |chalkboard| Tutorials + |plane| Getting Started + |book-reader| User Guide + |chalkboard| Tutorials |project-diagram| Concepts |rocket| Deployment - |book| API Reference + |book| API Reference |hands-helping| Community .. NOTE: the caption text is important for the sphinx theme to correctly render the nav header diff --git a/flyteidl/protos/docs/core/core.rst b/flyteidl/protos/docs/core/core.rst index 43bf4c9c6e..dd3cf71341 100644 --- a/flyteidl/protos/docs/core/core.rst +++ b/flyteidl/protos/docs/core/core.rst @@ -2206,7 +2206,7 @@ Sql represents a generic sql workload with a statement and dialect. :header: "Field", "Type", "Label", "Description" :widths: auto - "statement", ":ref:`ref_string`", "", "The actual query to run, the query can have templated parameters. We use Flyte's Golang templating format for Query templating. Refer to the templating documentation. https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/hive.html#sphx-glr-auto-integrations-external-services-hive-hive-py For example, insert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet select * from my_table where ds = '{{ .Inputs.ds }}'" + "statement", ":ref:`ref_string`", "", "The actual query to run, the query can have templated parameters. We use Flyte's Golang templating format for Query templating. For example, insert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet select * from my_table where ds = '{{ .Inputs.ds }}'" "dialect", ":ref:`ref_flyteidl.core.Sql.Dialect`", "", "" diff --git a/flyteidl/protos/flyteidl/core/tasks.proto b/flyteidl/protos/flyteidl/core/tasks.proto index ab96615be7..20a1fa0cbf 100644 --- a/flyteidl/protos/flyteidl/core/tasks.proto +++ b/flyteidl/protos/flyteidl/core/tasks.proto @@ -332,8 +332,6 @@ message K8sObjectMetadata { message Sql { // The actual query to run, the query can have templated parameters. // We use Flyte's Golang templating format for Query templating. - // Refer to the templating documentation. - // https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/hive.html#sphx-glr-auto-integrations-external-services-hive-hive-py // For example, // insert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet // select * diff --git a/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go b/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go index 99a3ccdf7a..9482b3df95 100644 --- a/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go +++ b/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go @@ -2,9 +2,9 @@ package webapi import ( "context" - pluginErrors "github.com/flyteorg/flyte/flyteplugins/go/tasks/errors" "time" + pluginErrors "github.com/flyteorg/flyte/flyteplugins/go/tasks/errors" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/webapi" "github.com/flyteorg/flyte/flytestdlib/cache" diff --git a/flyteplugins/go/tasks/plugins/webapi/agent/plugin.go b/flyteplugins/go/tasks/plugins/webapi/agent/plugin.go index 11ef7871b3..c4decd04ef 100644 --- a/flyteplugins/go/tasks/plugins/webapi/agent/plugin.go +++ b/flyteplugins/go/tasks/plugins/webapi/agent/plugin.go @@ -4,13 +4,13 @@ import ( "context" "encoding/gob" "fmt" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service" "time" "golang.org/x/exp/maps" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" flyteIdl "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service" pluginErrors "github.com/flyteorg/flyte/flyteplugins/go/tasks/errors" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core" diff --git a/flytepropeller/events/local_eventsink.go b/flytepropeller/events/local_eventsink.go index 39e3ff5cac..fdcd5408a4 100644 --- a/flytepropeller/events/local_eventsink.go +++ b/flytepropeller/events/local_eventsink.go @@ -7,8 +7,9 @@ import ( "os" "sync" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event" "github.com/golang/protobuf/proto" + + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event" ) type localSink struct { diff --git a/flytepropeller/pkg/compiler/test/compiler_test.go b/flytepropeller/pkg/compiler/test/compiler_test.go index 115d4bbf3e..ae0322b66b 100644 --- a/flytepropeller/pkg/compiler/test/compiler_test.go +++ b/flytepropeller/pkg/compiler/test/compiler_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "flag" - "github.com/flyteorg/flyte/flytepropeller/pkg/visualize" "io/ioutil" "os" "path/filepath" @@ -27,6 +26,7 @@ import ( "github.com/flyteorg/flyte/flytepropeller/pkg/compiler/common" "github.com/flyteorg/flyte/flytepropeller/pkg/compiler/errors" "github.com/flyteorg/flyte/flytepropeller/pkg/compiler/transformers/k8s" + "github.com/flyteorg/flyte/flytepropeller/pkg/visualize" ) var update = flag.Bool("update", false, "Update .golden files") diff --git a/flytepropeller/pkg/compiler/validators/utils_test.go b/flytepropeller/pkg/compiler/validators/utils_test.go index 29daed99ae..4a37f100dc 100644 --- a/flytepropeller/pkg/compiler/validators/utils_test.go +++ b/flytepropeller/pkg/compiler/validators/utils_test.go @@ -1,9 +1,9 @@ package validators import ( - "github.com/golang/protobuf/proto" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/flyteorg/flyte/flyteidl/clients/go/coreutils" diff --git a/flytepropeller/pkg/compiler/workflow_compiler_test.go b/flytepropeller/pkg/compiler/workflow_compiler_test.go index 1e1e99fb02..eaffec3e21 100644 --- a/flytepropeller/pkg/compiler/workflow_compiler_test.go +++ b/flytepropeller/pkg/compiler/workflow_compiler_test.go @@ -2,10 +2,10 @@ package compiler import ( "fmt" - "github.com/golang/protobuf/proto" "strings" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/sets" diff --git a/flytepropeller/pkg/controller/nodes/executor.go b/flytepropeller/pkg/controller/nodes/executor.go index 0c98aeeb5f..cf8c62cfad 100644 --- a/flytepropeller/pkg/controller/nodes/executor.go +++ b/flytepropeller/pkg/controller/nodes/executor.go @@ -791,7 +791,7 @@ func isTimeoutExpired(queuedAt *metav1.Time, timeout time.Duration) bool { return false } -func (c *nodeExecutor) isEligibleForRetry(nCtx interfaces.NodeExecutionContext, nodeStatus v1alpha1.ExecutableNodeStatus, err *core.ExecutionError) (currentAttempt, maxAttempts uint32, isEligible bool) { +func (c *nodeExecutor) isEligibleForRetry(nCtx interfaces.NodeExecutionContext, nodeStatus v1alpha1.ExecutableNodeStatus, err *core.ExecutionError) (currentAttempt uint32, maxAttempts uint32, isEligible bool) { if config.GetConfig().NodeConfig.IgnoreRetryCause { currentAttempt = nodeStatus.GetAttempts() + 1 } else { diff --git a/flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context.go b/flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context.go index 17bbce5398..bb987acbc2 100644 --- a/flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context.go +++ b/flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context.go @@ -5,6 +5,7 @@ import ( pluginsCore "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/utils" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/utils/secrets" + k8sUtils "github.com/flyteorg/flyte/flytepropeller/pkg/utils" ) const executionIdentityVariable = "execution-identity" @@ -60,7 +61,8 @@ func newTaskExecutionMetadata(tCtx pluginsCore.TaskExecutionMetadata, taskTmpl * id := tCtx.GetSecurityContext().RunAs.ExecutionIdentity if len(id) > 0 { - injectLabels[executionIdentityVariable] = id + sanitizedID := k8sUtils.SanitizeLabelValue(id) + injectLabels[executionIdentityVariable] = sanitizedID } return TaskExecutionMetadata{ diff --git a/flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context_test.go b/flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context_test.go index bf9ca1eadb..e3c6f10ab6 100644 --- a/flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context_test.go +++ b/flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context_test.go @@ -86,6 +86,25 @@ func Test_newTaskExecutionMetadata(t *testing.T) { assert.Equal(t, 2, len(actual.GetLabels())) assert.Equal(t, "test-exec-identity", actual.GetLabels()[executionIdentityVariable]) }) + t.Run("Inject exec identity K8s label sanitation", func(t *testing.T) { + + existingMetadata := &mocks.TaskExecutionMetadata{} + existingAnnotations := map[string]string{} + existingMetadata.OnGetAnnotations().Return(existingAnnotations) + + existingMetadata.OnGetSecurityContext().Return(core.SecurityContext{RunAs: &core.Identity{ExecutionIdentity: "name@company.com"}}) + + existingLabels := map[string]string{ + "existingLabel": "existingLabelValue", + } + existingMetadata.OnGetLabels().Return(existingLabels) + + actual, err := newTaskExecutionMetadata(existingMetadata, &core.TaskTemplate{}) + assert.NoError(t, err) + + assert.Equal(t, 2, len(actual.GetLabels())) + assert.Equal(t, "name-company-com", actual.GetLabels()[executionIdentityVariable]) + }) } func Test_newTaskExecutionContext(t *testing.T) {