Switching public images to ECR-backed images to avoid DockerHub rate …

…limits (flyteorg#1565)

.github/workflows/tests.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Build the documentation
         run: make docs
 
-  end-to-end:
+  end-to-end-kustomize:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
@@ -41,4 +41,31 @@ jobs:
           kubectl get pods -n kube-system
           echo "current-context:" $(kubectl config current-context)
           echo "environment-kubeconfig:" ${KUBECONFIG}
+          # launch flyte end2end
+          kubectl apply -f "./deployment/test/flyte_generated.yaml"
+          make end2end_execute
+
+  end-to-end-helm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Kustomize and diff
+        run: DELTA_CHECK=true make kustomize
+      - name: Helm and diff
+        run: DELTA_CHECK=true make helm
+      - uses: engineerd/[email protected]
+        with:
+          version: "v0.11.1"
+      - name: End2End
+        env:
+          DOCKER_USERNAME: ${{ github.actor }}
+          DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}"
+        run: |
+          kubectl cluster-info
+          kubectl get pods -n kube-system
+          echo "current-context:" $(kubectl config current-context)
+          echo "environment-kubeconfig:" ${KUBECONFIG}
+          # launch flyte end2end
+          make helm_update
+          make helm_install
           make end2end_execute

Makefile

@@ -57,14 +57,17 @@ stats:
 prepare_artifacts:
 	bash script/prepare_artifacts.sh
 
+.PHONE: helm_update
+helm_update:
+	helm dep update ./charts/flyte/
+
 .PHONY: helm_install
 helm_install:
-	helm install flyte --debug ./helm  -f helm/values-sandbox.yaml --create-namespace --namespace=flyte
+	helm install flyte --debug ./charts/flyte -f ./charts/flyte/values-sandbox.yaml --create-namespace --namespace=flyte
 
 .PHONY: helm_upgrade
 helm_upgrade:
-	helm upgrade flyte --debug ./helm  -f helm/values-sandbox.yaml --create-namespace --namespace=flyte
-	bash script/prepare_artifacts.sh
+	helm upgrade flyte --debug ./charts/flyte -f ./charts/flyte/values-sandbox.yaml --create-namespace --namespace=flyte
 
 .PHONY: docs
 docs:

charts/flyte-core/README.md

@@ -122,6 +122,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.additionalVolumes | list | `[]` |  |
 | flyteadmin.affinity | object | `{}` | affinity for Flyteadmin deployment |
 | flyteadmin.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files |
+| flyteadmin.deployRedoc | bool | `true` | Deploys a Redoc container in Flyteadmin's pod |
 | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment |
 | flyteadmin.image.tag | string | `"v0.6.28"` |  |

charts/flyte-core/templates/admin/deployment.yaml

@@ -20,7 +20,7 @@ spec:
       initContainers:
         {{- if .Values.postgres.enabled }}
         - name: check-db-ready
-          image: postgres:10.16-alpine
+          image: ecr.flyte.org/ubuntu/postgres:13-21.04_beta
           command:
             - sh
             - -c
@@ -113,6 +113,7 @@ spec:
           {{- with .Values.flyteadmin.additionalVolumeMounts -}}
           {{ toYaml . | nindent 8 }}
           {{- end }}
+        {{- if .Values.flyteadmin.deployRedoc -}}
         - command:
             - sh
             - -c
@@ -133,6 +134,7 @@ spec:
             limits:
               cpu: "0.1"
               memory: 200Mi
+        {{- end }}
       serviceAccountName: {{ template "flyteadmin.name" . }}
       volumes: {{- include "databaseSecret.volume" . | nindent 8 }}
         - emptyDir: {}

charts/flyte-core/templates/common/ingress.yaml

@@ -101,7 +101,7 @@ spec:
             pathType: ImplementationSpecific
             backend:
               serviceName: flyteadmin
-              servicePort: 80
+              servicePort: 87
           - path: /.well-known
             pathType: ImplementationSpecific
             backend:

charts/flyte-core/values-eks.yaml

@@ -9,7 +9,6 @@ userSettings:
   redisHostUrl: <REDIS_HOST_URL>
   redisHostKey: <REDIS_HOST_KEY>
 
-
 #
 # FLYTEADMIN
 #
@@ -49,6 +48,7 @@ flyteadmin:
             matchLabels:
               app.kubernetes.io/name: flyteadmin
           topologyKey: kubernetes.io/hostname
+  deployRedoc: false
 
 #
 # FLYTESCHEDULER

charts/flyte-core/values-gcp.yaml

@@ -1,3 +1,10 @@
+#
+# FlyteAdmin
+#
+
+flyteadmin:
+  deployRedoc: false
+
 #
 # COMMON
 #

charts/flyte-core/values.yaml

@@ -1,4 +1,3 @@
-
 # ---------------------------------------------------------------------
 # Core System settings
 # This section consists of Core components of Flyte and their deployment
@@ -60,6 +59,8 @@ flyteadmin:
   secrets: {}
   additionalVolumes: []
   additionalVolumeMounts: []
+  # -- Deploys a Redoc container in Flyteadmin's pod
+  deployRedoc: true
 
 #
 # FLYTESCHEDULER SETTINGS
@@ -68,7 +69,7 @@ flyteadmin:
 flytescheduler:
   image:
     # -- Docker image for Flytescheduler deployment
-    repository:  cr.flyte.org/flyteorg/flytescheduler
+    repository: cr.flyte.org/flyteorg/flytescheduler
     # -- Docker image tag
     tag: v0.6.28 # FLYTESCHEDULER_TAG
     # -- Docker image pull policy

charts/flyte/README.md

@@ -142,6 +142,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.additionalVolumes | list | `[]` |  |
 | flyteadmin.affinity | object | `{}` | affinity for Flyteadmin deployment |
 | flyteadmin.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files |
+| flyteadmin.deployRedoc | bool | `true` | Deploys a Redoc container in Flyteadmin's pod |
 | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment |
 | flyteadmin.image.tag | string | `"v0.6.28"` | Docker image tag |
@@ -198,12 +199,12 @@ helm install gateway bitnami/contour -n flyte
 | flytescheduler.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account |
 | flytescheduler.tolerations | list | `[]` | tolerations for Flytescheduler deployment |
 | kubernetes-dashboard | object | `{"enabled":false}` |  KUBERNETES DASHBOARD |
-| minio | object | `{"affinity":{},"enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"minio/minio","tag":"RELEASE.2020-12-16T05-05-17Z"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]}` |  MINIO SETTINGS |
+| minio | object | `{"affinity":{},"enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"ecr.flyte.org/bitnami/minio","tag":"2021.9.18-debian-10-r1"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]}` |  MINIO SETTINGS |
 | minio.affinity | object | `{}` | affinity for Minio deployment |
 | minio.enabled | bool | `true` | - enable or disable Minio deployment installation |
 | minio.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
-| minio.image.repository | string | `"minio/minio"` | Docker image for Minio deployment |
-| minio.image.tag | string | `"RELEASE.2020-12-16T05-05-17Z"` | Docker image tag |
+| minio.image.repository | string | `"ecr.flyte.org/bitnami/minio"` | Docker image for Minio deployment |
+| minio.image.tag | string | `"2021.9.18-debian-10-r1"` | Docker image tag |
 | minio.nodeSelector | object | `{}` | nodeSelector for Minio deployment |
 | minio.podAnnotations | object | `{}` | Annotations for Minio pods |
 | minio.replicaCount | int | `1` | Replicas count for Minio deployment |
@@ -212,12 +213,12 @@ helm install gateway bitnami/contour -n flyte
 | minio.resources.requests | object | `{"cpu":"10m","memory":"128Mi"}` | Requests are the minimum set of resources needed for this pod |
 | minio.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Minio |
 | minio.tolerations | list | `[]` | tolerations for Minio deployment |
-| postgres | object | `{"affinity":{},"enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"postgres","tag":"10.16"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"1000m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]}` |  POSTGRES SETTINGS |
+| postgres | object | `{"affinity":{},"enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"ecr.flyte.org/ubuntu/postgres","tag":"13-21.04_beta"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"1000m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]}` |  POSTGRES SETTINGS |
 | postgres.affinity | object | `{}` | affinity for Postgres deployment |
 | postgres.enabled | bool | `true` | - enable or disable Postgres deployment installation |
 | postgres.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
-| postgres.image.repository | string | `"postgres"` | Docker image for Postgres deployment |
-| postgres.image.tag | string | `"10.16"` | Docker image tag |
+| postgres.image.repository | string | `"ecr.flyte.org/ubuntu/postgres"` | Docker image for Postgres deployment |
+| postgres.image.tag | string | `"13-21.04_beta"` | Docker image tag |
 | postgres.nodeSelector | object | `{}` | nodeSelector for Postgres deployment |
 | postgres.podAnnotations | object | `{}` | Annotations for Postgres pods |
 | postgres.replicaCount | int | `1` | Replicas count for Postgres deployment |
@@ -237,12 +238,12 @@ helm install gateway bitnami/contour -n flyte
 | pytorchoperator.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Pytorchoperator |
 | pytorchoperator.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Pytorchoperator pods |
 | pytorchoperator.tolerations | list | `[]` | tolerations for Pytorchoperator deployment |
-| redis | object | `{"affinity":{},"enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"docker.io/bitnami/redis","tag":"4.0.2-r1"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"1000m","memory":"1Gi"},"requests":{"cpu":"10m","memory":"50Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]}` | -------------------------------------------- Sandbox Configuration Sandbox allows to run flyte without any cloud dependencies and can be run even locally on your laptop. This is achieved by replacing cloud service dependencies by k8s local alternatives. These may not be ideal for a high performance setup, but are great to try out flyte ----------------------------------------------- REDIS SETTINGS |
+| redis | object | `{"affinity":{},"enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"ecr.flyte.org/bitnami/redis","tag":"6.2.5-debian-10-r59"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"1000m","memory":"1Gi"},"requests":{"cpu":"10m","memory":"50Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]}` | -------------------------------------------- Sandbox Configuration Sandbox allows to run flyte without any cloud dependencies and can be run even locally on your laptop. This is achieved by replacing cloud service dependencies by k8s local alternatives. These may not be ideal for a high performance setup, but are great to try out flyte ----------------------------------------------- REDIS SETTINGS |
 | redis.affinity | object | `{}` | affinity for Redis Statefulset |
 | redis.enabled | bool | `true` | - enable or disable Redis Statefulset installation |
 | redis.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
-| redis.image.repository | string | `"docker.io/bitnami/redis"` | Docker image for Redis Statefulset |
-| redis.image.tag | string | `"4.0.2-r1"` | Docker image tag |
+| redis.image.repository | string | `"ecr.flyte.org/bitnami/redis"` | Docker image for Redis Statefulset |
+| redis.image.tag | string | `"6.2.5-debian-10-r59"` | Docker image tag |
 | redis.nodeSelector | object | `{}` | nodeSelector for Redis Statefulset |
 | redis.podAnnotations | object | `{}` | Annotations for Redis pods |
 | redis.replicaCount | int | `1` | Replicas count for Redis Statefulset |

charts/flyte/templates/admin/deployment.yaml

@@ -20,7 +20,7 @@ spec:
       initContainers:
         {{- if .Values.postgres.enabled }}
         - name: check-db-ready
-          image: postgres:10.16-alpine
+          image: ecr.flyte.org/ubuntu/postgres:13-21.04_beta
           command:
           - sh
           - -c
@@ -113,6 +113,7 @@ spec:
         {{- with .Values.flyteadmin.additionalVolumeMounts -}}
         {{ toYaml . | nindent 8 }}
         {{- end }}
+      {{- if .Values.flyteadmin.deployRedoc }}
       - command:
         - sh
         - -c
@@ -133,6 +134,7 @@ spec:
           limits:
             cpu: "0.1"
             memory: 200Mi
+      {{- end }}
       serviceAccountName: {{ template "flyteadmin.name" . }}
       volumes: {{- include "databaseSecret.volume" . | nindent 8 }}
         - emptyDir: {}

charts/flyte/templates/common/ingress.yaml

@@ -101,7 +101,7 @@ spec:
             pathType: ImplementationSpecific
             backend:
               serviceName: flyteadmin
-              servicePort: 80
+              servicePort: 87
           - path: /.well-known
             pathType: ImplementationSpecific
             backend:

charts/flyte/templates/datacatalog/deployment.yaml

@@ -20,7 +20,7 @@ spec:
       initContainers:
         {{- if .Values.postgres.enabled }}
         - name: check-db-ready
-          image: postgres:10.16-alpine
+          image: ecr.flyte.org/ubuntu/postgres:13-21.04_beta
           command:
             - sh
             - -c

charts/flyte/templates/minio/deployment.yaml

@@ -20,30 +20,22 @@ spec:
       - image: "{{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}"
         imagePullPolicy: "{{ .Values.minio.image.pullPolicy }}"
         name: minio
-        args:
-        - server
-        - /var/lib/minioql/data
         env:
-        - name: MINIO_ACCESS_KEY
-          value: minio
-        - name: MINIO_SECRET_KEY
-          value: miniostorage
+          - name: MINIO_ACCESS_KEY
+            value: minio
+          - name: MINIO_SECRET_KEY
+            value: miniostorage
+          - name: MINIO_DEFAULT_BUCKETS
+            value: my-s3-bucket
         ports:
         - containerPort: 9000
           name: minio
+        - containerPort: 9001
+          name: minio-console
         resources: {{ toYaml .Values.minio.resources | nindent 10 }}
         volumeMounts:
         - name: minio-storage
-          mountPath: /var/lib/minioql/data
-      initContainers:
-      - command:
-          - mkdir
-          - /var/lib/minioql/data/my-s3-bucket
-        image: "{{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}"
-        name: create-bucket
-        volumeMounts:
-        - name: minio-storage
-          mountPath: /var/lib/minioql/data
+          mountPath: /data
       volumes:
       - name: minio-storage
         emptyDir: {}

charts/flyte/templates/minio/service.yaml

@@ -18,5 +18,9 @@ spec:
     port: 9000
     protocol: TCP
     targetPort: minio
+  - name: minio-console
+    port: 9001
+    protocol: TCP
+    targetPort: minio-console
   selector: {{ include "minio.selectorLabels" . | nindent 4 }}
 {{- end }}

charts/flyte/values-eks.yaml

@@ -46,6 +46,7 @@ flyteadmin:
             matchLabels:
               app.kubernetes.io/name: flyteadmin
           topologyKey: kubernetes.io/hostname
+  deployRedoc: false
 
 #
 # FLYTESCHEDULER
@@ -187,7 +188,7 @@ common:
       kubernetes.io/ingress.class: alb
       alb.ingress.kubernetes.io/tags: service_instance=production
       alb.ingress.kubernetes.io/scheme: internet-facing
-      # -- This is the certificate arn of the cert imported in AWS certificate manager. 
+      # -- This is the certificate arn of the cert imported in AWS certificate manager.
       alb.ingress.kubernetes.io/certificate-arn: "{{ .Values.userSettings.certificateArn }}"
       alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
       alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}'

charts/flyte/values-gcp.yaml

@@ -1,3 +1,10 @@
+#
+# FlyteAdmin
+#
+
+flyteadmin:
+  deployRedoc: false
+
 #
 # COMMON
 #

charts/flyte/values-sandbox.yaml

@@ -17,7 +17,6 @@ flyteadmin: {} # use default config
 
 flytescheduler: {} # use default config
 
-
 #
 # DATACATALOG
 #
@@ -56,15 +55,15 @@ redis:
 postgres:
   image:
     # Override postgres image to use alpine based (rather smaller) docker image
-    tag: "10.16-alpine"
+    tag: "13-21.04_beta"
 
 #
 # MINIO
 #
 
 minio:
   service:
-    annotations: { }
+    annotations: {}
     type: NodePort
 
 #
@@ -202,7 +201,7 @@ configmap:
     plugins:
       logs:
         kubernetes-enabled: true
-        kubernetes-template-uri: "http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"
+        kubernetes-template-uri: 'http://localhost:30082/#/log/{{ "{{" }} .namespace {{ "}}" }}/{{ "{{" }} .podName {{ "}}" }}/pod?namespace={{ "{{" }} .namespace {{ "}}" }}'
 
 # ----------------------------------------------------------------
 # Optional Modules