Spark CRD + Operator v1beta2 (flyteorg#572)

* Spark v1beta2 * Update labels * update propeller * Update Propeller version * Bump propeller version * Fix end-to-end tests
Tom-Newton · Nov 17, 2020 · 860af42 · 860af42
1 parent 0936811
commit 860af42
Show file tree

Hide file tree

Showing 16 changed files with 30,091 additions and 61 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -11,7 +11,7 @@ jobs:
     - uses: actions/checkout@v1
     - name: Kustomize and diff
       run: DELTA_CHECK=true make kustomize
-    - uses: engineerd/setup-kind@v0.4.0
+    - uses: engineerd/setup-kind@v0.5.0
     - name: End2End
       env:
         DOCKER_USERNAME: ${{ github.actor }}

diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml
diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml
diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml
diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml
@@ -1067,7 +1067,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.4.8
+        image: docker.io/lyft/flytepropeller:v0.5.2
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:

diff --git a/kustomize/base/operators/spark/deployment.yaml b/kustomize/base/operators/spark/deployment.yaml
@@ -5,13 +5,12 @@ metadata:
   namespace: sparkoperator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.4.0-v1beta1
+    app.kubernetes.io/version: v2.4.5-v1beta2
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: sparkoperator
-      app.kubernetes.io/version: v2.4.0-v1beta1
   strategy:
     type: Recreate
   template:
@@ -22,7 +21,7 @@ spec:
         prometheus.io/path: "/metrics"
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.4.0-v1beta1
+        app.kubernetes.io/version: v2.4.5-v1beta2
     spec:
       serviceAccountName: sparkoperator
       volumes:
@@ -31,7 +30,7 @@ spec:
             secretName: spark-webhook-certs
       containers:
       - name: sparkoperator-unknown
-        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1beta1-0.9.0
+        image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5
         volumeMounts:
           - name: webhook-certs
             mountPath: /etc/webhook-certs

diff --git a/kustomize/base/operators/spark/kustomization.yaml b/kustomize/base/operators/spark/kustomization.yaml
@@ -1,6 +1,7 @@
 resources:
 - spark-operator.yaml
-- spark-dummy-crd.yaml
+- sparkapplications-crd.yaml
+- scheduledsparkapplications-crd.yaml
 - deployment.yaml
 - webhook.yaml
 

diff --git a/kustomize/base/operators/spark/scheduledsparkapplications-crd.yaml b/kustomize/base/operators/spark/scheduledsparkapplications-crd.yaml
diff --git a/kustomize/base/operators/spark/spark-dummy-crd.yaml b/kustomize/base/operators/spark/spark-dummy-crd.yaml
diff --git a/kustomize/base/operators/spark/spark-operator.yaml b/kustomize/base/operators/spark/spark-operator.yaml
@@ -39,7 +39,7 @@ rules:
   resources: ["mutatingwebhookconfigurations"]
   verbs: ["create", "get", "update", "delete"]
 - apiGroups: ["sparkoperator.k8s.io"]
-  resources: ["sparkapplications", "scheduledsparkapplications"]
+  resources: ["sparkapplications", "scheduledsparkapplications", "sparkapplications/status", "scheduledsparkapplications/status"]
   verbs: ["*"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1