[SDKTECHNO-214] Build spark python using new python image and add pyt…

…hon 3.10 for spark

technologies/job/spark/spark-2.4/innerContexts/python/spark-2.4-python-3.7/build.gradle.kts

@@ -30,7 +30,7 @@ tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
     dependsOn(":spark-2.4:testImage")
     this.buildArgs.put(
         "base_img",
-        "saagie/python:3.7-1.46.0"
+        "saagie/python:3.7-1.143.0_SDKTECHNO-214"
     )
     this.buildArgs.put(
         "spark_base_img",

technologies/job/spark/spark-2.4/innerContexts/python/spark-2.4-python-3.7/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 2.4-py-3.7
-dynamicVersion: 1.100.0_PRODUCT-3089
-version: 2.4-py-3.7-1.100.0
+dynamicVersion: 1.143.0_SDKTECHNO-214
+version: 2.4-py-3.7-1.143.0

technologies/job/spark/spark-3.0/innerContexts/python/spark-3.0-python-3.7/build.gradle.kts

@@ -30,7 +30,7 @@ tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
     dependsOn(":spark-3.0:testImage")
     this.buildArgs.put(
         "base_img",
-        "saagie/python:3.7-1.46.0"
+        "saagie/python:3.7-1.143.0_SDKTECHNO-214"
     )
     this.buildArgs.put(
         "spark_base_img",

technologies/job/spark/spark-3.0/innerContexts/python/spark-3.0-python-3.7/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.0-py-3.7
-dynamicVersion: 1.139.0_SDKTECHNO-207
-version: 3.0-py-3.7-1.139.0
+dynamicVersion: 1.143.0_SDKTECHNO-214
+version: 3.0-py-3.7-1.143.0

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.10/build.gradle.kts

@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019-2021.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+import com.saagie.technologies.readDockerInfo
+import com.saagie.technologies.getVersionForDocker
+
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()
+
+val dockerInfo = readDockerInfo(projectDir)
+
+tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
+    dependsOn(":spark-3.1:testImage")
+    this.buildArgs.put(
+        "base_img",
+        "saagie/python:3.10-1.143.0_SDKTECHNO-214"
+    )
+    this.buildArgs.put(
+        "spark_base_img",
+        "${dockerInfo?.image}:3.1-${this.project.getVersionForDocker()}"
+    )
+}
+

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.10/build.me

@@ -0,0 +1 @@
+04/11/2021

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.10/dockerInfo.yaml

@@ -0,0 +1,4 @@
+image: saagie/spark
+baseTag: 3.1-py-3.10
+dynamicVersion: 1.143.0_SDKTECHNO-214
+version: 3.1-py-3.10-1.143.0

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.10/image_test.yaml

@@ -0,0 +1,98 @@
+schemaVersion: "2.0.0"
+
+metadataTest:
+  env:
+    - key: LANG
+      value: "C.UTF-8"
+    - key: JAVA_HOME
+      value: "/usr/local/openjdk-11"
+    - key: SPARK_HOME
+      value: "/opt/spark"
+    - key: "HADOOP_CONF_DIR"
+      value: "/etc/hadoop/conf"
+
+fileExistenceTests:
+  - name: "entrypoint.sh"
+    path: "/opt/entrypoint.sh"
+    shouldExist: true
+    permissions: "-rwxr-xr-x"
+
+  - name: "kinit"
+    path: "/usr/bin/kinit"
+    shouldExist: true
+    permissions: "-rwxr-xr-x"
+
+fileContentTests:
+  - name: "entrypoint.sh"
+    path: "/opt/entrypoint.sh"
+    expectedContents:
+      [
+        'wget -nv \$PYSPARK_FILES',
+        'unzip -q \*\.zip',
+        'ln -s /sandbox/ /opt/spark/work-dir',
+        '--py-files=/sandbox/\*',
+        'exec /sbin/tini -s -- '
+      ]
+
+commandTests:
+  - name: "Workdir"
+    command: "pwd"
+    expectedOutput: ["/opt/spark/work-dir"]
+
+  - name: "Spark version"
+    command: "/opt/spark/bin/spark-submit"
+    args: ["--version"]
+    expectedError: ["version 3.1.*"]
+
+  - name: "python installation"
+    command: "which"
+    args: ["python"]
+    expectedOutput: ["/usr/local/bin/python"]
+
+  - name: "krb5-user installation"
+    command: "kinit"
+    expectedError: ['kinit: Program lacks support for encryption type while getting initial credentials']
+    exitCode: 1
+
+  - name: "ftp"
+    args: ["-h"]
+    command: "ftp"
+    exitCode: 0
+
+  - name: "wget"
+    args: ["--help"]
+    command: "wget"
+    exitCode: 0
+
+  - name: "curl"
+    args: ["--help"]
+    command: "curl"
+    exitCode: 0
+
+  - name: "unzip"
+    args: ["--help"]
+    command: "unzip"
+    exitCode: 0
+
+  - name: "tar"
+    args: ["--help"]
+    command: "tar"
+    exitCode: 0
+
+  - name: "telnet"
+    command: "which"
+    args: ["telnet"]
+    expectedOutput: ["/usr/bin/telnet"]
+    exitCode: 0
+
+  - name: "scp"
+    command: "which"
+    args: ["scp"]
+    expectedOutput: ["/usr/bin/scp"]
+    exitCode: 0
+
+  - name: "tini"
+    command: "/sbin/tini"
+    args: ["--version"]
+    expectedOutput: ["tini version 0.18.0.*"]
+    exitCode: 0

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.10/innerContext.yaml

@@ -0,0 +1,5 @@
+id: "3.10"
+label: "3.10"
+available: true
+trustLevel: stable
+recommended: true

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.7/build.gradle.kts

@@ -30,7 +30,7 @@ tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
     dependsOn(":spark-3.1:testImage")
     this.buildArgs.put(
         "base_img",
-        "saagie/python:3.7-1.46.0"
+        "saagie/python:3.7-1.143.0_SDKTECHNO-214"
     )
     this.buildArgs.put(
         "spark_base_img",

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.7/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.1-py-3.7
-dynamicVersion: 1.139.0_SDKTECHNO-207
-version: 3.1-py-3.7-1.139.0
+dynamicVersion: 1.143.0_SDKTECHNO-214
+version: 3.1-py-3.7-1.143.0

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.8/build.gradle.kts

@@ -30,7 +30,7 @@ tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
     dependsOn(":spark-3.1:testImage")
     this.buildArgs.put(
         "base_img",
-        "saagie/python:3.8-1.137.0"
+        "saagie/python:3.8-1.143.0_SDKTECHNO-214"
     )
     this.buildArgs.put(
         "spark_base_img",

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.8/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.1-py-3.8
-dynamicVersion: 1.139.0_SDKTECHNO-207
-version: 3.1-py-3.8-1.139.0
+dynamicVersion: 1.143.0_SDKTECHNO-214
+version: 3.1-py-3.8-1.143.0

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.9/build.gradle.kts

@@ -30,7 +30,7 @@ tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
     dependsOn(":spark-3.1:testImage")
     this.buildArgs.put(
         "base_img",
-        "saagie/python:3.9-1.137.0"
+        "saagie/python:3.9-1.143.0_SDKTECHNO-214"
     )
     this.buildArgs.put(
         "spark_base_img",

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.9/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.1-py-3.9
-dynamicVersion: 1.139.0_SDKTECHNO-207
-version: 3.1-py-3.9-1.139.0
+dynamicVersion: 1.143.0_SDKTECHNO-214
+version: 3.1-py-3.9-1.143.0