Sdktechno 264 - python 3.12

technologies/job/python/metadata.yaml

@@ -94,6 +94,33 @@ contexts:
       image: "saagie/python"
       baseTag: "3.11"
       version: "3.11-1.181.0_SDKTECHNO-235"
+  - id: "3.12"
+    label: "3.12"
+    available: true
+    recommended: true
+    trustLevel: stable
+    job:
+      features:
+        - type: COMMAND_LINE
+          label: Command line
+          mandatory: true
+          comment: Linux shell command to launch the job.
+          defaultValue: python {file} arg1 arg2
+        - type: ARTIFACT
+          label: Package
+          mandatory: true
+          comment: "Compatible upload file : .py or .zip (with optional requirements.txt for pip install)"
+        - type: SCHEDULER
+          label: Scheduled
+          mandatory: true
+        - type: AI_DESCRIPTION_GENERATOR
+          label: AI description generator enabled
+          mandatory: true
+          comment: Activation of the AI-based automatic description generation function.
+    dockerInfo:
+      image: "saagie/python"
+      baseTag: "3.12"
+      version: "3.12-1.183.0_SDKTECHNO-264"
   - id: "3.7"
     label: "3.7"
     available: true

technologies/job/python/python-3.12-base/Dockerfile

@@ -0,0 +1,49 @@
+FROM python:3.12.4-slim-bullseye
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV LC_ALL C
+ENV LD_RUN_PATH /usr/local/lib
+
+# LIGHT DEPENDENCIES START
+RUN apt-get update -qq && \
+    apt-get install -yqq --no-install-recommends \
+      ftp wget curl unzip telnet openssh-client git apt-transport-https ca-certificates \
+      krb5-user libpam-krb5 libpam-ccreds locales openjdk-11-jdk ca-certificates-java ant && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+# LIGHT DEPENDENCIES END
+
+# UPGRADE PIP
+RUN pip --no-cache-dir install --upgrade pip
+
+# Add Hadoop CLI
+RUN cd / \
+    && mkdir app \
+    && cd app \
+    && wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.6.5/hadoop-2.6.5.tar.gz \
+    && tar xvf hadoop-2.6.5.tar.gz \
+    && rm hadoop-2.6.5.tar.gz \
+    && rm -rf hadoop-2.6.5/etc/hadoop \
+    && ln -s /etc/hadoop/conf hadoop-2.6.5/etc/hadoop;
+
+# Fix Locale
+RUN sed -i '/^#.* en_US.* /s/^#//' /etc/locale.gen \
+    && locale-gen
+
+# Fix certificate issues
+RUN update-ca-certificates -f;
+
+ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
+ENV HADOOP_HOME=/app/hadoop-2.6.5
+ENV HADOOP_CONF_DIR=/etc/hadoop/conf
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/app/hadoop-2.6.5/lib/native:/usr/lib/jvm/java-11-openjdk-amd64/lib"
+ENV PATH "/app/hadoop-2.6.5/bin:${PATH}"
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+COPY entrypoint /entrypoint
+RUN chmod 755 /entrypoint
+
+WORKDIR /sandbox
+
+ENTRYPOINT  ["bash","/entrypoint"]

technologies/job/python/python-3.12-base/build.gradle.kts

@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019-2021.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()

technologies/job/python/python-3.12-base/dockerInfo.yaml

@@ -0,0 +1,4 @@
+image: saagie/python
+baseTag: 3.12-base
+dynamicVersion: 1.183.0_SDKTECHNO-264
+version: 3.12-base-1.183.0_SDKTECHNO-264

technologies/job/python/python-3.12-base/entrypoint

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+export CLASSPATH=`hadoop classpath --glob`
+
+if compgen -G "*.zip*" > /dev/null; then
+    unzip -q *.zip
+fi
+if test -f requirements.txt;
+then
+    echo "*******PIP INSTALL BEGIN*******"
+    pip install -r requirements.txt;
+    echo "*******PIP INSTALL END*******"
+fi;
+if [ $? != 0 ]; then echo "Error during 'pip install -f 'requirements.txt''";exit 1; fi;
+if test -f main_script;
+then sh ./main_script;
+else exec "$@"
+fi;

technologies/job/python/python-3.12-base/image_test.yaml

@@ -0,0 +1,100 @@
+schemaVersion: "2.0.0"
+
+metadataTest:
+  env:
+    - key: LANG
+      value: "en_US.UTF-8"
+    - key: LC_ALL
+      value: "en_US.UTF-8"
+    - key: LD_RUN_PATH
+      value: "/usr/local/lib"
+    - key: PATH
+      value: "/app/hadoop-2.6.5/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    - key: "HADOOP_CONF_DIR"
+      value: "/etc/hadoop/conf"
+    - key: "JAVA_HOME"
+      value: "/usr/lib/jvm/java-11-openjdk-amd64/"
+
+fileExistenceTests:
+  - name: "unzip"
+    path: "/usr/bin/unzip"
+    shouldExist: true
+    permissions: "-rwxr-xr-x"
+
+  - name: "entrypoint"
+    path: "/entrypoint"
+    shouldExist: true
+    permissions: "-rwxr-xr-x"
+
+  - name: "kinit"
+    path: "/usr/bin/kinit"
+    shouldExist: true
+    permissions: "-rwxr-xr-x"
+
+fileContentTests:
+  - name: "entrypoint"
+    path: "/entrypoint"
+    expectedContents:
+      [
+        "unzip -q \\*.zip",
+        "pip install -r requirements.txt",
+        "sh ./main_script",
+      ]
+
+commandTests:
+  - name: "java version"
+    command: "java"
+    args: [ "-version" ]
+    expectedError: [ 'openjdk version "11.*' ]
+
+  - name: "ftp"
+    args: ["-h"]
+    command: "ftp"
+    exitCode: 0
+
+  - name: "wget"
+    args: ["--help"]
+    command: "wget"
+    exitCode: 0
+
+  - name: "curl"
+    args: ["--help"]
+    command: "curl"
+    exitCode: 0
+
+  - name: "unzip"
+    args: ["--help"]
+    command: "unzip"
+    exitCode: 0
+
+  - name: "tar"
+    args: ["--help"]
+    command: "tar"
+    exitCode: 0
+
+  - name: "telnet"
+    command: "which"
+    args: ["telnet"]
+    expectedOutput: ["/usr/bin/telnet"]
+    exitCode: 0
+
+  - name: "scp"
+    command: "which"
+    args: ["scp"]
+    expectedOutput: ["/usr/bin/scp"]
+    exitCode: 0
+
+  - name: "python installation"
+    command: "which"
+    args: ["python"]
+    expectedOutput: ["/usr/local/bin/python"]
+
+  - name: "python version"
+    command: "python"
+    args: ["-V"]
+    expectedOutput: ["Python 3.12*"]
+
+  - name: "krb5-user installation"
+    command: "kinit"
+    expectedError: ['kinit: Program lacks support for encryption type while getting initial credentials']
+    exitCode: 1

technologies/job/python/python-3.12/Dockerfile

@@ -0,0 +1,35 @@
+ARG base_img
+
+FROM ${base_img} AS BASE_IMG
+
+# LIBS PART BEGIN
+RUN apt update -qq && apt install -qqy --no-install-recommends \
+      python3-pip \
+      gcc \
+      g++ \
+      libsasl2-2 \
+      libsasl2-modules-ldap \
+      build-essential \
+      unixodbc \
+      unixodbc-dev \
+      libpq-dev \
+      libsqlite3-dev \
+      libkrb5-dev \
+      libsasl2-dev \
+      libssl-dev \
+      libcurl4-openssl-dev \
+      libgeos-dev \
+      swig \
+      python3-matplotlib \
+      python3-lxml \
+      cmake \
+    && rm -rf /var/lib/apt/lists/*;
+# LIBS PART END
+
+# Need to read requirements.txt file sequentially to ensure consistent installation order
+COPY resources/requirements.txt /tmp/requirements.txt
+RUN grep -v '^#' /tmp/requirements.txt | xargs -L 1 pip --no-cache-dir install \
+    && rm -rf /root/.cachex \
+    && rm -rf /boot/.cache/pip \
+    && rm -rf ~/.cache/pip
+

technologies/job/python/python-3.12/build.gradle.kts

@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019-2021.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+import com.saagie.technologies.readDockerInfo
+import com.saagie.technologies.getVersionForDocker
+
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()
+
+val dockerInfo = readDockerInfo(projectDir)
+
+tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
+    dependsOn(":${this.project.name}-base:testImage")
+    this.buildArgs.put(
+            "base_img",
+            "${dockerInfo?.image}:${dockerInfo?.baseTag}-base-${this.project.getVersionForDocker()}"
+    )
+}

technologies/job/python/python-3.12/context.yaml

@@ -0,0 +1,23 @@
+id: "3.12"
+label: "3.12"
+available: true
+recommended: true
+trustLevel: stable
+job:
+  features:
+    - type: COMMAND_LINE
+      label: Command line
+      mandatory: true
+      comment: Linux shell command to launch the job.
+      defaultValue: python {file} arg1 arg2
+    - type: ARTIFACT
+      label: Package
+      mandatory: true
+      comment: "Compatible upload file : .py or .zip (with optional requirements.txt for pip install)"
+    - type: SCHEDULER
+      label: Scheduled
+      mandatory: true
+    - type: AI_DESCRIPTION_GENERATOR
+      label: AI description generator enabled
+      mandatory: true
+      comment: Activation of the AI-based automatic description generation function.

technologies/job/python/python-3.12/dockerInfo.yaml

@@ -0,0 +1,4 @@
+image: saagie/python
+baseTag: 3.12
+dynamicVersion: 1.183.0_SDKTECHNO-264
+version: 3.12-1.183.0_SDKTECHNO-264