From fa7108b45684c729c2e88b12d7fbf4bbb5a4dc65 Mon Sep 17 00:00:00 2001
From: Christian Tacke <58549698+ChristianTackeGSI@users.noreply.github.com>
Date: Wed, 24 May 2023 14:34:42 +0200
Subject: [PATCH] ci: Rewrite jobscript creation

Refactor job script creation into its own shell script.
This should improve general readability.

Also use the opportunity and create a dedicated, temporary
home directory for the CI run. This way all files created
in the home directory by the job do not end up in the real
one.

(cherry picked from commit e4ecb3c0ef8950089fa067fddcb69330eda020f6)
---
 Jenkinsfile                                | 16 +++------
 test/ci/slurm-create-jobscript.sh          | 41 ++++++++++++++++++++++
 slurm-submit.sh => test/ci/slurm-submit.sh |  0
 3 files changed, 46 insertions(+), 11 deletions(-)
 create mode 100755 test/ci/slurm-create-jobscript.sh
 rename slurm-submit.sh => test/ci/slurm-submit.sh (100%)

diff --git a/Jenkinsfile b/Jenkinsfile
index 97a0527907..292eaaeb3b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,17 +50,11 @@ def jobMatrix(String prefix, String type, List specs) {
             sh "cat ${jobscript}"
             sh "bash ${jobscript}"
           } else {
-            def containercmd = "singularity exec -B/shared ${env.SINGULARITY_CONTAINER_ROOT}/fairroot/${os}.${ver}.sif bash -l -c \\\"${ctestcmd}\\\""
-            sh """\
-              echo \"echo \\\"*** Job started at .......: \\\$(date -R)\\\"\" >> ${jobscript}
-              echo \"echo \\\"*** Job ID ...............: \\\${SLURM_JOB_ID}\\\"\" >> ${jobscript}
-              echo \"echo \\\"*** Compute node .........: \\\$(hostname -f)\\\"\" >> ${jobscript}
-              echo \"unset http_proxy\" >> ${jobscript}
-              echo \"unset HTTP_PROXY\" >> ${jobscript}
-              echo \"${containercmd}\" >> ${jobscript}
-            """
-            sh "cat ${jobscript}"
-            sh "./slurm-submit.sh \"FairRoot \${JOB_BASE_NAME} ${label}\" ${jobscript}"
+            def container = "${os}.${ver}.sif"
+            sh(label: "Create Slurm Job Script", script: """
+              exec test/ci/slurm-create-jobscript.sh "${label}" "${container}" "${jobscript}" ${ctestcmd}
+            """)
+            sh "./test/ci/slurm-submit.sh \"FairRoot \${JOB_BASE_NAME} ${label}\" ${jobscript}"
           }
           if (check == "warnings" || check == "doxygen") {
             discoverGitReferenceBuild()
diff --git a/test/ci/slurm-create-jobscript.sh b/test/ci/slurm-create-jobscript.sh
new file mode 100755
index 0000000000..3269dd1422
--- /dev/null
+++ b/test/ci/slurm-create-jobscript.sh
@@ -0,0 +1,41 @@
+#! /bin/bash
+
+if [ $# -lt 4 ]
+then
+	echo "*** Please call like: $0 LABEL CONTAINER JOBSH CTEST [CTESTARG ...]"
+	exit 1
+fi
+
+label="$1"; shift
+container="$1"; shift
+jobsh="$1"; shift
+# Convert commandline array into single string, with proper quoting
+ctestcmd="$(printf ' %q' "$@")"
+# Strip leading space
+ctestcmd="${ctestcmd:1}"
+
+echo "*** Creating job script ..: ${jobsh}"
+
+(
+	echo   '# #! /bin/bash'
+	echo   "# export LABEL=${label}"
+	echo   '  echo "*** Job started at .......: $(date -R)"'
+	echo   '  echo "*** Job ID ...............: $SLURM_JOB_ID"'
+	echo   '  echo "*** Compute node .........: $(hostname -f)"'
+	echo   "# source <(sed -e '/^#/d' -e '/^export/!s/^.*=/export &/' /etc/environment)"
+	printf '# ./test/test-start-container.sh %q %q\n' "${container}" "${ctestcmd}"
+	echo   '# retval=$?'
+	echo   '# mkdir -p build'
+	echo   '# echo $retval >"build/${LABEL}-last-exit-code"'
+	echo   '# exit $retval'
+	echo   'newhomedir="$(mktemp --tmpdir -d FairRoot-CI-home.XXXXXX)"'
+	echo   '  echo "*** Temporary homedir ....: $newhomedir"'
+	echo   '  unset http_proxy'
+	echo   '  unset HTTP_PROXY'
+	printf 'singularity exec -B/shared -H "${newhomedir}" %q bash -l -c %q\n' \
+		"${SINGULARITY_CONTAINER_ROOT}/fairroot/${container}" \
+		"${ctestcmd}"
+) >> "$jobsh"
+
+echo "*** Content:"
+sed -e 's/^/    /' "$jobsh"
diff --git a/slurm-submit.sh b/test/ci/slurm-submit.sh
similarity index 100%
rename from slurm-submit.sh
rename to test/ci/slurm-submit.sh