From 445d7eed4d06f4afedd03f867f1460329a7474c5 Mon Sep 17 00:00:00 2001 From: RodrigoSobral2000 Date: Mon, 25 Nov 2024 14:14:55 +0100 Subject: [PATCH] swancustomenvironments: Use api for logging Write to the log file in the api, instead of doing it in the script --- .../swancustomenvironments/scripts/builders/accpy.sh | 8 ++++---- .../swancustomenvironments/scripts/builders/mamba.sh | 2 +- .../swancustomenvironments/scripts/builders/venv.sh | 4 ++-- .../swancustomenvironments/scripts/makenv.sh | 11 +++++------ .../swancustomenvironments/serverextension.py | 12 +++++++++--- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/accpy.sh b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/accpy.sh index dace7c13..1c1c28d2 100644 --- a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/accpy.sh +++ b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/accpy.sh @@ -10,7 +10,7 @@ fi # Set up Acc-Py and create the environment source "${ACCPY_PATH}/base/${BUILDER_VERSION}/setup.sh" -acc-py venv ${ENV_PATH} 2>&1 | tee -a ${LOG_FILE} +acc-py venv ${ENV_PATH} 2>&1 # Activate the environment _log "Setting up the environment..." @@ -19,13 +19,13 @@ eval "${ACTIVATE_ENV_CMD}" # Install packages in the environment and the same ipykernel that the Jupyter server uses _log "Installing packages from ${REQ_PATH}..." -pip install -r "${REQ_PATH}" "ipykernel==${IPYKERNEL_VERSION}" ${NXCALS} ${SPARKMONITOR} ${SPARKCONNECTOR_DEPENDENCIES} 2>&1 | tee -a ${LOG_FILE} # TODO +pip install -r "${REQ_PATH}" "ipykernel==${IPYKERNEL_VERSION}" ${NXCALS} ${SPARKMONITOR} ${SPARKCONNECTOR_DEPENDENCIES} 2>&1 # TODO # -------------- HACK SECTION -------------- # Install SPARKCONNECTOR_DEPENDENCIES separately, install SparkConnector without its dependencies and change the configuration file # TODO: Remove this when the SparkConnector package gets properly updated if [ -n "${INSTALL_NXCALS}" ]; then - pip install ${SPARKCONNECTOR} --no-deps 2>&1 | tee -a ${LOG_FILE} - wget https://raw.githubusercontent.com/swan-cern/jupyter-extensions/refs/heads/swan-on-tn/SparkConnector/sparkconnector/configuration.py -O ${ENV_PATH}/lib/python3.11/site-packages/sparkconnector/configuration.py 2>&1 | tee -a ${LOG_FILE} + pip install ${SPARKCONNECTOR} --no-deps 2>&1 + wget https://raw.githubusercontent.com/swan-cern/jupyter-extensions/refs/heads/swan-on-tn/SparkConnector/sparkconnector/configuration.py -O ${ENV_PATH}/lib/python3.11/site-packages/sparkconnector/configuration.py 2>&1 fi diff --git a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/mamba.sh b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/mamba.sh index c3310b88..8a3efcaa 100644 --- a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/mamba.sh +++ b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/mamba.sh @@ -1,7 +1,7 @@ #!/bin/bash # Create the environment, install packages and the same ipykernel that the Jupyter server uses -mamba create -p ${ENV_PATH} --file ${REQ_PATH} "ipykernel==${IPYKERNEL_VERSION}" -y | tee -a ${LOG_FILE} +mamba create -p ${ENV_PATH} --file ${REQ_PATH} "ipykernel==${IPYKERNEL_VERSION}" -y # Activate the environment _log "Setting up the environment..." diff --git a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/venv.sh b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/venv.sh index b1124862..2e6f12d3 100644 --- a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/venv.sh +++ b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/venv.sh @@ -1,7 +1,7 @@ #!/bin/bash # Create the environment -python3 -m venv ${ENV_PATH} 2>&1 | tee -a ${LOG_FILE} +python3 -m venv ${ENV_PATH} 2>&1 # Activate the environment _log "Setting up the environment..." @@ -10,4 +10,4 @@ eval "${ACTIVATE_ENV_CMD}" # Install packages in the environment and the same ipykernel that the Jupyter server uses _log "Installing packages from ${REQ_PATH}..." -pip install -r "${REQ_PATH}" "ipykernel==${IPYKERNEL_VERSION}" 2>&1 | tee -a ${LOG_FILE} +pip install -r "${REQ_PATH}" "ipykernel==${IPYKERNEL_VERSION}" 2>&1 diff --git a/SwanCustomEnvironments/swancustomenvironments/scripts/makenv.sh b/SwanCustomEnvironments/swancustomenvironments/scripts/makenv.sh index af0024f7..3f88f092 100755 --- a/SwanCustomEnvironments/swancustomenvironments/scripts/makenv.sh +++ b/SwanCustomEnvironments/swancustomenvironments/scripts/makenv.sh @@ -14,12 +14,11 @@ if [ -n "${CURRENT_ENV_NAME}" ]; then exit 1 fi -LOG_FILE=/tmp/makenv.log # File to keep a backlog of this script output GIT_HOME="$HOME/SWAN_projects" # Path where git repositories are stored _log () { if [ "$*" == "ERROR:"* ] || [ "$*" == "WARNING:"* ] || [ "${JUPYTER_DOCKER_STACKS_QUIET}" == "" ]; then - echo "$@" 2>&1 | tee -a ${LOG_FILE} + echo "$@" 2>&1 fi } @@ -144,7 +143,7 @@ if [[ "$REPOSITORY" =~ $REPO_GIT_PATTERN ]]; then # Otherwise, use the existing one in the SWAN_projects folder if [ ! -d "${GIT_REPO_PATH}" ]; then _log "Cloning the repository from ${REPOSITORY}..." - git clone $REPOSITORY -q "${TMP_REPO_PATH}" 2>&1 | tee -a ${LOG_FILE} + git clone $REPOSITORY -q "${TMP_REPO_PATH}" 2>&1 if [ $? -ne 0 ]; then _log "ERROR: Failed to clone Git repository" && exit 1 fi @@ -174,12 +173,12 @@ HOME=/home/$USER source "${BUILDER_PATH}" # Install environment kernel. # Setting JUPYTER_PATH prevents ipykernel installation from complaining about non-found kernelspec -JUPYTER_PATH=${ENV_PATH}/share/jupyter python -m ipykernel install --name "${ENV_NAME}" --display-name "Python (${ENV_NAME})" --prefix "${ENV_PATH}" 2>&1 | tee -a ${LOG_FILE} +JUPYTER_PATH=${ENV_PATH}/share/jupyter python -m ipykernel install --name "${ENV_NAME}" --display-name "Python (${ENV_NAME})" --prefix "${ENV_PATH}" 2>&1 # Make sure the Jupyter server finds the new environment kernel in /home/$USER/.local # We modify the already existing Python3 kernel with the kernel.json of the environment KERNEL_JSON=${ENV_PATH}/share/jupyter/kernels/${ENV_NAME}/kernel.json -ln -f -s ${KERNEL_JSON} /home/$USER/.local/share/jupyter/kernels/python3/kernel.json | tee -a ${LOG_FILE} +ln -f -s ${KERNEL_JSON} /home/$USER/.local/share/jupyter/kernels/python3/kernel.json # For NXCALS, configure the environment kernel and terminal with some variables to # ensure the connection with the cluster works properly. @@ -202,7 +201,7 @@ if [ -n "${INSTALL_NXCALS}" ]; then --arg SPARK_DRIVER_EXTRA_JAVA_OPTIONS "${SPARK_DRIVER_EXTRA_JAVA_OPTIONS}" \ '. + {env: {$SPARK_HOME, $PYSPARK_PYTHON, $PATH, $VIRTUAL_ENV, $SPARK_DRIVER_EXTRA_JAVA_OPTIONS}}' \ ${KERNEL_JSON} > ${NEW_KERNEL} - mv -f ${NEW_KERNEL} ${KERNEL_JSON} 2>&1 | tee -a ${LOG_FILE} + mv -f ${NEW_KERNEL} ${KERNEL_JSON} 2>&1 # Terminal configuration # Only SPARK_HOME and PYSPARK_PYTHON are needed, since PATH and VIRTUAL_ENV are already diff --git a/SwanCustomEnvironments/swancustomenvironments/serverextension.py b/SwanCustomEnvironments/swancustomenvironments/serverextension.py index 23b21c1c..cdab3c73 100644 --- a/SwanCustomEnvironments/swancustomenvironments/serverextension.py +++ b/SwanCustomEnvironments/swancustomenvironments/serverextension.py @@ -13,6 +13,9 @@ class SwanCustomEnvironmentsApiHandler(APIHandler): # Path to the script used to create custom environments makenv_path = path.join(path.dirname(__file__), "scripts/makenv.sh") + # Path to the file where the log of the makenv.sh script is written + LOG_FILE = "/tmp/makenv.log" + @web.authenticated def get(self): """ @@ -37,9 +40,12 @@ def get(self): makenv_process = subprocess.Popen([self.makenv_path, *arguments], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - for line in iter(makenv_process.stdout.readline, b""): - self.write(line.decode('utf-8')) - self.flush() + with open(self.LOG_FILE, "w") as log_file: + for line in iter(makenv_process.stdout.readline, b""): + line = line.decode('utf-8') + log_file.write(line) # send log to file (for debugging) + self.write(line) # send log to frontend + self.flush() self.finish()