From 2fc2d4072a0e0943117b6f4e456100c1e79f3707 Mon Sep 17 00:00:00 2001 From: RodrigoSobral2000 Date: Mon, 9 Dec 2024 10:11:14 +0100 Subject: [PATCH] swancustomenvironments: Remove swan middle layer Keep ipykernel package definition consistent with the remaining extra packages. Due to the fact `venv-pack` (used by nxcals) can only ship one environment (user's), the packages installed in the middle layer won't be shipped, which leads to discrepancies in the results. For that reason, we remove this middle layer and ignore the extra packages installed by swan, ensuring they stay updated and reliable. In spite of this, the requirements file must not be modified, which means a temporary and modified version of requirements.txt has to be created and used for the installation --- .../scripts/builders/accpy.sh | 38 +++++++++---------- .../scripts/builders/mamba.sh | 2 +- .../scripts/builders/venv.sh | 14 +------ .../swancustomenvironments/scripts/makenv.sh | 3 +- 4 files changed, 21 insertions(+), 36 deletions(-) diff --git a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/accpy.sh b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/accpy.sh index 60b50758..b288afac 100644 --- a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/accpy.sh +++ b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/accpy.sh @@ -1,27 +1,10 @@ #!/bin/bash -# Create a middle layer for installing ipykernel, putting it apart from the user environment -uv venv $SWAN_ENV --seed 2>&1 -source $SWAN_ENV/bin/activate -uv pip install "ipykernel==${IPYKERNEL_VERSION}" -SWAN_PACKAGES_PATH=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') -deactivate - # If using NXCALS, we need to also install the Spark packages and their dependencies in the SWAN environment if [ -n "${USE_NXCALS}" ]; then SPARKCONNECTOR="sparkconnector==$(python -c 'import sparkconnector; print(sparkconnector.__version__)')" SPARKMONITOR="sparkmonitor==$(python -c 'import sparkmonitor; print(sparkmonitor.__version__)')" SPARKCONNECTOR_DEPENDENCIES="swanportallocator requests" # TODO: Remove swanportallocator and requests installation when the SparkConnector package gets properly updated - - # Activate the SWAN environment for installing the Spark packages - source $SWAN_ENV/bin/activate - uv pip install ${SPARKMONITOR} ${SPARKCONNECTOR_DEPENDENCIES} 2>&1 - - # -------------- HACK SECTION -------------- - # Install SPARKCONNECTOR_DEPENDENCIES separately, install SparkConnector without its dependencies and change the configuration file - # TODO: Remove this when the SparkConnector package gets properly updated - uv pip install ${SPARKCONNECTOR} --no-deps 2>&1 - wget https://raw.githubusercontent.com/swan-cern/jupyter-extensions/refs/heads/swan-on-tn/SparkConnector/sparkconnector/configuration.py -O ${SWAN_PACKAGES_PATH}/sparkconnector/configuration.py 2>&1 fi # Set up Acc-Py and create the environment @@ -41,15 +24,28 @@ if [ "${RESOLVED_REQ}" = true ]; then # Use the same pip configuration as the Acc-Py default pip ACCPY_PIP_CONF="-i $(pip config get global.index-url) --allow-insecure-host $(pip config get global.trusted-host)" uv pip install ${ACCPY_PIP_CONF} -r "${REQ_PATH}" 2>&1 + uv pip install ${ACCPY_PIP_CONF} ${IPYKERNEL} 2>&1 else pip install -r "${REQ_PATH}" 2>&1 + pip install ${IPYKERNEL} 2>&1 fi if [ $? -ne 0 ]; then exit 1 fi -# Inject middle layer packages into the user environment by adding a .pth file to -# the environment site-packages that contains the path to the middle layer site-packages -USER_PACKAGES_PATH=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') -echo ${SWAN_PACKAGES_PATH} > ${USER_PACKAGES_PATH}/$(basename $SWAN_ENV).pth +if [ -n "${USE_NXCALS}" ]; then + # For NXCALS, install the Spark packages and their dependencies, using the according tool for its resolution + if [ "${RESOLVED_REQ}" = true ]; then + uv pip install ${ACCPY_PIP_CONF} ${SPARKMONITOR} ${SPARKCONNECTOR_DEPENDENCIES} 2>&1 + uv pip install ${ACCPY_PIP_CONF} ${SPARKCONNECTOR} --no-deps 2>&1 + else + pip install ${SPARKMONITOR} ${SPARKCONNECTOR_DEPENDENCIES} 2>&1 + pip install ${SPARKCONNECTOR} --no-deps 2>&1 + fi + # -------------- HACK SECTION -------------- + # Install SPARKCONNECTOR_DEPENDENCIES separately, install SparkConnector without its dependencies and change the configuration file + # TODO: Remove this when the SparkConnector package gets properly updated + USER_PACKAGES_PATH=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') + wget https://raw.githubusercontent.com/swan-cern/jupyter-extensions/refs/heads/swan-on-tn/SparkConnector/sparkconnector/configuration.py -O ${USER_PACKAGES_PATH}/sparkconnector/configuration.py 2>&1 +fi \ No newline at end of file diff --git a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/mamba.sh b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/mamba.sh index d764765a..ca1d42af 100644 --- a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/mamba.sh +++ b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/mamba.sh @@ -1,7 +1,7 @@ #!/bin/bash # Create the environment, install packages and the same ipykernel that the Jupyter server uses -mamba create -p ${ENV_PATH} --file ${REQ_PATH} "ipykernel==${IPYKERNEL_VERSION}" -y +mamba create -p ${ENV_PATH} --file ${REQ_PATH} ${IPYKERNEL} -y if [ $? -ne 0 ]; then exit 1 fi diff --git a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/venv.sh b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/venv.sh index 74eefdbc..ade0aaa7 100755 --- a/SwanCustomEnvironments/swancustomenvironments/scripts/builders/venv.sh +++ b/SwanCustomEnvironments/swancustomenvironments/scripts/builders/venv.sh @@ -1,12 +1,5 @@ #!/bin/bash -# Create a middle layer for installing ipykernel, putting it apart from the user environment -uv venv $SWAN_ENV --seed 2>&1 -source $SWAN_ENV/bin/activate -uv pip install "ipykernel==${IPYKERNEL_VERSION}" -SWAN_PACKAGES_PATH=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') -deactivate - if [ "${RESOLVED_REQ}" = true ]; then uv venv ${ENV_PATH} --seed 2>&1 else @@ -24,14 +17,11 @@ eval "${ACTIVATE_ENV_CMD}" _log "Installing packages from ${REQ_PATH}..." if [ "${RESOLVED_REQ}" = true ]; then uv pip install -r "${REQ_PATH}" 2>&1 + uv pip install ${IPYKERNEL} 2>&1 else pip install -r "${REQ_PATH}" 2>&1 + pip install ${IPYKERNEL} 2>&1 fi if [ $? -ne 0 ]; then exit 1 fi - -# Inject middle layer packages into the user environment by adding a .pth file to -# the environment site-packages that contains the path to the middle layer site-packages -USER_PACKAGES_PATH=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') -echo ${SWAN_PACKAGES_PATH} > ${USER_PACKAGES_PATH}/$(basename $SWAN_ENV).pth diff --git a/SwanCustomEnvironments/swancustomenvironments/scripts/makenv.sh b/SwanCustomEnvironments/swancustomenvironments/scripts/makenv.sh index 52a43958..38484a7f 100755 --- a/SwanCustomEnvironments/swancustomenvironments/scripts/makenv.sh +++ b/SwanCustomEnvironments/swancustomenvironments/scripts/makenv.sh @@ -157,8 +157,7 @@ fi # Create and set up the environment ENV_PATH="/home/$USER/${ENV_NAME}" -SWAN_ENV="/home/$USER/swan" -IPYKERNEL_VERSION=$(python -c "import ipykernel; print(ipykernel.__version__)") +IPYKERNEL="ipykernel==$(python -c 'import ipykernel; print(ipykernel.__version__)')" if [ -f "${TMP_REPO_PATH}/requirements.txt" ]; then # Fully resolved requirements (requirements.txt) take precedence