python-modules: use venv to install environment (O2-1089) (alisw#2034)

Now that we can rely on python3, we can simply setup our modules in a
virtualenv environment, with the added advantage that we can pick up whatever
is already available from the system or any previously setup virtualenv,
allowing people to easily customize their Python-modules environment.

python-modules.sh

@@ -11,13 +11,17 @@ build_requires:
 prepend_path:
   PYTHONPATH: $PYTHON_MODULES_ROOT/share/python-modules/lib/python/site-packages
 ---
-#!/bin/bash -ex
 
-# Major.minor version of Python
-export PYVER=$(python3 -c 'import distutils.sysconfig; print(distutils.sysconfig.get_python_version())')
-
-# Ignore what is already in PYTHONPATH. We will set PYTHONPATH or PYTHONUSERBASE per command
+# A spurios PYTHONPATH can affect later commands
 unset PYTHONPATH
+# If we are in a virtualenv, assume that what you want to do is to copy
+# the same installation in your alibuild one.
+if [ ! "X$VIRTUAL_ENV" = X ]; then
+  # Once more to get the deactivate
+  . $VIRTUAL_ENV/bin/activate
+  pip freeze > system-requirements.txt
+  deactivate
+fi
 
 # PIP_REQUIREMENTS, PIP36_REQUIREMENTS, PIP38_REQUIREMENTS come from python-modules-list.sh
 echo $PIP_REQUIREMENTS | tr \  \\n > requirements.txt
@@ -35,16 +39,24 @@ esac
 # do not conflict with the underlying Python installation.
 PYTHON_MODULES_INSTALLROOT=$INSTALLROOT/share/python-modules
 mkdir -p $PYTHON_MODULES_INSTALLROOT
+
+# Create the virtualenv
+python3 -m venv --system-site-packages --symlinks $PYTHON_MODULES_INSTALLROOT
+. $PYTHON_MODULES_INSTALLROOT/bin/activate
+
 # Install setuptools upfront, since this seems to create issues now...
-env PYTHONUSERBASE="$PYTHON_MODULES_INSTALLROOT" python3 -m pip install --user -IU setuptools
-env PYTHONUSERBASE="$PYTHON_MODULES_INSTALLROOT" python3 -m pip install --user -IU wheel
+python3 -m pip install --user -IU setuptools
+python3 -m pip install --user -IU wheel
+
 # FIXME: required because of the newly introduced dependency on scikit-garden requires
 # a numpy to be installed separately
 # See also:
 #   https://github.com/scikit-garden/scikit-garden/issues/23
-grep RootInteractive requirements.txt && env PYTHONUSERBASE="$PYTHON_MODULES_INSTALLROOT" python3 -m pip install --user -IU numpy
-env PYTHONUSERBASE="$PYTHON_MODULES_INSTALLROOT" python3 -m pip install --user -IU -r requirements.txt
+grep RootInteractive requirements.txt && python3 -m pip install --user -IU numpy
+python3 -m pip install --user -IU -r requirements.txt
 
+# Major.minor version of Python
+export PYVER=$(python3 -c 'import distutils.sysconfig; print(distutils.sysconfig.get_python_version())')
 # Find the proper Python lib library and export it
 pushd "$PYTHON_MODULES_INSTALLROOT"
   if [[ -d lib64 ]]; then
@@ -57,15 +69,15 @@ pushd "$PYTHON_MODULES_INSTALLROOT"
   popd
   pushd bin
     # Fix shebangs: remove hardcoded Python path
-    sed -i.deleteme -e "1 s|^#!${PYTHON_MODULES_INSTALLROOT}/bin/\(.*\)$|#!/usr/bin/env \1|" * || true
-    rm -f *.deleteme || true
+    find . -type f -exec sed -i.deleteme -e "1 s|^#!${PYTHON_MODULES_INSTALLROOT}/bin/\(.*\)$|#!/usr/bin/env \1|" \;
+    find . -name "*.deleteme" -delete
   popd
 popd
 
 # Patch long shebangs (by default max is 128 chars on Linux)
 pushd "$PYTHON_MODULES_INSTALLROOT/bin"
-  sed -i.deleteme -e '1 s|^#!.*$|#!/usr/bin/env python3|' * || true
-  rm -f *.deleteme
+  find . -type f -exec sed -i.deleteme -e "1 s|^#!${PYTHON_MODULES_INSTALLROOT}/bin/\(.*\)$|#!/usr/bin/env \1|" \;
+  find . -name "*.deleteme" -delete
 popd
 
 # Remove useless stuff

python-system.sh

@@ -7,5 +7,5 @@ system_requirement_missing: |
    * On macOS: brew install python3
 system_requirement: ".*"
 system_requirement_check: |
-  python3 -c 'import sys; import sqlite3; sys.exit(1 if sys.version_info < (3,5) else 0)' && pip3 help
+  python3 -c 'import sys; import sqlite3; sys.exit(1 if sys.version_info < (3,5) else 0)'
 ---

python.sh

@@ -133,5 +133,5 @@ setenv PYTHONHOME \$PYTHON_ROOT
 prepend-path PYTHONPATH \$PYTHON_ROOT/lib/python/site-packages
 prepend-path PATH \$PYTHON_ROOT/bin
 prepend-path LD_LIBRARY_PATH \$PYTHON_ROOT/lib
-setenv SSL_CERT_FILE  [exec python3 -c "import certifi; print(certifi.where())"]
+setenv SSL_CERT_FILE  [exec \$PYTHON_ROOT/bin/python3 -c "import certifi; print(certifi.where())"]
 EoF