diff --git a/CHANGELOG.md b/CHANGELOG.md index f8a03f22d1..2e5990a8b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ **Note**: Numbers like (\#1234) point to closed Pull Requests on the fractal-server repository. +# 2.3.7 + +* Task collection: + * Introduce a new configuration variable `FRACTAL_MAX_PIP_VERSION` to pin task-collection pip (\#1675). # 2.3.6 @@ -21,6 +25,7 @@ * SSH features: * Add `FractalSSH.write_remote_file` method (\#1678). + # 2.3.4 * SSH SLURM runner: diff --git a/fractal_server/config.py b/fractal_server/config.py index e1a9feb54b..b0496f3b0f 100644 --- a/fractal_server/config.py +++ b/fractal_server/config.py @@ -553,6 +553,11 @@ def check_tasks_python(cls, values) -> None: Whether to include the v1 API. """ + FRACTAL_MAX_PIP_VERSION: str = "24.0" + """ + Maximum value at which to update `pip` before performing task collection. + """ + ########################################################################### # BUSINESS LOGIC ########################################################################### diff --git a/fractal_server/tasks/v2/_venv_pip.py b/fractal_server/tasks/v2/_venv_pip.py index 6324e4b303..7cde862cf9 100644 --- a/fractal_server/tasks/v2/_venv_pip.py +++ b/fractal_server/tasks/v2/_venv_pip.py @@ -2,7 +2,9 @@ from typing import Optional from ..utils import COLLECTION_FREEZE_FILENAME +from fractal_server.config import get_settings from fractal_server.logger import get_logger +from fractal_server.syringe import Inject from fractal_server.tasks.v2._TaskCollectPip import _TaskCollectPip from fractal_server.tasks.v2.utils import get_python_interpreter_v2 from fractal_server.utils import execute_command @@ -24,6 +26,7 @@ async def _pip_install( Returns: The location of the package. """ + settings = Inject(get_settings) logger = get_logger(logger_name) @@ -41,7 +44,10 @@ async def _pip_install( await execute_command( cwd=venv_path, - command=f"{pip} install --upgrade pip", + command=( + f"{pip} install --upgrade " + f"'pip<={settings.FRACTAL_MAX_PIP_VERSION}'" + ), logger_name=logger_name, ) await execute_command( diff --git a/fractal_server/tasks/v2/background_operations_ssh.py b/fractal_server/tasks/v2/background_operations_ssh.py index 3423232dae..3ba0f8dd75 100644 --- a/fractal_server/tasks/v2/background_operations_ssh.py +++ b/fractal_server/tasks/v2/background_operations_ssh.py @@ -174,6 +174,10 @@ def background_collect_pip_ssh( ("__PACKAGE__", task_pkg.package), ("__PYTHON__", python_bin), ("__INSTALL_STRING__", install_string), + ( + "__FRACTAL_MAX_PIP_VERSION__", + settings.FRACTAL_MAX_PIP_VERSION, + ), ] common_args = dict( diff --git a/fractal_server/tasks/v2/templates/_2_upgrade_pip.sh b/fractal_server/tasks/v2/templates/_2_upgrade_pip.sh index b37875cd10..dae92aa972 100644 --- a/fractal_server/tasks/v2/templates/_2_upgrade_pip.sh +++ b/fractal_server/tasks/v2/templates/_2_upgrade_pip.sh @@ -14,7 +14,7 @@ VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python # Upgrade pip write_log "START upgrade pip" -"$VENVPYTHON" -m pip install pip --upgrade +"$VENVPYTHON" -m pip install "pip<=__FRACTAL_MAX_PIP_VERSION__" --upgrade write_log "END upgrade pip" echo diff --git a/fractal_server/tasks/v2/templates/_4_pip_freeze.sh b/fractal_server/tasks/v2/templates/_4_pip_freeze.sh index 73cf14dc08..ef6e570a79 100644 --- a/fractal_server/tasks/v2/templates/_4_pip_freeze.sh +++ b/fractal_server/tasks/v2/templates/_4_pip_freeze.sh @@ -12,4 +12,4 @@ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__ VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python -"$VENVPYTHON" -m pip freeze +"$VENVPYTHON" -m pip freeze --all diff --git a/tests/v2/00_ssh/test_task_collection_ssh.py b/tests/v2/00_ssh/test_task_collection_ssh.py index ac31ff366b..1e1cf597f9 100644 --- a/tests/v2/00_ssh/test_task_collection_ssh.py +++ b/tests/v2/00_ssh/test_task_collection_ssh.py @@ -26,12 +26,15 @@ async def test_task_collection_ssh( parents=True, ) + CURRENT_FRACTAL_MAX_PIP_VERSION = "21.0" + current_py_version_underscore = current_py_version.replace(".", "_") PY_KEY = f"FRACTAL_TASKS_PYTHON_{current_py_version_underscore}" setting_overrides = { "FRACTAL_SLURM_WORKER_PYTHON": f"/usr/bin/python{current_py_version}", PY_KEY: f"/usr/bin/python{current_py_version}", "FRACTAL_SLURM_SSH_WORKING_BASE_DIR": remote_basedir, + "FRACTAL_MAX_PIP_VERSION": CURRENT_FRACTAL_MAX_PIP_VERSION, } override_settings_factory(**setting_overrides) @@ -54,6 +57,14 @@ async def test_task_collection_ssh( debug(state) assert state.data["status"] == "OK" + # Check that pip version contraint is valid + pip_version = next( + line + for line in state.data["freeze"].split("\n") + if line.startswith("pip") + ).split("==")[1] + assert pip_version == CURRENT_FRACTAL_MAX_PIP_VERSION + # Check that the remote folder exists (note: we can do it on the host # machine, because /tmp is shared with the container) venv_dir = Path(remote_basedir) / ".fractal/fractal-tasks-core1.0.2" diff --git a/tests/v2/03_api/test_api_task_collection.py b/tests/v2/03_api/test_api_task_collection.py index 6d9e5702c6..ca947c51af 100644 --- a/tests/v2/03_api/test_api_task_collection.py +++ b/tests/v2/03_api/test_api_task_collection.py @@ -5,6 +5,7 @@ import pytest from devtools import debug # noqa +from packaging.version import Version from fractal_server.app.models.v2 import CollectionStateV2 from fractal_server.app.models.v2 import TaskV2 @@ -40,6 +41,7 @@ async def test_task_collection_from_wheel( FRACTAL_TASKS_DIR=(tmp_path / "FRACTAL_TASKS_DIR"), FRACTAL_LOGGING_LEVEL=logging.CRITICAL, FRACTAL_TASKS_PYTHON_DEFAULT_VERSION=current_py_version, + FRACTAL_MAX_PIP_VERSION="20.0", ) settings = Inject(get_settings) @@ -78,6 +80,14 @@ async def test_task_collection_from_wheel( res = await client.get(f"{PREFIX}/collect/{state_id}/") assert res.status_code == 200 state = res.json() + pip_version = next( + line + for line in state["data"]["freeze"].split("\n") + if line.startswith("pip") + ).split("==")[1] + assert Version(pip_version) <= Version( + settings.FRACTAL_MAX_PIP_VERSION + ) data = state["data"] task_list = data["task_list"] for i, task in enumerate(task_list):