From 51514c56a707dcb6e1e5a5d343bf3351a23d9250 Mon Sep 17 00:00:00 2001 From: Alexandre Menasria Date: Wed, 30 Oct 2024 15:51:40 +0100 Subject: [PATCH 1/5] Revert "Revert "[CI] Select the right Python virtualenv on the macOS Gitlab runners (#30329)" (#30579)" This reverts commit 165d82003eb0a4db4f6c58b1d13fd115648630b0. --- .gitlab/common/macos.yml | 52 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/.gitlab/common/macos.yml b/.gitlab/common/macos.yml index 51146fd31c80b..2fdf2fde4386f 100644 --- a/.gitlab/common/macos.yml +++ b/.gitlab/common/macos.yml @@ -1,27 +1,49 @@ --- # This is the scripts to be executed on the Gitlab macOS runners before every job. # We don't have virtualization now so we need to clean the environment and install the proper dependencies before every job. +.list_go_versions_commands: &list_go_versions_commands + - | + echo "Don't forget to regularly delete Go unused versions. Here are the installed Go versions and their disk space on the runner:" + echo "Go:" + du -sh $HOME/.gimme/versions/* + echo "To remove a Go version please run:" + echo "gimme uninstall " + +.list_python_versions_commands: &list_python_versions_commands + - | + echo "Don't forget to regularly delete Python unused versions. Here are the installed Python versions and their disk space on the runner:" + echo "Python:" + du -sh $(pyenv root)/versions/* + echo "To remove a Python version please run:" + echo "pyenv uninstall -f " + +.select_python_env_commands: &select_python_env_commands + # Print a warning if the current Python version is different from the one in .python-version + # Select the virtualenv using the current Python version. Create it if it doesn't exist. + - | + PYTHON_REPO_VERSION=$(cat .python-version) + PYTHON_VERSION=$(python3 --version | awk '{print $2}' | sed 's/\.[0-9]*$//') + VENV_NAME="datadog-agent-python-$PYTHON_VERSION" + if [ "$PYTHON_REPO_VERSION" != "$PYTHON_VERSION" ]; then + echo "Warning: The current Python version $PYTHON_VERSION is different from $PYTHON_REPO_VERSION in .python-version." + echo "Installing Python $PYTHON_REPO_VERSION..." + fi + if ! pyenv virtualenvs --bare | grep -q "^${VENV_NAME}$"; then + pyenv virtualenv $PYTHON_VERSION $VENV_NAME + fi + pyenv activate $VENV_NAME + .macos_gitlab: before_script: # Selecting the current Go version - | eval $(gimme $(cat .go-version)) export GOPATH=$GOROOT - echo "Don't forget to regularly delete unused versions. Here are the installed versions and their memory usage on the runner:" - du -sh $HOME/.gimme/versions/* - # Remove the Python cache and env if the Python version changed - - | - PYTHON_REPO_VERSION=$(cat .python-version) - PYTHON_VERSION=$(python3 --version | awk '{print $2}' | sed 's/\.[0-9]*$//') - if [ "$PYTHON_REPO_VERSION" != "$PYTHON_VERSION" ]; then - echo "Python version $PYTHON_VERSION is different from $PYTHON_REPO_VERSION in .python-version. Cleaning the environment." - pyenv uninstall -f datadog-agent - echo "Installing Python $PYTHON_REPO_VERSION..." - pyenv virtualenv 3.11.8 datadog-agent - pyenv activate datadog-agent - else - echo "Python current version $PYTHON_VERSION is the same as .python-version. Keeping the existing environment." - fi + # Selecting the current Python version + - *select_python_env_commands + # List Python and Go existing environments and their disk space + - *list_go_versions_commands + - *list_python_versions_commands # Installing the job dependencies - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-github.txt - pyenv rehash From 2885e88a90a8b6341c34963d98062b7e3d1b8d30 Mon Sep 17 00:00:00 2001 From: Alexandre Menasria Date: Wed, 30 Oct 2024 15:56:18 +0100 Subject: [PATCH 2/5] Use Gitlab references instead of yaml references --- .gitlab/common/macos.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab/common/macos.yml b/.gitlab/common/macos.yml index 2fdf2fde4386f..e5e87ad4086ac 100644 --- a/.gitlab/common/macos.yml +++ b/.gitlab/common/macos.yml @@ -1,7 +1,7 @@ --- # This is the scripts to be executed on the Gitlab macOS runners before every job. # We don't have virtualization now so we need to clean the environment and install the proper dependencies before every job. -.list_go_versions_commands: &list_go_versions_commands +.list_go_versions_commands: - | echo "Don't forget to regularly delete Go unused versions. Here are the installed Go versions and their disk space on the runner:" echo "Go:" @@ -9,7 +9,7 @@ echo "To remove a Go version please run:" echo "gimme uninstall " -.list_python_versions_commands: &list_python_versions_commands +.list_python_versions_commands: - | echo "Don't forget to regularly delete Python unused versions. Here are the installed Python versions and their disk space on the runner:" echo "Python:" @@ -17,7 +17,7 @@ echo "To remove a Python version please run:" echo "pyenv uninstall -f " -.select_python_env_commands: &select_python_env_commands +.select_python_env_commands: # Print a warning if the current Python version is different from the one in .python-version # Select the virtualenv using the current Python version. Create it if it doesn't exist. - | @@ -40,10 +40,10 @@ eval $(gimme $(cat .go-version)) export GOPATH=$GOROOT # Selecting the current Python version - - *select_python_env_commands + - !reference [.select_python_env_commands] # List Python and Go existing environments and their disk space - - *list_go_versions_commands - - *list_python_versions_commands + - !reference [.list_go_versions_commands] + - !reference [.list_python_versions_commands] # Installing the job dependencies - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-github.txt - pyenv rehash From 67fba0a3989f8ea4ec9b9f499d541d0f3f20c7e0 Mon Sep 17 00:00:00 2001 From: Alexandre Menasria Date: Thu, 31 Oct 2024 11:44:22 +0100 Subject: [PATCH 3/5] Trigger pipeline From 48b1d5b68d9d8c7587b3f42d3493b23fcb542911 Mon Sep 17 00:00:00 2001 From: Alexandre Menasria Date: Thu, 31 Oct 2024 13:56:44 +0100 Subject: [PATCH 4/5] Test --- .gitlab/common/macos.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab/common/macos.yml b/.gitlab/common/macos.yml index e5e87ad4086ac..9220148a1dbc6 100644 --- a/.gitlab/common/macos.yml +++ b/.gitlab/common/macos.yml @@ -26,8 +26,15 @@ VENV_NAME="datadog-agent-python-$PYTHON_VERSION" if [ "$PYTHON_REPO_VERSION" != "$PYTHON_VERSION" ]; then echo "Warning: The current Python version $PYTHON_VERSION is different from $PYTHON_REPO_VERSION in .python-version." - echo "Installing Python $PYTHON_REPO_VERSION..." + echo "Using Python $PYTHON_VERSION" fi + echo $PYTHON_VERSION + echo $VENV_NAME + pyenv virtualenvs --bare + pyenv virtualenvs --bare | grep -q "^${VENV_NAME}$" + echo $? + ! pyenv virtualenvs --bare | grep -q "^${VENV_NAME}$" + echo $? if ! pyenv virtualenvs --bare | grep -q "^${VENV_NAME}$"; then pyenv virtualenv $PYTHON_VERSION $VENV_NAME fi From 2b8b510a2040c2d4e84aabd77d3b89867a1972db Mon Sep 17 00:00:00 2001 From: Alexandre Menasria Date: Thu, 31 Oct 2024 16:08:22 +0100 Subject: [PATCH 5/5] Revert "Test" This reverts commit 48b1d5b68d9d8c7587b3f42d3493b23fcb542911. --- .gitlab/common/macos.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.gitlab/common/macos.yml b/.gitlab/common/macos.yml index 9220148a1dbc6..e5e87ad4086ac 100644 --- a/.gitlab/common/macos.yml +++ b/.gitlab/common/macos.yml @@ -26,15 +26,8 @@ VENV_NAME="datadog-agent-python-$PYTHON_VERSION" if [ "$PYTHON_REPO_VERSION" != "$PYTHON_VERSION" ]; then echo "Warning: The current Python version $PYTHON_VERSION is different from $PYTHON_REPO_VERSION in .python-version." - echo "Using Python $PYTHON_VERSION" + echo "Installing Python $PYTHON_REPO_VERSION..." fi - echo $PYTHON_VERSION - echo $VENV_NAME - pyenv virtualenvs --bare - pyenv virtualenvs --bare | grep -q "^${VENV_NAME}$" - echo $? - ! pyenv virtualenvs --bare | grep -q "^${VENV_NAME}$" - echo $? if ! pyenv virtualenvs --bare | grep -q "^${VENV_NAME}$"; then pyenv virtualenv $PYTHON_VERSION $VENV_NAME fi