From f5719b304a089a15a6097df26ed3a2c4ee905e3c Mon Sep 17 00:00:00 2001
From: Lukas Piwowarski <lpiwowar@redhat.com>
Date: Fri, 10 Nov 2023 17:17:21 +0100
Subject: [PATCH] Support external tempest plugins

This patch adds support for the tempest container to be able to install
external plugins that are not part of the .rpm the container is
built with.

When a user defines the TEMPEST_EXTERNAL_PLUGIN_GIT_URLS then plugin(s)
defined by this variable will be downloaded and executed. The user
can also define TEMPEST_EXTERNAL_PLUGIN_CHANGE_URLS and
TEMPEST_EXTERNAL_PLUGIN_REFSPEC variable to download a change(s) for a
given plugin. This can be used for testing tests under developoment.

TEMPEST_EXTERNAL_PLUGIN_GIT_URLS
    - URL of a git repository that holds the plugin

TEMPEST_EXTERNAL_PLUGIN_CHANGE_URLS
    - URL from which a change should be downloaded for a given plugin

TEMPEST_EXTERNAL_PLUGIN_REFSPEC
    - A change that should be downloaded for a given plugin
---
 .../tcib/base/os/tempest/run_tempest.sh       | 205 +++++++++++++++---
 1 file changed, 180 insertions(+), 25 deletions(-)

diff --git a/container-images/tcib/base/os/tempest/run_tempest.sh b/container-images/tcib/base/os/tempest/run_tempest.sh
index 8aa5ecd7..3bad6a38 100644
--- a/container-images/tcib/base/os/tempest/run_tempest.sh
+++ b/container-images/tcib/base/os/tempest/run_tempest.sh
@@ -1,9 +1,96 @@
 #!/bin/sh
+# run_tempest.sh
+# ==============
+#
+# This script is executed inside the tempest containers defined in the tcib
+# repository (tempest, tempest-all and tempest-extra). The main purpose of this
+# script is executing the tempest command with correct arguments and preparing
+# the tempest.conf file. The execution of the script can be influenced by
+# setting values for environment variables which match the TEMPEST_* or
+# TEMPESTCONF_* regex.
+#
+#
+# TEMPESTCONF_* environment variables
+# -----------------------------------
+#
+# These variables define with which arguments should be the
+# discover-tempest-config command executed.
+#
+# Supported boolean arguments:
+#   --create, --insecure, --collect-timing, --no-default-deployer, --debug
+#   --verbose, --no-rng, --non-admin, --retry-image, --convert-to-raw
+#
+# Supported string arguments:
+#   --timeout, --out, --deployer-input, --test-accounts, --create-accounts-file
+#   --profile, --generate-profile, --image-disk-format, --image,
+#   --flavor-min-mem, --flavor-min-disk, --disk, --network-id
+#
+# If you want discover-tempest-config to be executed with any of the boolean arguments
+# mentioned above, then set corresponding environment variable to 'true'. The
+# name of the variable consists of the prefix 'TEMPESTCONF_' plus the name
+# of the argument. For example, if you want to run discover-tempest-config with
+# --non-admin argument then set TEMPESTCONF_NON_ADMIN=true. Similarly, in the case
+# of string arguments, when you want to run discover-tempest-config with
+# --timeout 600 then set TEMPESTCONF_TIMEOUT=600.
+#
+# To override values in tempest.conf generated by discover-tempest-config please
+# use TEMPESTCONF_OVERRIDES.
+#
+# TEMPEST_* environment ariables
+# ------------------------------
+#
+# These variables define behaviour of the part of the script which is
+# responsible for the tempest tests execution.
+#
+# Supported boolean arguments:
+#   --smoke, --parallel, --serial
+#
+# Supported string arguments:
+#   --include-list, --exclude-list, --concurrency, --worker-file,
+#
+# If you want the tempest command to be executed with any of the boolean
+# arguments mentioned above, then set the corresponding environment variable to
+# 'true'. The name of the variable consists of the prefix 'TEMPEST_' plus the
+# name of the argument. For example, if you want to run tempest with --smoke
+# argument then set TEMPEST_SMOKE=true. Similarly, in the case of string
+# arguments, when you want to run tempest witn --include-list=/path/to/list.txt
+# then set TEMPEST_INCLUDE_LIST=/path/to/list.txt
+#
+# The script, putting the above-mentioned tempest environment variables asside,
+# offers also the following extra variables: TEMPEST_EXTERNAL_PLUGIN_GIT_URL,
+# TEMPEST_EXTERNAL_PLUGIN_CHANGE_URL, TEMPEST_EXTERNAL_PLUGIN_REFSPEC. These
+# can be used to specify extra tempest plugins you want to be installed within
+# the container plus patches you want to be applied on top of the installed
+# plugins.
+#
+# For example, by setting the environment variables like in the example below
+# you ensure that the script downloads barbican-tempest-plugin and
+# neutron-tempest-plugin and installs neutron-tempest-plugin with
+# "refs/changes/97/896397/2" change from gerrit:
+#
+#
+# TEMPEST_EXTERNAL_PLUGIN_CHANGE_URL=\
+# "https://opendev.org/openstack/barbican-tempest-plugin.git,"\
+# "https://opendev.org/openstack/neutron-tempest-plugin.git"
+#
+# TEMPEST_EXTERNAL_PLUGIN_CHANGE_URL=\
+# "-,"\
+# "https://review.opendev.org/openstack/neutron-tempest-plugin"
+#
+# TEMPEST_EXTERNAL_PLUGIN_REFSPEC=\
+# "-,"\
+# "refs/changes/97/896397/2"
+#
+# Note, that if you specify these variables then tempest will have access only
+# to plugins which were specified by these variables. Meaning, if you consider
+# the example above, then tempest won't be able to execute tests from
+# octavia-tempest-plugin.
+
 set -x
 
 HOMEDIR=/var/lib/tempest
-TEMPEST_DIR=$HOMEDIR/openshift
 TEMPEST_PATH=$HOMEDIR/
+TEMPEST_DIR=$HOMEDIR/openshift
 CONCURRENCY="${CONCURRENCY:-}"
 TEMPESTCONF_ARGS=""
 TEMPEST_ARGS=""
@@ -34,7 +121,22 @@ TEMPEST_ARGS=""
 [[ ! -z ${TEMPESTCONF_FLAVOR_MIN_DISK} ]] && TEMPESTCONF_ARGS+="--flavor-min-disk ${TEMPESTCONF_FLAVOR_MIN_DISK} "
 [[ ! -z ${TEMPESTCONF_NETWORK_ID} ]] && TEMPESTCONF_ARGS+="--network-id ${TEMPESTCONF_NETWORK_ID} "
 
+TEMPESTCONF_OVERRIDES="$(echo ${TEMPESTCONF_OVERRIDES} | tr '\n' ' ') identity.v3_endpoint_type public"
+
 # Tempest arguments
+TEMPEST_EXTERNAL_PLUGIN_GIT_URL="${TEMPEST_EXTERNAL_PLUGIN_GIT_URL:-}"
+TEMPEST_EXTERNAL_PLUGIN_CHANGE_URL="${TEMPEST_EXTERNAL_PLUGIN_CHANGE_URL:-}"
+TEMPEST_EXTERNAL_PLUGIN_REFSPEC="${TEMPEST_EXTERNAL_PLUGIN_REFSPEC:-}"
+TEMPEST_EXTERNAL_PLUGIN_DIR=/var/lib/tempest/external-plugins
+
+# Convert comma separated lists to arrays
+OLD_IFS=$IFS
+IFS=","
+read -ra TEMPEST_EXTERNAL_PLUGIN_GIT_URL <<< $TEMPEST_EXTERNAL_PLUGIN_GIT_URL
+read -ra TEMPEST_EXTERNAL_PLUGIN_CHANGE_URL <<< $TEMPEST_EXTERNAL_PLUGIN_CHANGE_URL
+read -ra TEMPEST_EXTERNAL_PLUGIN_REFSPEC <<< $TEMPEST_EXTERNAL_PLUGIN_REFSPEC
+IFS=$OLD_IFS
+
 [[ ${TEMPEST_SMOKE} == true ]] && TEMPEST_ARGS+="--smoke "
 [[ ${TEMPEST_PARALLEL:=true} == true ]] && TEMPEST_ARGS+="--parallel "
 [[ ${TEMPEST_SERIAL} == true ]] && TEMPEST_ARGS+="--serial "
@@ -58,13 +160,82 @@ if [[ ! -z ${TEMPESTCONF_REMOVE} ]]; then
     done <<< "$TEMPESTCONF_REMOVE"
 fi
 
-TEMPESTCONF_OVERRIDES="$(echo ${TEMPESTCONF_OVERRIDES} | tr '\n' ' ') identity.v3_endpoint_type public"
 
 if [ -n "$CONCURRENCY" ] && [ -z ${TEMPEST_CONCURRENCY} ]; then
     TEMPEST_ARGS+="--concurrency ${CONCURRENCY} "
 fi
 
-pushd $HOMEDIR
+function run_git_tempest {
+    mkdir -p $TEMPEST_EXTERNAL_PLUGIN_DIR
+    pushd $TEMPEST_EXTERNAL_PLUGIN_DIR
+
+    python3 -m venv .venv
+    source ./.venv/bin/activate
+
+    for plugin_index in "${!TEMPEST_EXTERNAL_PLUGIN_GIT_URL[@]}"; do
+        git_url=${TEMPEST_EXTERNAL_PLUGIN_GIT_URL[plugin_index]}
+        change_url=${TEMPEST_EXTERNAL_PLUGIN_CHANGE_URL[plugin_index]}
+        refspec=${TEMPEST_EXTERNAL_PLUGIN_REFSPEC[plugin_index]}
+        plugin_name=$(basename -s .git $git_url)
+
+        git clone $git_url
+
+        if [[ ! -z $change_url ]] && [[ ! -z $refspec ]] || \
+           [[ $change_url != "-" ]] && [[ $refspec != "-" ]]; then
+            pushd $plugin_name
+            git fetch $change_url $refspec
+            git checkout FETCH_HEAD
+            popd
+        fi
+
+        pip install ./$plugin_name
+    done
+
+    pushd $HOMEDIR
+    tempest init openshift
+    pushd $TEMPEST_DIR
+
+    discover-tempest-config ${TEMPESTCONF_ARGS} ${TEMPESTCONF_OVERRIDES}
+    tempest run ${TEMPEST_ARGS}
+    RETURN_VALUE=$?
+
+    deactivate
+
+    popd
+    popd
+    popd
+}
+
+function run_rpm_tempest {
+    pushd $HOMEDIR
+    tempest init openshift
+    pushd $TEMPEST_DIR
+
+    discover-tempest-config ${TEMPESTCONF_ARGS} ${TEMPESTCONF_OVERRIDES}
+    tempest run ${TEMPEST_ARGS}
+    RETURN_VALUE=$?
+
+    popd
+    popd
+}
+
+function generate_test_results {
+    pushd $TEMPEST_DIR
+
+    echo "Generate subunit"
+    stestr last --subunit > ${TEMPEST_PATH}testrepository.subunit || true
+
+    echo "Generate subunit xml file"
+    subunit2junitxml ${TEMPEST_PATH}testrepository.subunit > ${TEMPEST_PATH}tempest_results.xml || true
+
+    echo "Generate html result"
+    subunit2html ${TEMPEST_PATH}testrepository.subunit ${TEMPEST_PATH}stestr_results.html || true
+
+    echo Copying logs file
+    cp -rf ${TEMPEST_DIR}/* ${TEMPEST_PATH}
+
+    popd
+}
 
 export OS_CLOUD=default
 
@@ -85,28 +256,12 @@ if [ ! -f ${TEMPEST_PATH}exclude.txt ] && [ -z ${TEMPEST_EXCLUDE_LIST} ]; then
     touch ${TEMPEST_PATH}exclude.txt
 fi
 
-tempest init openshift
-
-pushd $TEMPEST_DIR
-discover-tempest-config ${TEMPESTCONF_ARGS} ${TEMPESTCONF_OVERRIDES}
-
-tempest run ${TEMPEST_ARGS}
-
-RETURN_VALUE=$?
-
-echo "Generate subunit"
-stestr last --subunit > ${TEMPEST_PATH}testrepository.subunit || true
-
-echo "Generate subunit xml file"
-subunit2junitxml ${TEMPEST_PATH}testrepository.subunit > ${TEMPEST_PATH}tempest_results.xml || true
-
-echo "Generate html result"
-subunit2html ${TEMPEST_PATH}testrepository.subunit ${TEMPEST_PATH}stestr_results.html || true
+if [ -z $TEMPEST_EXTERNAL_PLUGIN_GIT_URL ]; then
+    run_rpm_tempest
+else
+    run_git_tempest
+fi
 
-echo Copying logs file
-cp -rf ${TEMPEST_DIR}/* ${TEMPEST_PATH}
+generate_test_results
 
 exit ${RETURN_VALUE}
-
-popd
-popd