[#502] experimental test harness using a single container

irods/test/demo.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo "$0 running"
+echo args:
+for arg in $*; do
+    echo $((++x)): "[$arg]"
+done
+
+exit 118

irods/test/demo_A.sh

@@ -0,0 +1 @@
+demo.sh

irods/test/demo_B.sh

@@ -0,0 +1 @@
+demo.sh

irods/test/demo_hook.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "-- HOOK RUNNING --"
+command "/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" $*

irods/test/harness/000_install-irods.Dockerfile

@@ -0,0 +1,14 @@
+FROM ubuntu:18.04
+COPY install.sh /
+ARG irods_package_version
+ENV IRODS_PACKAGE_VERSION "$irods_package_version"
+RUN for phase in initialize install-essential-packages add-package-repo; do \
+        bash /install.sh --w=$phase 0; \
+    done
+RUN /install.sh 4
+COPY start_postgresql_and_irods.sh /
+RUN apt install -y sudo
+RUN useradd -ms/bin/bash testuser
+RUN echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers
+RUN apt install -y faketime
+CMD bash /start_postgresql_and_irods.sh

irods/test/harness/001_bats-python3.Dockerfile

@@ -0,0 +1,5 @@
+from install-irods
+run apt update; apt install -y python3-pip bats
+run python3 -m pip install --upgrade pip
+run python3 -m pip install virtualenv
+run python3 -m virtualenv /py3

irods/test/harness/002_ssl-and-pam.Dockerfile

@@ -0,0 +1 @@
+FROM bats-python3

irods/test/harness/003_prc-with-py3_12_2.Dockerfile

@@ -0,0 +1,14 @@
+from install-irods
+run apt update
+run apt install -y wget build-essential libssl-dev zlib1g-dev
+run apt install wget build-essential
+run wget https://www.python.org/ftp/python/3.12.2/Python-3.12.2.tar.xz
+run tar xf Python-3.12.2.tar.xz
+workdir /Python-3.12.2
+run ./configure --prefix /root/python --with-ensurepip=install
+run make -j
+run mkdir /root/python
+run make install
+workdir /
+run /root/python/bin/python3 -m pip install python-irodsclient
+run chmod a+rx /root

irods/test/harness/README.txt

@@ -0,0 +1,46 @@
+SAMPLE RUNS
+
+To build required images
+------------------------
+Examples
+
+    1) ./build-docker.sh 
+       DEFAULT: build  single-node system based on latest iRODS release
+
+    2) IRODS_PACKAGE_VERSION="4.2.12-1~bionic" NO_CACHE='1' ./build-docker.sh 
+       Build (ignoring docker cache) single-node system based on specified package version string.
+
+simple examples
+---------------
+./docker_container_driver.sh  tests/test_1.sh 
+./docker_container_driver.sh  tests/test_2.sh 
+
+Any script in a subdirectory of the repo (mounted at /prc within the container) can be
+executed and will be able to find other scripts and source include files within the tree.
+[See "experiment.sh" example below.]
+
+Examples of options in driver script
+------------------------------------
+
+  1. To start container and run test script:
+     C=$(  ./docker_container_driver.sh -c -L -u testuser  ../scripts/experiment.sh )
+
+  2. To manually examine results afterward:
+     docker exec -it $C bash 
+
+
+Demo / Demo hook  / args
+------------------------
+
+$ ~/python-irodsclient/irods/test/harness$ ./docker_container_driver.sh  ../demo.sh 
+ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[irods/test/demo.sh]
+image=[ssl-and-pam]
+.......-- HOOK RUNNING --
+/prc/irods/test/demo.sh running
+args:
+1: [arg1]
+2: [arg2]
+Killed: 1358fbff6eadac24f0915ffb414f0367deedc84b0c3e4de69a23bd3a8726298f
+daniel@prec3431:~/python-irodsclient/irods/test/harness$ echo $?
+118
+

irods/test/harness/build-docker.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# IRODS_PACKAGE_VERSION if defined is like "4.3.1-0~bionic"
+# (but contains no '~' suffix for irods versions <= 4.2.10)
+
+BASE=$(basename "$0")
+DIR=$(realpath "$(dirname "$0")")
+cd "$DIR"
+DOCKER=docker
+for dockerfile in [0-9]*.Dockerfile; do 
+    image_name=${dockerfile#[0-9]*_}
+    image_name=${image_name%.Dockerfile}
+    if [ "$image_name" = "install-irods" ];then
+        package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"}
+    else
+        package_version_option=""
+    fi
+    $DOCKER build -f $dockerfile -t $image_name . $package_version_option \
+                                                  ${NO_CACHE+"--no-cache"} || 
+                                                  { STATUS=$?; echo "*** Failure while building [$image_name]"; exit $STATUS; }
+done

irods/test/harness/docker_container_driver.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+KILL_TEST_CONTAINER=1
+RUN_AS_USER=""
+ECHO_CONTAINER=""
+
+EXPLICIT_WORKDIR=""
+while [[ $1  = -* ]]; do
+    if [ "$1" = -c ]; then
+        ECHO_CONTAINER=1
+        shift
+    fi
+    if [ "$1" = -L ]; then
+        KILL_TEST_CONTAINER=0
+        shift
+    fi
+    if [ "$1" = -u ]; then
+        RUN_AS_USER="$2"
+        shift 2
+    fi
+    if [ "$1" = -w ]; then
+        EXPLICIT_WORKDIR="$2"
+        shift 2
+    fi
+done
+
+if [ "$1" = "" ]; then
+    echo >&2 "Usage: $0 [options] /path/to/script"
+    echo >&2 "With options: [-L] to leak, [-u username] to run as non-root user"
+    exit 1
+fi
+
+DIR=$(dirname $0)
+. "$DIR"/test_script_parameters
+
+testscript=${1}
+
+testscript_basename=$(basename "$testscript")
+arglist=${wrapper_arglist[$testscript_basename]}  # arglist dominated by symbolic link name if any
+
+if [ -L "$testscript" ]; then
+    testscript=$(realpath "$testscript")
+    testscript_basename=$(basename "$testscript")
+fi
+
+original_testscript_abspath=$(realpath "$testscript")
+
+wrapped=${wrappers["$testscript_basename"]}
+
+if [ -n "$wrapped" ]; then
+    # wrapped is assumed to contain a leading path element relative to the referencing script's containing directory
+    testscript="$(dirname "$testscript")/$wrapped"
+    testscript_basename=$(basename "$testscript")
+fi
+
+testscript_abspath=$(realpath "$testscript")
+
+cd "$DIR"
+
+image=${images[$testscript_basename]}
+
+if [ -z "$RUN_AS_USER" ]; then
+    RUN_AS_USER=${user[$testscript_basename]}
+fi
+
+# Tests are run as testuser by default
+: ${RUN_AS_USER:='testuser'}
+
+WORKDIR=""
+if [ -n "$EXPLICIT_WORKDIR" ]; then
+    WORKDIR="$EXPLICIT_WORKDIR"
+else
+    WORKDIR=${workdirs[$RUN_AS_USER]}
+fi
+
+reporoot=$(./print_repo_root_location)
+ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$(realpath --relative-to $reporoot "$original_testscript_abspath")
+
+echo "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT]" 
+INNER_MOUNT=/prc
+
+# Start the container.
+echo image="[$image]"
+CONTAINER=$(docker run -d -v $reporoot:$INNER_MOUNT:ro --rm $image)
+
+# Wait for iRODS and database to start up.
+TIME0=$(date +%s)
+while :; do
+    [ `date +%s` -gt $((TIME0 + 30)) ] && { echo >&2 "Waited too long for DB and iRODS to start"; exit 124; }
+    sleep 1 
+    docker exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null
+    [ $? -ne 0 ] && { echo -n . >&2; continue; }
+    break
+done
+
+docker exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \
+            ${WORKDIR:+"-w$WORKDIR"} \
+	    -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \
+            $CONTAINER \
+            $INNER_MOUNT/$(realpath --relative-to $reporoot "$testscript_abspath") \
+            $arglist
+STATUS=$?
+
+if [ $((0+KILL_TEST_CONTAINER)) -ne 0 ]; then
+    echo >&2 'Killed:' $(docker stop --time=0 $CONTAINER)
+fi
+
+[ -n "$ECHO_CONTAINER" ] && echo $CONTAINER
+exit $STATUS

irods/test/harness/install.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+
+IRODS_HOME=/var/lib/irods
+DEV_HOME="$HOME"
+: ${DEV_REPOS:="$DEV_HOME/github"}
+
+add_package_repo()
+{
+      local R="/etc/apt/sources.list.d/renci-irods.list"
+      echo >&2 "... installing package repo"
+      sudo apt update
+      sudo apt install -y lsb-release apt-transport-https
+      wget -qO - https://packages.irods.org/irods-signing-key.asc | sudo apt-key add - && \
+      echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+          sudo tee "$R"
+      sudo apt update
+}
+
+DIST_NAME=$(lsb_release -sc)
+
+: ${IRODS_VSN:=4.3.1-0~$DIST_NAME}
+
+while [[ "$1" = -* ]]; do
+  ARG="$1"
+  shift
+  case $ARG in
+    --i=* | --irods=* |\
+    --irods-version=*) IRODS_PACKAGE_VERSION=${ARG#*=};;
+    --w=* | --with=* | --with-options=* ) withopts=${ARG#*=} ;;
+    -v) VERBOSE=1;;
+  esac
+done
+
+
+run_phase() {
+
+ local PHASE=$1
+ local with_opts=" $2 "
+
+ case "$PHASE" in
+
+ 0)
+
+    if [[ $with_opts = *\ initialize\ * ]]; then
+        apt-get -y update
+        apt-get  install -y apt-transport-https wget lsb-release sudo jq
+    fi
+
+    if [[ $with_opts = *\ sudo-without-pw\ * ]]; then
+      if [ `id -u` = 0 -a "${USER:-root}" = root ] ; then
+        echo >&2 "root authorization for 'sudo' is automatic - no /etc/sudoers modification needed"
+      else
+        if [ -f "/etc/sudoers" ]; then
+           if [ -n "$USER" ] ; then
+             # add a line with our USER name to /etc/sudoers if not already there
+             sudo su -c "sed -n '/^\s*[^#]/p' /etc/sudoers | grep '^$USER\s*ALL=(ALL)\s*NOPASSWD:\s*ALL\s*$' >/dev/null" || \
+             sudo su -c "echo '$USER ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers"
+           else
+             echo >&2 "user login is '$USER' - can this be right?"
+           fi
+        else
+           echo >&2 "WARNING - Could not modify sudoers files"
+           echo -n >&2 "           (hit 'Enter' to continue)"
+           read key
+        fi
+      fi # not root
+    fi # with-opts
+
+    #------ (needed for both package install and build from source)
+
+    if [[ $with_opts = *\ install-essential-packages\ * ]]; then
+
+        if ! dpkg -l tzdata >/dev/null 2>&1 ; then
+          sudo su - root -c \
+           "env DEBIAN_FRONTEND=noninteractive bash -c 'apt-get install -y tzdata'"
+        fi
+        sudo apt-get update
+        sudo apt-get install -y software-properties-common postgresql
+        sudo apt-get update && \
+        sudo apt-get install -y libfuse2 unixodbc rsyslog ################### less python-pip 
+    fi
+
+
+    if [[ $with_opts = *\ add-package-repo\ * ]]; then
+            add_package_repo -f
+    fi
+
+
+    if [[ $with_opts = *\ create-db\ * ]]; then
+    sudo su - postgres -c "
+        { dropdb --if-exists ICAT
+          dropuser --if-exists irods ; } >/dev/null 2>&1"
+    sudo su - postgres -c "psql <<\\
+________
+        CREATE DATABASE \"ICAT\";
+        CREATE USER irods WITH PASSWORD 'testpassword';
+        GRANT ALL PRIVILEGES ON DATABASE \"ICAT\" to irods;
+________"
+    echo >&2 "-- status of create-db =  $? -- "
+    fi
+    ;;
+
+ 4)
+   sudo apt install -y irods-{dev,runtime}${IRODS_PACKAGE_VERSION:+"=$IRODS_PACKAGE_VERSION"}
+   if [[ $with_opts != *\ basic\ * ]]; then
+     sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_PACKAGE_VERSION:+"=$IRODS_PACKAGE_VERSION"}
+   fi
+ ;;
+
+ 5)
+ if [ ! "$IRODS_VSN" '<' "4.3" ]; then
+    PYTHON=python3
+ else
+    PYTHON=python2
+ fi
+ sudo $PYTHON /var/lib/irods/scripts/setup_irods.py < /var/lib/irods/packaging/localhost_setup_postgres.input
+ ;;
+
+ *) echo >&2 "unrecognized phase: '$PHASE'." ; QUIT=1 ;;
+ esac
+ return $?
+}
+
+#-------------------------- main
+
+QUIT=0
+while [ $# -gt 0 ] ; do
+  ARG=$1 ; shift
+  NOP="" ; run_phase $ARG " $withopts "; sts=$?
+  [ $QUIT != 0 ] && break
+  [ -n "$NOP" ] && continue
+  echo -n "== $ARG == "
+  if [ $sts -eq 0 ]; then
+    echo Y >&2
+  else
+    [ $quit_on_phase_err ] && { echo >&2 "N - quitting"; exit 1; }
+    echo N >&2
+  fi
+done

irods/test/harness/print_repo_root_location

@@ -0,0 +1,5 @@
+#!/bin/bash
+# The following line needs be kept updated to reflect true position relative to repository root,
+# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved.
+REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=../../..
+realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT"