From a1bc98e4a74cb3697de48ce5511a1d5a504c68a2 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Tue, 30 Jan 2024 11:07:48 -0500 Subject: [PATCH 1/4] [_502] experimental test harness using containers setupssl chgs -q [_502] fixes/enhancements to test harness (inclusive misc__ commits) misc__ pre: make irods install simpler, independent of other repos run the test script using exec (The docker run command initiates an iRODS startup process via CMD). now able to define the irods version to be installed. fix relative paths; sample BATS test setupssl changes pam and ssl setup funcs build target version of Python (will parameterize version later) refactor of test containers experiment.sh -- call setupssl.py in container tighter checking in driver/test-runner corrections enable ageing out pam pw (postgresql only) misc__: add/refine options - workdir etc options include dummy_script called from experiment.sh move to test-script-parameters more utility funcs directory name change misc__ post: update json files for test restore correction to update json funcs chmod +x login_auth_test.py failing/script-to-script call demo implement wrapping virtual env provide path to original script; allow args (testnames) delete script not used include overlooked ssl-and-pam Dockerfile wrap/args demo get rid of troublesome recursive symlink abspath for orig script; -t better for early docker versions username is now param for setting up pam with iinit setup conditions properly for tests adjustments to keys. arglist is now affected by symlink basename of argument if any. changes to login_auth test changes for login_auth_test nr 2 allow different irods versions include symlink change to test script params only separate container run for env-modifying test test001 working include the other symlink In testing don't depend on strong primes for DH param generation. 362-test-harness setupssl-q-nr2 --- irods/test/demo.sh | 9 + irods/test/demo_A.sh | 1 + irods/test/demo_B.sh | 1 + irods/test/demo_hook.sh | 3 + .../test/harness/000_install-irods.Dockerfile | 12 ++ .../test/harness/001_bats-python3.Dockerfile | 5 + irods/test/harness/002_ssl-and-pam.Dockerfile | 4 + .../harness/003_prc-with-py3_12_2.Dockerfile | 14 ++ irods/test/harness/README.txt | 46 +++++ irods/test/harness/build-docker.sh | 21 ++ irods/test/harness/docker_container_driver.sh | 109 ++++++++++ irods/test/harness/install.sh | 139 +++++++++++++ irods/test/harness/print_repo_root_location | 5 + .../harness/start_postgresql_and_irods.sh | 20 ++ irods/test/harness/test_script_parameters | 41 ++++ irods/test/harness/tests/test_1.sh | 6 + irods/test/harness/tests/test_2.sh | 6 + irods/test/harness/tests/test_3.bats | 21 ++ irods/test/login_auth_test.sh | 39 ++++ irods/test/login_auth_test_1.py | 1 + irods/test/login_auth_test_2.py | 1 + irods/test/pam.bats/funcs | 108 ---------- irods/test/scripts/demo_script | 2 + irods/test/scripts/experiment.sh | 6 + irods/test/scripts/funcs | 190 ++++++++++++++++++ irods/test/scripts/pam.bats/fail.sh | 10 + .../test001_pam_password_expiration.bats | 2 +- irods/test/scripts/update_json_for_test | 55 +++++ 28 files changed, 768 insertions(+), 109 deletions(-) create mode 100755 irods/test/demo.sh create mode 120000 irods/test/demo_A.sh create mode 120000 irods/test/demo_B.sh create mode 100755 irods/test/demo_hook.sh create mode 100644 irods/test/harness/000_install-irods.Dockerfile create mode 100644 irods/test/harness/001_bats-python3.Dockerfile create mode 100644 irods/test/harness/002_ssl-and-pam.Dockerfile create mode 100644 irods/test/harness/003_prc-with-py3_12_2.Dockerfile create mode 100644 irods/test/harness/README.txt create mode 100755 irods/test/harness/build-docker.sh create mode 100755 irods/test/harness/docker_container_driver.sh create mode 100755 irods/test/harness/install.sh create mode 100755 irods/test/harness/print_repo_root_location create mode 100755 irods/test/harness/start_postgresql_and_irods.sh create mode 100644 irods/test/harness/test_script_parameters create mode 100755 irods/test/harness/tests/test_1.sh create mode 100755 irods/test/harness/tests/test_2.sh create mode 100755 irods/test/harness/tests/test_3.bats create mode 100755 irods/test/login_auth_test.sh create mode 120000 irods/test/login_auth_test_1.py create mode 120000 irods/test/login_auth_test_2.py delete mode 100644 irods/test/pam.bats/funcs create mode 100755 irods/test/scripts/demo_script create mode 100755 irods/test/scripts/experiment.sh create mode 100644 irods/test/scripts/funcs create mode 100755 irods/test/scripts/pam.bats/fail.sh rename irods/test/{pam.bats => scripts}/test001_pam_password_expiration.bats (97%) mode change 100644 => 100755 create mode 100644 irods/test/scripts/update_json_for_test diff --git a/irods/test/demo.sh b/irods/test/demo.sh new file mode 100755 index 00000000..7ef5b7c2 --- /dev/null +++ b/irods/test/demo.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +echo "$0 running" +echo args: +for arg in $*; do + echo $((++x)): "[$arg]" +done + +exit 118 diff --git a/irods/test/demo_A.sh b/irods/test/demo_A.sh new file mode 120000 index 00000000..52239f28 --- /dev/null +++ b/irods/test/demo_A.sh @@ -0,0 +1 @@ +demo.sh \ No newline at end of file diff --git a/irods/test/demo_B.sh b/irods/test/demo_B.sh new file mode 120000 index 00000000..52239f28 --- /dev/null +++ b/irods/test/demo_B.sh @@ -0,0 +1 @@ +demo.sh \ No newline at end of file diff --git a/irods/test/demo_hook.sh b/irods/test/demo_hook.sh new file mode 100755 index 00000000..8a9f855f --- /dev/null +++ b/irods/test/demo_hook.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo "-- HOOK RUNNING --" +command "/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" $* diff --git a/irods/test/harness/000_install-irods.Dockerfile b/irods/test/harness/000_install-irods.Dockerfile new file mode 100644 index 00000000..3d21517c --- /dev/null +++ b/irods/test/harness/000_install-irods.Dockerfile @@ -0,0 +1,12 @@ +FROM ubuntu:18.04 +COPY install.sh / +ARG irods_package_version +ENV IRODS_PACKAGE_VERSION "$irods_package_version" +RUN for phase in initialize install-essential-packages add-package-repo; do \ + bash /install.sh --w=$phase 0; \ + done +RUN /install.sh 4 +COPY start_postgresql_and_irods.sh / +RUN useradd -ms/bin/bash user +RUN apt install -y faketime +CMD bash /start_postgresql_and_irods.sh diff --git a/irods/test/harness/001_bats-python3.Dockerfile b/irods/test/harness/001_bats-python3.Dockerfile new file mode 100644 index 00000000..1c1e7b69 --- /dev/null +++ b/irods/test/harness/001_bats-python3.Dockerfile @@ -0,0 +1,5 @@ +from install-irods +run apt update; apt install -y python3-pip bats +run python3 -m pip install --upgrade pip +run python3 -m pip install virtualenv +run python3 -m virtualenv /py3 diff --git a/irods/test/harness/002_ssl-and-pam.Dockerfile b/irods/test/harness/002_ssl-and-pam.Dockerfile new file mode 100644 index 00000000..5682169b --- /dev/null +++ b/irods/test/harness/002_ssl-and-pam.Dockerfile @@ -0,0 +1,4 @@ +FROM bats-python3 +RUN apt install -y sudo +RUN useradd -ms/bin/bash testuser +RUN echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers diff --git a/irods/test/harness/003_prc-with-py3_12_2.Dockerfile b/irods/test/harness/003_prc-with-py3_12_2.Dockerfile new file mode 100644 index 00000000..9a77a104 --- /dev/null +++ b/irods/test/harness/003_prc-with-py3_12_2.Dockerfile @@ -0,0 +1,14 @@ +from install-irods +run apt update +run apt install -y wget build-essential libssl-dev zlib1g-dev +run apt install wget build-essential +run wget https://www.python.org/ftp/python/3.12.2/Python-3.12.2.tar.xz +run tar xf Python-3.12.2.tar.xz +workdir /Python-3.12.2 +run ./configure --prefix /root/python --with-ensurepip=install +run make -j +run mkdir /root/python +run make install +workdir / +run /root/python/bin/python3 -m pip install python-irodsclient +run chmod a+rx /root diff --git a/irods/test/harness/README.txt b/irods/test/harness/README.txt new file mode 100644 index 00000000..31f84896 --- /dev/null +++ b/irods/test/harness/README.txt @@ -0,0 +1,46 @@ +SAMPLE RUNS + +To build required images +------------------------ +Examples + + 1) ./build-docker.sh + DEFAULT: build single-node system based on latest iRODS release + + 2) IRODS_PACKAGE_VERSION="4.2.12-1~bionic" NO_CACHE='1' ./build-docker.sh + Build (ignoring docker cache) single-node system based on specified package version string. + +simple examples +--------------- +./docker_container_driver.sh tests/test_1.sh +./docker_container_driver.sh tests/test_2.sh + +Any script in a subdirectory of the repo (mounted at /prc within the container) can be +executed and will be able to find other scripts and source include files within the tree. +[See "experiment.sh" example below.] + +Examples of options in driver script +------------------------------------ + + 1. To start container and run test script: + C=$( ./docker_container_driver.sh -c -L -u testuser ../scripts/experiment.sh ) + + 2. To manually examine results afterward: + docker exec -it $C bash + + +Demo / Demo hook / args +------------------------ + +$ ~/python-irodsclient/irods/test/harness$ ./docker_container_driver.sh ../demo.sh +ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[irods/test/demo.sh] +image=[ssl-and-pam] +.......-- HOOK RUNNING -- +/prc/irods/test/demo.sh running +args: +1: [arg1] +2: [arg2] +Killed: 1358fbff6eadac24f0915ffb414f0367deedc84b0c3e4de69a23bd3a8726298f +daniel@prec3431:~/python-irodsclient/irods/test/harness$ echo $? +118 + diff --git a/irods/test/harness/build-docker.sh b/irods/test/harness/build-docker.sh new file mode 100755 index 00000000..252b87fe --- /dev/null +++ b/irods/test/harness/build-docker.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# IRODS_PACKAGE_VERSION if defined is like "4.3.1-0~bionic" +# (but contains no '~' suffix for irods versions <= 4.2.10) + +BASE=$(basename "$0") +DIR=$(realpath "$(dirname "$0")") +cd "$DIR" +DOCKER=docker +for dockerfile in [0-9]*.Dockerfile; do + image_name=${dockerfile#[0-9]*_} + image_name=${image_name%.Dockerfile} + if [ "$image_name" = "install-irods" ];then + package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"} + else + package_version_option="" + fi + $DOCKER build -f $dockerfile -t $image_name . $package_version_option \ + ${NO_CACHE+"--no-cache"} || + { STATUS=$?; echo "*** Failure while building [$image_name]"; exit $STATUS; } +done diff --git a/irods/test/harness/docker_container_driver.sh b/irods/test/harness/docker_container_driver.sh new file mode 100755 index 00000000..c340360f --- /dev/null +++ b/irods/test/harness/docker_container_driver.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +KILL_TEST_CONTAINER=1 +RUN_AS_USER="" +ECHO_CONTAINER="" + +EXPLICIT_WORKDIR="" +while [[ $1 = -* ]]; do + if [ "$1" = -c ]; then + ECHO_CONTAINER=1 + shift + fi + if [ "$1" = -L ]; then + KILL_TEST_CONTAINER=0 + shift + fi + if [ "$1" = -u ]; then + RUN_AS_USER="$2" + shift 2 + fi + if [ "$1" = -w ]; then + EXPLICIT_WORKDIR="$2" + shift 2 + fi +done + +if [ "$1" = "" ]; then + echo >&2 "Usage: $0 [options] /path/to/script" + echo >&2 "With options: [-L] to leak, [-u username] to run as non-root user" + exit 1 +fi + +DIR=$(dirname $0) +. "$DIR"/test_script_parameters + +testscript=${1} + +testscript_basename=$(basename "$testscript") +arglist=${wrapper_arglist[$testscript_basename]} # arglist dominated by symbolic link name if any + +if [ -L "$testscript" ]; then + testscript=$(realpath "$testscript") + testscript_basename=$(basename "$testscript") +fi + +original_testscript_abspath=$(realpath "$testscript") + +wrapped=${wrappers["$testscript_basename"]} + +if [ -n "$wrapped" ]; then + # wrapped is assumed to contain a leading path element relative to the referencing script's containing directory + testscript="$(dirname "$testscript")/$wrapped" + testscript_basename=$(basename "$testscript") +fi + +testscript_abspath=$(realpath "$testscript") + +cd "$DIR" + +image=${images[$testscript_basename]} + +if [ -z "$RUN_AS_USER" ]; then + RUN_AS_USER=${user[$testscript_basename]} +fi + +# Tests are run as testuser by default +: ${RUN_AS_USER:='testuser'} + +WORKDIR="" +if [ -n "$EXPLICIT_WORKDIR" ]; then + WORKDIR="$EXPLICIT_WORKDIR" +else + WORKDIR=${workdirs[$RUN_AS_USER]} +fi + +reporoot=$(./print_repo_root_location) +ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$(realpath --relative-to $reporoot "$original_testscript_abspath") + +echo "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT]" +INNER_MOUNT=/prc + +# Start the container. +echo image="[$image]" +CONTAINER=$(docker run -d -v $reporoot:$INNER_MOUNT:ro --rm $image) + +# Wait for iRODS and database to start up. +TIME0=$(date +%s) +while :; do + [ `date +%s` -gt $((TIME0 + 30)) ] && { echo >&2 "Waited too long for DB and iRODS to start"; exit 124; } + sleep 1 + docker exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null + [ $? -ne 0 ] && { echo -n . >&2; continue; } + break +done + +docker exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \ + ${WORKDIR:+"-w$WORKDIR"} \ + -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \ + $CONTAINER \ + $INNER_MOUNT/$(realpath --relative-to $reporoot "$testscript_abspath") \ + $arglist +STATUS=$? + +if [ $((0+KILL_TEST_CONTAINER)) -ne 0 ]; then + echo >&2 'Killed:' $(docker stop --time=0 $CONTAINER) +fi + +[ -n "$ECHO_CONTAINER" ] && echo $CONTAINER +exit $STATUS diff --git a/irods/test/harness/install.sh b/irods/test/harness/install.sh new file mode 100755 index 00000000..cd840a98 --- /dev/null +++ b/irods/test/harness/install.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +IRODS_HOME=/var/lib/irods +DEV_HOME="$HOME" +: ${DEV_REPOS:="$DEV_HOME/github"} + +add_package_repo() +{ + local R="/etc/apt/sources.list.d/renci-irods.list" + echo >&2 "... installing package repo" + sudo apt update + sudo apt install -y lsb-release apt-transport-https + wget -qO - https://packages.irods.org/irods-signing-key.asc | sudo apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\ + sudo tee "$R" + sudo apt update +} + +DIST_NAME=$(lsb_release -sc) + +: ${IRODS_VSN:=4.3.1-0~$DIST_NAME} + +while [[ "$1" = -* ]]; do + ARG="$1" + shift + case $ARG in + --i=* | --irods=* |\ + --irods-version=*) IRODS_PACKAGE_VERSION=${ARG#*=};; + --w=* | --with=* | --with-options=* ) withopts=${ARG#*=} ;; + -v) VERBOSE=1;; + esac +done + + +run_phase() { + + local PHASE=$1 + local with_opts=" $2 " + + case "$PHASE" in + + 0) + + if [[ $with_opts = *\ initialize\ * ]]; then + apt-get -y update + apt-get install -y apt-transport-https wget lsb-release sudo jq + fi + + if [[ $with_opts = *\ sudo-without-pw\ * ]]; then + if [ `id -u` = 0 -a "${USER:-root}" = root ] ; then + echo >&2 "root authorization for 'sudo' is automatic - no /etc/sudoers modification needed" + else + if [ -f "/etc/sudoers" ]; then + if [ -n "$USER" ] ; then + # add a line with our USER name to /etc/sudoers if not already there + sudo su -c "sed -n '/^\s*[^#]/p' /etc/sudoers | grep '^$USER\s*ALL=(ALL)\s*NOPASSWD:\s*ALL\s*$' >/dev/null" || \ + sudo su -c "echo '$USER ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers" + else + echo >&2 "user login is '$USER' - can this be right?" + fi + else + echo >&2 "WARNING - Could not modify sudoers files" + echo -n >&2 " (hit 'Enter' to continue)" + read key + fi + fi # not root + fi # with-opts + + #------ (needed for both package install and build from source) + + if [[ $with_opts = *\ install-essential-packages\ * ]]; then + + if ! dpkg -l tzdata >/dev/null 2>&1 ; then + sudo su - root -c \ + "env DEBIAN_FRONTEND=noninteractive bash -c 'apt-get install -y tzdata'" + fi + sudo apt-get update + sudo apt-get install -y software-properties-common postgresql + sudo apt-get update && \ + sudo apt-get install -y libfuse2 unixodbc rsyslog ################### less python-pip + fi + + + if [[ $with_opts = *\ add-package-repo\ * ]]; then + add_package_repo -f + fi + + + if [[ $with_opts = *\ create-db\ * ]]; then + sudo su - postgres -c " + { dropdb --if-exists ICAT + dropuser --if-exists irods ; } >/dev/null 2>&1" + sudo su - postgres -c "psql <<\\ +________ + CREATE DATABASE \"ICAT\"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE \"ICAT\" to irods; +________" + echo >&2 "-- status of create-db = $? -- " + fi + ;; + + 4) + sudo apt install -y irods-{dev,runtime}${IRODS_PACKAGE_VERSION:+"=$IRODS_PACKAGE_VERSION"} + if [[ $with_opts != *\ basic\ * ]]; then + sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_PACKAGE_VERSION:+"=$IRODS_PACKAGE_VERSION"} + fi + ;; + + 5) + if [ ! "$IRODS_VSN" '<' "4.3" ]; then + PYTHON=python3 + else + PYTHON=python2 + fi + sudo $PYTHON /var/lib/irods/scripts/setup_irods.py < /var/lib/irods/packaging/localhost_setup_postgres.input + ;; + + *) echo >&2 "unrecognized phase: '$PHASE'." ; QUIT=1 ;; + esac + return $? +} + +#-------------------------- main + +QUIT=0 +while [ $# -gt 0 ] ; do + ARG=$1 ; shift + NOP="" ; run_phase $ARG " $withopts "; sts=$? + [ $QUIT != 0 ] && break + [ -n "$NOP" ] && continue + echo -n "== $ARG == " + if [ $sts -eq 0 ]; then + echo Y >&2 + else + [ $quit_on_phase_err ] && { echo >&2 "N - quitting"; exit 1; } + echo N >&2 + fi +done diff --git a/irods/test/harness/print_repo_root_location b/irods/test/harness/print_repo_root_location new file mode 100755 index 00000000..adcf3fc7 --- /dev/null +++ b/irods/test/harness/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=../../.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/irods/test/harness/start_postgresql_and_irods.sh b/irods/test/harness/start_postgresql_and_irods.sh new file mode 100755 index 00000000..292dfb01 --- /dev/null +++ b/irods/test/harness/start_postgresql_and_irods.sh @@ -0,0 +1,20 @@ +#!/bin/bash +service postgresql start +x=${DB_WAIT_SEC:-20} +while [ $x -ge 0 ] && { ! $SUDO su - postgres -c "psql -c '\l' >/dev/null 2>&1" || x=""; } +do + [ -z "$x" ] && break + echo >&2 "$((x--)) secs til database timeout"; sleep 1 +done +[ -z "$x" ] || { echo >&2 "Error -- database didn't start" ; exit 1; } +if ! id -u irods >/dev/null 2>&1 ; then + /install.sh --w=create-db 0 + VERSION_file=$(ls /var/lib/irods/{VERSION,version}.json.dist 2>/dev/null) + IRODS_VSN=$(jq -r '.irods_version' $VERSION_file) /install.sh 5 +fi +su - irods -c '~/irodsctl restart' +pgrep irodsServer +STATUS=$? +echo "($STATUS)" >/tmp/irods_status +[ $STATUS -eq 0 ] || exit 125 +tail -f /dev/null diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters new file mode 100644 index 00000000..a5f3675b --- /dev/null +++ b/irods/test/harness/test_script_parameters @@ -0,0 +1,41 @@ +# keys for Arglist refer to argument given, which could be a symlink. + +declare -A wrapper_arglist=( + [demo.sh]="arg1 arg2" + [demo_A.sh]="arg1-a arg2-a" + [login_auth_test.py]="TestLogins" + [login_auth_test_1.py]="-v TestAnonymousUser TestMiscellaneous" + [login_auth_test_2.py]="-v TestWithSSL" +) + +# keys for Wrapper refer to argument after resolution of any symlinks + +declare -A wrappers=( + [login_auth_test.py]=./login_auth_test.sh + [PRC_issue_362.bats]=./login_auth_test.sh + [test001_pam_password_expiration.bats]=../login_auth_test.sh + [demo.sh]=./demo_hook.sh +) + +# keys for Image and User refer to the basename after resolution to a wrapper if one is used + +declare -A images=( + [test_1.sh]=install-irods + [test_2.sh]=bats-python3 + [test_3.bats]=bats-python3 + [experiment.sh]=ssl-and-pam + [fail.sh]=ssl-and-pam + [login_auth_test.sh]=ssl-and-pam + [demo_hook.sh]=ssl-and-pam +) + +declare -A user=( +) + +# keys for WorkDir refer to user + +declare -A workdirs=( + [testuser]=/home/testuser + [irods]=/var/lib/irods + [root]=/ +) diff --git a/irods/test/harness/tests/test_1.sh b/irods/test/harness/tests/test_1.sh new file mode 100755 index 00000000..798302f1 --- /dev/null +++ b/irods/test/harness/tests/test_1.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +run() { + echo dir of this = $(realpath "$(dirname "${BASH_SOURCE[0]}")/repo") +} + +echo hello_there diff --git a/irods/test/harness/tests/test_2.sh b/irods/test/harness/tests/test_2.sh new file mode 100755 index 00000000..f5b9279f --- /dev/null +++ b/irods/test/harness/tests/test_2.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +run() { + echo dir of this = $(realpath "$(dirname "${BASH_SOURCE[0]}")/repo") +} + +echo later_alligator diff --git a/irods/test/harness/tests/test_3.bats b/irods/test/harness/tests/test_3.bats new file mode 100755 index 00000000..18be2ad0 --- /dev/null +++ b/irods/test/harness/tests/test_3.bats @@ -0,0 +1,21 @@ +#!/usr/bin/env bats + +. "$BATS_TEST_DIRNAME"/../setup_pam_and_ssl.funcs + +setup() { + echo setup >>/tmp/log + setup_preconnect_preference DONT_CARE + python3 "$BATS_TEST_DIRNAME"/repo/irods/test/setupssl.py +: +} + +teardown() { + echo teardown >>/tmp/log +: +} + +@test mytest { + grep acPreConn /etc/irods/core.re >>/tmp/log + echo test proper >>/tmp/log +: +} diff --git a/irods/test/login_auth_test.sh b/irods/test/login_auth_test.sh new file mode 100755 index 00000000..97c2de85 --- /dev/null +++ b/irods/test/login_auth_test.sh @@ -0,0 +1,39 @@ +#!/bin/bash +. $(dirname $0)/scripts/funcs +. $(dirname $0)/scripts/update_json_for_test + +IRODS_SERVICE_ACCOUNT_ENV_FILE=~irods/.irods/irods_environment.json +LOCAL_ACCOUNT_ENV_FILE=~/.irods/irods_environment.json + +setup_preconnect_preference DONT_CARE + +add_irods_to_system_pam_configuration + +# set up /etc/irods/ssl directory and files +set_up_ssl sudo -q + +sudo useradd -ms/bin/bash alissa +sudo chpasswd <<<"alissa:test123" + +update_json_file $IRODS_SERVICE_ACCOUNT_ENV_FILE \ + "$(newcontent $IRODS_SERVICE_ACCOUNT_ENV_FILE ssl_keys)" + +# This is mostly so we can call python3 as just "python" +activate_virtual_env_with_prc_installed >/dev/null 2>&1 || { echo >&2 "couldn't set up virtual environment"; exit 1; } + +# Set up testuser with rods+SSL so we never have to run login_auth_tests.py as the service account. +iinit_as_rods >/dev/null 2>&1 || { echo >&2 "couldn't iinit as rods"; exit 2; } +update_json_file $LOCAL_ACCOUNT_ENV_FILE \ + "$(newcontent $LOCAL_ACCOUNT_ENV_FILE ssl_keys encrypt_keys)" + +original_script=/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT +# Run tests. +if [ -x "$original_script" ]; then + command "$original_script" $* +elif [[ $original_script =~ \.py$ ]]; then + python "$original_script" $* +elif [[ $original_script =~ \.bats$ ]]; then + bats "$original_script" +else + echo >&2 "I don't know how to run this: original_script=[$original_script]" +fi diff --git a/irods/test/login_auth_test_1.py b/irods/test/login_auth_test_1.py new file mode 120000 index 00000000..dfa92c61 --- /dev/null +++ b/irods/test/login_auth_test_1.py @@ -0,0 +1 @@ +login_auth_test.py \ No newline at end of file diff --git a/irods/test/login_auth_test_2.py b/irods/test/login_auth_test_2.py new file mode 120000 index 00000000..dfa92c61 --- /dev/null +++ b/irods/test/login_auth_test_2.py @@ -0,0 +1 @@ +login_auth_test.py \ No newline at end of file diff --git a/irods/test/pam.bats/funcs b/irods/test/pam.bats/funcs deleted file mode 100644 index 30539a03..00000000 --- a/irods/test/pam.bats/funcs +++ /dev/null @@ -1,108 +0,0 @@ -dot_to_space() { - sed 's/\./ /g'<<<"$1" -} - -CLEANUP=$':\n' - -GT() { (return 1); echo $?; } -LT() { (return -1); echo $?; } -EQ() { (return 0); echo $?; } - -compare_int_tuple() { - local x=($1) y=($2) - local lx=${#x[@]} ly=${#y[@]} - local i maxlen=$((lx > ly ? lx : ly)) - for ((i=0;i ~/.irods/irods_environment.json - iinit <<<"$1" 2>/dev/tty -} - -_end_pam_environment_and_password() { - rm -fr ~/.irods - mv ~/.irods.$$ ~/.irods -} - -setup_pam_login_for_alice() { - sudo useradd alice --create-home - local PASSWD=${1:-test123} - sudo chpasswd <<<"alice:$PASSWD" - iadmin mkuser alice rodsuser - _begin_pam_environment_and_password "$PASSWD" -} - -finalize_pam_login_for_alice() { - _end_pam_environment_and_password - iadmin rmuser alice - sudo userdel alice --remove -} - -test_specific_cleanup() { - eval "$CLEANUP" -} diff --git a/irods/test/scripts/demo_script b/irods/test/scripts/demo_script new file mode 100755 index 00000000..7202d9cf --- /dev/null +++ b/irods/test/scripts/demo_script @@ -0,0 +1,2 @@ +#!/bin/bash +echo "hello from script [$0]" diff --git a/irods/test/scripts/experiment.sh b/irods/test/scripts/experiment.sh new file mode 100755 index 00000000..7333969a --- /dev/null +++ b/irods/test/scripts/experiment.sh @@ -0,0 +1,6 @@ +#!/bin/bash +DIR=$(dirname $0) +. $DIR/funcs +cd "$DIR" +set_up_ssl sudo +add_irods_to_system_pam_configuration diff --git a/irods/test/scripts/funcs b/irods/test/scripts/funcs new file mode 100644 index 00000000..7ca7b36d --- /dev/null +++ b/irods/test/scripts/funcs @@ -0,0 +1,190 @@ +up_from_script_dir() { + local x incr="" + for ((x=0;x<${1:-0};x++)); do incr+="/.."; done + realpath "$(dirname ${BASH_SOURCE[0]})""$incr" +} + +# Sample usages: +# By user irods: set_up_ssl "" "-q" +# By sudo enabled user: set_up_ssl "sudo" "-q" +set_up_ssl() { + local SUDO=${1:-""} + local OPTS=${2:-""} + $SUDO su - irods -c "python3 $(up_from_script_dir 1)/setupssl.py $OPTS" +} + +# Clears out environment and resets to rodsadmin 'rods'. +# Meant mostly to allow initial steps by a rodsadminfor setting up tests. + +iinit_as_rods() { + rm -fr ~/.irods + iinit <<<$(hostname)$'\n1247\nrods\ntempZone\nrods' +} + +dot_to_space() { + sed 's/\./ /g'<<<"$1" +} + +CLEANUP=$':\n' + +GT() { (return 1); echo $?; } +LT() { (return -1); echo $?; } +EQ() { (return 0); echo $?; } + +compare_int_tuple() { + local x=($1) y=($2) + local lx=${#x[@]} ly=${#y[@]} + local i maxlen=$((lx > ly ? lx : ly)) + for ((i=0;i ~/.irods/irods_environment.json + + # TODO: check: it seems /dev/tty won't work if docker exec is not invoked with -t + if [ -n "$1" ]; then + iinit <<<"$1" 2>/tmp/iinit_as_alice.log + fi +} + +_end_pam_environment_and_password() { + rm -fr ~/.irods + mv ~/.irods.$$ ~/.irods +} + +setup_pam_login_for_user() { + local user=${2:-alice} + sudo useradd $user --create-home + local PASSWD=${1:-test123} + sudo chpasswd <<<"$user:$PASSWD" + iadmin mkuser $user rodsuser + _begin_pam_environment_and_password "$PASSWD" $user +} + +setup_pam_login_for_alice() { + setup_pam_login_for_user "$1" alice +} + +finalize_pam_login_for_user() { + local USER=${1} + _end_pam_environment_and_password + iadmin rmuser "$USER" + sudo userdel "$USER" --remove +} + +finalize_pam_login_for_alice() { + finalize_pam_login_for_user alice +} + +test_specific_cleanup() { + eval "$CLEANUP" +} + +# PostgreSQL only +age_out_pam_password() { + local id=$(iquest %s "select USER_ID where USER_NAME = '$1'") + local offset=$(sudo su - postgres -c "psql -t ICAT -c 'select pass_expiry_ts from r_user_password where user_id = $id'") + local mtime=$(sudo su - postgres -c "psql -t ICAT -c 'select modify_ts from r_user_password where user_id = $id'") + mtime=$(sed 's/^\s*0//' <<<"$mtime") + ((offset+=1)) + local new_time=$((mtime - offset)) + sudo su - postgres -c "psql ICAT -c 'update r_user_password set create_ts=$new_time, modify_ts=$new_time where user_id=$id'" +} + +call_irodsctl() { + local arg=${1:-restart} + sudo su - irods -c "./irodsctl $arg" +} + +add_irods_to_system_pam_configuration() { + local tempfile=/tmp/irods-pam-config.$$ + cat <<-EOF >$tempfile + auth required pam_env.so + auth sufficient pam_unix.so + auth requisite pam_succeed_if.so uid >= 500 quiet + auth required pam_deny.so + EOF + sudo chown root.root $tempfile + sudo mv $tempfile /etc/pam.d/irods +} + +setup_preconnect_preference() { + sudo su irods -c "sed -i.orig 's/\(^\s*acPreConnect.*CS_NEG\)\([A-Z_]*\)/\1_$1/' /etc/irods/core.re" +} + +# requires image to descend from bats-python3 +activate_virtual_env_with_prc_installed() +{ + # install python client using copy of /prc so that bdist doesn't build in the readonly mount + sudo su - -c "source /py3/bin/activate && cp -rp /prc /prc-copy && \ + pip install '/prc-copy[tests]' && sudo rm -fr /prc-copy" + source /py3/bin/activate +} diff --git a/irods/test/scripts/pam.bats/fail.sh b/irods/test/scripts/pam.bats/fail.sh new file mode 100755 index 00000000..b475a121 --- /dev/null +++ b/irods/test/scripts/pam.bats/fail.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +DIR=$(dirname $0) +. $DIR/funcs +cd "$DIR"; echo >&2 -n -- ; pwd +#echo "setting up" +$(up_from_script_dir 1)/demo_script +#set_up_ssl sudo +id -un +exit 12 diff --git a/irods/test/pam.bats/test001_pam_password_expiration.bats b/irods/test/scripts/test001_pam_password_expiration.bats old mode 100644 new mode 100755 similarity index 97% rename from irods/test/pam.bats/test001_pam_password_expiration.bats rename to irods/test/scripts/test001_pam_password_expiration.bats index 7bb98718..190a09aa --- a/irods/test/pam.bats/test001_pam_password_expiration.bats +++ b/irods/test/scripts/test001_pam_password_expiration.bats @@ -48,7 +48,7 @@ print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) HOME_COLLECTION=$(ipwd) sleep 9 OUTPUT=$($PYTHON -c "$SCRIPT" 2>&1 >/dev/null || true) - grep 'RuntimeError: Time To Live' <<<"$OUTPUT" + grep 'CAT_PASSWORD_EXPIRED' <<<"$OUTPUT" # Test that the $SCRIPT, when run with proper settings, can successfully reset the password. diff --git a/irods/test/scripts/update_json_for_test b/irods/test/scripts/update_json_for_test new file mode 100644 index 00000000..3dc84dba --- /dev/null +++ b/irods/test/scripts/update_json_for_test @@ -0,0 +1,55 @@ +#!/bin/bash +declare -A ssl_keys=( + [irods_client_server_negotiation]='"request_server_negotiation"' + [irods_client_server_policy]='"CS_NEG_REQUIRE"' + [irods_ssl_ca_certificate_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_chain_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_key_file]='"/etc/irods/ssl/irods.key"' + [irods_ssl_dh_params_file]='"/etc/irods/ssl/dhparams.pem"' + [irods_ssl_verify_server]='"cert"' +) + +declare -A pam_keys=( + [irods_authentication_scheme]="\"$(pam_auth_string)\"" +) +declare -A encrypt_keys=( + [irods_encryption_key_size]=16 + [irods_encryption_salt_size]=8 + [irods_encryption_num_hash_rounds]=16 + [irods_encryption_algorithm]='"AES-256-CBC"' +) + +declare -A RESTORE_FILES=() + +update_json_file() { + local file=$1 content=$2 + local bn=$(basename "$file") + local orig=/tmp/$bn.orig.$$ + local newfile=/tmp/$bn.new.$$ + echo "$content" >"$newfile" + sudo chmod --reference "$file" "$newfile" + sudo chown --reference "$file" "$newfile" + { sudo mv "$file" "$orig" && sudo mv "$newfile" "$file"; } || return 1 + RESTORE_FILES["$file"]="$orig" +} + +restore_json_files() { + local kk + for kk in ${!RESTORE_FILES[@]};do + sudo mv -f "${RESTORE_FILES["$kk"]}" "$kk" + done +} + +newcontent () { + local file=$1 + shift + local j=$(sudo cat "$file") + while [ $# -gt 0 ]; do + eval ' + for kk in ${!'$1'[@]}; do + j=$(jq ".$kk=${'$1'[$kk]}" <<<"$j") + done' + shift + done + echo "$j" +} From 2f3aefd44029e81b7447ebe031dfd688d6151684 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Sun, 11 Aug 2024 00:01:34 -0400 Subject: [PATCH 2/4] testuser universal for all containers --- irods/test/harness/000_install-irods.Dockerfile | 4 +++- irods/test/harness/002_ssl-and-pam.Dockerfile | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/irods/test/harness/000_install-irods.Dockerfile b/irods/test/harness/000_install-irods.Dockerfile index 3d21517c..7071aa1e 100644 --- a/irods/test/harness/000_install-irods.Dockerfile +++ b/irods/test/harness/000_install-irods.Dockerfile @@ -7,6 +7,8 @@ RUN for phase in initialize install-essential-packages add-package-repo; do \ done RUN /install.sh 4 COPY start_postgresql_and_irods.sh / -RUN useradd -ms/bin/bash user +RUN apt install -y sudo +RUN useradd -ms/bin/bash testuser +RUN echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers RUN apt install -y faketime CMD bash /start_postgresql_and_irods.sh diff --git a/irods/test/harness/002_ssl-and-pam.Dockerfile b/irods/test/harness/002_ssl-and-pam.Dockerfile index 5682169b..810d10e8 100644 --- a/irods/test/harness/002_ssl-and-pam.Dockerfile +++ b/irods/test/harness/002_ssl-and-pam.Dockerfile @@ -1,4 +1 @@ FROM bats-python3 -RUN apt install -y sudo -RUN useradd -ms/bin/bash testuser -RUN echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers From 3e40db544e9cccb6ca935caf638b16c53d9471c6 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Wed, 14 Aug 2024 15:04:59 -0400 Subject: [PATCH 3/4] login_auth*.py adjustments --- irods/test/harness/test_script_parameters | 5 +++-- irods/test/login_auth_test_1.py | 2 +- irods/test/login_auth_test_2.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters index a5f3675b..77ddb303 100644 --- a/irods/test/harness/test_script_parameters +++ b/irods/test/harness/test_script_parameters @@ -11,7 +11,7 @@ declare -A wrapper_arglist=( # keys for Wrapper refer to argument after resolution of any symlinks declare -A wrappers=( - [login_auth_test.py]=./login_auth_test.sh + [login_auth_test_must_run_manually.py]=./login_auth_test.sh [PRC_issue_362.bats]=./login_auth_test.sh [test001_pam_password_expiration.bats]=../login_auth_test.sh [demo.sh]=./demo_hook.sh @@ -20,12 +20,13 @@ declare -A wrappers=( # keys for Image and User refer to the basename after resolution to a wrapper if one is used declare -A images=( + [login_auth_test.sh]=bats-python3 [test_1.sh]=install-irods [test_2.sh]=bats-python3 [test_3.bats]=bats-python3 [experiment.sh]=ssl-and-pam [fail.sh]=ssl-and-pam - [login_auth_test.sh]=ssl-and-pam + [login_auth_test_must_run_manually.py]=ssl-and-pam [demo_hook.sh]=ssl-and-pam ) diff --git a/irods/test/login_auth_test_1.py b/irods/test/login_auth_test_1.py index dfa92c61..23402ed8 120000 --- a/irods/test/login_auth_test_1.py +++ b/irods/test/login_auth_test_1.py @@ -1 +1 @@ -login_auth_test.py \ No newline at end of file +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_2.py b/irods/test/login_auth_test_2.py index dfa92c61..23402ed8 120000 --- a/irods/test/login_auth_test_2.py +++ b/irods/test/login_auth_test_2.py @@ -1 +1 @@ -login_auth_test.py \ No newline at end of file +login_auth_test_must_run_manually.py \ No newline at end of file From 8cc6ee48bd696884dfa33f9405441675f1520032 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Sat, 21 Sep 2024 12:28:20 -0400 Subject: [PATCH 4/4] [__502] scriptdir --- irods/test/scripts/funcs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/irods/test/scripts/funcs b/irods/test/scripts/funcs index 7ca7b36d..91b659c5 100644 --- a/irods/test/scripts/funcs +++ b/irods/test/scripts/funcs @@ -1,7 +1,8 @@ +SCRIPTDIR=${BASH_SOURCE[0]} up_from_script_dir() { local x incr="" for ((x=0;x<${1:-0};x++)); do incr+="/.."; done - realpath "$(dirname ${BASH_SOURCE[0]})""$incr" + realpath "$(dirname "$SCRIPTDIR")""$incr" } # Sample usages: