diff --git a/.travis.yml b/.travis.yml index 8045880..145a330 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,9 +38,8 @@ script: - tests/comparators/run_all - wft4galaxy -f examples/change_case/workflow-test.yml --server ${GALAXY_URL} --api-key ${GALAXY_API_KEY} - export GALAXY_URL=http://${GALAXY_ADDRESS} # use the address of the Galaxy container within the Docker network -- branch=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}} -- tests/docker/run-all.sh --branch ${branch} --server ${GALAXY_URL} --api-key ${GALAXY_API_KEY} --network ${GALAXY_NETWORK} +- tests/docker/run-all.sh --server ${GALAXY_URL} --api-key ${GALAXY_API_KEY} --network ${GALAXY_NETWORK} --local-copy --image-owner travis-ci after_script: -- docker rm -f ${GALAXY_CONTAINER_NAME} \ No newline at end of file +- docker rm -f ${GALAXY_CONTAINER_NAME} diff --git a/examples/change_case/workflow-test.yml b/examples/change_case/workflow-test.yml index c1cdb1b..2e0dea7 100644 --- a/examples/change_case/workflow-test.yml +++ b/examples/change_case/workflow-test.yml @@ -38,6 +38,15 @@ # Enable to print yet more information #enable_debug: True +# Maximum number of retries after a connection failure +# max_retries: 10 + +# Delay between consecutive retries (in seconds) +# retry_delay: 1 + +# Wait time between consecutive checks of the workflow state (in seconds) +# polling_interval: 1 + ########################################################################################## # Workflow tests ########################################################################################## diff --git a/setup.py b/setup.py index 946cca4..9ff5b70 100644 --- a/setup.py +++ b/setup.py @@ -145,7 +145,7 @@ def run(self): name='wft4galaxy', description='Utility module for testing Galaxy workflows', url='https://github.com/phnmnl/wft4galaxy', - version='0.1', + version='0.3', install_requires=requirements, package_data={'wft4galaxy': ['wft4galaxy.properties'], 'templates': ['*']}, packages=["wft4galaxy", "wft4galaxy.comparators", "wft4galaxy.app", "templates"], diff --git a/tests/docker/test-image.sh b/tests/docker/test-image.sh index ea4deea..d389cbb 100755 --- a/tests/docker/test-image.sh +++ b/tests/docker/test-image.sh @@ -11,150 +11,170 @@ image_root_path="${script_path}/../../utils/docker" # print help function print_usage(){ - ( echo "USAGE: $0 [--server URL] [--api-key API-KEY] [--network ADDRESS] " - echo " [-r|--image-registry REGISTRY] [-o|--image-owner OWNER] [-n|--image-name NAME] [-t|--image-tag TAG]" - echo " {minimal,develop}" - echo "" - echo "If no options related to the Docker image are provided, this script will try to get the" - echo "required image information from the local repository itself") >&2 + echo "USAGE: $0 [--server URL] [--api-key API-KEY] [--network ADDRESS] { any options build-image.sh accepts }" >&2 } -# init argument variables -image_type='' +function error() { + if [[ $# -gt 0 ]]; then + echo "$@" >&2 + else + echo "Sorry. There's been an error!" >&2 + fi + exit 2 +} + +function usage_error() { + if [[ $# -gt 0 ]]; then + echo "$@" >&2 + fi + print_usage + exit 2 +} + +function parse_args() { + # this function parses the arguments it receives in $@ and sets variables + while test $# -gt 0 + do + case "$1" in + -h|--help) + print_usage + exit 0 + ;; + --server ) + [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; } + GALAXY_URL="$2" + shift + ;; + --api-key) + [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; } + GALAXY_API_KEY="$2" + shift + ;; + --network ) + [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; } + GALAXY_NETWORK="$2" + shift + ;; + --debug ) + debug="--debug" + shift + ;; + *) + OTHER_ARGS+=("${1}") + ;; + esac + shift + done + + # check required options + local settings=(GALAXY_URL GALAXY_API_KEY) + for s in ${settings[@]}; do + if [[ -z ${!s} ]]; then + echo "No ${s} provided." >&2 + exit 99 + fi + done +} + +function extract_build_image_info() { # args: (filename, tag) + if [[ $# -ne 2 ]]; then + error "BUG! extract_build_image_tag called with $# arguments (expected 2)" + fi + local filename="${1}" + local tag="${2}" + + local sed_expr="/${tag}/s/\s*${tag}\s*:\s*\([^,]*\),\?\s*/\1/p" + local value="$(sed -n -e "${sed_expr}" "${filename}")" + echo "Extracted ${tag}: '${value}'" &>2 + echo "${value}" + + return 0 +} + +########## main ########### + +# absolute path of the current script +script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# initialize variables to default values, if any GALAXY_URL='' GALAXY_API_KEY='' GALAXY_NETWORK='' -repo_url='' -GIT_BRANCH='' -repo_branch='' -# options -opts="$@" +# other options to be passed to build-image.sh +OTHER_ARGS=() # disable debug debug="" -# parse args -while test $# -gt 0 -do - case "$1" in - -h|--help) - print_usage - exit 0 - ;; - --server ) - [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; } - GALAXY_URL="$2" - shift - ;; - --api-key) - [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; } - GALAXY_API_KEY="$2" - shift - ;; - --network ) - [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; } - GALAXY_NETWORK="$2" - shift - ;; - --url|--repo-url) - [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; } - repo_url="--url $2" - shift - ;; - --branch|--repo-branch) - [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; } - GIT_BRANCH=$2 - repo_branch="--branch $2" - shift - ;; - --debug ) - debug="--debug" - shift - ;; - --*) - print_usage - exit -1 - ;; - *) - # support only the first argument; skip all remaining - if [[ -z ${image_type} ]]; then - image_type=${1} - fi - ;; - esac - shift -done - -# check required options -settings=(image_type GALAXY_URL GALAXY_API_KEY) -for s in ${settings[@]}; do - if [[ -z ${!s} ]]; then - echo "No ${s} provided." >&2 - exit 99 - fi -done - -# check image type -if [[ -z ${image_type} ]]; then - image_type="minimal" - echo "No image type provided. Using the default : ${image_type}">&2 -elif [[ ${image_type} != "minimal" && ${image_type} != "develop" ]]; then - echo -e "\nERROR: '${image_type}' not supported! Use 'minimal' or 'develop'." - exit 99 +# parse args. An empty $@ will result in the empty string "" being passed, which is ok +# (we'll raise an error later for lack of mandatory arguments) +parse_args "${@:-}" + +# copy the wft4galaxy script +docker_runner="${script_path}/../../wft4galaxy/app/docker_runner.py" + +echo "Building docker image" >&2 +build_image="${script_path}/../../utils/docker/build-image.sh" + +# Use a temporary file to capture the name of the image being created +# `mktemp -t` is deprecated on Linux, but it gives us compatibility with both Linux and MacOS +tmpfile=$(mktemp -t wft4galaxy-test-image.XXXXX) +trap "rm -f '${tmpfile}'" EXIT # remove that file on exit + +# fail if any step in the pipe files (we especially care about build-image.sh :-) ) +set -o pipefail +# We want to capture the last section of the stdout from build-image.sh where it's +# going to print the name of the docker image it created. At the same time, we +# want the output to go to the console so that we can see what's going on. >() +# is the syntax for bash's process substitution: tee writes to an "anonymous" +# pipe which is connected to sed's stdin. The sed command will delete everything +# up to and including the line with the marker + +if [[ ${#OTHER_ARGS[@]} -eq 0 ]]; then + # This works around bash's quirky behaviour of raising an error if nounset is on and + # you expand an empty (but declared) array + "${build_image}" | tee >(sed -e '1,/^=== build-image.sh complete ===$/d' > "${tmpfile}") else - export IMAGE_TYPE="${image_type}" + "${build_image}" "${OTHER_ARGS[@]}" | tee >(sed -e '1,/^=== build-image.sh complete ===$/d' > "${tmpfile}") fi -# extract git info & Docker image name -source ${image_root_path}/set-git-repo-info.sh ${repo_url} ${repo_branch} -source ${image_root_path}/set-docker-image-info.sh +image_repo="$(extract_build_image_info "${tmpfile}" image_repository)" +image_tag="$(extract_build_image_info "${tmpfile}" image_tag)" -# download wft4galaxy script -owner=${GIT_OWNER:-"phnmnl"} -branch=${GIT_BRANCH:-"develop"} -curl -s https://raw.githubusercontent.com/${owner}/wft4galaxy/${branch}/utils/docker/install.sh | bash /dev/stdin --repo "${owner}/wft4galaxy" --branch ${branch} . -echo "Downloaded 'wft4galaxy-docker' Github repository: ${owner}/wft4galaxy (branch: ${branch})" >&2 +# array of arguments for the docker run +cmd_args=(--server "${GALAXY_URL}" --api-key "${GALAXY_API_KEY}" --skip-update) -# switch the Docker image context -cd ${image_root_path} > /dev/null - -# build docker image -echo "${image_root_path}/${image_type}/build.sh ${opts}" >&2 -"${image_root_path}/${image_type}/build.sh" ${repo_branch} -cd - > /dev/null - -# set optional arguments -cmd_other_opts="--repository ${IMAGE_REPOSITORY} --skip-update ${debug}" -if [[ -n ${IMAGE_TAG} ]]; then - cmd_other_opts="${cmd_other_opts} --tag ${IMAGE_TAG}" +if [[ -n "${image_repo}" ]]; then + cmd_args+=(--repository "${image_repo}") +fi +if [[ -n "${image_tag}" ]]; then + cmd_args+=(--tag "${image_tag}") fi -if [[ -n ${GALAXY_NETWORK} ]]; then - cmd_other_opts="${cmd_other_opts} --network ${GALAXY_NETWORK}" +if [[ -n "${debug}" ]]; then + cmd_args+=("${debug}") fi +if [[ -n "${GALAXY_NETWORK}" ]]; then + cmd_args+=(--network "${GALAXY_NETWORK}") +fi + +# finally, add the test definition argument +cmd_args+=(-f "${script_path}/../../examples/change_case/workflow-test.yml") # uncomment for debug #echo "Trying to contact Galaxy sever..." -#docker run --rm --network ${GALAXY_NETWORK} \ +#docker run --rm --network "${GALAXY_NETWORK}" \ # ubuntu bash -c "apt-get update && apt-get install -y iputils-ping && timeout 5 ping 172.18.0.22" -# build cmd -base_cmd="./wft4galaxy-docker ${cmd_other_opts} --server ${GALAXY_URL} --api-key ${GALAXY_API_KEY}" -cmd="${base_cmd} -f examples/change_case/workflow-test.yml" -echo -e "CMD: ${cmd}\n">&2 - -# turn off command error checking +# now run the tests +echo -e "CMD: ${docker_runner} ${cmd_args[@]}\n" >&2 +# first turn off command error checking set +o errexit - -# run test -${cmd} +"${docker_runner}" "${cmd_args[@]}" exit_code=$? -# cleanup -rm -f wft4galaxy-docker - -if [ ${exit_code} -ne 0 ]; then - echo "Test failed (exit code: ${exit_code}" >&2 +if [[ ${exit_code} -ne 0 ]]; then + echo "Test failed (exit code: ${exit_code}" >&2 fi exit ${exit_code} diff --git a/utils/docker/develop/bashrc b/utils/docker/bashrc similarity index 100% rename from utils/docker/develop/bashrc rename to utils/docker/bashrc diff --git a/utils/docker/build-image.sh b/utils/docker/build-image.sh index a216bcc..95adb50 100755 --- a/utils/docker/build-image.sh +++ b/utils/docker/build-image.sh @@ -1,89 +1,195 @@ #!/usr/bin/env bash +set -o nounset +set -o errexit function print_usage(){ - ( echo "USAGE: $0 [-o|--image-owner OWNER] [-n|--image-name NAME] [-t|--image-tag TAG] < minimal | develop >" + ( echo "USAGE: $0 [ --image REG/OWNER/NAME[:TAG] ] [ --image-repository OWNER/NAME[:TAG] ] \\" + echo " [-o|--image-owner OWNER] [-n|--image-name NAME] [-t|--image-tag TAG] \\" + echo " [ --local-copy | --repo-url URL --repo-branch BRANCH ] < minimal | develop >" echo echo "If no arguments are provided, this script will try to get the" echo "required repository information from the local repository itself") >&2 } -# parse args -while test $# -gt 0 -do - case "$1" in - -h|--help) - print_usage - exit 0 - ;; - -r|--image-registry) - export IMAGE_REGISTRY=$2 - shift - ;; - -o|--image-owner) - export IMAGE_OWNER=$2 - shift - ;; - -n|--image-name) - export IMAGE_NAME=$2 - shift - ;; - -t|--image-tag) - export IMAGE_TAG=$2 - shift - ;; - --url|--repo-url) - repo_url="--url $2" - shift - ;; - --branch|--repo-branch) - repo_branch="--branch $2" - shift - ;; - --*) - print_usage - exit 99 - ;; - *) - # support only the first argument; skip all remaining - if [[ -z ${image_type} ]]; then - image_type=${1} - fi - ;; - esac - shift -done - -if [[ -z ${image_type} ]]; then - image_type="minimal" - echo "No image type provided. Using the default : ${image_type}">&2 -elif [[ ${image_type} != "minimal" && ${image_type} != "develop" ]]; then - echo -e "\nERROR: '${image_type}' not supported! Use 'minimal' or 'develop'." - exit 99 -else - export IMAGE_TYPE="${image_type}" -fi + +function error() { + if [[ $# -gt 0 ]]; then + echo "$@" >&2 + else + echo "Sorry. There's been an error!" >&2 + fi + exit 2 +} + +function usage_error() { + if [[ $# -gt 0 ]]; then + echo "$@" >&2 + fi + print_usage + exit 2 +} + +function parse_args() { + # this function parses the arguments it receives in $@ and sets variables + local image_type="" + while test $# -gt 0 + do + case "$1" in + -h|--help) + print_usage + exit 0 + ;; + --image) + IMAGE="${2}" + shift + ;; + --image-repository) + IMAGE_REPOSITORY="${2}" + shift + ;; + -r|--image-registry) + IMAGE_REGISTRY=$2 + shift + ;; + -o|--image-owner) + IMAGE_OWNER=$2 + shift + ;; + -n|--image-name) + IMAGE_NAME=$2 + shift + ;; + -t|--image-tag) + IMAGE_TAG=$2 + shift + ;; + --url|--repo-url) + repo_url="--url $2" + shift + ;; + --branch|--repo-branch) + repo_branch="--branch $2" + shift + ;; + --local-copy) + local_copy="true" + ;; + --*) + print_usage + exit 99 + ;; + *) + # support only the first argument; skip all remaining + if [[ -z "${image_type}" ]]; then + image_type="${1}" + fi + ;; + esac + shift + done + + if [[ -z "${image_type}" ]]; then + echo "No image type provided. Using the default: ${IMAGE_TYPE}" >&2 + elif [[ ${image_type} != "minimal" && ${image_type} != "develop" ]]; then + echo -e "\nERROR: '${image_type}' not supported! Use 'minimal' or 'develop'." + exit 99 + else + IMAGE_TYPE="${image_type}" + fi + + if [[ "${local_copy}" = "true" && ( -n "${repo_url}" || -n "${repo_branch}" ) ]]; then + usage_error "--local-copy is mutually exclusive with repository url and/or branch" + fi + + if [[ "${local_copy}" = "true" && ( -z "${IMAGE}" && -z "${IMAGE_REPOSITORY}" && -z "${IMAGE_OWNER}" ) ]]; then + echo "Sorry. For local mode you need to specify an --image-owner or an --image-repository or an --image" >&2 + exit 1 + fi + + if [[ "${local_copy}" == "true" ]]; then + echo "Buildind an image using the current local project state" >&2 + echo "=*=*=*=*=*=* WARNING: Your Docker image could include uncommitted changes! *=*=*=*=*=*=" >&2 + else + echo "Building an image using a wft4galaxy pulled from a git repository" >&2 + fi +} + +function cd_to_project_root() { + local src_dir_root="${script_path}/../../" + # Try to verify that we have the right directory + if [[ ! -d "${src_dir_root}/wft4galaxy" || ! -d "${src_dir_root}/utils" ]]; then + error "I think the wft4galaxy root directory should be '${src_dir_root}' but I can't \n" \ + "find the 'wft4galaxy' and 'utils' subdirectories there. Aborting!" + fi + + cd "${src_dir_root}" +} + +function build_local_image() { + if [[ -z "${IMAGE}" && -z "${IMAGE_REPOSITORY}" && -z "${IMAGE_OWNER}" ]]; then + echo "BUG! Trying to build a local image without image information" >&2 + exit 2 + fi + + # assemble any missing image info + source ${script_path}/set-docker-image-info.sh + + cd_to_project_root + docker build . -f "${script_path}/${IMAGE_TYPE}/Dockerfile.local" -t ${IMAGE} +} + +function build_repo_image() { + # set git && image info + source ${script_path}/set-git-repo-info.sh ${repo_url} ${repo_branch} + source ${script_path}/set-docker-image-info.sh + + cd_to_project_root + docker build . -f "${script_path}/${IMAGE_TYPE}/Dockerfile.git" \ + --build-arg git_branch=${GIT_BRANCH} --build-arg git_url=${GIT_HTTPS} -t ${IMAGE} +} + +########## main ########### # absolute path of the current script script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# absolute path of the image folder -image_path="${script_path}/${image_type}" -# check whether the image type exists -if [[ ! -d ${image_path} ]]; then - echo -e "\nThe image type '${image_type}' doen't exist!!!" - exit 99 -fi - -# set git && image info -source ${script_path}/set-git-repo-info.sh ${repo_url} ${repo_branch} -source ${script_path}/set-docker-image-info.sh +# initialize variables to default values, if any +IMAGE="" +IMAGE_REPOSITORY="" +IMAGE_REGISTRY="" +IMAGE_OWNER="" +IMAGE_NAME="" +IMAGE_TAG="" +IMAGE_TYPE="minimal" +repo_url="" +repo_branch="" +local_copy="false" -# Need to cd into this script's directory because image-config assumes it's running within it -cd "${image_path}" > /dev/null +# parse args. An empty $@ will result in the empty string "" being passed, which is ok +parse_args "${@:-}" -# build the Docker image -docker build --build-arg git_branch=${GIT_BRANCH} --build-arg git_url=${GIT_HTTPS} -t ${IMAGE} . +if [[ "${local_copy}" = "true" ]]; then + build_local_image +else + build_repo_image +fi # restore the original path cd - > /dev/null + +echo "Built image: ${IMAGE}" >&2 + +# Don't modify this output without updating test-image.sh!" +Template="{ +image : %s, +image_repository : %s, +image_registry : %s, +image_owner : %s, +image_name : %s, +image_tag : %s, +image_type : %s +}\n" +echo "=== build-image.sh complete ===" +printf "${Template}" "${IMAGE}" "${IMAGE_REPOSITORY}" "${IMAGE_REGISTRY}" \ + "${IMAGE_OWNER}" "${IMAGE_NAME}" "${IMAGE_TAG}" "${IMAGE_TYPE}" diff --git a/utils/docker/develop/Dockerfile b/utils/docker/develop/Dockerfile.git similarity index 90% rename from utils/docker/develop/Dockerfile rename to utils/docker/develop/Dockerfile.git index 395b635..1277116 100644 --- a/utils/docker/develop/Dockerfile +++ b/utils/docker/develop/Dockerfile.git @@ -47,7 +47,7 @@ RUN echo "Installing dependencies" >&2 \ && rm -rf /var/cache/apk/* # setup bash custom prompt (PS1) -ADD bashrc /root/.bashrc +ADD utils/docker/bashrc /root/.bashrc # build a tutorial folder within the user's home RUN mkdir ${HOME}/tutorial && \ @@ -59,8 +59,8 @@ RUN mkdir ${HOME}/tutorial && \ WORKDIR /root # add container entrypoint -COPY entrypoint.sh /bin/entrypoint.sh -COPY entrypoint-argparser.sh /usr/local/bin/entrypoint-argparser.sh +COPY utils/docker/develop/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY utils/docker/entrypoint-argparser.sh /usr/local/bin/entrypoint-argparser.sh # set the entrypoint ENTRYPOINT ["entrypoint.sh"] diff --git a/utils/docker/develop/Dockerfile.local b/utils/docker/develop/Dockerfile.local new file mode 100644 index 0000000..44e469e --- /dev/null +++ b/utils/docker/develop/Dockerfile.local @@ -0,0 +1,57 @@ +FROM alpine:3.6 + +# metadata +MAINTAINER PhenoMeNal-H2020 Project ( phenomenal-h2020-users@googlegroups.com ) + +# set the term var +ENV TERM xterm-256color + +# wft4galaxy path +ENV WFT4GALAXY_PATH /opt/wft4galaxy + +# Copy the wft4galaxy project, skipping hidden files (such as the .git repo) +COPY "." "${WFT4GALAXY_PATH}" + +# install base packages +RUN echo "Installing dependencies" >&2 \ + && apk update && apk add \ + bash \ + build-base \ + gcc \ + git \ + iputils \ + make \ + nano \ + py-lxml \ + py-pip \ + py-setuptools \ + python \ + python-dev \ + vim \ + && pip install --upgrade pip \ + && pip install ipython jupyter bash_kernel \ + && cd ${WFT4GALAXY_PATH} \ + && pip install -r requirements.txt \ + && python -m bash_kernel.install \ + && echo "Building and installing wft4galaxy" >&2 \ + && python setup.py install \ + && rm -rf /var/cache/apk/* + +# setup bash custom prompt (PS1) +ADD utils/docker/bashrc /root/.bashrc + +# build a tutorial folder within the user's home +RUN mkdir ${HOME}/tutorial && \ + ln -s ${WFT4GALAXY_PATH}/examples ${HOME}/tutorial/examples && \ + ln -s ${WFT4GALAXY_PATH}/docs/notebooks ${HOME}/tutorial/notebooks && \ + ln -s ${WFT4GALAXY_PATH}/docs/images ${HOME}/tutorial/images + +# update the working dir +WORKDIR /root + +# add container entrypoint +COPY utils/docker/develop/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY utils/docker/entrypoint-argparser.sh /usr/local/bin/entrypoint-argparser.sh + +# set the entrypoint +ENTRYPOINT ["entrypoint.sh"] diff --git a/utils/docker/develop/build-local.sh b/utils/docker/develop/build-local.sh deleted file mode 100755 index 28a66d6..0000000 --- a/utils/docker/develop/build-local.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -# Use this script to build an image with the version of wft4galaxy that's in the -# current project. This is useful for development. -# - -set -o errexit -set -o nounset - -function log() { - echo $@ >&2 -} - -function usage() { - log "$(basename $0) [ IMAGE_NAME ]" -} - -# work_dir: temporary work directory that will be deleted when the script exits -work_dir= - -function cleanup() { - if [ -n "${work_dir}" ]; then - log "Cleaning up ${work_dir}" - rm -rf "${work_dir}" - fi -} - -if [ $# -ne 1 ]; then - usage - exit 1 -fi - -image_name="${1}" - -# absolute path of the current script -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -proj_dir="${script_dir}/../../../" - -# On exit, for whatever reason, try to clean up -trap "cleanup" EXIT INT ERR - -work_dir="$(mktemp -d)" - -log "copying project dir to work directory ${work_dir}" -if [ $(ls "${proj_dir}" | grep 'wft4galaxy\|setup.py' | wc -l) -lt 2 ] ; then - log "There seems to be a problem with the project directory ${proj_dir}" - log "We're not seeing the expected wft4galaxy directory and the setup.py" - log "file. Is something wrong?" - exit 1 -fi - -cp -a ${proj_dir}/* "${work_dir}" -cp ${script_dir}/bashrc "${work_dir}" -cp ${script_dir}/*.sh "${work_dir}" - -sed_script="${work_dir}/Dockerfile.sed" -# subtle thing: when ADDing multiple things to a directory, the directory's -# path must end with a slash -cat < "${sed_script}" -/git *clone .*\${WFT4GALAXY/d -/echo "Cloning wft4galaxy:/d -/^RUN *echo *"Installing dependencies"/i\\ -ADD . "\${WFT4GALAXY_PATH}\/" -END - -sed -f "${sed_script}" "${proj_dir}/utils/docker/minimal/Dockerfile" > "${work_dir}/Dockerfile" - -log "New Dockerfile" -cat "${work_dir}/Dockerfile" - -cd "${work_dir}" -docker build -t ${image_name} . - -exit 0 diff --git a/utils/docker/develop/entrypoint-argparser.sh b/utils/docker/develop/entrypoint-argparser.sh deleted file mode 100755 index 14b145e..0000000 --- a/utils/docker/develop/entrypoint-argparser.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# parse arguments -while [ -n "$1" ]; do - # Copy so we can modify it (can't modify $1) - OPT="$1" - # Detect argument termination - if [ x"$OPT" = x"--" ]; then - shift - for OPT ; do - OTHER_OPTS="$OTHER_OPTS \"$OPT\"" - done - break - fi - # Parse current opt - while [ x"$OPT" != x"-" ] ; do - case "$OPT" in - bash | ipython | jupyter | wft4galaxy | runtest | wizard ) - ENTRYPOINT="$1" - ;; - --server=* ) - GALAXY_SERVER="${OPT#*=}" - shift - ;; - --server ) - GALAXY_SERVER="$2" - shift - ;; - --api-key=* ) - GALAXY_API_KEY="${OPT#*=}" - shift - ;; - --api-key ) - GALAXY_API_KEY="$2" - shift - ;; - * ) - OTHER_OPTS="$OTHER_OPTS $OPT" - break - ;; - esac - # Check for multiple short options - # NOTICE: be sure to update this pattern to match valid options - NEXTOPT="${OPT#-[cfr]}" # try removing single short opt - if [ x"$OPT" != x"$NEXTOPT" ] ; then - OPT="-$NEXTOPT" # multiple short opts, keep going - else - break # long form, exit inner loop - fi - done - # move to the next param - shift -done - -# set BIOBLEND -export GALAXY_URL=${GALAXY_SERVER} -export GALAXY_API_KEY=${GALAXY_API_KEY} - -# export wft4galaxy arguments -export ENTRYPOINT -export ENTRYPOINT_ARGS=${OTHER_OPTS} \ No newline at end of file diff --git a/utils/docker/develop/entrypoint.sh b/utils/docker/develop/entrypoint.sh index 8bcaf37..eb56592 100755 --- a/utils/docker/develop/entrypoint.sh +++ b/utils/docker/develop/entrypoint.sh @@ -2,20 +2,20 @@ source entrypoint-argparser.sh "$@" -if [[ ! ${ENTRYPOINT} =~ ^(bash|ipython|jupyter|wft4galaxy|runtest|wizard)$ ]]; then - echo -e "\nERROR: Command \"${ENTRYPOINT_ARGS} \" not supported !" +if [[ ! "${ENTRYPOINT}" =~ ^(bash|ipython|jupyter|wft4galaxy|runtest|wizard)$ ]]; then + echo -e "\nERROR: Command \"${ENTRYPOINT} \" not supported !" echo -e " Supported commands: bash | ipython| jupyter | wft4galaxy | runtest | wizard \n" exit 99 fi -if [[ ${ENTRYPOINT} == "wft4galaxy" ]] || [[ ${ENTRYPOINT} == "runtest" ]]; then - wft4galaxy ${ENTRYPOINT_ARGS} -elif [[ ${ENTRYPOINT} == "wizard" ]]; then - wft4galaxy-wizard ${ENTRYPOINT_ARGS} -elif [[ ${ENTRYPOINT} == "ipython" ]]; then - ipython ${ENTRYPOINT_ARGS} -elif [[ ${ENTRYPOINT} == "jupyter" ]]; then - ipython notebook --ip=$(hostname) --no-browser --allow-root ${ENTRYPOINT_ARGS} +if [[ "${ENTRYPOINT}" == "wft4galaxy" ]] || [[ "${ENTRYPOINT}" == "runtest" ]]; then + wft4galaxy "${ENTRYPOINT_ARGS[@]}" +elif [[ "${ENTRYPOINT}" == "wizard" ]]; then + wft4galaxy-wizard "${ENTRYPOINT_ARGS[@]}" +elif [[ "${ENTRYPOINT}" == "ipython" ]]; then + ipython "${ENTRYPOINT_ARGS[@]}" +elif [[ "${ENTRYPOINT}" == "jupyter" ]]; then + ipython notebook --ip=$(hostname) --no-browser --allow-root "${ENTRYPOINT_ARGS[@]}" else - /bin/bash ${ENTRYPOINT_ARGS} -fi \ No newline at end of file + /bin/bash "${ENTRYPOINT_ARGS[@]}" +fi diff --git a/utils/docker/minimal/entrypoint-argparser.sh b/utils/docker/entrypoint-argparser.sh similarity index 78% rename from utils/docker/minimal/entrypoint-argparser.sh rename to utils/docker/entrypoint-argparser.sh index 14b145e..677e428 100755 --- a/utils/docker/minimal/entrypoint-argparser.sh +++ b/utils/docker/entrypoint-argparser.sh @@ -1,5 +1,9 @@ #!/bin/bash +# Collect arguments to be passed on to the next program in an array, rather than +# a simple string. This choice lets us deal with arguments that contain spaces. +ENTRYPOINT_ARGS=() + # parse arguments while [ -n "$1" ]; do # Copy so we can modify it (can't modify $1) @@ -8,7 +12,8 @@ while [ -n "$1" ]; do if [ x"$OPT" = x"--" ]; then shift for OPT ; do - OTHER_OPTS="$OTHER_OPTS \"$OPT\"" + # append to array + ENTRYPOINT_ARGS+=("$OPT") done break fi @@ -35,7 +40,8 @@ while [ -n "$1" ]; do shift ;; * ) - OTHER_OPTS="$OTHER_OPTS $OPT" + # append to array + ENTRYPOINT_ARGS+=("$OPT") break ;; esac @@ -53,9 +59,9 @@ while [ -n "$1" ]; do done # set BIOBLEND -export GALAXY_URL=${GALAXY_SERVER} -export GALAXY_API_KEY=${GALAXY_API_KEY} +export GALAXY_URL="${GALAXY_SERVER}" +export GALAXY_API_KEY="${GALAXY_API_KEY}" # export wft4galaxy arguments export ENTRYPOINT -export ENTRYPOINT_ARGS=${OTHER_OPTS} \ No newline at end of file +export ENTRYPOINT_ARGS diff --git a/utils/docker/minimal/Dockerfile b/utils/docker/minimal/Dockerfile.git similarity index 88% rename from utils/docker/minimal/Dockerfile rename to utils/docker/minimal/Dockerfile.git index 43c2a82..5f23a1a 100644 --- a/utils/docker/minimal/Dockerfile +++ b/utils/docker/minimal/Dockerfile.git @@ -22,16 +22,16 @@ ENV WFT4GALAXY_BRANCH ${git_branch:-develop} # install base packages RUN echo "Installing dependencies" >&2 \ && apk update && apk add \ + bash \ build-base \ gcc \ - make \ git \ - py-pip \ - bash \ + make \ python \ - py-setuptools \ python-dev \ py-lxml \ + py-pip \ + py-setuptools \ && pip install --upgrade pip \ && echo "Cloning wft4galaxy: branch=${WFT4GALAXY_BRANCH} url=${WFT4GALAXY_REPOSITORY_URL}" >&2 \ && git clone --single-branch --depth 1 -b ${WFT4GALAXY_BRANCH} ${WFT4GALAXY_REPOSITORY_URL} ${WFT4GALAXY_PATH} \ @@ -45,20 +45,20 @@ RUN echo "Installing dependencies" >&2 \ gcc \ make \ git \ - py-pip \ python-dev \ + py-pip \ && rm -rf ${WFT4GALAXY_PATH} \ && rm -rf /var/cache/apk/* # setup bash custom prompt (PS1) -ADD bashrc /root/.bashrc +ADD utils/docker/bashrc /root/.bashrc # update the working dir WORKDIR /root # add container entrypoint -COPY entrypoint.sh /bin/entrypoint.sh -COPY entrypoint-argparser.sh /usr/local/bin/entrypoint-argparser.sh +COPY utils/docker/minimal/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY utils/docker/entrypoint-argparser.sh /usr/local/bin/entrypoint-argparser.sh # set the entrypoint ENTRYPOINT ["entrypoint.sh"] diff --git a/utils/docker/minimal/Dockerfile.local b/utils/docker/minimal/Dockerfile.local new file mode 100644 index 0000000..79dcda7 --- /dev/null +++ b/utils/docker/minimal/Dockerfile.local @@ -0,0 +1,56 @@ +FROM alpine:3.6 + +# metadata +MAINTAINER PhenoMeNal-H2020 Project ( phenomenal-h2020-users@googlegroups.com ) +# +# set the term var +ENV TERM xterm-256color + +# wft4galaxy path +ENV WFT4GALAXY_PATH /opt/wft4galaxy + +# Copy the wft4galaxy project, skipping hidden files (such as the .git repo) +COPY "." "${WFT4GALAXY_PATH}" + +# install base packages +RUN echo "Installing dependencies" >&2 \ + && apk update && apk add \ + bash \ + build-base \ + gcc \ + git \ + make \ + py-lxml \ + py-pip \ + py-setuptools \ + python \ + python-dev \ + && pip install --upgrade pip \ + && cd ${WFT4GALAXY_PATH} \ + && pip install -r requirements.txt \ + && echo "Building and installing wft4galaxy" >&2 \ + && python setup.py install \ + && echo "Removing build-time dependencies" >&2 \ + && apk del \ + build-base \ + gcc \ + git \ + make \ + py-pip \ + python-dev \ + && rm -rf /var/cache/apk/* + +# Don't bother removing /opt/wft4galaxy since we've already committed that step + +# setup bash custom prompt (PS1) +ADD utils/docker/bashrc /root/.bashrc + +# update the working dir +WORKDIR /root + +# add container entrypoint +COPY utils/docker/minimal/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY utils/docker/entrypoint-argparser.sh /usr/local/bin/entrypoint-argparser.sh + +# set the entrypoint +ENTRYPOINT ["entrypoint.sh"] diff --git a/utils/docker/minimal/bashrc b/utils/docker/minimal/bashrc deleted file mode 100644 index 4e5bcf2..0000000 --- a/utils/docker/minimal/bashrc +++ /dev/null @@ -1,7 +0,0 @@ -parse_git_branch() { - git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/' -} - -export PS1="\[\033[36m\]\u\[\033[m\]@\[\033[32m\]\h:\[\033[33;1m\]\w\[\033[m\]\[\e[1;31m\]$(parse_git_branch)\[\e[0m\]$ " -export CLICOLOR=1 -export LSCOLORS=ExFxBxDxCxegedabagacad \ No newline at end of file diff --git a/utils/docker/minimal/entrypoint.sh b/utils/docker/minimal/entrypoint.sh index 1d0b136..e567fec 100755 --- a/utils/docker/minimal/entrypoint.sh +++ b/utils/docker/minimal/entrypoint.sh @@ -2,16 +2,16 @@ source entrypoint-argparser.sh "$@" -if [[ ! ${ENTRYPOINT} =~ ^(bash|wft4galaxy|runtest|wizard)$ ]]; then +if [[ ! "${ENTRYPOINT}" =~ ^(bash|wft4galaxy|runtest|wizard)$ ]]; then echo -e "\nERROR: Command \"${ENTRYPOINT_ARGS} \" not supported !" echo -e " Supported commands: bash | wft4galaxy | runtest | wizard \n" exit 99 fi -if [[ ${ENTRYPOINT} == "bash" ]]; then - /bin/bash ${ENTRYPOINT_ARGS} -elif [[ ${ENTRYPOINT} == "wizard" ]]; then - wft4galaxy-wizard ${ENTRYPOINT_ARGS} +if [[ "${ENTRYPOINT}" == "bash" ]]; then + /bin/bash "${ENTRYPOINT_ARGS[@]}" +elif [[ "${ENTRYPOINT}" == "wizard" ]]; then + wft4galaxy-wizard "${ENTRYPOINT_ARGS[@]}" else - wft4galaxy ${ENTRYPOINT_ARGS} + wft4galaxy "${ENTRYPOINT_ARGS[@]}" fi diff --git a/utils/docker/set-docker-image-info.sh b/utils/docker/set-docker-image-info.sh index 8d2ea01..8b69d7b 100755 --- a/utils/docker/set-docker-image-info.sh +++ b/utils/docker/set-docker-image-info.sh @@ -43,14 +43,14 @@ if [[ -n ${IMAGE} && ${IMAGE} =~ ${pattern} || -n ${IMAGE_REPOSITORY} && ${IMAGE IMAGE_REPOSITORY="${IMAGE_REGISTRY}/${IMAGE_OWNER}/${IMAGE_NAME}" fi -else # neither IMAGE nor IMAGE_REPOSITORY are setted +else # neither IMAGE nor IMAGE_REPOSITORY are set # set image owner if [[ -z "${IMAGE_OWNER}" ]]; then # map the git phnmnl repository to the Crs4 DockerHub repository - if [[ ${GIT_OWNER} == "phnmnl" ]]; then + if [[ ${GIT_OWNER:-} == "phnmnl" ]]; then IMAGE_OWNER="crs4" else - IMAGE_OWNER="${GIT_OWNER}" + IMAGE_OWNER="${GIT_OWNER:-}" # Do we absolutely need an owner? If so, there are cases where we'd need to raise an error here fi fi @@ -65,11 +65,10 @@ else # neither IMAGE nor IMAGE_REPOSITORY are setted # if image tag isn't set, trying getting it from git if [[ -z "${IMAGE_TAG}" ]]; then - if [[ -n ${GIT_BRANCH} || -n ${GIT_TAG} ]]; then + if [[ -n "${GIT_BRANCH:-}" ]]; then IMAGE_TAG="${GIT_BRANCH}" - if [[ -n ${GIT_TAG} ]]; then - IMAGE_TAG="${GIT_TAG}" # preference to git tag name over branch - fi + elif [[ -n "${GIT_TAG:-}" ]]; then + IMAGE_TAG="${GIT_TAG}" # preference to git tag name over branch fi fi diff --git a/utils/docker/set-git-repo-info.sh b/utils/docker/set-git-repo-info.sh index 2b0a8f9..97c7f7c 100755 --- a/utils/docker/set-git-repo-info.sh +++ b/utils/docker/set-git-repo-info.sh @@ -90,6 +90,9 @@ if [[ -d .git || -d $(git rev-parse --git-dir 2> /dev/null) ]]; then echo "Getting git repository URL from local repository" >&2 first_remote=$(git remote | head -n 1) echo "Using git remote '${first_remote}'" >&2 + if [ "${first_remote}" != "origin" ]; then + echo "=*=*=*=*=*=*= WARNING: automatically choosing first remote in list: ${first_remote}" >&2 + fi git_url=$(git config --get remote.${first_remote}.url) fi diff --git a/wft4galaxy/app/docker_runner.py b/wft4galaxy/app/docker_runner.py index a568406..a9acfe4 100755 --- a/wft4galaxy/app/docker_runner.py +++ b/wft4galaxy/app/docker_runner.py @@ -416,8 +416,8 @@ def run(self, options): if options.debug: cmd.append("--debug") # Galaxy settings server (redundant) - cmd += ["--server ", options.server] - cmd += ["--api-key ", options.api_key] + cmd += ["--server", options.server] + cmd += ["--api-key", options.api_key] # configuration file cmd += ["-f", options.file if options.entrypoint in ("generate-test", "generate-template") diff --git a/wft4galaxy/app/runner.py b/wft4galaxy/app/runner.py index fcb331f..208b855 100644 --- a/wft4galaxy/app/runner.py +++ b/wft4galaxy/app/runner.py @@ -20,6 +20,13 @@ def __init__(self, prog, indent_increment=2, max_help_position=40, width=None): super(_CustomFormatter, self).__init__(prog, indent_increment, max_help_position, width) +def _check_positive(value): + int_value = int(value) + if int_value <= 0: + raise _argparse.ArgumentTypeError("%s is an invalid positive int value" % value) + return int_value + + def _make_parser(): parser = _argparse.ArgumentParser(add_help=True, formatter_class=_CustomFormatter) parser.add_argument("test", help="Workflow Test Name", nargs="*") @@ -33,10 +40,14 @@ def _make_parser(): parser.add_argument('--disable-cleanup', help='Disable cleanup', action='store_true', default=None) parser.add_argument('--output-format', choices=OutputFormat, help='Choose output type', default=OutputFormat.text) - parser.add_argument('--xunit-file', default=None, metavar="FILE_PATH", help='Set the path of the xUnit report file (absolute or relative to the output folder)') parser.add_argument('-o', '--output', dest="output_folder", metavar="PATH", help='Path of the output folder') + + parser.add_argument('--max-retries', type=_check_positive, help='Max number of retries', default=None) + parser.add_argument('--retry-delay', type=_check_positive, help='Delay between retries in seconds', default=None) + parser.add_argument('--polling-interval', type=_check_positive, help='Delay between polling requests in seconds', default=None) + return parser @@ -54,56 +65,11 @@ def _parse_cli_arguments(parser, cmd_args): return args -def _configure_test(galaxy_url, galaxy_api_key, suite, output_folder, tests, - enable_logger, enable_debug, disable_cleanup, disable_assertions): - # configure `galaxy_url` - suite.galaxy_url = galaxy_url or suite.galaxy_url or _os.environ.get(_common.ENV_KEY_GALAXY_URL) - if not suite.galaxy_url: - raise _common.TestConfigError("Galaxy URL not defined! Use --server or the environment variable {} " - "or specify it in the test configuration".format(_common.ENV_KEY_GALAXY_URL)) - # configure `galaxy_api_key` - suite.galaxy_api_key = galaxy_api_key \ - or suite.galaxy_api_key \ - or _os.environ.get(_common.ENV_KEY_GALAXY_API_KEY) - if not suite.galaxy_api_key: - raise _common.TestConfigError("Galaxy API key not defined! Use --api-key or the environment variable {} " - "or specify it in the test configuration".format(_common.ENV_KEY_GALAXY_API_KEY)) - # configure `output_folder` - suite.output_folder = output_folder \ - or suite.output_folder \ - or _core.WorkflowTestCase.DEFAULT_OUTPUT_FOLDER - - if enable_logger is not None: - suite.enable_logger = enable_logger - - if enable_debug is not None: - suite.enable_debug = enable_debug - - if disable_cleanup is not None: - suite.disable_cleanup = disable_cleanup - - if disable_assertions is not None: - suite.disable_assertions = disable_assertions - - for test_config in suite.workflow_tests.values(): - test_config.disable_cleanup = suite.disable_cleanup - test_config.disable_assertions = suite.disable_assertions - - # enable the logger with the proper detail level - if suite.enable_logger or suite.enable_debug: - _common.LoggerManager.update_log_level(_logging.DEBUG if suite.enable_debug else _logging.INFO) - - # log Python version - _logger.debug("Python version: %s", _sys.version) - - # log the current configuration - _logger.info("Configuration: %s", suite) - - def run_tests(filename, galaxy_url=None, galaxy_api_key=None, enable_logger=None, enable_debug=None, disable_cleanup=None, disable_assertions=None, + max_retries=None, retry_delay=None, polling_interval=None, output_folder=None, enable_xunit=False, xunit_file=None, tests=None): """ Run a workflow test suite defined in a configuration file. @@ -126,15 +92,16 @@ def run_tests(filename, # load suite configuration suite = _core.WorkflowTestSuite.load(filename, output_folder=output_folder) # FIXME: do we need output_folder here ? - _configure_test(galaxy_url=galaxy_url, galaxy_api_key=galaxy_api_key, - suite=suite, tests=tests, output_folder=output_folder, - enable_logger=enable_logger, enable_debug=enable_debug, - disable_cleanup=disable_cleanup, disable_assertions=disable_assertions) + # log the current configuration + _logger.info("Configuration: %s", suite) # run the configured test suite - result = suite.run(galaxy_url=galaxy_url, galaxy_api_key=galaxy_api_key, verbosity=2, + result = suite.run(galaxy_url=galaxy_url, galaxy_api_key=galaxy_api_key, verbosity=2, tests=tests, enable_xunit=enable_xunit or (xunit_file != None), xunit_file=xunit_file, - enable_logger=enable_logger, enable_debug=enable_debug, disable_cleanup=disable_cleanup) + output_folder=output_folder, + enable_logger=enable_logger, enable_debug=enable_debug, + disable_cleanup=disable_cleanup, disable_assertions=disable_assertions, + max_retries=max_retries, retry_delay=retry_delay, polling_interval=polling_interval) # compute exit code exit_code = len([r for r in result.test_case_results if r.failed()]) _logger.debug("wft4galaxy.run_tests exiting with code: %s", exit_code) @@ -153,6 +120,9 @@ def main(): show_logger_name=True if options.debug else False ) + # log Python version + _logger.debug("Python version: %s", _sys.version) + # run tests and collect exit code code = run_tests(filename=options.file, galaxy_url=options.galaxy_url, @@ -161,6 +131,9 @@ def main(): enable_logger=options.enable_logger, enable_debug=options.debug, disable_cleanup=options.disable_cleanup, + max_retries=options.max_retries, + retry_delay=options.retry_delay, + polling_interval=options.polling_interval, enable_xunit=(options.output_format == OutputFormat.xunit), xunit_file=options.xunit_file, tests=options.test) diff --git a/wft4galaxy/common.py b/wft4galaxy/common.py index 468ab51..1ddcc5b 100644 --- a/wft4galaxy/common.py +++ b/wft4galaxy/common.py @@ -8,16 +8,17 @@ import logging as _logging import datetime as _datetime -# bioblend dependencies +# BioBlend dependency from bioblend.galaxy.objects import GalaxyInstance as ObjGalaxyInstance # Galaxy ENV variable names ENV_KEY_GALAXY_URL = "GALAXY_URL" ENV_KEY_GALAXY_API_KEY = "GALAXY_API_KEY" -# configure logger - - +# http settings +MAX_RETRIES = 1 +RETRY_DELAY = 10 +POLLING_INTERVAL = 10 # map `StandardError` to `Exception` to allow compatibility both with Python2 and Python3 RunnerStandardError = Exception @@ -139,55 +140,6 @@ def makedirs(path, check_if_exists=False): raise OSError(e.message) -def configure_env_galaxy_server_instance(config, options, base_config=None): - config["galaxy_url"] = options.galaxy_url \ - or base_config and base_config.get("galaxy_url") \ - or _os.environ.get(ENV_KEY_GALAXY_URL) - if not config["galaxy_url"]: - raise TestConfigError("Galaxy URL not defined! Use --server or the environment variable {} " - "or specify it in the test configuration".format(ENV_KEY_GALAXY_URL)) - - config["galaxy_api_key"] = options.galaxy_api_key \ - or base_config and base_config.get("galaxy_api_key") \ - or _os.environ.get(ENV_KEY_GALAXY_API_KEY) - if not config["galaxy_api_key"]: - raise TestConfigError("Galaxy API key not defined! Use --api-key or the environment variable {} " - "or specify it in the test configuration".format(ENV_KEY_GALAXY_API_KEY)) - - -def get_galaxy_instance(galaxy_url=None, galaxy_api_key=None): - """ - Private utility function to instantiate and configure a :class:`bioblend.GalaxyInstance` - - :type galaxy_url: str - :param galaxy_url: the URL of the Galaxy server - - :type galaxy_api_key: str - :param galaxy_api_key: a registered Galaxy API KEY - - :rtype: :class:`bioblend.objects.GalaxyInstance` - :return: a new :class:`bioblend.objects.GalaxyInstance` instance - """ - # configure `galaxy_url` - if galaxy_url is None: - if ENV_KEY_GALAXY_URL not in _os.environ: - raise TestConfigError("Galaxy URL not defined! Use --server or the environment variable {} " - "or specify it in the test configuration".format(ENV_KEY_GALAXY_URL)) - else: - galaxy_url = _os.environ[ENV_KEY_GALAXY_URL] - - # configure `galaxy_api_key` - if galaxy_api_key is None: - if ENV_KEY_GALAXY_API_KEY not in _os.environ: - raise TestConfigError("Galaxy API key not defined! Use --api-key or the environment variable {} " - "or specify it in the test configuration".format(ENV_KEY_GALAXY_API_KEY)) - else: - galaxy_api_key = _os.environ[ENV_KEY_GALAXY_API_KEY] - - # initialize the galaxy instance - return ObjGalaxyInstance(galaxy_url, galaxy_api_key) - - class WorkflowLoader(object): """ Singleton utility class responsible for loading (unloading) workflows to (from) a Galaxy server. @@ -318,3 +270,91 @@ def unload_workflows(self): raise RuntimeError("WorkflowLoader not initialized") for _, wf in _iteritems(self._workflows): self.unload_workflow(wf.id) + + +# GalaxyInstance wrapper +class GalaxyInstance(ObjGalaxyInstance): + def __init__(self, url, api_key=None, email=None, password=None, + max_retries=MAX_RETRIES, retry_delay=RETRY_DELAY, polling_interval=POLLING_INTERVAL): + super(GalaxyInstance, self).__init__(url, api_key, email, password) + if max_retries is not None: + self.max_retries = max_retries + if retry_delay is not None: + self.retry_delay = retry_delay + if polling_interval is not None: + self.polling_interval = polling_interval + + @property + def max_retries(self): + return self.gi.max_get_attempts + + @max_retries.setter + def max_retries(self, v): + self.gi.max_get_attempts = v + + @property + def retry_delay(self): + return self.gi.get_retry_delay + + @retry_delay.setter + def retry_delay(self, v): + self.gi.get_retry_delay = v + + @property + def polling_interval(self): + return self._polling_interval + + @polling_interval.setter + def polling_interval(self, interval): + self._polling_interval = interval + + +def configure_env_galaxy_server_instance(config, options, base_config=None): + config["galaxy_url"] = options.galaxy_url \ + or base_config and base_config.get("galaxy_url") \ + or _os.environ.get(ENV_KEY_GALAXY_URL) + if not config["galaxy_url"]: + raise TestConfigError("Galaxy URL not defined! Use --server or the environment variable {} " + "or specify it in the test configuration".format(ENV_KEY_GALAXY_URL)) + + config["galaxy_api_key"] = options.galaxy_api_key \ + or base_config and base_config.get("galaxy_api_key") \ + or _os.environ.get(ENV_KEY_GALAXY_API_KEY) + if not config["galaxy_api_key"]: + raise TestConfigError("Galaxy API key not defined! Use --api-key or the environment variable {} " + "or specify it in the test configuration".format(ENV_KEY_GALAXY_API_KEY)) + + +def get_galaxy_instance(galaxy_url=None, galaxy_api_key=None, + max_retries=MAX_RETRIES, retry_delay=RETRY_DELAY, polling_interval=POLLING_INTERVAL): + """ + Private utility function to instantiate and configure a :class:`bioblend.GalaxyInstance` + + :type galaxy_url: str + :param galaxy_url: the URL of the Galaxy server + + :type galaxy_api_key: str + :param galaxy_api_key: a registered Galaxy API KEY + + :rtype: :class:`bioblend.objects.GalaxyInstance` + :return: a new :class:`bioblend.objects.GalaxyInstance` instance + """ + # configure `galaxy_url` + if galaxy_url is None: + if ENV_KEY_GALAXY_URL not in _os.environ: + raise TestConfigError("Galaxy URL not defined! Use --server or the environment variable {} " + "or specify it in the test configuration".format(ENV_KEY_GALAXY_URL)) + else: + galaxy_url = _os.environ[ENV_KEY_GALAXY_URL] + + # configure `galaxy_api_key` + if galaxy_api_key is None: + if ENV_KEY_GALAXY_API_KEY not in _os.environ: + raise TestConfigError("Galaxy API key not defined! Use --api-key or the environment variable {} " + "or specify it in the test configuration".format(ENV_KEY_GALAXY_API_KEY)) + else: + galaxy_api_key = _os.environ[ENV_KEY_GALAXY_API_KEY] + + # initialize the galaxy instance + return GalaxyInstance(galaxy_url, galaxy_api_key, + max_retries=max_retries, retry_delay=retry_delay, polling_interval=polling_interval) diff --git a/wft4galaxy/core.py b/wft4galaxy/core.py index f6e34a2..edb2197 100644 --- a/wft4galaxy/core.py +++ b/wft4galaxy/core.py @@ -5,6 +5,7 @@ from past.builtins import basestring as _basestring import os as _os +import sys as _sys import logging as _logging from json import dumps as _json_dumps from uuid import uuid1 as _uuid1 @@ -538,17 +539,21 @@ def dump(filename, worflow_tests_config, file_format=FileFormats.YAML): def run(self, galaxy_url=None, galaxy_api_key=None, output_folder=None, enable_xunit=False, xunit_file=None, verbosity=0, - enable_logger=None, enable_debug=None, disable_cleanup=None): + enable_logger=None, enable_debug=None, disable_cleanup=None, + max_retries=None, retry_delay=None, polling_interval=None): _common.LoggerManager.configure_logging( _logging.DEBUG if enable_debug is True else _logging.INFO if enable_logger is True else _logging.ERROR) import wft4galaxy.runner as _runner return _runner.WorkflowTestsRunner( - galaxy_url, galaxy_api_key).run(self, verbosity=verbosity, - output_folder=output_folder or self.output_folder, - report_format="xunit" if enable_xunit else None, - report_filename=xunit_file, - enable_logger=enable_logger, enable_debug=enable_debug, - disable_cleanup=disable_cleanup) + galaxy_url, galaxy_api_key, + max_retries=max_retries, retry_delay=retry_delay, + polling_interval=polling_interval).run(self, verbosity=verbosity, + output_folder=output_folder or self.output_folder, + report_format="xunit" if enable_xunit else None, + report_filename=xunit_file, + enable_logger=enable_logger, + enable_debug=enable_debug, + disable_cleanup=disable_cleanup) class WorkflowTestSuite(object): @@ -558,7 +563,8 @@ class WorkflowTestSuite(object): def __init__(self, galaxy_url=None, galaxy_api_key=None, output_folder=WorkflowTestCase.DEFAULT_OUTPUT_FOLDER, - enable_logger=True, enable_debug=False, disable_cleanup=False, disable_assertions=False): + enable_logger=True, enable_debug=False, disable_cleanup=False, disable_assertions=False, + max_retries=None, retry_delay=None, polling_interval=None): """ Create an instance of :class:`WorkflowTestSuite`. @@ -578,6 +584,10 @@ def __init__(self, galaxy_url=None, galaxy_api_key=None, self.disable_cleanup = disable_cleanup self.disable_assertions = disable_assertions self.output_folder = output_folder + self.max_retries = max_retries + self.retry_delay = retry_delay + self.polling_interval = polling_interval + # instantiate the dict for worklofws self._workflows = {} @@ -641,7 +651,10 @@ def load(filename, output_folder=None): disable_assertions=file_configuration.get("disable_assertions", False), output_folder=output_folder \ or file_configuration.get("output_folder") \ - or WorkflowTestCase.DEFAULT_OUTPUT_FOLDER + or WorkflowTestCase.DEFAULT_OUTPUT_FOLDER, + max_retries=file_configuration.get("max_retries", None), + retry_delay=file_configuration.get("retry_delay", None), + polling_interval=file_configuration.get("polling_interval", None) ) for wf_name, wf_config in _iteritems(file_configuration.get("workflows")): wf_base_path = _os.path.join(base_path, wf_config.get("base_path", "")) @@ -659,17 +672,20 @@ def load(filename, output_folder=None): def run(self, galaxy_url=None, galaxy_api_key=None, tests=None, output_folder=None, enable_xunit=False, xunit_file=None, verbosity=0, - enable_logger=None, enable_debug=None, disable_cleanup=None): + enable_logger=None, enable_debug=None, disable_cleanup=None, disable_assertions=None, + max_retries=None, retry_delay=None, polling_interval=None): + # configure logger _common.LoggerManager.configure_logging( _logging.DEBUG if enable_debug is True else _logging.INFO if enable_logger is True else _logging.ERROR) + import wft4galaxy.runner as _runner - return _runner.WorkflowTestsRunner( - galaxy_url, galaxy_api_key).run(self, filter=tests, verbosity=verbosity, - output_folder=output_folder or self.output_folder, - report_format="xunit" if enable_xunit else None, - report_filename=xunit_file, - enable_logger=enable_logger, enable_debug=enable_debug, - disable_cleanup=disable_cleanup) + return _runner.WorkflowTestsRunner(galaxy_url, galaxy_api_key, max_retries=max_retries, + retry_delay=retry_delay, polling_interval=polling_interval) \ + .run(self, filter=tests, verbosity=verbosity, + output_folder=output_folder or self.output_folder, + report_format="xunit" if enable_xunit else None, report_filename=xunit_file, + enable_logger=enable_logger, enable_debug=enable_debug, disable_cleanup=disable_cleanup, + max_retries=max_retries, retry_delay=retry_delay, polling_interval=polling_interval) class WorkflowTestResult(object): diff --git a/wft4galaxy/runner.py b/wft4galaxy/runner.py index 09e01de..4a62a08 100644 --- a/wft4galaxy/runner.py +++ b/wft4galaxy/runner.py @@ -9,6 +9,7 @@ import logging as _logging import unittest as _unittest from uuid import uuid1 as _uuid1 + try: from StringIO import StringIO as _StringIO except ImportError: @@ -55,12 +56,15 @@ class WorkflowTestsRunner(): """ def __init__(self, galaxy_url=None, galaxy_api_key=None, + max_retries=None, retry_delay=None, polling_interval=None, output_folder='.', stream=_sys.stderr, descriptions=True, verbosity=1, elapsed_times=True): self.galaxy_api_key = galaxy_api_key # create Galaxy instance - self._galaxy_instance = _common.get_galaxy_instance(galaxy_url, galaxy_api_key) + self._galaxy_instance = _common.get_galaxy_instance(galaxy_url, galaxy_api_key, + max_retries=max_retries, retry_delay=retry_delay, + polling_interval=polling_interval) # create WorkflowLoader self._workflow_loader = _common.WorkflowLoader.get_instance(self._galaxy_instance) @@ -74,7 +78,8 @@ def __init__(self, galaxy_url=None, galaxy_api_key=None, descriptions=descriptions, elapsed_times=elapsed_times) def _setup(self, test, output_folder=None, verbosity=2, - disable_assertions=None, disable_cleanup=None, enable_logger=None, enable_debug=None): + disable_assertions=None, disable_cleanup=None, enable_logger=None, enable_debug=None, + max_retries=None, retry_delay=None, polling_interval=None): """ Update runner configuration accordingly to the test configuration""" if enable_logger is not None: @@ -86,6 +91,12 @@ def _setup(self, test, output_folder=None, verbosity=2, if disable_assertions is not None: test.disable_assertions = disable_assertions + # update http properties + self._galaxy_instance.max_retries = max_retries or getattr(test, "max_retries", None) or _common.MAX_RETRIES + self._galaxy_instance.retry_delay = retry_delay or getattr(test, "retry_delay", None) or _common.RETRY_DELAY + self._galaxy_instance.polling_interval = polling_interval \ + or getattr(test, "polling_interval", None) or _common.POLLING_INTERVAL + # update verbosity level self._runner.verbosity = verbosity @@ -113,7 +124,8 @@ def _make_wrappers(self, test, filter=None, output_folder=None, def run(self, test, filter=None, stream=_sys.stderr, verbosity=2, output_folder=None, output_suffix=None, report_format=None, report_filename=None, - disable_assertions=None, disable_cleanup=None, enable_logger=None, enable_debug=None): + disable_assertions=None, disable_cleanup=None, enable_logger=None, enable_debug=None, + max_retries=None, retry_delay=None, polling_interval=None): """ Run a single test case or a suite of test cases. """ @@ -123,7 +135,8 @@ def run(self, test, filter=None, stream=_sys.stderr, verbosity=2, # update configuration self._setup(test, output_folder=output_folder, verbosity=verbosity, disable_assertions=disable_assertions, disable_cleanup=disable_cleanup, - enable_logger=enable_logger, enable_debug=enable_debug) + enable_logger=enable_logger, enable_debug=enable_debug, + max_retries=max_retries, retry_delay=retry_delay, polling_interval=polling_interval) # prepare wrappers self._logger.debug("Creating unittest wrappers...") @@ -541,8 +554,13 @@ def run_test(self, base_path=None, inputs=None, params=None, expected_outputs=No datamap[label].append(history.upload_dataset(dataset_filename)) # run the workflow + _logger.debug("About to launch workflow.") + _logger.debug("history: %r", history) + _logger.debug("datamap: %r", datamap) + _logger.debug("params: %r", params) _logger.info("Workflow '%s' (id: %s) running ...", workflow.name, workflow.id) - outputs, output_history = workflow.run(datamap, history, params=params, wait=True, polling_interval=0.5) + outputs, output_history = workflow.run(datamap, history, params=params, wait=True, + polling_interval=self._galaxy_instance.polling_interval) _logger.info("Workflow '%s' (id: %s) executed", workflow.name, workflow.id) # check outputs