diff --git a/cicd/crabserver_pypi/env.sh b/cicd/crabserver_pypi/env.sh index cd328c1278..797cea0bbd 100755 --- a/cicd/crabserver_pypi/env.sh +++ b/cicd/crabserver_pypi/env.sh @@ -11,7 +11,10 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) if [[ ${1} == '-g' ]]; then - eval "$("${SCRIPT_DIR}"/manage.py env -g)" + out="$("${SCRIPT_DIR}"/manage.py env -g)" else - eval "$("${SCRIPT_DIR}"/manage.py env -c)" + out="$("${SCRIPT_DIR}"/manage.py env -c)" fi +echo "will source these variables: " +echo "${out}" +eval "${out}" diff --git a/cicd/crabserver_pypi/manage.py b/cicd/crabserver_pypi/manage.py index 2d5e33d2f2..6e89947ae7 100755 --- a/cicd/crabserver_pypi/manage.py +++ b/cicd/crabserver_pypi/manage.py @@ -5,7 +5,9 @@ """ import argparse import os +from pathlib import Path +MANAGE_SH_PATH = Path(__file__).parent / Path('manage.sh') class EnvDefault(argparse.Action): # pylint: disable=too-few-public-methods """ @@ -24,8 +26,13 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) myparser = argparse.ArgumentParser(description='crab service process controller') +# SERVICE, for taskworker +myparser.add_argument('-s', dest='service', action=EnvDefault, envvar='SERVICE', + default='', + help='Name of the service to run. Only use in Publisher (Publisher_schedd, Publisher_rucio)') subparsers = myparser.add_subparsers(dest='command', required=True, help='command to run') + parserStart = subparsers.add_parser('start', help='start the service') groupModeStart = parserStart.add_mutually_exclusive_group(required=True) @@ -36,14 +43,10 @@ def __call__(self, parser, namespace, values, option_string=None): parserStart.add_argument('-d', dest='debug', action='store_const', const='t', default='', help='Enable debug mode (foreground)') -# env $SERVICE -parserStart.add_argument('-s', dest='service', action=EnvDefault, envvar='SERVICE', - default='', - help='Name of the service to run. Only use in Publisher (Publisher_schedd, Publisher_rucio)') parserStop = subparsers.add_parser('stop', help='show service status (exit non-zero if service does not start)') parserStatus = subparsers.add_parser('status', - help='start the service') + help='status of the service (pid, value of PYTHONPATH)') parserEnv = subparsers.add_parser('env', help='print environment variable for sourcing it locally.') # Wa: it actually the same as groupModeStart but I do not know how to reuse it @@ -69,6 +72,9 @@ def __call__(self, parser, namespace, values, option_string=None): #import sys #sys.exit(0) -# re exec the ./manage.sh, equivalent to `exec ./manage.sh` in shell script +# re exec the ./manage.sh, equivalent to `exec ./manage.sh` in shell script. # os.execle(filepath, arg0, arg1, ..., argN, env_dict) -os.execle('./manage.sh','./manage.sh', env) +# arg0 is usually the exec path or simply the exec name. For example, +# path = "/usr/bin/python3" +# os.execle(path, "python3", "-c", "print('CRAB!')", {"HOME": "/home/run"}) +os.execle(MANAGE_SH_PATH, MANAGE_SH_PATH, env) diff --git a/cicd/crabserver_pypi/manage.sh b/cicd/crabserver_pypi/manage.sh index c98a7cf1eb..2b4e8edaac 100755 --- a/cicd/crabserver_pypi/manage.sh +++ b/cicd/crabserver_pypi/manage.sh @@ -14,9 +14,14 @@ if [[ -z ${COMMAND+x} || -z ${MODE+x} || -z ${DEBUG+x} || -z ${SERVICE+x} ]]; th fi script_env() { - # app path - PYTHONPATH=${PYTHONPATH:-/data/srv/current/lib/python/site-packages} - # secrets + # path where we install crab code + APP_PATH="${APP_PATH:-/data/srv/current/lib/python/site-packages/}" + if [[ $MODE = 'fromGH' ]]; then + PYTHONPATH=/data/repos/CRABServer/src/python:/data/repos/WMCore/src/python:${PYTHONPATH:-} + else + PYTHONPATH="${APP_PATH}":"${PYTHONPATH:-}" + fi + # secrets CRABServerAuth.py PYTHONPATH=/data/srv/current/auth/crabserver:${PYTHONPATH} # export PYTHONPATH export PYTHONPATH @@ -37,7 +42,7 @@ script_env() { export X509_USER_CERT=${X509_USER_CERT:-/data/srv/current/auth/crabserver/dmwm-service-cert.pem} export X509_USER_KEY=${X509_USER_KEY:-/data/srv/current/auth/crabserver/dmwm-service-key.pem} - if [[ "${DEBUG:-}" == true ]]; then + if [[ -n "${DEBUG:-}" ]]; then # this will direct WMCore/REST/Main.py to run in the foreground rather than as a demon # allowing among other things to insert pdb calls in the crabserver code and debug interactively export DONT_DAEMONIZE_REST=True @@ -63,8 +68,21 @@ stop_srv() { } status_srv() { + # The `wmc-httpd -s` output when there is process running, + # crabserver is RUNNING, PID 85298 + # Otherwise (with exit 1) + # crabserver is NOT RUNNING + # Note that PID is actually PGID. script_env - wmc-httpd -s -d $STATEDIR $CFGFILE + rc=0 + out="$(wmc-httpd -s -d $STATEDIR $CFGFILE)" || rc=$? + echo "${out}" + pgid=$(echo "${out}" | awk '{print $NF}') + if [[ "${pgid}" =~ ^[0-9]+$ ]]; then + pid=$(pgrep -g "${pgid}" | head -n1) + cat /proc/"${pid}"/environ | tr '\0' '\n' | grep PYTHONPATH + fi + exit "${rc}" } env_eval() { diff --git a/cicd/crabtaskworker_pypi/Publisher/manage.sh b/cicd/crabtaskworker_pypi/Publisher/manage.sh index bd9d8fa471..223d1341e2 100755 --- a/cicd/crabtaskworker_pypi/Publisher/manage.sh +++ b/cicd/crabtaskworker_pypi/Publisher/manage.sh @@ -81,11 +81,16 @@ start_srv() { stop_srv() { nIter=1 + maxIter=60 # 600 secs => 10 mins # check if publisher is still processing by look at the pb logs while _isPublisherBusy; do # exit loop if publisher process is gone if [[ -z $(_getPublisherPid) ]]; then break; + elif [[ $nIter -gt $maxIter ]]; then + echo "Error: execeed maximum waiting time (10 mins)" + echo "crab-publisher is still running ${stillrunpid}" + exit 1 fi [[ $nIter = 1 ]] && echo "Waiting for MasterPublisher to complete cycle: ." nIter=$((nIter+1)) @@ -110,8 +115,14 @@ stop_srv() { } status_srv() { - >&2 echo "Error: Not implemented." - exit 1 + pid=$(_getPublisherPid) + if [[ -z ${pid} ]]; then + echo "Publisher's Master process is not running." + exit 1 + fi + echo "Publisher's Master process is running with PID ${pid}" + cat /proc/"${pid}"/environ | tr '\0' '\n' | grep PYTHONPATH + exit 0 } diff --git a/cicd/crabtaskworker_pypi/TaskWorker/manage.sh b/cicd/crabtaskworker_pypi/TaskWorker/manage.sh index 415f789f83..64f8df9386 100755 --- a/cicd/crabtaskworker_pypi/TaskWorker/manage.sh +++ b/cicd/crabtaskworker_pypi/TaskWorker/manage.sh @@ -96,8 +96,14 @@ stop_srv() { } status_srv() { - >&2 echo "Error: Not implemented." - exit 1 + pid=$(_getMasterWorkerPid) + if [[ -z ${pid} ]]; then + echo "TaskWorker's Master process is not running." + exit 1 + fi + echo "TaskWorker's Master process is running with PID ${pid}" + cat /proc/"${pid}"/environ | tr '\0' '\n' | grep PYTHONPATH + exit 0 } env_eval() {