Various fix for process controller scripts (manage.sh/manage.py/etc.) (…

…#8737)

cicd/crabserver_pypi/env.sh

@@ -11,7 +11,10 @@
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 if [[ ${1} == '-g' ]]; then
-    eval "$("${SCRIPT_DIR}"/manage.py env -g)"
+    out="$("${SCRIPT_DIR}"/manage.py env -g)"
 else
-    eval "$("${SCRIPT_DIR}"/manage.py env -c)"
+    out="$("${SCRIPT_DIR}"/manage.py env -c)"
 fi
+echo "will source these variables: "
+echo "${out}"
+eval "${out}"

cicd/crabserver_pypi/manage.py

@@ -5,7 +5,9 @@
 """
 import argparse
 import os
+from pathlib import Path
 
+MANAGE_SH_PATH = Path(__file__).parent / Path('manage.sh')
 
 class EnvDefault(argparse.Action): # pylint: disable=too-few-public-methods
     """
@@ -24,8 +26,13 @@ def __call__(self, parser, namespace, values, option_string=None):
         setattr(namespace, self.dest, values)
 
 myparser = argparse.ArgumentParser(description='crab service process controller')
+# SERVICE, for taskworker
+myparser.add_argument('-s', dest='service', action=EnvDefault, envvar='SERVICE',
+                         default='',
+                         help='Name of the service to run. Only use in Publisher (Publisher_schedd, Publisher_rucio)')
 
 subparsers = myparser.add_subparsers(dest='command', required=True, help='command to run')
+
 parserStart = subparsers.add_parser('start',
                                     help='start the service')
 groupModeStart = parserStart.add_mutually_exclusive_group(required=True)
@@ -36,14 +43,10 @@ def __call__(self, parser, namespace, values, option_string=None):
 parserStart.add_argument('-d', dest='debug', action='store_const', const='t',
                          default='',
                          help='Enable debug mode (foreground)')
-# env $SERVICE
-parserStart.add_argument('-s', dest='service', action=EnvDefault, envvar='SERVICE',
-                         default='',
-                         help='Name of the service to run. Only use in Publisher (Publisher_schedd, Publisher_rucio)')
 parserStop = subparsers.add_parser('stop',
                                    help='show service status (exit non-zero if service does not start)')
 parserStatus = subparsers.add_parser('status',
-                                    help='start the service')
+                                    help='status of the service (pid, value of PYTHONPATH)')
 parserEnv = subparsers.add_parser('env',
                                   help='print environment variable for sourcing it locally.')
 # Wa: it actually the same as groupModeStart but I do not know how to reuse it
@@ -69,6 +72,9 @@ def __call__(self, parser, namespace, values, option_string=None):
 #import sys
 #sys.exit(0)
 
-# re exec the ./manage.sh, equivalent to `exec ./manage.sh` in shell script
+# re exec the ./manage.sh, equivalent to `exec ./manage.sh` in shell script.
 # os.execle(filepath, arg0, arg1, ..., argN, env_dict)
-os.execle('./manage.sh','./manage.sh', env)
+# arg0 is usually the exec path or simply the exec name. For example,
+#   path = "/usr/bin/python3"
+#   os.execle(path, "python3", "-c", "print('CRAB!')", {"HOME": "/home/run"})
+os.execle(MANAGE_SH_PATH, MANAGE_SH_PATH, env)

cicd/crabserver_pypi/manage.sh

@@ -14,9 +14,14 @@ if [[ -z ${COMMAND+x} || -z ${MODE+x} || -z ${DEBUG+x} || -z ${SERVICE+x} ]]; th
 fi
 
 script_env() {
-    # app path
-    PYTHONPATH=${PYTHONPATH:-/data/srv/current/lib/python/site-packages}
-    # secrets
+    # path where we install crab code
+    APP_PATH="${APP_PATH:-/data/srv/current/lib/python/site-packages/}"
+    if [[ $MODE = 'fromGH' ]]; then
+        PYTHONPATH=/data/repos/CRABServer/src/python:/data/repos/WMCore/src/python:${PYTHONPATH:-}
+    else
+        PYTHONPATH="${APP_PATH}":"${PYTHONPATH:-}"
+    fi
+    # secrets CRABServerAuth.py
     PYTHONPATH=/data/srv/current/auth/crabserver:${PYTHONPATH}
     # export PYTHONPATH
     export PYTHONPATH
@@ -37,7 +42,7 @@ script_env() {
     export X509_USER_CERT=${X509_USER_CERT:-/data/srv/current/auth/crabserver/dmwm-service-cert.pem}
     export X509_USER_KEY=${X509_USER_KEY:-/data/srv/current/auth/crabserver/dmwm-service-key.pem}
 
-    if [[ "${DEBUG:-}" == true ]]; then
+    if [[ -n "${DEBUG:-}" ]]; then
         # this will direct WMCore/REST/Main.py to run in the foreground rather than as a demon
         # allowing among other things to insert pdb calls in the crabserver code and debug interactively
         export DONT_DAEMONIZE_REST=True
@@ -63,8 +68,21 @@ stop_srv() {
 }
 
 status_srv() {
+    # The `wmc-httpd -s` output when there is process running,
+    #   crabserver is RUNNING, PID 85298
+    # Otherwise (with exit 1)
+    #   crabserver is NOT RUNNING
+    # Note that PID is actually PGID.
     script_env
-    wmc-httpd -s -d $STATEDIR $CFGFILE
+    rc=0
+    out="$(wmc-httpd -s -d $STATEDIR $CFGFILE)" || rc=$?
+    echo "${out}"
+    pgid=$(echo "${out}" | awk '{print $NF}')
+    if [[ "${pgid}" =~ ^[0-9]+$ ]]; then
+        pid=$(pgrep -g "${pgid}" | head -n1)
+        cat /proc/"${pid}"/environ | tr '\0' '\n' | grep PYTHONPATH
+    fi
+    exit "${rc}"
 }
 
 env_eval() {

cicd/crabtaskworker_pypi/Publisher/manage.sh

@@ -81,11 +81,16 @@ start_srv() {
 
 stop_srv() {
   nIter=1
+  maxIter=60 # 600 secs => 10 mins
   # check if publisher is still processing by look at the pb logs
   while _isPublisherBusy;  do
       # exit loop if publisher process is gone
       if [[ -z $(_getPublisherPid) ]]; then
          break;
+      elif [[ $nIter -gt $maxIter ]]; then
+          echo "Error: execeed maximum waiting time (10 mins)"
+          echo "crab-publisher is still running ${stillrunpid}"
+          exit 1
       fi
       [[ $nIter = 1 ]] && echo "Waiting for MasterPublisher to complete cycle: ."
       nIter=$((nIter+1))
@@ -110,8 +115,14 @@ stop_srv() {
 }
 
 status_srv() {
-    >&2 echo "Error: Not implemented."
-    exit 1
+    pid=$(_getPublisherPid)
+    if [[ -z ${pid} ]]; then
+        echo "Publisher's Master process is not running."
+        exit 1
+    fi
+    echo "Publisher's Master process is running with PID ${pid}"
+    cat /proc/"${pid}"/environ | tr '\0' '\n' | grep PYTHONPATH
+    exit 0
 }
 
 

cicd/crabtaskworker_pypi/TaskWorker/manage.sh

@@ -96,8 +96,14 @@ stop_srv() {
 }
 
 status_srv() {
-    >&2 echo "Error: Not implemented."
-    exit 1
+    pid=$(_getMasterWorkerPid)
+    if [[ -z ${pid} ]]; then
+        echo "TaskWorker's Master process is not running."
+        exit 1
+    fi
+    echo "TaskWorker's Master process is running with PID ${pid}"
+    cat /proc/"${pid}"/environ | tr '\0' '\n' | grep PYTHONPATH
+    exit 0
 }
 
 env_eval() {