diff --git a/README.md b/README.md index c452c7d..29b8de2 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,15 @@ udp_client ``` The `udp_client` provides options to specify host, port and number of documents to be used. + +### Production procedure +To manage all services (including node exporter and process monitoring) you can use `udp-collector.sh` script: +``` +$ udp-collector.sh start|stop|restart|status +``` + + +Before starting it you should compile your executables for `process_exporter` (source code [here](https://github.com/dmwm/cmsweb-exporters/blob/master/process_exporter.go)) and `node_exporter` (source code [here](https://github.com/prometheus/node_exporter/tree/master)) and put them to the same directory. You can then start your `udp-collector` with the surroinding monitoring by running the following command: +``` +$ udp-collector.sh start +``` \ No newline at end of file diff --git a/process_monitor.sh b/process_monitor.sh new file mode 100755 index 0000000..0f28764 --- /dev/null +++ b/process_monitor.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Author: Valentin Kuznetsov +# process_monitor.sh script starts new process_exporter for given +# pattern and prefix. It falls into infinitive loop with given interval +# and restart process_exporter for our pattern process. + +usage="Usage: process_monitor.sh
" +if [ $# -ne 4 ]; then + echo $usage + echo "" + echo "For example, start UNIX process monitoring on address :18883 with 15 sec interval" + echo "" + echo "# monitor process with given scitoken pattern" + echo "process_monitor.sh \".*scitoken\" scitoken \":18883\" 15" + echo "" + echo "# monitor crabserver via provide pid file" + echo "process_monitor.sh /data/srv/state/crabserver/pid scitoken \":18883\" 15" + exit 1 +fi +if [ "$1" == "-h" ] || [ "$1" == "-help" ] || [ "$1" == "--help" ]; then + echo $usage + exit 1 +fi + +# setup our input parameters +pat=$1 +prefix=$2 +address=$3 +interval=$4 +while : +do + # start with empty pid + pid="" + # if pat is an existing file, e.g. /data/srv/state/reqmgr2/pid + # we'll use it to find out group pid + if [ -f "$pat" ]; then + gid=`cat $pat` + pid=`ps -g $gid | grep -v PID | awk '{print $1}' | sort | tail -1` + if [ -z "$pid" ]; then + pid=`ps -p $gid | grep -v PID | awk '{print $1}' | sort | tail -1` + fi + else + # find pid of our pattern + pid=`ps auxw | grep "$pat" | grep -v grep | grep -v process_exporter | grep -v process_monitor | grep -v rotatelogs | awk '{print $2}'` + fi + if [ -z "$pid" ]; then + echo "No pattern '$pat' found" + sleep $interval + continue + fi + + # check if process_exporter is running + out=`ps auxw | grep "process_exporter -pid $pid -prefix $prefix" | grep -v grep` + if [ -n "$out" ]; then + echo "Found existing process_exporter: $out" + sleep $interval + continue + fi + + # find if there is existing process_exporter process + out=`ps auxw | grep "process_exporter -pid [0-9]* -prefix $prefix" | grep -v grep` + if [ -n "$out" ]; then + echo "Killing existing process_exporter: $out" + prevpid=`echo "$out" | awk '{print $2}'` + kill -9 $prevpid + fi + + # start new process_exporter process + echo "Starting: process_exporter -pid=$pid -prefix $prefix" + #nohup process_exporter -pid $pid -prefix $prefix -address $address 2>&1 1>& /dev/null < /dev/null & + process_exporter -pid $pid -prefix $prefix -address $address & + + # sleep our interval for next iteration + sleep $interval +done