fault-tolerant

#!/bin/bash

router=c11node28
target=c11node10
#target=10.5.147.210
ganglia=10.244.36.20

#postcopy=on
postcopy=off
#masterpost=on
masterpost=off

#prefix=1vm-r3
#prefix=2vms-r2
prefix=4vms-r1

ram=2048
vcpu=2

# boot up vm 
preptime=90

# sleep interval
sleeptime=5
#ports=( 4446 4447 4448 4449 )
ports=( 4446 )
#ports=( 4447 4448 4449 )
nports=${#ports[@]}
##nports=1
#nports=4
remain=$nports
#migspeed=30
migspeed=125

# for cpu utilization
PREV_TOTAL=0
PREV_IDLE=0

function startcluster() {
	# generate a random mac address for the qemu nic
	#mac=`printf 'DE:AD:BE:EF:%02X:%02X\n' $((RANDOM%256)) $((RANDOM%256))`

	# filename ram vcpu telnetport vncport listenport postcopy
	# master 10.244.36.132 (old 10.244.39.141)
	mac=DE:AD:BE:EF:0C:0F
#	./runvm-daemon master.qcow2 $ram $vcpu 4446 1 $mac

	# slave1 10.244.37.1
	mac=DE:AD:BE:EF:FA:3A
#	./runvm-daemon slave1.qcow2 $ram $vcpu 4447 2 $mac

	# slave2 10.244.38.71
	mac=DE:AD:BE:EF:FF:57
#	./runvm-daemon slave2.qcow2 $ram $vcpu 4448 3 $mac

	# slave3 10.244.38.70
	mac=DE:AD:BE:EF:A6:75
	#mac=DE:AD:BE:EF:92:C8
	./runvm-daemon slave3.qcow2 $ram $vcpu 4449 4 $mac
}

function migrate() {
	./ncmigrate master.qcow2 $ram $vcpu 4446 1 5446 $masterpost $migspeed
	#./ncmigrate slave1.qcow2 $ram $vcpu 4447 2 5447 $postcopy $migspeed
	#./ncmigrate slave2.qcow2 $ram $vcpu 4448 3 5448 $postcopy $migspeed
	#./ncmigrate slave3.qcow2 $ram $vcpu 4449 4 5449 $postcopy $migspeed
}

function check() {
	miglog=$prefix.log
	migtxt=$prefix.mig
	rm -rf $miglog
	#while [ $remain -gt 0 ]; 
	while [ 1 ]; 
	do
		#remain=${#ports[@]}
		remain=$nports
		for port in ${ports[@]}; 
		do
			info=`./checkstatus localhost $port`
			if [[ -z $info ]]; then
				(( remain -- ))
				continue
			fi

			status=`echo "$info" | grep -a complete | awk '{print $3}'`
			#echo $status

			if [[ $status == *completed* ]]; then
				echo $port
				echo "$info"
				echo $info >> $miglog

				./stopvm localhost $port
				#./stopvm $target $port
				(( remain -- ))
				continue
			fi

		#	./checkstatus $target $port
		done

		echo remaining: $remain
		if [ $remain -le 0 ]; then
			break
		fi
		sleep $sleeptime
	done
	# convert mac format to unix
	tr '\r' '\n' < $miglog > $migtxt
}

function finish() {
	for port in ${ports[@]}; 
	do
		echo $port

		./stopvm localhost $port
#		./stopvm $target $port
	done
}

# average cpu usage since the last measurement
function cpusage() {
  CPU=(`grep '^cpu ' /proc/stat`) # Get the total CPU statistics.
  unset CPU[0]                          # Discard the "cpu" prefix.
  IDLE=${CPU[4]}                        # Get the idle CPU time.
  # Calculate the total CPU time.
  TOTAL=0
  for VALUE in "${CPU[@]}"; do
    let "TOTAL=$TOTAL+$VALUE"
  done

  # Calculate the CPU usage since we last checked.
  let "DIFF_IDLE=${IDLE}-${PREV_IDLE}"
  let "DIFF_TOTAL=${TOTAL}-${PREV_TOTAL}"
  let "DIFF_USAGE=(1000*(${DIFF_TOTAL}-${DIFF_IDLE})/${DIFF_TOTAL}+5)/10"
  #echo -e "$DIFF_USAGE"
  echo -e "CPU: $DIFF_USAGE %"
  #echo -e "\rCPU: $DIFF_USAGE %  \b\b"

  # Remember the total and idle CPU times for the next check.
  PREV_TOTAL="$TOTAL"
  PREV_IDLE="$IDLE"
}

function iperftest() {
	logsave 1vm-r1.iperf iperf -c c11node10 -f M -t 30
}

function download() {
	min=3min
	wget -O $prefix.src.net.png "http://10.244.36.20/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=c11node9&s=by+name&g=network_report"
	wget -O $prefix.src.net.csv "http://10.244.36.20/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=c11node9&s=by+name&g=network_report&csv=1"
	wget -O $prefix.src.cpu.png "http://10.244.36.20/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=c11node9&s=by+name&g=cpu_report"
	wget -O $prefix.src.cpu.csv "http://10.244.36.20/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=c11node9&s=by+name&g=cpu_report&csv=1"
	wget -O $prefix.dest.net.png "http://10.244.36.20/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=${target}&s=by+name&g=network_report"
	wget -O $prefix.dest.net.csv "http://10.244.36.20/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=${target}&s=by+name&g=network_report&csv=1"
	wget -O $prefix.dest.cpu.png "http://10.244.36.20/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=${target}&s=by+name&g=cpu_report"
	wget -O $prefix.dest.cpu.csv "http://10.244.36.20/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=${target}&s=by+name&g=cpu_report&csv=1"

	for node in master slave1 slave2 slave3
        do
                for resource in cpu network
                do
                        #for filetype in png csv
                        #do
                                wget -O $prefix.$node.$resource.png "http://${ganglia}/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=${node}&s=by+name&g=${resource}_report"
                                wget -O $prefix.$node.$resource.csv "http://${ganglia}/ganglia/graph.php?r=${min}&z=medium&c=Hadoop&h=${node}&s=by+name&g=${resource}_report&csv=1"
                        #done
                done
        done
}

function migratecluster() {
	dstatfile=$prefix.src.dstat
	srccsv=$dstatfile.csv
	dstatdest=$prefix.dest.dstat
	destcsv=$dstatdest.csv
	dstatrouter=$prefix.router.dstat
	routercsv=$dstatrouter.csv
	
        dstatcmd="dstat -cnmd -N eth0,eth1,total --noheaders -o "
        #$dstatcmd $srccsv > $dstatfile &
	#ssh $target $dstatcmd $destcsv > $dstatdest &
	#ssh $router $dstatcmd $routercsv > $dstatrouter &

#	sleep $preptime
	migrate 
	check

        # stop dstat
        pgrep -fl dstat| awk '{print $1}' | xargs kill -9
        #ssh $target "pgrep -fl dstat| awk '{print $1}' | xargs kill -9"
	
	# delete the first 5 lines
	#sed -i '1,5d' $dstatfile

	#download

	#scp $target:/root/$destcsv ./
	#scp $router:/root/$routercsv ./	

        # plot host cpu/net usage
	# fix to add both send/receive
        plotcpu="./plotcpu"
        plotnet="./plotnet"
        #$plotcpu $dstatfile
        #$plotnet $dstatfile
        #$plotcpu $dstatdest
        #$plotnet $dstatdest
}

cpusage

#startcluster
migratecluster
#migratecluster > migrate.log
#finish

cpusage