init-cluster

#!/bin/bash
# default version of hadoop
HADOOP_VERSION=2.8.2
# default version of spark
SPARK_VERSION=2.2.0
# compile option to create image
COMPILE=false

# get opts
usage() { echo "Usage: $0 -n NODES [-h HADOOP_VERSION] [-s SPARK_VERSION] [-c true]" 1>&2; exit 1; }
while getopts ":n:h::s::c::" o; do
    case "${o}" in
        n)
            NODES=${OPTARG}
            ;;
        h)
            HADOOP_VERSION=${OPTARG}
            ;;
        s)
            SPARK_VERSION=${OPTARG}
            ;;
        c)
            COMPILE=${OPTARG}
            ;;
        *)
            usage
            ;;
    esac
done
shift $((OPTIND-1))

# check if NODES is passed and is a number
RE='^[0-9]+$'
if [ -z "${NODES}" ] ; then
    usage
fi
if ! [[ $NODES =~ $RE ]] ; then
   usage
fi

# check previous nodes number of cluster
if [[ -f nodes ]]; then
  CURRENT_NODES=$(cat nodes)
fi

# check if docker hadoop image exists
if [ "$CURRENT_NODES" == "" ] || \
   [ "$NODES" != "$CURRENT_NODES" ] || \
   [ "$(docker images -q $(whoami)/hadoop:$HADOOP_VERSION 2> /dev/null)" == "" ] ; then
  # create new docker hadoop image
  if [ $COMPILE == true ]; then
      cd hadoop-compile-build
      ./build.sh $NODES $HADOOP_VERSION
      cd ..
  else
      cd hadoop-build
      ./build.sh $NODES $HADOOP_VERSION
      cd ..
  fi
fi

# start hadoop master container
docker rm -f hadoop-master &> /dev/null
echo "create hadoop-master container..."
docker run -itd \
                --net=hadoop \
                -p 50070:50070 \
                -p 8088:8088 \
                --name hadoop-master \
                --hostname hadoop-master \
                $(whoami)/hadoop:$HADOOP_VERSION

# start hadoop slaves containers
i=1
while [ $i -lt $NODES ]
do
	docker rm -f hadoop-slave$i &> /dev/null
	echo "create hadoop-slave$i container..."
	docker run -itd \
	                --net=hadoop \
	                --name hadoop-slave$i \
	                --hostname hadoop-slave$i \
	                $(whoami)/hadoop:$HADOOP_VERSION
	i=$(( $i + 1 ))
done

# check if docker spark image exists
if [ "$(docker images -q $(whoami)/hadoop-spark:$SPARK_VERSION 2> /dev/null)" == "" ] ; then
  # create new docker spark image
  cd spark-build
  ./build.sh $HADOOP_VERSION $SPARK_VERSION
  cd ..
fi

# start spark container
docker rm -f spark &> /dev/null
echo "create spark container..."
docker run -itd \
                --net=hadoop \
                --name spark \
                --hostname spark \
                $(whoami)/hadoop-spark:$SPARK_VERSION

# update nodes number
echo $NODES > nodes

# start hadoop inside container
echo "start hadoop cluster..."
docker exec -it hadoop-master /root/start-hadoop.sh > /dev/null

echo "hadoop cluster with $NODES nodes started successfully!"