01-bootkube-gen-up

#!/bin/bash
# Copyright 2017 Charter DNA Team (CTEC).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
### Declare colors to use during the running of this script:
declare -r GREEN="\033[0;32m"
declare -r RED="\033[0;31m"
declare -r YELLOW="\033[0;33m"

function echo_green {
  echo -e "${GREEN}$1"; tput sgr0
}
function echo_red {
  echo -e "${RED}$1"; tput sgr0
}
function echo_yellow {
  echo -e "${YELLOW}$1"; tput sgr0
}

### Add IP Address of Hosts in /etc/hosts:
HOST_IFACE=$(ip route | grep "^default" | head -1 | awk '{ print $5 }')
LOCAL_IP=$(ip addr | awk "/inet/ && /${HOST_IFACE}/{sub(/\/.*$/,\"\",\$2); print \$2}")
cat << EOF | sudo tee -a /etc/hosts
${LOCAL_IP} $(hostname)
EOF

### NEW INSTALLATIONS:
echo_green "\nPhase I: Installing system prerequisites:"
pkg="build-essential curl docker.io ethtool git lldpd socat traceroute vim ceph-common nfs-common"

for pkg in $pkg; do
    if sudo dpkg --get-selections | grep -q "^$pkg[[:space:]]*install$" >/dev/null; then
        echo -e "${pkg} is already installed"
    else
        sudo apt-get update && sudo apt-get -qq install ${pkg}
        echo "Successfully installed ${pkg}"
    fi
done


### PREPARE THE ENVIRONMENT:
sudo service docker restart
sudo service docker enable
source /opt/demo/.bootkube_env
echo "export KUBE_MASTER=${KUBE_IP}" >> .bootkube_env
source /opt/demo/.bootkube_env
echo_green "\nPhase II: Using the following variables during the deployment:"
printf "PROXY_ENABLED: ${PROXY_ENABLED}
BOOTKUBE_DIR: ${BOOTKUBE_DIR}
TMPDIR: ${TMPDIR}
CNI_VERSION: ${CNI_VERSION}
HELM_VERSION: ${HELM_VERSION}
BOOTKUBE_VERSION: ${BOOTKUBE_VERSION}
KUBERNETES_VERSION: ${KUBERNETES_VERSION}
KUBE_SDN: ${KUBE_SDN}
KUBE_POD_CIDR: ${KUBE_POD_CIDR}
KUBE_SVC_CIDR: ${KUBE_SVC_CIDR}
ETCD_SVC_ADDR: ${ETCD_SVC_ADDR}
ETCD_SVC_PORT: ${ETCD_SVC_PORT}
ETCD_SVC_PEER: ${ETCD_SVC_PEER}
HOST_SUB_CIDR: ${HOST_SUB_CIDR}
KUBE_HW: ${KUBE_HW}
KUBE_DNS_API: ${KUBE_DNS_API}
NSERVER01: ${NSERVER01}
NSERVER02: ${NSERVER02}
NSERVER03: ${NSERVER03}
NSEARCH01: ${NSEARCH01}
NSEARCH02: ${NSEARCH02}
KUBE_ARCH: ${KUBE_ARCH}
KUBE_IMAGE: ${KUBE_IMAGE}
KUBE_IP: ${KUBE_IP}
KUBE_MASTER: ${KUBE_MASTER} \n \n"

### PREPARE: /etc/resolv.conf
echo_green "\nPhase III: Preparing system DNS:"
sudo cp /etc/resolv.conf ${BOOTKUBE_DIR}/backups/

### PREPARE: /etc/resolv.conf
sudo -E bash -c "cat <<EOF > /etc/resolvconf/resolv.conf.d/head
nameserver ${NSERVER01}
EOF"
sudo -E bash -c "cat <<EOF > /etc/resolvconf/resolv.conf.d/base
search kubernetes ${KUBE_DNS_API} ${NSEARCH01} ${NSEARCH02}
EOF"
sudo resolvconf -u

### PREPARE: /etc/hosts with idempotency (hostess):
### DOWNLOAD: bootkube
if [ ! -e /usr/local/bin/hostess ]; then
    sudo wget -O /usr/local/bin/hostess https://github.com/cbednarski/hostess/releases/download/v0.2.0/hostess_linux_${KUBE_ARCH}
    sudo chmod +x /usr/local/bin/hostess
fi
sudo hostess fix
sudo hostess add ${KUBE_DNS_API} ${KUBE_MASTER}
sudo hostess add kubernetes ${KUBE_MASTER}
sudo hostess fix

### PREPARE: /etc/systemd/system/kubelet.service
echo_green "\nPhase VI: Deploying the Kubelet systemd unit:"
printf "Kubelet deployed to host:\n"
echo "[Unit]
Description=Kubernetes Kubelet
Documentation=https://kubernetes.io/docs/admin/kubelet/
[Service]
EnvironmentFile=/etc/environment
ExecStartPre=/bin/mkdir -p /etc/kubernetes
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/bin/mkdir -p /opt/cni/bin
ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
ExecStartPre=/bin/mkdir -p /var/lib/cni
ExecStart=/usr/local/bin/kubelet \\
  --allow-privileged \\
  --cloud-provider= \\
  --cluster_domain=cluster.local \\
  --cluster_dns='${NSERVER01}','${NSERVER02}','${NSERVER03}' \\
  --cni-conf-dir=/etc/cni/net.d \\
  --exit-on-lock-contention \\
  --hostname-override='${KUBE_IP}' \\
  --kubeconfig=/etc/kubernetes/kubeconfig \\
  --lock-file=/var/run/lock/kubelet.lock \\
  --minimum-container-ttl-duration=3m0s \\
  --node-labels=node-role.kubernetes.io/master \\
  --node-ip='${KUBE_IP}' \\
  --network-plugin=cni \\
  --pod-manifest-path=/etc/kubernetes/manifests \\
  --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \\
  --require-kubeconfig \\
  --v=2
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/kubelet.service

### DOWNLOAD: bootkube
echo_green "\nPhase V: Downloading Bootkube required binaries:"
mkdir -p ${TMPDIR}

### DOWNLOAD: bootkube
if [[ ! -e '${TMPDIR}'/'${BOOTKUBE_VERSION}'-bootkube.tgz && ! -e /usr/local/bin/bootkube ]]; then
    wget -O ${TMPDIR}/${BOOTKUBE_VERSION}-bootkube.tgz https://github.com/kubernetes-incubator/bootkube/releases/download/${BOOTKUBE_VERSION}/bootkube.tar.gz
    tar zxvf ${TMPDIR}/${BOOTKUBE_VERSION}-bootkube.tgz -C ${TMPDIR}/
    mv ${TMPDIR}/bin/ ${TMPDIR}/${BOOTKUBE_VERSION}-bootkube-${KUBE_ARCH}
    chmod +x ${TMPDIR}/${BOOTKUBE_VERSION}-bootkube-${KUBE_ARCH}/linux/bootkube
    sudo cp ${TMPDIR}/${BOOTKUBE_VERSION}-bootkube-${KUBE_ARCH}/linux/bootkube /usr/local/bin/
fi

### DOWNLOAD: kubectl
if [[ ! -e '${TMPDIR}'/'${KUBERNETES_VERSION}'-kubectl-${KUBE_ARCH} && ! -e /usr/local/bin/kubectl ]]; then
    wget -O ${TMPDIR}/${KUBERNETES_VERSION}-kubectl-${KUBE_ARCH} http://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/${KUBE_ARCH}/kubectl
    chmod +x ${TMPDIR}/${KUBERNETES_VERSION}-kubectl-${KUBE_ARCH}
    sudo cp ${TMPDIR}/${KUBERNETES_VERSION}-kubectl-${KUBE_ARCH} /usr/local/bin/kubectl
fi

### DOWNLOAD: kubelet
if [[ ! -e '${TMPDIR}'/'${KUBERNETES_VERSION}'-kubelet-${KUBE_ARCH} && ! -e /usr/local/bin/kubelet ]]; then
    wget -O ${TMPDIR}/${KUBERNETES_VERSION}-kubelet-${KUBE_ARCH} http://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/${KUBE_ARCH}/kubelet
    chmod +x ${TMPDIR}/${KUBERNETES_VERSION}-kubelet-${KUBE_ARCH}
    sudo cp ${TMPDIR}/${KUBERNETES_VERSION}-kubelet-${KUBE_ARCH} /usr/local/bin/kubelet
fi

### DOWNLOAD: cni
if [[ ! -e '${TMPDIR}'/'${CNI_VERSION}'-cni-${KUBE_ARCH}.tgz && ! -e /opt/cni/bin ]]; then
    wget -O ${TMPDIR}/${CNI_VERSION}-cni-${KUBE_ARCH}.tgz https://github.com/containernetworking/cni/releases/download/${CNI_VERSION}/cni-${KUBE_ARCH}-${CNI_VERSION}.tgz
    sudo mkdir -p /opt/cni/bin
    sudo tar -xf ${TMPDIR}/${CNI_VERSION}-cni-${KUBE_ARCH}.tgz -C /opt/cni/bin/
fi

### CLEANUP:
cd ${BOOTKUBE_DIR}
sudo rm -rf ${TMPDIR}/linux-${KUBE_ARCH}
echo_green "\nComplete!"

### RENDER ASSETS:
echo_green "\nPhase VI: Running Bootkube to render the Kubernetes assets:"
bootkube render --asset-dir=${BOOTKUBE_DIR}/.bootkube --experimental-self-hosted-etcd --etcd-servers=http://10.3.0.15:12379 --api-servers=https://${KUBE_DNS_API}:${KUBE_SVC_PORT},https://${KUBE_IP}:${KUBE_SVC_PORT} --pod-cidr=${KUBE_POD_CIDR} --service-cidr=${KUBE_SVC_CIDR}
sudo rm -rf $BOOTKUBE_DIR/.bootkube/manifests/kube-flannel*
if [ ${KUBE_SDN} = "kube-router" ]; then
   echo_yellow "\nSDN variable is set to kube-router. Proceeding to remove kube-proxy manifests for Bootkube!" && sudo rm -rf ${BOOTKUBE_DIR}/.bootkube/manifests/kube-proxy.yaml
else
   echo_green "\nSDN is set to ${KUBE_SDN}. No additional changes are required for ${KUBE_SDN}!"
fi

### DEPLOY KUBERNETES SELF-HOSTED CLUSTER:
echo_green "\nPhase VII: Preparing the environment for Kubernetes to run for the first time:"
sudo systemctl daemon-reload
sudo systemctl enable kubelet.service
sudo systemctl restart kubelet.service

sudo cp ${BOOTKUBE_DIR}/.bootkube/auth/kubeconfig /etc/kubernetes/
sudo cp ${BOOTKUBE_DIR}/.bootkube/tls/ca.crt /etc/kubernetes/ca.crt

### Ensuring that kubectl has a proper configuration file in the $USER/.kube directory, and backing up the old config file if required:
if [[ ! -e ~/.kube/config ]]; then
     echo_green "Copying Kubernetes config to ${HOME}/.kube/config" && sudo mkdir -p ~/.kube && sudo cp /etc/kubernetes/kubeconfig ~/.kube/config
 else
     echo_yellow "Moving old ${HOME}/.kube/config to ${HOME}/.kube/config.backup" && sudo cp ~/.kube/config ~/.kube/config.backup && sudo cp /etc/kubernetes/kubeconfig ~/.kube/config
fi
sudo chmod 644 ~/.kube/config

### Enabling Kubernetes CronJob Support:
echo_green "\nPhase VIII: Running conditional options to enable Kubernetes CronJobs. Below are enabled options for the API Server:"
awk '/etcd-servers/{print;print "        - --runtime-config=batch/v2alpha1=true";next}1' ${BOOTKUBE_DIR}/.bootkube/manifests/kube-apiserver.yaml > ${BOOTKUBE_DIR}/.bootkube/manifests/kube-apiserver-custom.yaml
rm -rf ${BOOTKUBE_DIR}/.bootkube/manifests/kube-apiserver.yaml
cat ${BOOTKUBE_DIR}/.bootkube/manifests/kube-apiserver-custom.yaml | grep "        - --"


echo_green "\nPhase IX: Running Bootkube start to bring up the temporary Kubernetes self-hosted control plane:"
nohup sudo bash -c 'bootkube start --asset-dir='${BOOTKUBE_DIR}'/.bootkube' >${BOOTKUBE_DIR}/log/bootkube-start.log 2>&1 &

### WAIT FOR KUBERNETES ENVIRONMENT TO COME UP:
echo -e -n "Waiting for master components to start..."
while true; do
  running_count=$(sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig get pods -n kube-system --no-headers 2>/dev/null | grep "Running" | grep "boot" | wc -l)
  ### Expect 4 bootstrap components for a truly "Running" state: etcd, apiserver, controller, and scheduler:
  if [ "${running_count}" -ge 4 ]; then
    break
  fi
  echo -n "."
  sleep 1
done
echo_green "SUCCESS"
echo_green "Cluster created!"
echo ""
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig cluster-info

sleep 10

### WAIT FOR KUBERNETES API TO COME UP CLEANLY, THEN APPLY FOLLOWING LABELS AND MANIFESTS:
echo_green "\nPhase X: Cluster created, and now deploying requested SDN along with additional labels:"
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig label node --all node-role.kubernetes.io/${KUBE_SDN}-node=true --overwrite
sed "s?KUBE_POD_CIDR?${KUBE_POD_CIDR}?g; s?ETCD_SVC_ADDR?${ETCD_SVC_ADDR}?g; s?ETCD_SVC_PORT?${ETCD_SVC_PORT}?g; s?ETCD_SVC_PEER?${ETCD_SVC_PEER}?g; s?KUBE_SVC_PORT?${KUBE_SVC_PORT}?g" ${BOOTKUBE_DIR}/deploy-sdn/${KUBE_SDN}/calico-${KUBE_SDN_VERSION}-v1.7.5.yaml  >  ${BOOTKUBE_DIR}/deploy-sdn/${KUBE_SDN}/calico-${KUBE_SDN_VERSION}-bootkube-v1.7.5.yaml

sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig apply -f ${BOOTKUBE_DIR}/deploy-sdn/${KUBE_SDN}/calico-${KUBE_SDN_VERSION}-bootkube-v1.7.5.yaml
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig taint nodes --all node-role.kubernetes.io/master- || true

echo_green "\nPhase XI: Writing Kubernetes environment cluster-info logs:"
### WAIT FOR KUBERNETES ENVIRONMENT TO COME UP:
echo -e -n "Waiting for all services to be in a running state..."
while true; do
  creating_count=$(sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig get pods -n kube-system --no-headers 2>/dev/null | egrep "i(Pending|ContainerCreating)" | grep "kube" | wc -l)
  ### Expect all components to be out of a "Pending" state before collecting log data (this includes CrashLoopBackOff states):
  if [ "${creating_count}" -eq 0 ]; then
    break
  fi
  echo -n "."
  sleep 1
done
export CLUSTER_TIMESTAMP="$(date +%s)"
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig cluster-info dump --all-namespaces --output-directory "${BOOTKUBE_DIR}/log/cluster-info-${CLUSTER_TIMESTAMP}"
echo_yellow "\nCluster logs can be found in ${BOOTKUBE_DIR}/log/cluster-info-${CLUSTER_TIMESTAMP}"
echo_green "\nCOMPLETE!\n"

echo_green "\nPhase XII: Creating a proper ${HOME}/.kube/config file for Armada use:"
### WAIT FOR KUBERNETES ENVIRONMENT TO COME UP:
sudo mv ${HOME}/.kube/config /opt/demo/backups/config
sudo kubectl config set-cluster local --server=https://${KUBE_MASTER}:${KUBE_SVC_PORT} --certificate-authority=${BOOTKUBE_DIR}/.bootkube/tls/ca.crt
sudo kubectl config set-context default --cluster=local --user=kubelet
sudo kubectl config use-context default
sudo kubectl config set-credentials kubelet --client-certificate=${BOOTKUBE_DIR}/.bootkube/tls/kubelet.crt  --client-key=${BOOTKUBE_DIR}/.bootkube/tls/kubelet.key
sudo chown -R ${USER} ${HOME}/.kube

echo_green "\nPhase XIII: Installing Helm:"
### DOWNLOAD: Helm
if [[ ! -e '$TMPDIR'/'$HELM_VERSION'-helm-$KUBE_ARCH.tgz && ! -e /usr/local/bin/helm ]]; then
    wget -O $TMPDIR/$HELM_VERSION-helm-$KUBE_ARCH.tgz https://storage.googleapis.com/kubernetes-helm/helm-$HELM_VERSION-linux-$KUBE_ARCH.tar.gz
    tar zxvf $TMPDIR/$HELM_VERSION-helm-$KUBE_ARCH.tgz -C $TMPDIR/
    mv $TMPDIR/linux-$KUBE_ARCH/ $TMPDIR/$HELM_VERSION-helm-$KUBE_ARCH
    chmod +x $TMPDIR/$HELM_VERSION-helm-$KUBE_ARCH/helm
    sudo cp $TMPDIR/$HELM_VERSION-helm-$KUBE_ARCH/helm /usr/local/bin/
fi