From 4f8e288d2b2c0c8537137fbd79bcf2c383e3503d Mon Sep 17 00:00:00 2001
From: Nikhil Malik <nikhil.malik@netlox.io>
Date: Tue, 24 Sep 2024 17:49:39 +0900
Subject: [PATCH] gh-87 : Updating scripts inline with PR #808

---
 .../node_scripts/common.sh                    | 81 +++++++++++--------
 .../node_scripts/master.sh                    |  7 +-
 .../yaml/kubeadm-config.yaml                  |  5 ++
 3 files changed, 56 insertions(+), 37 deletions(-)

diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh
index c01ad688f..ded6b8b5d 100644
--- a/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh
+++ b/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh
@@ -15,6 +15,7 @@ cat <<EOF | sudo tee /etc/systemd/resolved.conf.d/dns_servers.conf
 DNS=${DNS_SERVERS}
 EOF
 
+VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
 sudo systemctl restart systemd-resolved
 
 # disable swap
@@ -23,18 +24,56 @@ sudo swapoff -a
 # keeps the swap off during reboot
 (crontab -l 2>/dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true
 sudo apt-get update -y
-# Install CRI-O Runtime
 
-VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
-CRIO_VERSION=1.27
-# Create the .conf file to load the modules at bootup
-cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
-overlay
-br_netfilter
+sudo apt install -y snapd
+#sudo snap install docker
+
+sudo apt install -y docker.io
+sudo usermod -aG docker vagrant
+
+# cri-docker Install
+VER=$(curl -s https://api.github.com/repos/Mirantis/cri-dockerd/releases/latest|grep tag_name | cut -d '"' -f 4|sed 's/v//g')
+echo $VER
+wget -q https://github.com/Mirantis/cri-dockerd/releases/download/v${VER}/cri-dockerd-${VER}.amd64.tgz --retry-connrefused  --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 --continue
+tar xvf cri-dockerd-${VER}.amd64.tgz
+sudo mv cri-dockerd/cri-dockerd /usr/local/bin/
+
+# cri-docker Version Check
+cri-dockerd --version
+
+wget -q https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.service --retry-connrefused  --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 --continue
+wget -q https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.socket --retry-connrefused  --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 --continue
+sudo mv cri-docker.socket cri-docker.service /etc/systemd/system/
+sudo sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service
+
+sudo systemctl daemon-reload
+sudo systemctl enable cri-docker.service
+sudo systemctl enable --now cri-docker.socket
+
+# cri-docker Active Check
+sudo systemctl restart docker && sudo systemctl restart cri-docker
+sudo systemctl status cri-docker.socket --no-pager 
+
+# Docker cgroup Change Require to Systemd
+sudo mkdir -p /etc/docker || true
+cat <<EOF | sudo tee /etc/docker/daemon.json
+{
+  "exec-opts": ["native.cgroupdriver=systemd"],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "storage-driver": "overlay2"
+}
 EOF
 
-sudo modprobe overlay
-sudo modprobe br_netfilter
+sudo systemctl restart docker && sudo systemctl restart cri-docker
+sudo docker info | grep Cgroup
+
+# Kernel Forwarding 
+cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
+br_netfilter
+EOF
 
 # Install ipvs related modules
 sudo modprobe ip_vs
@@ -54,30 +93,8 @@ EOF
 
 sudo sysctl --system
 
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
-deb https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
-EOF
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
-deb http://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
-EOF
-
-curl -L https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
-curl -L https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
-
-sudo apt-get update
-sudo apt-get install cri-o cri-o-runc -y
-
-cat >> /etc/default/crio << EOF
-${ENVIRONMENT}
-EOF
-sudo systemctl daemon-reload
-sudo systemctl enable crio --now
-
-echo "CRI runtime installed successfully"
-
-sudo apt-get update
 sudo apt-get install -y apt-transport-https ca-certificates curl gpg
-curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --no-tty --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
 echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
 sudo apt-get update -y
diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh
index a5cdecaac..d4b836aef 100644
--- a/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh
+++ b/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh
@@ -6,10 +6,6 @@ set -euxo pipefail
 
 NODENAME=$(hostname -s)
 
-sudo kubeadm config images pull
-
-echo "Preflight Check Passed: Downloaded All Required Images"
-
 #sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap
 sudo kubeadm init --ignore-preflight-errors Swap --config /vagrant/yaml/kubeadm-config.yaml
 
@@ -33,7 +29,8 @@ cp -i /etc/kubernetes/admin.conf $config_path/config
 touch $config_path/join.sh
 chmod +x $config_path/join.sh
 
-kubeadm token create --print-join-command > $config_path/join.sh
+join_cmd=`kubeadm token create --print-join-command`
+echo $join_cmd "--cri-socket /var/run/cri-dockerd.sock" > $config_path/join.sh
 
 # Install Calico Network Plugin
 
diff --git a/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml b/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml
index 245a62553..20a7207ac 100644
--- a/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml
+++ b/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml
@@ -14,6 +14,7 @@ nodeRegistration:
   imagePullPolicy: IfNotPresent
   name: master
   taints: null
+  criSocket: unix:///var/run/cri-dockerd.sock
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 certificatesDir: /etc/kubernetes/pki
@@ -67,3 +68,7 @@ metricsBindAddress: 127.0.0.1:10249
 nodePortAddresses: null
 oomScoreAdj: -999
 portRange: ""
+---
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+cgroupDriver: systemd