Merge pull request #523 from nik-netlox/main

BFD failover detection in L2 with connection sync CICD test case

cicd/k8s-calico-ipvs2-ha-ka-sync/README

@@ -0,0 +1,22 @@
+## Test Case Description
+
+This scenario will demonstrate LoxiLB in L2 HA mode(clustering). The setup will have 2 LoxiLB nodes, K8s(1 Master Nodes & 2 Worker Nodes) cluster with Calico CNI in ipvs mode. LoxiLB will be running as external Service LB. Workloads will be spawned in all the cluster nodes.
+
+Highlights of this demo:
+1) HA in L2 Mode
+2) Sub-second HA failover detection
+3) Connection sync
+
+Client will be connected to the LoxiLB with L2 network. Master LoxiLB will announce the Service IP to the client and cluster nodes.
+
+Service CIDR will also be a Virtual IP, a subnet of the K8s cluster network.
+
+In scenarios where LoxiLB runs outside of the cluster in HA mode, it is advised to create LB services in fullnat mode for ease of connectivity.
+
+If you wish to create this scenario in your lab then install Vagrant and follow the steps below:
+
+1. Run ./config.sh to setup the K8s cluster, client and LoxiLB nodes
+
+2. Run ./validation.sh to run the TCP HA test or ./validation_with_sctp.sh to run TCP & SCTP HA Test. Test Results will be displayed at the end.
+
+3. Run ./rmconfig.sh to cleanup the setup.

cicd/k8s-calico-ipvs2-ha-ka-sync/Vagrantfile

@@ -0,0 +1,95 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+require "yaml"
+settings = YAML.load_file "yaml/settings.yaml"
+
+workers = settings["nodes"]["workers"]["count"]
+loxilbs = (ENV['LOXILBS'] || "2").to_i
+
+Vagrant.configure("2") do |config|
+
+  if Vagrant.has_plugin?("vagrant-vbguest")
+    config.vbguest.auto_update = false
+  end
+   config.vm.define "host" do |host|
+    host.vm.hostname = 'host'
+    host.vm.box = settings["software"]["cluster"]["box"]
+    host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
+    host.vm.provision :shell, :path => "node_scripts/host.sh"
+    host.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 2048]
+        vbox.customize ["modifyvm", :id, "--cpus", 2]
+    end
+  end
+
+
+
+  (1..loxilbs).each do |node_number|
+    config.vm.define "llb#{node_number}" do |loxilb|
+      loxilb.vm.box = settings["software"]["loxilb"]["box"]["name"]
+      loxilb.vm.box_version = settings["software"]["loxilb"]["box"]["version"]
+      loxilb.vm.hostname = "llb#{node_number}"
+      ip = node_number + 251
+      loxilb.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
+      loxilb.vm.provision :shell, :path => "node_scripts/loxilb#{node_number}.sh"
+      loxilb.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 6000]
+        vbox.customize ["modifyvm", :id, "--cpus", 4]
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+      end
+    end
+  end
+
+  config.vm.define "master" do |master|
+    master.vm.box = settings["software"]["cluster"]["box"]
+    master.vm.hostname = 'master'
+    master.vm.network :private_network, ip: settings["network"]["control_ip"], :netmask => "255.255.255.0"
+    master.vm.provision "shell",
+      env: {
+        "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
+        "ENVIRONMENT" => settings["environment"],
+        "KUBERNETES_VERSION" => settings["software"]["kubernetes"],
+        "OS" => settings["software"]["os"]
+      },
+      path: "node_scripts/common.sh"
+    master.vm.provision "shell",
+      env: {
+        "CALICO_VERSION" => settings["software"]["calico"],
+        "CONTROL_IP" => settings["network"]["control_ip"],
+        "POD_CIDR" => settings["network"]["pod_cidr"],
+        "SERVICE_CIDR" => settings["network"]["service_cidr"]
+      },
+      path: "node_scripts/master.sh"
+
+    master.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 4096]
+        vbox.customize ["modifyvm", :id, "--cpus", 2]
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+    end
+  end
+
+  (1..workers).each do |node_number|
+    config.vm.define "worker#{node_number}" do |worker|
+      worker.vm.box = settings["software"]["cluster"]["box"]
+      worker.vm.hostname = "worker#{node_number}"
+      ip = node_number + 200
+      worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
+      worker.vm.provision "shell",
+        env: {
+          "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
+          "ENVIRONMENT" => settings["environment"],
+          "KUBERNETES_VERSION" => settings["software"]["kubernetes"],
+          "OS" => settings["software"]["os"]
+        },
+        path: "node_scripts/common.sh"
+      worker.vm.provision "shell", path: "node_scripts/worker.sh"
+
+      worker.vm.provider :virtualbox do |vbox|
+          vbox.customize ["modifyvm", :id, "--memory", 4096]
+          vbox.customize ["modifyvm", :id, "--cpus", 2]
+          vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+      end
+    end
+  end
+end

cicd/k8s-calico-ipvs2-ha-ka-sync/config.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+VMs=$(vagrant global-status  | grep -i virtualbox)
+while IFS= read -a VMs; do
+    read -a vm <<< "$VMs"
+    cd ${vm[4]} 2>&1>/dev/null
+    echo "Destroying ${vm[1]}"
+    vagrant destroy -f ${vm[1]}
+    cd - 2>&1>/dev/null
+done <<< "$VMs"
+
+vagrant up
+
+for((i=1; i<=60; i++))
+do
+    fin=1
+    pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE")
+
+    while IFS= read -a pods; do
+        read -a pod <<< "$pods"
+        if [[ ${pod[3]} != *"Running"* ]]; then
+            echo "${pod[1]} is not UP yet"
+            fin=0
+        fi
+    done <<< "$pods"
+    if [ $fin == 1 ];
+    then
+        break;
+    fi
+    echo "Will try after 10s"
+    sleep 10
+done
+
+sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1
+
+#Create fullnat Service
+vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null
+vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_fullnat.yml' 2> /dev/null
+#vagrant ssh master -c 'kubectl apply -f https://raw.githubusercontent.com/loxilb-io/kube-loxilb/main/manifest/mesh/loxilb-mesh.yml' 2> /dev/null

cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+extIP=$(cat /vagrant/extIP)
+
+code=0
+
+echo Service IP: $extIP
+
+neigh=$( ip neigh | grep $extIP )
+
+if [[ ! -z $neigh && $neigh != *"FAILED"* ]]; then
+    echo "Host route [OK]"
+else
+    echo "Host route [NOK]"
+fi
+echo -e "\n*********************************************"
+echo "Testing Service"
+echo "*********************************************"
+
+# iperf client accessing fullnat service
+stdbuf -oL nohup iperf -c 192.168.80.5 -p 56002 -t 60 -i 1 -b 100M &> iperff.out &
+echo "iperf client started"
+echo "phase-1 done"

cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation2.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+pkill iperf
+
+iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
+
+if [[ $iperff_res != 0 ]]; then
+    echo -e "K8s-calico-ipvs2-ha-ka-sync TCP\t\t(fullnat)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs2-ha-ka-sync TCP\t\t(fullnat)\t[FAILED]"
+    code=1
+fi
+
+rm *.out
+exit $code

cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation2_with_sctp.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+for((i=0;i<50;i++))
+do
+echo "snd=100" 1> sd1.pipe
+sleep 1
+done
+
+
+echo "stats" 1> sd1.pipe
+
+echo "shutdown" 1> sd1.pipe
+
+pkill iperf
+pkill sctp_darn
+
+iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
+
+sdf_res=$(grep -i "Client: Sending packets.(100000/100000)" sdf.out)
+
+sdf_res1=$(grep -i "packets sent" sdf.out | xargs | cut -d ' ' -f 3)
+sdf_res2=$(grep -i "packets rec" sdf.out | xargs | cut -d ' ' -f 3)
+
+if [[ $iperff_res != 0 ]]; then
+    echo -e "K8s-calico-ipvs2-ha-ka-sync TCP\t\t(fullnat)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs2-ha-ka-sync TCP\t\t(fullnat)\t[FAILED]"
+    code=1
+fi
+
+if [[ $sdf_res1 != 0 && $sdf_res2 != 0 && $sdf_res1 == $sdf_res2 ]]; then
+    echo -e "K8s-calico-ipvs2-ha-ka-sync SCTP\t(fullnat)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs2-ha-ka-sync SCTP\t(fullnat)\t[FAILED]"
+    code=1
+fi
+
+rm *.pipe
+rm *.out
+exit $code

cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation_with_sctp.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+extIP=$(cat /vagrant/extIP)
+
+code=0
+
+echo Service IP: $extIP
+
+neigh=$( ip neigh | grep $extIP )
+
+if [[ ! -z $neigh && $neigh != *"FAILED"* ]]; then
+    echo "Host route [OK]"
+else
+    echo "Host route [NOK]"
+fi
+echo -e "\n*********************************************"
+echo "Testing Service"
+echo "*********************************************"
+
+# iperf client accessing fullnat service
+stdbuf -oL nohup iperf -c  192.168.80.5 -p 56002 -t 100 -i 1 -b 100M &> iperff.out &
+
+echo "iperf client started"
+
+sleep 1
+
+mkfifo sd1.pipe
+
+sleep infinity > sd1.pipe &
+
+stdbuf -oL nohup sctp_darn -H 192.168.80.9 -h 192.168.80.5 -p 56004 -s -I < sd1.pipe &> sdf.out &
+
+echo "sctp_test client started"
+
+sleep 2
+for((i=0;i<30;i++))
+do
+echo "snd=100" 1> sd1.pipe
+sleep 1
+done
+echo "phase-1 done"
+exit 0

cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/common.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# Common setup for all servers (Control Plane and Nodes)
+
+set -euxo pipefail
+
+# Variable Declaration
+
+# DNS Setting
+if [ ! -d /etc/systemd/resolved.conf.d ]; then
+	sudo mkdir /etc/systemd/resolved.conf.d/
+fi
+cat <<EOF | sudo tee /etc/systemd/resolved.conf.d/dns_servers.conf
+[Resolve]
+DNS=${DNS_SERVERS}
+EOF
+
+sudo systemctl restart systemd-resolved
+
+# disable swap
+sudo swapoff -a
+
+# keeps the swap off during reboot
+(crontab -l 2>/dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true
+sudo apt-get update -y
+# Install CRI-O Runtime
+
+VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
+
+# Create the .conf file to load the modules at bootup
+cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
+overlay
+br_netfilter
+EOF
+
+sudo modprobe overlay
+sudo modprobe br_netfilter
+
+# Install ipvs related modules
+sudo modprobe ip_vs
+sudo modprobe ip_vs_rr
+sudo modprobe ip_vs_wrr
+sudo modprobe ip_vs_sh
+#sudo modprobe nf_conntrack_ipv4
+
+sudo sysctl net.ipv4.vs.sloppy_sctp=1
+
+# Set up required sysctl params, these persist across reboots.
+cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
+net.bridge.bridge-nf-call-iptables  = 1
+net.ipv4.ip_forward                 = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+EOF
+
+sudo sysctl --system
+
+cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
+EOF
+cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
+deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
+EOF
+
+curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+
+sudo apt-get update
+sudo apt-get install cri-o cri-o-runc -y
+
+cat >> /etc/default/crio << EOF
+${ENVIRONMENT}
+EOF
+sudo systemctl daemon-reload
+sudo systemctl enable crio --now
+
+echo "CRI runtime installed successfully"
+
+sudo apt-get update
+sudo apt-get install -y apt-transport-https ca-certificates curl
+curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+
+echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+sudo apt-get update -y
+sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
+sudo apt-get update -y
+sudo apt-get install -y jq
+sudo apt-get install -y ipvsadm
+
+local_ip="$(ip --json a s | jq -r '.[] | if .ifname == "eth1" then .addr_info[] | if .family == "inet" then .local else empty end else empty end')"
+cat > /etc/default/kubelet << EOF
+KUBELET_EXTRA_ARGS=--node-ip=$local_ip
+${ENVIRONMENT}
+EOF

cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/host.sh

@@ -0,0 +1,5 @@
+# Install Bird to work with k3s
+sudo apt-get update
+sudo apt-get -y install lksctp-tools iperf
+
+echo "Host is up"

cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb1.sh

@@ -0,0 +1,9 @@
+export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+
+apt-get update
+apt-get install -y software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=192.168.80.253 --self=0 --ka=192.168.80.253

cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb2.sh

@@ -0,0 +1,9 @@
+export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+
+apt-get update
+apt-get install -y software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=192.168.80.252 --self=1 --ka=192.168.80.252