diff --git a/.github/workflows/k8s-calico-ipvs3-ha.yml b/.github/workflows/k8s-calico-ipvs3-ha.yml index 45076a612..09d88e298 100644 --- a/.github/workflows/k8s-calico-ipvs3-ha.yml +++ b/.github/workflows/k8s-calico-ipvs3-ha.yml @@ -1,8 +1,8 @@ name: K8s-Calico-Cluster-IPVS3-HA-Sanity-CI on: - #schedule: + schedule: # Runs "At 17:00 UTC every day-of-week" - #- cron: '0 17 * * *' + - cron: '0 17 * * *' workflow_dispatch: inputs: testName: diff --git a/cicd/k8s-calico-ipvs3-ha/README b/cicd/k8s-calico-ipvs3-ha/README index 1e6e9dc91..b21a14a52 100644 --- a/cicd/k8s-calico-ipvs3-ha/README +++ b/cicd/k8s-calico-ipvs3-ha/README @@ -14,6 +14,6 @@ If you wish to create this scenario in your lab then install Vagrant and follow 1. Run ./config.sh to setup the K8s cluster, client and LoxiLB nodes -2. Run ./validation.sh to run the HA test. Test Results will be displayed at the end. +2. Run ./validation.sh to run the TCP HA test or ./validation_with_sctp.sh to run TCP & SCTP HA Test. Test Results will be displayed at the end. 3. Run ./rmconfig.sh to cleanup the setup. diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation.sh b/cicd/k8s-calico-ipvs3-ha/host_validation.sh index 814bb14b4..30817fb4b 100755 --- a/cicd/k8s-calico-ipvs3-ha/host_validation.sh +++ b/cicd/k8s-calico-ipvs3-ha/host_validation.sh @@ -2,6 +2,7 @@ extIP=$(cat /vagrant/extIP) code=0 + echo Service IP: $extIP numECMP=$(birdc show route | grep $extIP -A 3 | grep via | wc -l) @@ -18,26 +19,10 @@ echo "Testing Service" echo "*********************************************" # iperf client accessing fullnat service -stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M &> iperff.out & +stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 60 -i 1 -b 100M &> iperff.out & # iperf client accessing default service -stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out & - -mkfifo sd1.pipe -mkfifo sd2.pipe - -sleep infinity > sd1.pipe & -sleep infinity > sd2.pipe & - -sleep 1 - -stdbuf -oL nohup sctp_darn -H 192.168.90.9 -h 20.20.20.1 -p 56004 -s -I < sd1.pipe &> sdf.out & -stdbuf -oL nohup sctp_darn -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -I < sd2.pipe &> sdd.out & +stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 60 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out & -sleep 2 -for((i=0;i<30;i++)) -do -echo "snd=100" >> sd1.pipe -echo "snd=100" >> sd2.pipe -sleep 1 -done +echo "iperf client started" +echo "phase-1 done" diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation2.sh b/cicd/k8s-calico-ipvs3-ha/host_validation2.sh index 11e870001..b29b337c5 100755 --- a/cicd/k8s-calico-ipvs3-ha/host_validation2.sh +++ b/cicd/k8s-calico-ipvs3-ha/host_validation2.sh @@ -1,30 +1,10 @@ #!/bin/bash -for((i=0;i<50;i++)) -do -echo "snd=100" >> sd1.pipe -echo "snd=100" >> sd2.pipe -sleep 1 -done - - -echo "stats" >> sd1.pipe -echo "stats" >> sd2.pipe - -echo "shutdown" >> sd1.pipe -echo "shutdown" >> sd2.pipe pkill iperf -pkill sctp_darn iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7) iperfd_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7) -sdf_res1=$(grep -i "packets sent" sdf.out | xargs | cut -d ' ' -f 3) -sdf_res2=$(grep -i "packets rec" sdf.out | xargs | cut -d ' ' -f 3) - -sdd_res1=$(grep -i "packets sent" sdd.out | xargs | cut -d ' ' -f 3) -sdd_res2=$(grep -i "packets rec" sdd.out | xargs | cut -d ' ' -f 3) - if [[ $iperff_res != 0 ]]; then echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[OK]" else @@ -39,19 +19,5 @@ else code=1 fi -if [[ $sdf_res1 != 0 && $sdf_res2 != 0 && $sdf_res1 == $sdf_res2 ]]; then - echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[OK]" -else - echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[FAILED]" - code=1 -fi - -if [[ $sdd_res1 != 0 && $sdd_res2 != 0 && $sdd_res1 == $sdd_res2 ]]; then - echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[OK]" -else - echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[FAILED]" - code=1 -fi - -rm *.out *.pipe +rm *.out exit $code diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation2_with_sctp.sh b/cicd/k8s-calico-ipvs3-ha/host_validation2_with_sctp.sh new file mode 100755 index 000000000..5c44702be --- /dev/null +++ b/cicd/k8s-calico-ipvs3-ha/host_validation2_with_sctp.sh @@ -0,0 +1,60 @@ +#!/bin/bash +for((i=0;i<50;i++)) +do +echo "snd=100" 1> sd1.pipe +echo "snd=100" 1> sd2.pipe +sleep 1 +done + + +echo "stats" 1> sd1.pipe +echo "stats" 1> sd2.pipe + +echo "shutdown" 1> sd1.pipe +echo "shutdown" 1> sd2.pipe + +pkill iperf +pkill sctp_darn + +iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7) +iperfd_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7) + +sdf_res=$(grep -i "Client: Sending packets.(100000/100000)" sdf.out) +sdd_res=$(grep -i "Client: Sending packets.(100000/100000)" sdd.out) + +sdf_res1=$(grep -i "packets sent" sdf.out | xargs | cut -d ' ' -f 3) +sdf_res2=$(grep -i "packets rec" sdf.out | xargs | cut -d ' ' -f 3) + +sdd_res1=$(grep -i "packets sent" sdd.out | xargs | cut -d ' ' -f 3) +sdd_res2=$(grep -i "packets rec" sdd.out | xargs | cut -d ' ' -f 3) + +if [[ $iperff_res != 0 ]]; then + echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[OK]" +else + echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[FAILED]" + code=1 +fi + +if [[ $iperfd_res != 0 ]]; then + echo -e "K8s-calico-ipvs3-ha TCP\t\t(default\t[OK]" +else + echo -e "K8s-calico-ipvs3-ha TCP\t\t(default)\t[FAILED]" + code=1 +fi + +if [[ $sdf_res1 != 0 && $sdf_res2 != 0 && $sdf_res1 == $sdf_res2 ]]; then + echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[OK]" +else + echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[FAILED]" + code=1 +fi + +if [[ $sdd_res1 != 0 && $sdd_res2 != 0 && $sdd_res1 == $sdd_res2 ]]; then + echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[OK]" +else + echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[FAILED]" + code=1 +fi +rm *.pipe +rm *.out +exit $code diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation_with_sctp.sh b/cicd/k8s-calico-ipvs3-ha/host_validation_with_sctp.sh new file mode 100755 index 000000000..ff29729b4 --- /dev/null +++ b/cicd/k8s-calico-ipvs3-ha/host_validation_with_sctp.sh @@ -0,0 +1,50 @@ +#!/bin/bash +extIP=$(cat /vagrant/extIP) + +code=0 + +echo Service IP: $extIP + +numECMP=$(birdc show route | grep $extIP -A 3 | grep via | wc -l) + +birdc show route | grep $extIP -A 3 + +if [ $numECMP == "2" ]; then + echo "Host route [OK]" +else + echo "Host route [NOK]" +fi +echo -e "\n*********************************************" +echo "Testing Service" +echo "*********************************************" + +# iperf client accessing fullnat service +stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M &> iperff.out & + +# iperf client accessing default service +stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out & + +echo "iperf client started" + +sleep 1 + +mkfifo sd1.pipe +mkfifo sd2.pipe + +sleep infinity > sd1.pipe & +sleep infinity > sd2.pipe & + +stdbuf -oL nohup sctp_darn -H 192.168.90.9 -h 20.20.20.1 -p 56004 -s -I < sd1.pipe &> sdf.out & +stdbuf -oL nohup sctp_darn -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -I < sd2.pipe &> sdd.out & + +echo "sctp_test client started" + +sleep 2 +for((i=0;i<30;i++)) +do +echo "snd=100" 1> sd1.pipe +echo "snd=100" 1> sd2.pipe +sleep 1 +done +echo "phase-1 done" +exit 0 diff --git a/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh b/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh index cc999d846..2bae00c97 100644 --- a/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh +++ b/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh @@ -7,4 +7,5 @@ add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu apt-get update apt-get install -y docker-ce docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest -b --cluster=192.168.80.253 --self=0 +docker cp loxilb:/usr/local/sbin/loxicmd ./ #docker exec -dt loxilb /root/loxilb-io/loxilb/loxilb -b --cluster=192.168.80.253 --self=0 diff --git a/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh b/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh index 522ff30a9..bf2744b38 100644 --- a/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh +++ b/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh @@ -7,5 +7,7 @@ add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu apt-get update apt-get install -y docker-ce docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest -b --cluster=192.168.80.252 --self=1 +docker cp loxilb:/usr/local/sbin/loxicmd ./ #docker exec -dt loxilb /root/loxilb-io/loxilb/loxilb -b --cluster=192.168.80.252 --self=1 + diff --git a/cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh b/cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh index 8a4ee7477..a5cdecaac 100644 --- a/cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh +++ b/cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh @@ -40,6 +40,9 @@ kubeadm token create --print-join-command > $config_path/join.sh curl https://raw.githubusercontent.com/projectcalico/calico/v${CALICO_VERSION}/manifests/calico.yaml -O kubectl apply -f calico.yaml +kubectl patch configmap/calico-config -n kube-system --type merge \ + -p '{"data":{"veth_mtu": "8900"}}' +kubectl rollout restart daemonset calico-node -n kube-system sudo -i -u vagrant bash << EOF whoami diff --git a/cicd/k8s-calico-ipvs3-ha/validation.sh b/cicd/k8s-calico-ipvs3-ha/validation.sh index 0ddc714fe..ee9716394 100755 --- a/cicd/k8s-calico-ipvs3-ha/validation.sh +++ b/cicd/k8s-calico-ipvs3-ha/validation.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -eo pipefail source ../common.sh echo k8s-calico-ipvs3-ha @@ -47,18 +48,18 @@ echo "************************************************************************** echo -e "\nLB List" echo -e "\n---- LLB1 ----" echo "******************************************************************************" -vagrant ssh llb1 -c 'sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null +vagrant ssh llb1 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null echo "******************************************************************************" echo -e "\n---- LLB2 ----" -vagrant ssh llb2 -c 'sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null +vagrant ssh llb2 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null echo "******************************************************************************" echo -e "\nEP List" echo -e "\n---- LLB1 ----" echo "******************************************************************************" -vagrant ssh llb1 -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null +vagrant ssh llb1 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null echo "******************************************************************************" echo -e "\n---- LLB2 ----" -vagrant ssh llb2 -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null +vagrant ssh llb2 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null echo "******************************************************************************" echo -e "\nTEST RESULTS" echo "******************************************************************************" @@ -78,14 +79,17 @@ echo -e " BACKUP\t: $backup\n" vagrant ssh host -c 'sudo /vagrant/host_validation.sh' 2> /dev/null +sleep 15 +echo -e "phase-2 begins..\n" + count=1 sync=0 while [[ $count -le 5 ]] ; do echo -e "\nStatus at MASTER:$master\n" -vagrant ssh $master -c "sudo docker exec -it loxilb loxicmd get ct | grep est" 2> /dev/null +vagrant ssh $master -c "sudo ./loxicmd get ct | grep est" 2> /dev/null echo -e "\nStatus at BACKUP:$backup\n" -vagrant ssh $backup -c "sudo docker exec -it loxilb loxicmd get ct | grep est" 2> /dev/null +vagrant ssh $backup -c "sudo ./loxicmd get ct | grep est" 2> /dev/null nres1=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) nres2=$(curl -sX 'GET' 'http://192.168.80.253:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) @@ -102,17 +106,14 @@ done if [[ $sync == 0 ]]; then echo -e "\nConnection Sync failed\n" - vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_darn; sudo rm -rf *.pipe *.out' + vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_test; sudo rm -rf *.out' exit 1 fi echo "Restarting MASTER:$master.." vagrant ssh $master -c 'sudo docker restart loxilb' 2> /dev/null -sleep 10 - -vagrant ssh host -c 'sudo /vagrant/host_validation2.sh' 2> /dev/null +sleep 50 sudo rm extIP - -exit $code +vagrant ssh host -c 'sudo /vagrant/host_validation2.sh' 2> /dev/null diff --git a/cicd/k8s-calico-ipvs3-ha/validation_with_sctp.sh b/cicd/k8s-calico-ipvs3-ha/validation_with_sctp.sh new file mode 100755 index 000000000..393eaa395 --- /dev/null +++ b/cicd/k8s-calico-ipvs3-ha/validation_with_sctp.sh @@ -0,0 +1,119 @@ +#!/bin/bash +set -eo pipefail +source ../common.sh +echo k8s-calico-ipvs3-ha + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +for((i=0; i<120; i++)) +do + extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-default") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find tcp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +#sleep 30 + +echo Service IP : $extIP +echo $extIP > extIP +echo -e "\nEnd Points List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get endpoints -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nSVC List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get svc' 2> /dev/null +echo "******************************************************************************" +echo -e "\nPod List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nLB List" +echo -e "\n---- LLB1 ----" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\n---- LLB2 ----" +vagrant ssh llb2 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nEP List" +echo -e "\n---- LLB1 ----" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\n---- LLB2 ----" +vagrant ssh llb2 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nTEST RESULTS" +echo "******************************************************************************" + +master="llb1" +backup="llb2" + +state=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/cistate/all' -H 'accept: application/json') + +if [[ $state == *"BACKUP"* ]]; then + master="llb2" + backup="llb1" +fi + +echo -e "\n MASTER\t: $master" +echo -e " BACKUP\t: $backup\n" + +vagrant ssh host -c 'sudo /vagrant/host_validation_with_sctp.sh' 2> /dev/null + +#sleep 15 +echo -e "phase-2 begins..\n" + +count=1 +sync=0 +while [[ $count -le 5 ]] ; do +echo -e "\nStatus at MASTER:$master\n" +vagrant ssh $master -c "sudo ./loxicmd get ct | grep est" 2> /dev/null + +echo -e "\nStatus at BACKUP:$backup\n" +vagrant ssh $backup -c "sudo ./loxicmd get ct | grep est" 2> /dev/null + +nres1=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) +nres2=$(curl -sX 'GET' 'http://192.168.80.253:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + +if [[ $nres1 == $nres2 ]]; then + echo -e "\nConnections sync successful!!!\n" + sync=1 + break; +fi +echo -e "\nConnections sync pending.. Let's wait a little more..\n" +count=$(( $count + 1 )) +sleep 2 +done + +if [[ $sync == 0 ]]; then + echo -e "\nConnection Sync failed\n" + vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_test; sudo rm -rf *.out' + exit 1 +fi + +echo "Restarting MASTER:$master.." +vagrant ssh $master -c 'sudo docker restart loxilb' 2> /dev/null + +sleep 30 + +sudo rm extIP +vagrant ssh host -c 'sudo /vagrant/host_validation2_with_sctp.sh' 2> /dev/null diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml index 44de88b52..0359fcb70 100644 --- a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml +++ b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml @@ -5,6 +5,7 @@ metadata: annotations: loxilb.io/lbmode: "default" loxilb.io/liveness: "yes" + loxilb.io/probetype: "ping" spec: externalTrafficPolicy: Local loadBalancerClass: loxilb.io/loxilb @@ -29,5 +30,6 @@ spec: imagePullPolicy: Always #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"] command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"] + #command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"] ports: - containerPort: 9999 diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml index 87beb9d08..e010624f5 100644 --- a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml +++ b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml @@ -5,6 +5,7 @@ metadata: annotations: loxilb.io/lbmode: "fullnat" loxilb.io/liveness: "yes" + loxilb.io/probetype: "ping" spec: externalTrafficPolicy: Local loadBalancerClass: loxilb.io/loxilb @@ -29,5 +30,6 @@ spec: imagePullPolicy: Always #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"] command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"] + #command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"] ports: - containerPort: 9999 diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml b/cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml index c3def4e05..2742671bb 100644 --- a/cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml +++ b/cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml @@ -3,8 +3,9 @@ kind: Service metadata: name: tcp-lb-default annotations: - loxilb.io/liveness: "yes" loxilb.io/lbmode: "default" + #loxilb.io/liveness: "yes" + #loxilb.io/probetype: "ping" spec: externalTrafficPolicy: Local loadBalancerClass: loxilb.io/loxilb diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml b/cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml index d47eb27c3..0125b69b7 100644 --- a/cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml +++ b/cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml @@ -5,6 +5,7 @@ metadata: annotations: loxilb.io/liveness: "yes" loxilb.io/lbmode: "fullnat" + loxilb.io/probetype: "ping" spec: externalTrafficPolicy: Local loadBalancerClass: loxilb.io/loxilb diff --git a/cicd/k8s-calico-ipvs3/config.sh b/cicd/k8s-calico-ipvs3/config.sh index 09b5c1767..7dcbcfb3e 100755 --- a/cicd/k8s-calico-ipvs3/config.sh +++ b/cicd/k8s-calico-ipvs3/config.sh @@ -36,3 +36,30 @@ sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_onearm.yml' 2> /dev/null vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_onearm.yml' 2> /dev/null vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_onearm.yml' 2> /dev/null + +for((i=1; i<=60; i++)) +do + fin=1 + pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE") + + while IFS= read -a pods; do + read -a pod <<< "$pods" + if [[ ${pod[3]} != *"Running"* ]]; then + echo "${pod[1]} is not UP yet" + fin=0 + fi + done <<< "$pods" + if [ $fin == 1 ]; + then + break; + fi + echo "Will try after 10s" + sleep 10 +done + +if [[ $fin == 0 ]]; then +echo "Cluster is not ready" +exit 1 +fi +echo "Cluster is ready" + diff --git a/cicd/k8s-calico-ipvs3/node_scripts/host.sh b/cicd/k8s-calico-ipvs3/node_scripts/host.sh index 0018bc1c1..5b5efd476 100755 --- a/cicd/k8s-calico-ipvs3/node_scripts/host.sh +++ b/cicd/k8s-calico-ipvs3/node_scripts/host.sh @@ -1,4 +1,5 @@ # Install Bird to work with k3s +sudo apt-get update sudo apt-get -y install bird2 lksctp-tools sleep 5