diff --git a/cicd/common.sh b/cicd/common.sh index 9978c11f7..47923f1ca 100644 --- a/cicd/common.sh +++ b/cicd/common.sh @@ -13,6 +13,7 @@ hns="sudo ip netns " hexist="$vrn$hn" lxdocker="ghcr.io/loxilb-io/loxilb:latest" cluster_opts="" +ka_opts="" var=$(lsb_release -r | cut -f2) if [[ $var == *"22.04"* ]];then lxdocker="ghcr.io/loxilb-io/loxilb:latestu22" @@ -104,24 +105,9 @@ spawn_docker_host() { fi if [[ ! -z ${ka+x} ]]; then sudo mkdir -p /etc/shared/$dname/ - if [[ "$ka" == "in" ]];then - ka_opts="-k in" - if [[ ! -z "$kpath" ]]; then - ka_conf="-v $kpath:/etc/keepalived/" - fi - fi - docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config $ka_conf --name $dname $lxdocker + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config --name $dname $lxdocker get_llb_peerIP $dname docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts - - if [[ "$ka" == "out" ]];then - ka_opts="-k out" - if [[ ! -z "$kpath" ]]; then - ka_conf="-v $kpath:/container/service/keepalived/assets/" - fi - - docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit --network=container:$dname $ka_conf -v /etc/shared/$dname:/etc/shared --name ka_$dname osixia/keepalived:2.0.20 - fi else docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts @@ -170,6 +156,7 @@ get_llb_peerIP() { llb2IP="$A.$B.$C.$((D+1))" fi cluster_opts=" --cluster=$llb2IP --self=0" + ka_opts=" --ka=$llb2IP" elif [[ "$1" == "llb2" ]]; then llb2IP=$(docker inspect --format='{{.NetworkSettings.IPAddress}}' llb2) if [[ "lb$llb2IP" == "lb" ]];then @@ -179,6 +166,7 @@ get_llb_peerIP() { llb1IP="$A.$B.$C.$((D-1))" fi cluster_opts=" --cluster=$llb1IP --self=1" + ka_opts=" --ka=$llb1IP" fi } diff --git a/cicd/k3s-sctpmh/config.sh b/cicd/k3s-sctpmh/config.sh index 45bb94a16..dc5b7c815 100755 --- a/cicd/k3s-sctpmh/config.sh +++ b/cicd/k3s-sctpmh/config.sh @@ -7,8 +7,8 @@ echo "#########################################" echo "Spawning all hosts" echo "#########################################" -spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config --with-ka in --ka-config $(pwd)/keepalived_config1 -spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config --with-ka in --ka-config $(pwd)/keepalived_config2 +spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config --with-ka in +spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config --with-ka in spawn_docker_host --dock-type host --dock-name ep1 spawn_docker_host --dock-type host --dock-name ep2 spawn_docker_host --dock-type host --dock-name ep3 @@ -99,7 +99,8 @@ $hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11 #add_route llb2 1.1.1.0/24 11.11.11.254 # Route back to user -sudo ip route add 11.11.11.0/24 via 12.12.12.1 +sudo ip route add 11.11.11.0/24 via 14.14.14.1 +sudo ip route add 123.123.123.0/24 via 14.14.14.1 # Change default route in llb1 $hexec llb1 ip route del default @@ -195,11 +196,9 @@ $dexec llb1 loxicmd get lb -o wide echo "llb1: loxicmd get ep -o wide" echo "****************************" $dexec llb1 loxicmd get ep -o wide -$dexec llb1 cat /etc/shared/keepalive.state echo "llb2: loxicmd get lb -o wide" echo "****************************" $dexec llb2 loxicmd get lb -o wide echo "llb2: loxicmd get ep -o wide" echo "****************************" $dexec llb2 loxicmd get ep -o wide -$dexec llb2 cat /etc/shared/keepalive.state diff --git a/cicd/k3s-sctpmh/nginx-svc-lb1.yml b/cicd/k3s-sctpmh/nginx-svc-lb1.yml index 5bcf72fd4..c2925268a 100644 --- a/cicd/k3s-sctpmh/nginx-svc-lb1.yml +++ b/cicd/k3s-sctpmh/nginx-svc-lb1.yml @@ -2,6 +2,9 @@ apiVersion: v1 kind: Service metadata: name: nginx-lb1 + annotations: + loxilb.io/lbmode: "fullnat" + loxilb.io/staticIP : "11.11.11.11" spec: externalTrafficPolicy: Local loadBalancerClass: loxilb.io/loxilb diff --git a/cicd/k3s-sctpmh/nginx.yml b/cicd/k3s-sctpmh/nginx.yml index 8f1374908..e11223ef8 100644 --- a/cicd/k3s-sctpmh/nginx.yml +++ b/cicd/k3s-sctpmh/nginx.yml @@ -2,6 +2,9 @@ apiVersion: v1 kind: Pod metadata: name: nginx + annotations: + loxilb.io/lbmode: "fullnat" + loxilb.io/staticIP : "11.11.11.11" labels: app.kubernetes.io/name: proxy spec: diff --git a/cicd/k3s-sctpmh/validation.sh b/cicd/k3s-sctpmh/validation.sh index 0c9071a90..16433c954 100755 --- a/cicd/k3s-sctpmh/validation.sh +++ b/cicd/k3s-sctpmh/validation.sh @@ -125,6 +125,25 @@ sleep 10 # Wait for cluster to be ready wait_cluster_ready_full +for((i=0; i<120; i++)) +do + extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb1") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find nginx-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + port=${strarr[4]} + break + fi + echo "No external LB allocated" + sleep 1 +done + out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) if [[ ${out} == *"Welcome to nginx"* ]]; then echo "cluster-k3s TCP service nginx-lb del+add (kube-loxilb) [OK]" diff --git a/loxinet/cluster.go b/loxinet/cluster.go index 899106bce..995cf85d9 100644 --- a/loxinet/cluster.go +++ b/loxinet/cluster.go @@ -90,7 +90,7 @@ func (ci *CIStateH) startBFDProto() { time.Sleep(KAInitTiVal * time.Second) bs := bfd.StructNew(3784) - err := bs.BFDAddRemote(ci.RemoteIP.String(), 3784, bfd.BFDMinSysTXIntervalUs, 3, "Default", ci) + err := bs.BFDAddRemote(ci.RemoteIP.String(), 3784, bfd.BFDMinSysTXIntervalUs, 3, cmn.CIDefault, ci) if err != nil { tk.LogIt(tk.LogCritical, "KA - Cant add BFD remote\n") }