Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

K8s-calico-incluster CICD test added #793

Merged
merged 2 commits into from
Sep 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions .github/workflows/k8s-calico-incluster.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: K8s-Calico-Incluster-Sanity-CI
on:
schedule:
# Runs "At 11:00 UTC every day-of-week"
- cron: '0 11 * * *'
workflow_dispatch:
inputs:
testName:
description: 'Test Run-Name'
required: true
default: 'k8s-calico-incluster'
jobs:
test-runner:
name: k8s-calico-incluster-sanity
runs-on: [self-hosted, large]
if: github.repository == 'loxilb-io/loxilb'
&& github.event.inputs.tagName == ''
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive

- name: Run the test
run: |
cd cicd/k8s-calico-incluster
./config.sh
./validation.sh
cd -

- name: Clean test-bed
if: success() || failure()
run: |
cd cicd/k8s-calico-incluster || true
./rmconfig.sh
cd -
24 changes: 12 additions & 12 deletions cicd/k8s-calico-incluster/Vagrantfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,32 +5,29 @@ require "yaml"
settings = YAML.load_file "yaml/settings.yaml"

workers = settings["nodes"]["workers"]["count"]
loxilbs = (ENV['LOXILBS'] || "2").to_i

Vagrant.configure("2") do |config|

if Vagrant.has_plugin?("vagrant-vbguest")
config.vbguest.auto_update = false
end

config.vm.box = settings["software"]["cluster"]["box"]["name"]
config.vm.box_version = settings["software"]["cluster"]["box"]["version"]

config.vm.define "host" do |host|
config.vm.define "host" do |host|
host.vm.hostname = 'host1'
host.vm.network :private_network, ip: settings["network"]["client_ip"], :netmask => "255.255.255.0"
host.vm.box = settings["software"]["cluster"]["box"]
host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0"
host.vm.provision :shell, :path => "node_scripts/host.sh"
host.vm.provider :virtualbox do |vbox|
vbox.customize ["modifyvm", :id, "--memory", 2048]
vbox.customize ["modifyvm", :id, "--cpus", 1]
vbox.customize ["modifyvm", :id, "--cpus", 2]
end
end

config.vm.define "master" do |master|
master.vm.hostname = 'master1'
master.vm.box = settings["software"]["cluster"]["box"]
master.vm.hostname = 'master'
master.vm.network :private_network, ip: settings["network"]["control_ip"], :netmask => "255.255.255.0"
master.vm.network :private_network, ip: settings["network"]["control_ip2"], :netmask => "255.255.255.0"
master.vm.provision "shell",
env: {
"DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
Expand All @@ -50,14 +47,16 @@ Vagrant.configure("2") do |config|

master.vm.provider :virtualbox do |vbox|
vbox.customize ["modifyvm", :id, "--memory", 4096]
vbox.customize ["modifyvm", :id, "--cpus", 3]
vbox.customize ["modifyvm", :id, "--cpus", 2]
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
end
end

(1..workers).each do |node_number|
config.vm.define "worker#{node_number}" do |worker|
worker.vm.box = settings["software"]["cluster"]["box"]
worker.vm.hostname = "worker#{node_number}"
ip = node_number + 100
ip = node_number + 200
worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
worker.vm.provision "shell",
env: {
Expand All @@ -70,8 +69,9 @@ Vagrant.configure("2") do |config|
worker.vm.provision "shell", path: "node_scripts/worker.sh"

worker.vm.provider :virtualbox do |vbox|
vbox.customize ["modifyvm", :id, "--memory", 2048]
vbox.customize ["modifyvm", :id, "--memory", 4096]
vbox.customize ["modifyvm", :id, "--cpus", 2]
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
end
end
end
Expand Down
34 changes: 32 additions & 2 deletions cicd/k8s-calico-incluster/config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,38 @@ do
sleep 10
done

# Create fullnat Services
sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1

#Create fullnat Service
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_onearm.yml' 2> /dev/null
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_onearm.yml' 2> /dev/null
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_onearm.yml' 2> /dev/null
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_fullnat.yml' 2> /dev/null
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_fullnat.yml' 2> /dev/null
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp.yml' 2> /dev/null

for((i=1; i<=60; i++))
do
fin=1
pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE")

while IFS= read -a pods; do
read -a pod <<< "$pods"
if [[ ${pod[3]} != *"Running"* ]]; then
echo "${pod[1]} is not UP yet"
fin=0
fi
done <<< "$pods"
if [ $fin == 1 ];
then
echo "Cluster is ready"
break;
fi
echo "Will try after 10s"
sleep 10
done

if [[ $fin == 0 ]]; then
echo "Cluster is not ready"
exit 1
fi
30 changes: 20 additions & 10 deletions cicd/k8s-calico-incluster/node_scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ sudo apt-get update -y
# Install CRI-O Runtime

VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"

CRIO_VERSION=1.27
# Create the .conf file to load the modules at bootup
cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
overlay
Expand All @@ -36,6 +36,15 @@ EOF
sudo modprobe overlay
sudo modprobe br_netfilter

# Install ipvs related modules
sudo modprobe ip_vs
sudo modprobe ip_vs_rr
sudo modprobe ip_vs_wrr
sudo modprobe ip_vs_sh
#sudo modprobe nf_conntrack_ipv4

sudo sysctl net.ipv4.vs.sloppy_sctp=1

# Set up required sysctl params, these persist across reboots.
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
Expand All @@ -46,14 +55,14 @@ EOF
sudo sysctl --system

cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
deb https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
EOF
cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
deb http://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
EOF

curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
curl -L https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
curl -L https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -

sudo apt-get update
sudo apt-get install cri-o cri-o-runc -y
Expand All @@ -67,14 +76,15 @@ sudo systemctl enable crio --now
echo "CRI runtime installed successfully"

sudo apt-get update
sudo apt-get install -y apt-transport-https ca-certificates curl
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get update -y
sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
sudo apt-get install -y kubelet kubectl kubeadm
sudo apt-get update -y
sudo apt-get install -y jq
sudo apt-get install -y ipvsadm

local_ip="$(ip --json a s | jq -r '.[] | if .ifname == "eth1" then .addr_info[] | if .family == "inet" then .local else empty end else empty end')"
cat > /etc/default/kubelet << EOF
Expand Down
13 changes: 4 additions & 9 deletions cicd/k8s-calico-incluster/node_scripts/host.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,5 @@
sudo su
sudo apt-get install -y lksctp-tools socat
wget https://github.com/loxilb-io/loxilb/raw/main/cicd/common/sctp_client
wget https://github.com/loxilb-io/loxilb/raw/main/cicd/common/udp_client
chmod 777 sctp_client
chmod 777 udp_client
echo "123.123.123.1 k8s-svc" >> /etc/hosts
ifconfig eth2 mtu 1450
ip route add 123.123.123.0/24 via 192.168.90.10
# Install Bird to work with k3s
sudo apt-get update
sudo apt-get -y install socat lksctp-tools

echo "Host is up"
4 changes: 0 additions & 4 deletions cicd/k8s-calico-incluster/node_scripts/loxilb.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,3 @@ add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu
apt-get update
apt-get install -y docker-ce
docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest
echo alias loxicmd=\"sudo docker exec -it loxilb loxicmd\" >> ~/.bashrc
echo alias loxilb=\"sudo docker exec -it loxilb \" >> ~/.bashrc

echo $LOXILB_IP > /vagrant/loxilb-ip
10 changes: 4 additions & 6 deletions cicd/k8s-calico-incluster/node_scripts/master.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,13 @@ sudo kubeadm config images pull

echo "Preflight Check Passed: Downloaded All Required Images"

sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap
#sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap
sudo kubeadm init --ignore-preflight-errors Swap --config /vagrant/yaml/kubeadm-config.yaml

mkdir -p "$HOME"/.kube
sudo cp -i /etc/kubernetes/admin.conf "$HOME"/.kube/config
sudo chown "$(id -u)":"$(id -g)" "$HOME"/.kube/config

curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh -

# Save Configs to shared /Vagrant location

# For Vagrant re-runs, check if there is existing configs in the location and delete it for saving new configuration.
Expand Down Expand Up @@ -54,6 +53,5 @@ EOF
kubectl apply -f https://raw.githubusercontent.com/techiescamp/kubeadm-scripts/main/manifests/metrics-server.yaml

# Install loxilb
kubectl apply -f /vagrant/yaml/loxilb.yml
kubectl apply -f /vagrant/yaml/loxilb-peer.yml
kubectl apply -f /vagrant/yaml/kube-loxilb.yml
kubectl apply -f /vagrant/yaml/kube-loxilb.yaml
kubectl apply -f /vagrant/yaml/loxilb.yaml
Binary file added cicd/k8s-calico-incluster/udp_client
Binary file not shown.
55 changes: 31 additions & 24 deletions cicd/k8s-calico-incluster/validation.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ fi

# Set space as the delimiter
IFS=' '

alloc=0
for((i=0; i<120; i++))
do
extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-fullnat")
Expand All @@ -19,16 +19,32 @@ do
sleep 1
continue
fi
if [[ ${strarr[3]} != *"none"* ]]; then
if [[ ${strarr[3]} != *"none"* || ${strarr[3]} != *"pending"* ]]; then
extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
alloc=1
break
fi
echo "No external LB allocated"
sleep 1
done

if [[ $alloc != 1 ]]; then
echo "No external LB allocated. Check kube-loxilb and loxilb logs"
echo "******************************************************************************"
vagrant ssh master -c 'kubectl get endpoints -A' 2> /dev/null
echo "******************************************************************************"
echo -e "\nSVC List"
echo "******************************************************************************"
vagrant ssh master -c 'kubectl get svc' 2> /dev/null
echo "******************************************************************************"
echo -e "\nPod List"
echo "******************************************************************************"
vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null
exit 1
fi

## Any routing updates ??
sleep 30
#sleep 30

echo Service IP : $extIP
echo -e "\nEnd Points List"
Expand All @@ -45,30 +61,33 @@ vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null

echo -e "\nTEST RESULTS"
echo "******************************************************************************"
mode=( "fullnat" )
tcp_port=( 57002 )
udp_port=( 57003 )
sctp_port=( 57004 )
mode=( "onearm" "fullnat" )
tcp_port=( 56002 57002 )
udp_port=( 56003 57003 )
sctp_port=( 56004 57004 )
code=0
for ((i=0;i<1;i++)); do
out=$(vagrant ssh host -c "curl -s --connect-timeout 10 http://$extIP:${tcp_port[i]}")
echo $out

for ((i=0;i<=1;i++)); do
out=$(vagrant ssh host -c "curl -s --connect-timeout 10 http://$extIP:${tcp_port[i]}" 2> /dev/null)
#echo $out
if [[ ${out} == *"nginx"* ]]; then
echo -e "K8s-calico-incluster TCP\t(${mode[i]})\t[OK]"
else
echo -e "K8s-calico-incluster TCP\t(${mode[i]})\t[FAILED]"
code=1
fi

out=$(vagrant ssh host -c "timeout 5 ./udp_client $extIP ${udp_port[i]}")
out=$(vagrant ssh host -c "timeout 5 /vagrant/udp_client $extIP ${udp_port[i]}" 2> /dev/null)
#echo $out
if [[ ${out} == *"Client"* ]]; then
echo -e "K8s-calico-incluster UDP\t(${mode[i]})\t[OK]"
else
echo -e "K8s-calico-incluster UDP\t(${mode[i]})\t[FAILED]"
code=1
fi

out=$(vagrant ssh host -c "socat -T10 - SCTP:$extIP:${sctp_port[i]},bind=192.168.90.9")
out=$(vagrant ssh host -c "socat -T10 - SCTP:$extIP:${sctp_port[i]}" 2> /dev/null)
#echo $out
if [[ ${out} == *"server"* ]]; then
echo -e "K8s-calico-incluster SCTP\t(${mode[i]})\t[OK]"
else
Expand All @@ -77,16 +96,4 @@ else
fi
done

mode=( "default" )
sctp_port=( 55004 )
code=0

out=$(vagrant ssh host -c "socat -T10 - SCTP:$extIP:${sctp_port[0]},bind=192.168.90.9")
if [[ ${out} == *"server"* ]]; then
echo -e "K8s-calico-incluster SCTP\t(${mode[0]})\t[OK]"
else
echo -e "K8s-calico-incluster SCTP\t(${mode[0]})\t[FAILED]"
code=1
fi

exit $code
Loading
Loading