Merge pull request loxilb-io#904 from TrekkieCoder/main

loxilb-io/kube-loxilb#212 added cicd for dual-stack

.github/workflows/dual-stack.yml

@@ -0,0 +1,33 @@
+name: K3s-Dual-Stack-Sanity-CI
+on:
+ schedule:
+ # Runs "At 8:00 UTC every day-of-week"
+ - cron: '0 8 * * *'
+ workflow_dispatch:
+    inputs:
+      testName:
+        description: 'Test Run-Name'     
+        required: true
+        default: 'k3s-dual-stack'
+jobs:
+  build:
+    name: k3s-dual-stack-sanity
+    runs-on: [self-hosted, large]
+    if: github.repository == 'loxilb-io/loxilb' 
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Run K3s Dual-Stack CICD
+        run: |
+         cd cicd/k3s-flannel-cluster-dual-stack
+         ./config.sh
+         ./validation.sh
+         ./rmconfig.sh
+         cd -
+      - name: Clean test-bed
+        if: success() || failure()
+        run: |
+          cd cicd/k3s-flannel-cluster-dual-stack/ || true
+          ./rmconfig.sh
+          cd -

cicd/k3s-flannel-cluster-dual-stack/Vagrantfile

@@ -0,0 +1,48 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+workers = (ENV['WORKERS'] || "1").to_i
+box_name = (ENV['VAGRANT_BOX'] || "bento/ubuntu-22.04" )
+box_version = "202407.23.0"
+Vagrant.configure("2") do |config|
+  config.vm.box = "#{box_name}"
+  config.vm.box_version = "#{box_version}"
+
+  if Vagrant.has_plugin?("vagrant-vbguest")
+    config.vbguest.auto_update = false
+  end
+
+  config.vm.define "host" do |host|
+    host.vm.hostname = 'host'
+    host.vm.network :private_network, ip: "192.168.80.8", :netmask => "255.255.255.0"
+    host.vm.network :private_network, ip: "192.168.90.8", :netmask => "255.255.255.0"
+    host.vm.provision :shell, :path => "host.sh"
+    host.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 4000]
+        vbox.customize ["modifyvm", :id, "--cpus", 2]
+    end
+  end
+
+  config.vm.define "loxilb" do |loxilb|
+    loxilb.vm.hostname = 'llb1'
+    #loxilb.vm.network "forwarded_port", guest: 55002, host: 5502, protocol: "tcp"
+    loxilb.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
+    loxilb.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0"
+    loxilb.vm.provision :shell, :path => "loxilb.sh"
+    loxilb.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 6000]
+        vbox.customize ["modifyvm", :id, "--cpus", 4]
+    end
+  end
+
+
+  config.vm.define "master" do |master|
+    master.vm.hostname = 'master'
+    master.vm.network :private_network, ip: "192.168.80.10", :netmask => "255.255.255.0"
+    master.vm.provision :shell, :path => "master.sh"
+    master.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 4096]
+        vbox.customize ["modifyvm", :id, "--cpus", 2]
+    end
+  end
+end

cicd/k3s-flannel-cluster-dual-stack/config.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+vagrant global-status  | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f
+vagrant up

cicd/k3s-flannel-cluster-dual-stack/host.sh

@@ -0,0 +1,12 @@
+ip addr add 3ffe:cafe::8/64 dev eth2
+
+sysctl net.ipv6.conf.all.proxy_ndp=1
+sysctl net.ipv6.conf.default.proxy_ndp=1 
+sysctl net.ipv6.conf.eth2.proxy_ndp=1 
+
+apt-get update
+apt-get install -y software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce

cicd/k3s-flannel-cluster-dual-stack/host_validation.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+extIP=$(cat /vagrant/extIP)
+
+mode="default"
+tcp_port=55002
+
+code=0
+echo TCP Service IP: $extIP
+
+ip -6 route list match $extIP | grep $extIP -A 2
+
+echo -e "\n*********************************************"
+echo "Testing Service"
+echo "*********************************************"
+for((i=0;i<20;i++))
+do
+  out=$(curl -s --connect-timeout 10 http://[$extIP]:$tcp_port)
+  if [[ ${out} == *"Welcome to nginx"* ]]; then
+    echo -e "dual-stack TCP\t($mode)\t[OK]"
+  else
+    echo -e "dual-stack TCP\t($mode)\t[FAILED]"
+    code=1
+  fi
+done

cicd/k3s-flannel-cluster-dual-stack/kube-loxilb.yml

@@ -0,0 +1,132 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - namespaces
+      - services/status
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+      - update
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-loxilb
+subjects:
+  - kind: ServiceAccount
+    name: kube-loxilb
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+  labels:
+    app: loxilb
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: loxilb
+  template:
+    metadata:
+      labels:
+        app: loxilb
+    spec:
+      hostNetwork: true
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-loxilb
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: kube-loxilb
+        image: ghcr.io/loxilb-io/kube-loxilb:latest
+        imagePullPolicy: Always
+        command:
+        - /bin/kube-loxilb
+        args:
+        - --loxiURL=http://192.168.80.9:11111
+        - --cidrPools=defaultPool=123.123.123.1/24
+        - --setRoles=0.0.0.0
+        #- --monitor
+        #- --setBGP
+        #- --setLBMode=1
+        #- --config=/opt/loxilb/agent/kube-loxilb.conf
+        - --v=4 
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW"]

cicd/k3s-flannel-cluster-dual-stack/loxilb.sh

@@ -0,0 +1,25 @@
+export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+
+ip addr add 2001:cafe:43::2/112 dev eth1
+ip addr add 3ffe:cafe::9/64 dev eth2
+
+apt-get update
+apt-get install -y software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --entrypoint /root/loxilb-io/loxilb/loxilb --name loxilb ghcr.io/loxilb-io/loxilb:latest
+
+#sleep 20
+#curl -X 'POST' \
+#  'http://127.0.0.1:11111/netlox/v1/config/cistate' \
+#  -H 'accept: application/json' \
+#  -H 'Content-Type: application/json' \
+#  -d '{
+#  "instance": "default",
+#  "state": "MASTER",
+#  "vip": "0.0.0.0"
+#}'
+
+echo $LOXILB_IP > /vagrant/loxilb-ip

cicd/k3s-flannel-cluster-dual-stack/master.sh

@@ -0,0 +1,16 @@
+export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+ip addr add 2001:cafe:43::4/112 dev eth1
+ip -6 route add default via 2001:cafe:43::2
+echo '2001:cafe:43::4 master master' | sudo tee -a /etc/hosts
+#echo '192.168.80.10 master master' | sudo tee -a /etc/hosts
+
+curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="v1.25.16+k3s4" INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --cluster-cidr=2001:cafe:42::/56,192.169.0.0/16 --service-cidr=2001:cafe:43::/112,172.16.0.0/24 --disable-network-policy --node-ip=2001:cafe:43::4,192.168.80.10 --node-external-ip=2001:cafe:43::4,192.168.80.10 --flannel-ipv6-masq" sh -
+
+echo $MASTER_IP > /vagrant/master-ip
+sudo cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
+sudo cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
+sudo sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /vagrant/k3s.yaml
+sudo kubectl apply -f /vagrant/kube-loxilb.yml
+sleep 60
+sudo kubectl apply -f /vagrant/nginx6.yml
+/vagrant/wait_ready.sh

cicd/k3s-flannel-cluster-dual-stack/nginx.yml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-lb1
+  annotations:
+    loxilb.io/lbmode: "onearm"
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    what: nginx-test
+  ports:
+    - port: 55002
+      targetPort: 80 
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-test
+  labels:
+    what: nginx-test
+spec:
+  containers:
+    - name: nginx-test
+      image: ghcr.io/loxilb-io/nginx:stable
+      ports:
+        - containerPort: 80

cicd/k3s-flannel-cluster-dual-stack/nginx6.yml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-lb1-ipv6
+  annotations:
+   # If there is a need to do liveness check from loxilb
+   loxilb.io/liveness: "no"
+   # Specify LB mode - one of default, onearm or fullnat
+   loxilb.io/lbmode: "default"
+   # Specify loxilb IPAM mode - one of ipv4, ipv6 or ipv6to4
+   loxilb.io/ipam: "ipv6"
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    what: nginx-test-ipv6
+  ports:
+    - port: 55002
+      targetPort: 80
+  type: LoadBalancer
+  ipFamilies:
+    - IPv6
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-test-ipv6
+  labels:
+    what: nginx-test-ipv6
+spec:
+  containers:
+    - name: nginx-test-ipv6
+      image: ghcr.io/loxilb-io/nginx:stable
+      ports:
+        - containerPort: 80
+

cicd/k3s-flannel-cluster-dual-stack/rmconfig.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+vagrant destroy -f host
+vagrant destroy -f master
+vagrant destroy -f loxilb