Fix kind-setup regression, netifaces, ARM build changes, use 10.20.0.…

…0/16 CIDR for defaults (#655)

* Fix regression in kind cluster setup

* Use netifaces module to get default interface IPv4 address

* Fix to compile XDP for ARM-64, use 10.20 private CIDR

* Fix github actions workflow to use kind v0.11.0 until Mizar is caught up to latest

.github/workflows/e2e_tests.yaml

@@ -41,10 +41,16 @@ jobs:
         run: python3 -m pip install --upgrade pip
       - name: Install python packages
         run: sudo pip3 install setuptools netaddr docker grpcio grpcio-tools kubernetes
-      - name: install kind
+      # TODO: Update mizar to work with latest k8s version (and consequently latest kind versions)
+      #- name: install kind
+      #  run: |
+      #    ver=$(curl -s https://api.github.com/repos/kubernetes-sigs/kind/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+      #    curl -Lo kind "https://github.com/kubernetes-sigs/kind/releases/download/$ver/kind-$(uname)-amd64"
+      #    chmod +x ./kind
+      #    sudo mv ./kind /usr/local/bin
+      - name: install kind v0.11.0
         run: |
-          ver=$(curl -s https://api.github.com/repos/kubernetes-sigs/kind/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
-          curl -Lo kind "https://github.com/kubernetes-sigs/kind/releases/download/$ver/kind-$(uname)-amd64"
+          curl -Lo kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.11.0/kind-$(uname)-amd64"
           chmod +x ./kind
           sudo mv ./kind /usr/local/bin
       - name: Install kubectl

Makefile

@@ -37,9 +37,6 @@ SANITIZE += -fno-omit-frame-pointer
 ## CFLAGS
 CFLAGS += -I.
 CFLAGS += -Ilib/usr/include
-ifeq (${ARCH},aarch64)
-  CFLAGS += -I/usr/include/tirpc
-endif
 CFLAGS += -g -O3 -DDEBUG -Llib
 CFLAGS += -std=c11
 CFLAGS += -D_POSIX_C_SOURCE
@@ -64,19 +61,12 @@ LDFLAGS += -l:libbpf.a
 LDFLAGS += -l:libelf.a
 LDFLAGS += -lz
 LDFLAGS += -lnsl
-ifeq (${ARCH},aarch64)
-  LDFLAGS += -ltirpc
-endif
 LDFLAGS += -static-liblsan
 LDFLAGS += -static-libubsan
 $(info LDFLAGS=$(LDFLAGS))
 
 LLC=llc-7 -march=bpf -filetype=obj
 CLANG=clang-7
-ifeq (${ARCH},aarch64)
-  LLC=llc -march=bpf -filetype=obj
-  CLANG=clang
-endif
 CLANGFLAGS= -I.\
 			-Wno-unused-value -Wno-pointer-sign\
 			-Wno-compare-distinct-pointer-types \

bootstrap.sh

@@ -39,10 +39,6 @@ function install-dev-packages {
   sudo apt-get install -y \
     ${llvm_pkg} \
     ${clang_pkg}
-  if [[ "$(arch)" == "aarch64" ]]; then
-    sudo apt-get install -y \
-      libtirpc-dev
-  fi
 }
 
 function install-python {
@@ -114,7 +110,9 @@ function install-kind {
   echo ""
   echo "Installing kind ..."
   pushd /tmp
-  ver=$(curl -s https://api.github.com/repos/kubernetes-sigs/kind/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+  #TODO: Update Mizar to move forward to latest K8s and KinD versions
+  #ver=$(curl -s https://api.github.com/repos/kubernetes-sigs/kind/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+  ver="v0.11.0"
   curl -Lo kind "https://github.com/kubernetes-sigs/kind/releases/download/$ver/kind-$(uname)-${cpu_arch}"
   chmod +x kind
   sudo mv kind /usr/local/bin
@@ -201,15 +199,13 @@ function main {
 if [[ "$(arch)" == "x86_64" ]]; then
   main
 elif [[ "$(arch)" == "aarch64" ]]; then
-  cat /etc/os-release | grep VERSION_ID | grep "2[1-9].04"
+  cat /etc/os-release | grep VERSION_ID | grep "20.04"
   if [ $? -eq 0 ]; then
     cpu_arch="arm64"
-    llvm_pkg="llvm"
-    clang_pkg="clang"
     main
   else
-    echo "CPU architecture $(arch) not supported below Ubuntu 22.04"
+    echo "bootstrap.sh: CPU architecture $(arch) not supported for this OS version"
   fi
 else
-  echo "CPU architecture $(arch) not supported."
+  echo "bootstrap.sh: CPU architecture $(arch) not supported."
 fi

etc/docker/daemon.Dockerfile

@@ -22,5 +22,6 @@
 FROM mizarnet/python_base:latest
 COPY . /var/mizar/
 RUN apt update && apt-get install -y iproute2
+RUN pip3 install netifaces
 RUN pip3 install /var/mizar/
 CMD mizard

etc/docker/node-init.sh

@@ -53,6 +53,7 @@ nsenter -t 1 -m -u -n -i apt-get update -y && nsenter -t 1 -m -u -n -i apt-get i
     python$pyversion-dev \
     python3-apt \
     python3-testresources \
+    libaio-dev \
     libcmocka-dev \
     python3-pip && \
 nsenter -t 1 -m -u -n -i python3 -m pip install --upgrade pip
@@ -65,6 +66,7 @@ fi
 nsenter -t 1 -m -u -n -i mkdir -p /opt/cni/bin && \
 nsenter -t 1 -m -u -n -i mkdir -p /etc/cni/net.d && \
 nsenter -t 1 -m -u -n -i cp -f /var/mizar/build/bin/mizarcni /opt/cni/bin/mizarcni && \
+nsenter -t 1 -m -u -n -i pip3 install --upgrade netifaces && \
 nsenter -t 1 -m -u -n -i pip3 install --upgrade protobuf && \
 nsenter -t 1 -m -u -n -i pip3 install --ignore-installed /var/mizar/ && \
 nsenter -t 1 -m -u -n -i ln -snf /sys/fs/bpf /bpffs && \

etc/docker/python_base.Dockerfile

@@ -24,6 +24,7 @@ RUN apt-get update -y
 RUN apt-get install -y net-tools
 RUN apt-get install -y ethtool
 RUN apt-get install -y iproute2
+RUN apt-get install -y libcmocka-dev
 RUN apt-get install -y sudo
 RUN python3 -m pip install --upgrade pip
 RUN pip3 install PyYAML

k8s/kind/kindnode.Dockerfile

@@ -27,6 +27,7 @@ RUN apt-get install -y vim
 RUN apt-get install -y rpcbind
 RUN apt-get install -y rsyslog
 RUN apt-get install -y libelf-dev
+RUN apt-get install -y libcmocka-dev
 RUN apt-get install -y iproute2
 RUN apt-get install -y net-tools
 RUN apt-get install -y iputils-ping

k8s/kind/kindnode.Dockerfile-arm64

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2020 The Authors.
+
+# Authors: Vinay Kulkarni <@vinaykul>
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:The above copyright
+# notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.THE SOFTWARE IS PROVIDED "AS IS",
+# WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+# FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+# THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+FROM kindest/node-arm64:v1.18.20
+RUN apt-get update -y
+RUN apt-get install -y apt-utils
+RUN apt-get install -y sudo
+RUN apt-get install -y vim
+RUN apt-get install -y rpcbind
+RUN apt-get install -y rsyslog
+RUN apt-get install -y libaio-dev
+RUN apt-get install -y libcmocka-dev
+RUN apt-get install -y libelf-dev
+RUN apt-get install -y iproute2
+RUN apt-get install -y net-tools
+RUN apt-get install -y iputils-ping
+RUN apt-get install -y ethtool
+RUN apt-get install -y curl
+RUN apt-get install -y python3
+RUN apt-get install -y python3-pip
+RUN apt-get install -y tcpdump
+RUN python3 -m pip install --upgrade pip
+RUN pip3 install PyYAML
+RUN pip3 install kopf
+RUN pip3 install netaddr
+RUN pip3 install ipaddress
+RUN pip3 install pyroute2
+RUN pip3 install rpyc
+RUN pip3 install kubernetes==11.0.0
+RUN pip3 install luigi==2.8.12
+RUN pip3 install grpcio
+RUN pip3 install protobuf
+RUN pip3 install fs
+RUN mkdir -p /var/mizar/
+RUN mkdir -p /opt/cni/bin
+RUN mkdir -p /etc/cni/net.d
+RUN ln -snf /sys/fs/bpf /bpffs

kernelupdate.sh

@@ -73,5 +73,5 @@ function check-and-install-mizar-kernel {
 if [[ "$(arch)" == "x86_64" ]]; then
   check-and-install-mizar-kernel
 else
-  echo "CPU architecture $(arch) not supported."
+  echo "kernel-update.sh: CPU architecture $(arch) not supported."
 fi

mizar/common/constants.py

@@ -100,11 +100,11 @@ class OBJ_DEFAULTS:
     default_ep_net = 'net0'
     default_ep_type = 'simple'
     default_vpc_vni = '1'
-    default_vpc_ip = '20.0.0.0'
-    default_vpc_prefix = '8'
-    default_net_ip = '20.0.0.0'
-    default_net_gw = '20.0.0.1'
-    default_net_prefix = '8'
+    default_vpc_ip = '10.20.0.0'
+    default_vpc_prefix = '16'
+    default_net_ip = '10.20.0.0'
+    default_net_gw = '10.20.0.1'
+    default_net_prefix = '16'
     default_n_bouncers = 1
     default_n_dividers = 1
     default_host_ep_prefix = "32"

mizar/daemon/app.py

@@ -22,6 +22,7 @@
 import time
 import subprocess
 import json
+import netifaces as ni
 from google.protobuf import empty_pb2
 from concurrent import futures
 from mizar.daemon.interface_service import InterfaceServer
@@ -83,11 +84,8 @@ def init(benchmark=False):
     output = r.stdout.read().decode().strip()
     logging.info("Setup done")
 
-    cmd = 'nsenter -t 1 -m -u -n -i ip addr show ' + \
-        f'''{default_itf}''' + ' | grep "inet\\b" | awk \'{print $2}\''
-    r = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
-    nodeipmask = r.stdout.read().decode().strip()
-    nodeip = nodeipmask.split("/")[0]
+    nodeip = ni.ifaddresses(default_itf)[ni.AF_INET][0]['addr']
+    logging.info("default_itf: '{}' nodeip: '{}'".format(default_itf, nodeip))
 
     cmd = "nsenter -t 1 -m -u -n -i ip link set dev " + \
         f'''{default_itf}''' + " xdpgeneric off"
@@ -126,7 +124,7 @@ def init(benchmark=False):
         return
 
     # Setup mizar bridge, update routes, and load EDT TC eBPF program
-    logging.info("Node IP: {}".format(nodeipmask))
+    logging.info("Node IP: {}".format(nodeip))
 
     brcmd = f'''nsenter -t 1 -m -u -n -i sysctl -w net.bridge.bridge-nf-call-iptables=0 && \
         nsenter -t 1 -m -u -n -i ip link add {CONSTANTS.MIZAR_BRIDGE} type bridge && \

mizar/obj/tests/test_networkpolicy_1_1.yaml

@@ -32,7 +32,7 @@ spec:
   ingress:
   - from:
     - ipBlock:
-        cidr: 20.0.0.9/32
+        cidr: 10.20.0.9/32
     ports:
     - protocol: TCP
       port: 8000

mizar/obj/tests/test_pod_arktos_vpc1.yaml

@@ -25,7 +25,7 @@ metadata:
   labels:
     arktos.futurewei.com/network: my-net-mizar-1
   annotations:
-    arktos.futurewei.com/nic: "[{\"name\": \"eth0\", \"ip\": \"20.0.0.14\", \"subnet\": \"net0\"}]"
+    arktos.futurewei.com/nic: "[{\"name\": \"eth0\", \"ip\": \"10.20.0.14\", \"subnet\": \"net0\"}]"
 spec:
   containers:
   - name: arktospodvpc1net1

pkg/util/netutil/netutil_test.go

@@ -52,9 +52,9 @@ func Test_ActivateInterface(t *testing.T) {
 				hostVeth.Name,
 				netNSName,
 				ifName,
-				"8",
-				"20.0.0.81",
-				"20.0.0.1")
+				"16",
+				"10.20.0.81",
+				"10.20.0.1")
 			So(err, ShouldBeNil)
 			So(log, ShouldEqual, "Interface 'eth-894cad92' already UP.")
 		})
@@ -94,16 +94,16 @@ func createNetNS() (ns.NetNS, string) {
 func Test_ParseCIDR(t *testing.T) {
 	Convey("Subject: netutil.ParseCIDR", t, func() {
 		Convey("Given correct input, get expected result", func() {
-			ip, ipNet, err := netutil.ParseCIDR("20.0.0.8/16")
-			So(ip.String(), ShouldEqual, "20.0.0.8")
-			So(ipNet.String(), ShouldEqual, "20.0.0.0/16")
+			ip, ipNet, err := netutil.ParseCIDR("10.20.0.8/16")
+			So(ip.String(), ShouldEqual, "10.20.0.8")
+			So(ipNet.String(), ShouldEqual, "10.20.0.0/16")
 			So(err, ShouldBeNil)
 		})
 
 		Convey("Given no mask, get expected result", func() {
-			ip, ipNet, err := netutil.ParseCIDR("20.0.0.8")
-			So(ip.String(), ShouldEqual, "20.0.0.8")
-			So(ipNet.String(), ShouldEqual, "20.0.0.8/32")
+			ip, ipNet, err := netutil.ParseCIDR("10.20.0.8")
+			So(ip.String(), ShouldEqual, "10.20.0.8")
+			So(ipNet.String(), ShouldEqual, "10.20.0.8/32")
 			So(err, ShouldBeNil)
 		})
 
@@ -119,12 +119,12 @@ func Test_ParseCIDR(t *testing.T) {
 func Test_ParseIP(t *testing.T) {
 	Convey("Subject: netutil.ParseIP", t, func() {
 		Convey("Given correct input, get expected result", func() {
-			ip := netutil.ParseIP("20.0.0.8")
-			So(ip.String(), ShouldEqual, "20.0.0.8")
+			ip := netutil.ParseIP("10.20.0.8")
+			So(ip.String(), ShouldEqual, "10.20.0.8")
 		})
 
 		Convey("Given wrong input, get nil", func() {
-			ip := netutil.ParseIP("20.0.0.8/16")
+			ip := netutil.ParseIP("10.20.0.8/16")
 			So(ip, ShouldBeNil)
 		})
 	})