From b89fada14f3b307a73224e17c28622c3c7a6302f Mon Sep 17 00:00:00 2001
From: Mikhail Avramenko <avramenkomihail15@gmail.com>
Date: Fri, 20 Aug 2021 10:30:59 +0700
Subject: [PATCH] added performance testing infrastructure

Signed-off-by: Mikhail Avramenko <avramenkomihail15@gmail.com>
---
 examples/performance-testing/.gitignore       |   1 +
 .../Kernel2Vxlan2Kernel/README.md             | 168 ++++++++++++++++++
 .../Kernel2Vxlan2Kernel/test.sh               | 145 +++++++++++++++
 examples/performance-testing/namespace.yaml   |   5 +
 4 files changed, 319 insertions(+)
 create mode 100644 examples/performance-testing/.gitignore
 create mode 100644 examples/performance-testing/Kernel2Vxlan2Kernel/README.md
 create mode 100644 examples/performance-testing/Kernel2Vxlan2Kernel/test.sh
 create mode 100644 examples/performance-testing/namespace.yaml

diff --git a/examples/performance-testing/.gitignore b/examples/performance-testing/.gitignore
new file mode 100644
index 000000000000..60b8c45b1693
--- /dev/null
+++ b/examples/performance-testing/.gitignore
@@ -0,0 +1 @@
+**/kustomization.yaml
\ No newline at end of file
diff --git a/examples/performance-testing/Kernel2Vxlan2Kernel/README.md b/examples/performance-testing/Kernel2Vxlan2Kernel/README.md
new file mode 100644
index 000000000000..8701ca063dc7
--- /dev/null
+++ b/examples/performance-testing/Kernel2Vxlan2Kernel/README.md
@@ -0,0 +1,168 @@
+# Test kernel to vxlan to kernel connection
+
+This example shows that NSC and NSE on the different nodes could find and work with each other.
+
+NSC and NSE are using the `kernel` mechanism to connect to its local forwarder.
+Forwarders are using the `vxlan` mechanism to connect with each other.
+
+## Requires
+
+Make sure that you have completed steps from [basic](../../basic) or [memory](../../memory) setup.
+
+## Run
+
+Create test namespace:
+```bash
+NAMESPACE=($(kubectl create -f ../namespace.yaml)[0])
+NAMESPACE=${NAMESPACE:10}
+```
+
+Register namespace in `spire` server:
+```bash
+kubectl exec -n spire spire-server-0 -- \
+/opt/spire/bin/spire-server entry create \
+-spiffeID spiffe://example.org/ns/${NAMESPACE}/sa/default \
+-parentID spiffe://example.org/ns/spire/sa/spire-agent \
+-selector k8s:ns:${NAMESPACE} \
+-selector k8s:sa:default
+```
+
+Get nodes exclude control-plane:
+```bash
+NODES=($(kubectl get nodes -o go-template='{{range .items}}{{ if not .spec.taints  }}{{index .metadata.labels "kubernetes.io/hostname"}} {{end}}{{end}}'))
+```
+
+Setup for multiple clients
+```bash
+TEST_CLIENTS_N=8
+mkdir ns-clients
+RESOURCES_STR=""
+PATCHES_STR=""
+for (( i = 1; i <= TEST_CLIENTS_N; i++ )); do
+    cp ../../../apps/nsc-kernel/nsc.yaml ./ns-clients/nsc-kernel-${i}.yaml
+    sed -i "s/nsc-kernel/nsc-kernel-${i}/g" ./ns-clients/nsc-kernel-${i}.yaml
+    RESOURCES_STR="${RESOURCES_STR}- ./ns-clients/nsc-kernel-${i}.yaml"$'\n'
+    PATCHES_STR="${PATCHES_STR}- patch-nsc-${i}.yaml"$'\n'
+    echo "$RESOURCES_STR"
+    echo "$PATCHES_STR"
+done
+pow=1 
+val=2
+while (( TEST_CLIENTS_N >= val )); do
+    let pow=pow+1
+    let val=val*2
+done
+echo "$val"
+echo "$pow"
+```
+
+Create customization file:
+```bash
+cat > kustomization.yaml <<EOF
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: ${NAMESPACE}
+
+resources:
+${RESOURCES_STR}
+bases:
+- ../../../apps/nse-kernel
+
+patchesStrategicMerge:
+- patch-nse.yaml
+${PATCHES_STR}
+EOF
+```
+
+Create NSCs patches:
+```bash
+for (( i = 1; i <= TEST_CLIENTS_N; i++ )); do
+cat > patch-nsc-${i}.yaml <<EOF
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nsc-kernel-{i}
+spec:
+  template:
+    spec:
+      containers:
+        - name: nsc
+          env:
+            - name: NSM_NETWORK_SERVICES
+              value: kernel://icmp-responder/nsm-1
+        - name: iperf3
+          image: ubuntu
+          imagePullPolicy: IfNotPresent
+          command: [ "/bin/bash" ]
+          args: [ "-c", "while true; do echo hello; sleep 10;done" ]
+      nodeSelector:
+        kubernetes.io/hostname: ${NODES[0]}
+EOF
+done
+```
+Create NSE patch:
+```bash
+cat > patch-nse.yaml <<EOF
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nse-kernel
+spec:
+  template:
+    spec:
+      containers:
+        - name: nse
+          env:
+            - name: NSM_CIDR_PREFIX
+              value: 172.16.1.100/$((32-pow))
+        - name: iperf3
+          image: ubuntu
+          imagePullPolicy: IfNotPresent
+          command: [ "/bin/bash" ]
+          args: [ "-c", "while true; do echo hello; sleep 10;done" ]
+      nodeSelector:
+        kubernetes.io/hostname: ${NODES[1]}
+EOF
+```
+
+Deploy NSC and NSE:
+```bash
+kubectl apply -k .
+```
+
+Wait for applications ready:
+```bash
+kubectl wait --for=condition=ready --timeout=1m pod -l app=nsc-kernel -n ${NAMESPACE}
+```
+```bash
+kubectl wait --for=condition=ready --timeout=1m pod -l app=nse-kernel -n ${NAMESPACE}
+```
+
+Find NSC and NSE pods by labels:
+```bash
+NSC=$(kubectl get pods -l app=nsc-kernel -n ${NAMESPACE} --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+```
+```bash
+NSE=$(kubectl get pods -l app=nse-kernel -n ${NAMESPACE} --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+```
+
+Ping from NSC to NSE:
+```bash
+kubectl exec ${NSC} -n ${NAMESPACE} -- ping -c 4 172.16.1.100
+```
+
+Ping from NSE to NSC:
+```bash
+kubectl exec ${NSE} -n ${NAMESPACE} -- ping -c 4 172.16.1.101
+```
+
+## Cleanup
+
+Delete ns:
+```bash
+kubectl delete ns ${NAMESPACE}
+```
\ No newline at end of file
diff --git a/examples/performance-testing/Kernel2Vxlan2Kernel/test.sh b/examples/performance-testing/Kernel2Vxlan2Kernel/test.sh
new file mode 100644
index 000000000000..55a5df99f9fc
--- /dev/null
+++ b/examples/performance-testing/Kernel2Vxlan2Kernel/test.sh
@@ -0,0 +1,145 @@
+NAMESPACE=($(kubectl create -f ../namespace.yaml)[0])
+NAMESPACE=${NAMESPACE:10}
+
+kubectl exec -n spire spire-server-0 -- \
+/opt/spire/bin/spire-server entry create \
+-spiffeID spiffe://example.org/ns/${NAMESPACE}/sa/default \
+-parentID spiffe://example.org/ns/spire/sa/spire-agent \
+-selector k8s:ns:${NAMESPACE} \
+-selector k8s:sa:default
+
+NODES=($(kubectl get nodes -o go-template='{{range .items}}{{ if not .spec.taints  }}{{index .metadata.labels "kubernetes.io/hostname"}} {{end}}{{end}}'))
+
+TEST_CLIENTS_N=1
+mkdir ns-clients
+RESOURCES_STR=""
+PATCHES_STR=""
+for (( i = 1; i <= TEST_CLIENTS_N; i++ )); do
+    cp ../../../apps/nsc-kernel/nsc.yaml ./ns-clients/nsc-kernel-${i}.yaml
+    gsed -i "s/nsc-kernel/nsc-kernel-${i}/g" ./ns-clients/nsc-kernel-${i}.yaml
+    RESOURCES_STR="${RESOURCES_STR}- ./ns-clients/nsc-kernel-${i}.yaml"$'\n'
+    PATCHES_STR="${PATCHES_STR}- patch-nsc-${i}.yaml"$'\n'
+done
+pow=1
+val=2
+while (( TEST_CLIENTS_N >= val )); do
+    ((pow=pow+1))
+    ((val=val*2))
+done
+
+#kubectl create configmap iperf-script --from-file=iperf.sh
+
+cat > kustomization.yaml <<EOF
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: ${NAMESPACE}
+
+resources:
+${RESOURCES_STR}
+bases:
+- ../../../apps/nse-kernel
+
+patchesStrategicMerge:
+- patch-nse.yaml
+${PATCHES_STR}
+EOF
+
+for (( i = 1; i <= TEST_CLIENTS_N; i++ )); do
+cat > patch-nsc-"${i}".yaml <<EOF
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nsc-kernel-${i}
+spec:
+  template:
+    spec:
+      containers:
+        - name: nsc
+          env:
+            - name: NSM_NETWORK_SERVICES
+              value: kernel://icmp-responder/nsm-1
+        - name: iperf3
+          image: ubuntu
+          imagePullPolicy: IfNotPresent
+          command: [ "/bin/bash" ]
+          args: [ "-c", "while true; do echo hello; sleep 10;done" ]
+      nodeSelector:
+        kubernetes.io/hostname: ${NODES[0]}
+EOF
+done
+
+cat > patch-nse.yaml <<EOF
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nse-kernel
+spec:
+  template:
+    spec:
+      containers:
+        - name: nse
+          env:
+            - name: NSM_CIDR_PREFIX
+              value: 172.16.1.100/$((32-pow))
+        - name: iperf3
+          image: ubuntu
+          imagePullPolicy: IfNotPresent
+          command: [ "/bin/bash", "-c" ]
+          args:
+            - apt-get update;
+              apt-get install -y iproute2;
+              apt-get install -y iperf3;
+
+              IP_ADDRS_LEN_OLD=0;
+              while [ true ];
+              do IP_ADDRS=($(ip addr | egrep '172.16.1.[0-9]{1,3}/[0-9]{1,2}' -o));
+              IP_ADDRS_LEN_NEW=${#IP_ADDRS[@]};
+              echo $IP_ADDRS_LEN_NEW;
+
+              IDX_START=0;
+              if ((IP_ADDRS_LEN_OLD != 0));
+              then IDX_START=$((IP_ADDRS_LEN_OLD - 1));
+              fi;
+
+              echo $IDX_START;
+
+              IP_ADDRS_DIFF=$((IP_ADDRS_LEN_NEW - IP_ADDRS_LEN_OLD));
+              echo $IP_ADDRS_DIFF;
+
+              if (( IP_ADDRS_DIFF > 0));
+              then for (( i = IDX_START; i < IP_ADDRS_LEN_NEW; i++ ));
+              do IP_ADDR=$(echo "${IP_ADDRS[i]}" | sed 's=/[0-9]\{1,2\}==g');
+              echo $IP_ADDR;
+              iperf3 -s -D --pidfile /tmp/iperf.pid --bind ${IP_ADDR};
+              sleep 5;
+              if [ -f /tmp/iperf.pid ];
+              then echo "iperf daemon sucesfully started on ${IP_ADDR}";
+              rm /tmp/iperf.pid;
+              fi;
+              done;
+              fi;
+              sleep 5;
+
+              IP_ADDRS_LEN_OLD=${IP_ADDRS_LEN_NEW};
+              done;
+      nodeSelector:
+        kubernetes.io/hostname: ${NODES[1]}
+EOF
+
+kubectl apply -k .
+
+kubectl wait --for=condition=ready --timeout=1m pod -l app=nsc-kernel-1 -n ${NAMESPACE}
+
+kubectl wait --for=condition=ready --timeout=1m pod -l app=nse-kernel -n ${NAMESPACE}
+
+NSC1=$(kubectl get pods -l app=nsc-kernel-1 -n ${NAMESPACE} --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+
+NSE=$(kubectl get pods -l app=nse-kernel -n ${NAMESPACE} --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+
+kubectl exec ${NSC1} -n ${NAMESPACE} -- ping -c 4 172.16.1.100
+
+kubectl exec ${NSE} -n ${NAMESPACE} -- ping -c 4 172.16.1.101
diff --git a/examples/performance-testing/namespace.yaml b/examples/performance-testing/namespace.yaml
new file mode 100644
index 000000000000..7919ccab9bec
--- /dev/null
+++ b/examples/performance-testing/namespace.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  generateName: ns-