diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index cbcc85d..0da9706 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -18,9 +18,8 @@ package e2e import ( "context" - "fmt" - "os/exec" "strings" + "time" "github.com/onsi/ginkgo" appsv1 "k8s.io/api/apps/v1" @@ -36,30 +35,22 @@ const ( patuCgroupSockRecv = "patu_recvmsg4" ) -// runCommand runs the cmd and returns the combined stdout and stderr -func runCommand(cmd ...string) (string, error) { - output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output) - } - return string(output), nil -} - -// This test validates that the patu bpf programs are loaded +// This test Validates patu ebpf programs are loaded and unloaded with installation, deletion and re-installation /* This test does the following: 1. Verify sock_ops named patu_sockops program is loaded 2. Verify sk_msg program named patu_skmsg is loaded 3. Verify cgroup_sock_addr program named patu_sendmsg4 is loaded 4. Verify cgroup_sock_addr program named patu_recvmsg4 is loaded + 5. Uninstall Patu using the installer + 6. Verify none of the Patu bpf programs are loaded after uninstallation + 7. Reinstall Patu using the installer + 8. Verify all of the Patu bpf programs are loaded after re-installation */ -var _ = ginkgo.Describe("Validate patu ebpf programs are loaded", func() { +var _ = ginkgo.Describe("Validate patu ebpf programs are loaded and unloaded with installation, deletion and re-installation", func() { var bpfOut string - var err error + f := newPrivelegedTestFramework("bpf-installer-deployment") ginkgo.BeforeEach(func() { - bpfOut, err = runCommand("sudo", "bpftool", "prog", "list") - if err != nil { - framework.Failf("failed to run bpftool cmd: %v", err) - } + bpfOut = getBpfList() }) ginkgo.It("Verify Patu is loaded properly by examining the loaded bpf programs", func() { @@ -83,6 +74,57 @@ var _ = ginkgo.Describe("Validate patu ebpf programs are loaded", func() { if !strings.Contains(bpfOut, patuCgroupSockRecv) { framework.Failf("The program %s was not found\n", patuCgroupSockRecv) } + + ginkgo.By("5. Uninstall Patu using the installer") + patuDeleteStdOut, err := runCommand("patu-installer", "delete", "cni") + if err != nil { + framework.Failf("failed to run patu uninstall: %v", err) + } + framework.Logf("Patu uninstall output: %s", patuDeleteStdOut) + time.Sleep(15 * time.Second) + + ginkgo.By("6. Verify none of the Patu bpf programs are loaded after uninstallation") + // reread loaded bpf programs + bpfOut = getBpfList() + if strings.Contains(bpfOut, patuSockOps) { + framework.Failf("The program %s was not unloaded by the uninstaller\n", patuSockOps) + } + if strings.Contains(bpfOut, patuSkMsg) { + framework.Failf("The program %s was not unloaded by the uninstaller\n", patuSkMsg) + } + if strings.Contains(bpfOut, patuCgroupSockSend) { + framework.Failf("The program %s was not unloaded by the uninstaller\n", patuCgroupSockSend) + } + if strings.Contains(bpfOut, patuCgroupSockRecv) { + framework.Failf("The program %s was not unloaded by the uninstaller\n", patuCgroupSockRecv) + } + + ginkgo.By("7. Reinstall Patu using the installer") + patuInstallStdOut, err := runCommand("patu-installer", "apply", "cni") + if err != nil { + framework.Failf("failed to run patu-installer: %v", err) + } + framework.Logf("Patu install output: %s", patuInstallStdOut) + time.Sleep(15 * time.Second) + // verify the daemonset is loaded + err = checkDaemonStatus(f, "patu", "kube-system") + framework.ExpectNoError(err) + + ginkgo.By("8. Verify all of the Patu bpf programs are loaded after re-installation") + // reread loaded bpf programs + bpfOut = getBpfList() + if !strings.Contains(bpfOut, patuSockOps) { + framework.Failf("The program %s was not reloaded by the installer\n", patuSockOps) + } + if !strings.Contains(bpfOut, patuSkMsg) { + framework.Failf("The program %s was not reloaded by the installer\n", patuSkMsg) + } + if !strings.Contains(bpfOut, patuCgroupSockSend) { + framework.Failf("The program %s was not reloaded by the installer\n", patuCgroupSockSend) + } + if !strings.Contains(bpfOut, patuCgroupSockRecv) { + framework.Failf("The program %s was not reloaded by the installer\n", patuCgroupSockRecv) + } }) }) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index e43c584..db8809f 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -17,20 +17,34 @@ package e2e import ( + "context" "fmt" "net" + "os/exec" + "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" admissionapi "k8s.io/pod-security-admission/api" ) +// newPrivelegedTestFramework creates a new ginkgo framework func newPrivelegedTestFramework(basename string) *framework.Framework { f := framework.NewDefaultFramework(basename) f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged return f } -// Get the IP address of a pod in the specified namespace +// runCommand runs the cmd and returns the combined stdout and stderr +func runCommand(cmd ...string) (string, error) { + output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output) + } + return string(output), nil +} + +// getPodAddress Get the IP address of a pod in the specified namespace func getPodAddress(podName, namespace string) string { podIP, err := framework.RunKubectl(namespace, "get", "pods", podName, "--template={{.status.podIP}}") if err != nil { @@ -39,8 +53,30 @@ func getPodAddress(podName, namespace string) string { return podIP } +// curlConnectivity Curls a target address and port func curlConnectivity(ip, port string, timeout int) []string { curl := fmt.Sprintf("curl -g --connect-timeout %v http://%s", timeout, net.JoinHostPort(ip, port)) cmd := []string{"/bin/sh", "-c", curl} return cmd } + +// getBpfList retrieve the bpf program list +func getBpfList() string { + bpfOut, err := runCommand("sudo", "bpftool", "prog", "list") + if err != nil { + framework.Failf("failed to run bpftool cmd: %v", err) + } + return bpfOut +} + +func checkDaemonStatus(f *framework.Framework, dsName, ns string) error { + ds, err := f.ClientSet.AppsV1().DaemonSets(ns).Get(context.TODO(), dsName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("Could not get daemon set from v1") + } + desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady + if desired != scheduled && desired != ready { + return fmt.Errorf("Error in daemon status. DesiredScheduled: %d, CurrentScheduled: %d, Ready: %d", desired, scheduled, ready) + } + return nil +} diff --git a/test/scripts/setup-kind.sh b/test/scripts/setup-kind.sh index 6dc4e7f..06e4a19 100755 --- a/test/scripts/setup-kind.sh +++ b/test/scripts/setup-kind.sh @@ -500,7 +500,6 @@ EOF # Copy installer script and deployment files for installer in e2e mkdir -p $HOME/work/patu/patu/test/e2e/patu/deploy cp $HOME/work/patu/patu/deploy/* $HOME/work/patu/patu/test/e2e/patu/deploy/ - cp $HOME/work/patu/patu/scripts/installer/patu-installer $HOME/work/patu/patu/test/e2e/ # Install Kubeproxy backend matrix if [ "${backend}" == "kubeproxy" ]; then