Merge pull request #1071 from Tal-or/refactor_e2e_deployment

e2e: refactor NRO deployment

test/e2e/install/install_test.go

@@ -71,8 +71,9 @@ var _ = Describe("[Install] continuousIntegration", func() {
 
 	Context("with a running cluster with all the components", func() {
 		It("[test_id:47574][tier0] should perform overall deployment and verify the condition is reported as available", func() {
-			deployedObj := deploy.OverallDeployment()
-			nname := client.ObjectKeyFromObject(deployedObj.NroObj)
+			deployer := deploy.NewForPlatform(configuration.Plat)
+			nroObj := deployer.Deploy(context.TODO())
+			nname := client.ObjectKeyFromObject(nroObj)
 			Expect(nname.Name).ToNot(BeEmpty())
 
 			By("checking that the condition Available=true")
@@ -97,15 +98,15 @@ var _ = Describe("[Install] continuousIntegration", func() {
 			if err != nil {
 				logRTEPodsLogs(e2eclient.Client, e2eclient.K8sClient, context.TODO(), updatedNROObj, "NRO never reported available")
 			}
-			Expect(err).NotTo(HaveOccurred(), "NRO never reported available")
+			Expect(err).ToNot(HaveOccurred(), "NRO never reported available")
 
 			By("checking the NRT CRD is deployed")
 			_, err = crds.GetByName(e2eclient.Client, crds.CrdNRTName)
-			Expect(err).NotTo(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
 
 			By("checking the NRO CRD is deployed")
 			_, err = crds.GetByName(e2eclient.Client, crds.CrdNROName)
-			Expect(err).NotTo(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
 
 			By("checking Daemonset is up&running")
 			Eventually(func() bool {
@@ -139,7 +140,7 @@ var _ = Describe("[Install] continuousIntegration", func() {
 
 			By("checking DaemonSet pods are running with correct SELinux context")
 			ds, err := getDaemonSetByOwnerReference(updatedNROObj.UID)
-			Expect(err).NotTo(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
 			rteContainer, err := findContainerByName(*ds, containerNameRTE)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(rteContainer.SecurityContext.SELinuxOptions.Type).To(Equal(selinux.RTEContextType), "container %s is running with wrong selinux context", rteContainer.Name)
@@ -159,7 +160,7 @@ var _ = Describe("[Install] durability", Serial, func() {
 
 	Context("with deploying NUMAResourcesOperator with wrong name", func() {
 		It("should do nothing", func() {
-			nroObj := objects.TestNRO(objects.EmptyMatchLabels())
+			nroObj := objects.TestNRO(objects.NROWithMCPSelector(objects.EmptyMatchLabels()))
 			nroObj.Name = "wrong-name"
 
 			err := e2eclient.Client.Create(context.TODO(), nroObj)
@@ -190,22 +191,22 @@ var _ = Describe("[Install] durability", Serial, func() {
 	})
 
 	Context("with a running cluster with all the components and overall deployment", func() {
-		var deployedObj deploy.NroDeployment
+		var deployer deploy.Deployer
+		var nroObj *nropv1.NUMAResourcesOperator
 
 		BeforeEach(func() {
-			deployedObj = deploy.OverallDeployment()
+			deployer = deploy.NewForPlatform(configuration.Plat)
+			nroObj = deployer.Deploy(context.TODO())
 		})
 
 		AfterEach(func() {
-			deploy.TeardownDeployment(deployedObj, 5*time.Minute)
+			deployer.Teardown(context.TODO(), 5*time.Minute)
 		})
 
 		It("[test_id:47587][tier1] should restart RTE DaemonSet when image is updated in NUMAResourcesOperator", func() {
 			By("getting up-to-date NRO object")
-			nroKey := client.ObjectKeyFromObject(deployedObj.NroObj)
-			Expect(nroKey.Name).NotTo(BeEmpty())
+			nroKey := objects.NROObjectKey()
 
-			nroObj := &nropv1.NUMAResourcesOperator{}
 			immediate := true
 			err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 10*time.Minute, immediate, func(ctx context.Context) (bool, error) {
 				err := e2eclient.Client.Get(ctx, nroKey, nroObj)
@@ -278,11 +279,10 @@ var _ = Describe("[Install] durability", Serial, func() {
 		})
 
 		It("should be able to delete NUMAResourceOperator CR and redeploy without polluting cluster state", func() {
-			nname := client.ObjectKeyFromObject(deployedObj.NroObj)
+			nname := client.ObjectKeyFromObject(nroObj)
 			Expect(nname.Name).NotTo(BeEmpty())
 
-			nroObj := &nropv1.NUMAResourcesOperator{}
-			err := e2eclient.Client.Get(context.TODO(), nname, nroObj)
+			err := e2eclient.Client.Get(context.TODO(), objects.NROObjectKey(), nroObj)
 			Expect(err).ToNot(HaveOccurred())
 
 			By("waiting for the DaemonSet to be created..")
@@ -301,7 +301,7 @@ var _ = Describe("[Install] durability", Serial, func() {
 			By("checking there are no leftovers")
 			// by taking the ns from the ds we're avoiding the need to figure out in advanced
 			// at which ns we should look for the resources
-			mf, err := rte.GetManifests(configuration.Plat, configuration.PlatVersion, ds.Namespace, true, true)
+			mf, err := rte.GetManifests(configuration.Plat, configuration.PlatVersion, ds.Namespace, true, annotations.IsCustomPolicyEnabled(nroObj.Annotations))
 			Expect(err).ToNot(HaveOccurred())
 
 			Eventually(func() bool {
@@ -321,8 +321,8 @@ var _ = Describe("[Install] durability", Serial, func() {
 			}).WithTimeout(5 * time.Minute).WithPolling(10 * time.Second).Should(BeTrue())
 
 			By("redeploy with other parameters")
-			nroObjRedep := objects.TestNRO(objects.EmptyMatchLabels())
-			nroObjRedep.Spec = *deployedObj.NroObj.Spec.DeepCopy()
+			nroObjRedep := objects.TestNRO(objects.NROWithMCPSelector(objects.EmptyMatchLabels()))
+			nroObjRedep.Spec = *nroObj.Spec.DeepCopy()
 			// TODO change to an image which is test dedicated
 			nroObjRedep.Spec.ExporterImage = e2eimages.RTETestImageCI
 

test/e2e/must-gather/must_gather_suite_test.go

@@ -17,6 +17,7 @@
 package mustgather
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"testing"
@@ -65,12 +66,12 @@ var _ = ginkgo.BeforeSuite(func() {
 		ginkgo.By("Fetching up cluster data")
 
 		var err error
-		deployment, err = deploy.GetDeploymentWithSched()
+		deployment, err = deploy.GetDeploymentWithSched(context.TODO())
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 		return
 	}
 	ginkgo.By("Setting up the cluster")
-	deployment.NroDeployment = deploy.OverallDeployment()
+	deployment.Deploy(context.TODO())
 	deployment.NroSchedObj = deploy.DeployNROScheduler()
 })
 
@@ -81,7 +82,7 @@ var _ = ginkgo.AfterSuite(func() {
 	}
 	ginkgo.By("tearing down the cluster")
 	deploy.TeardownNROScheduler(deployment.NroSchedObj, 5*time.Minute)
-	deploy.TeardownDeployment(deployment.NroDeployment, 5*time.Minute)
+	deployment.Teardown(context.TODO(), 5*time.Minute)
 })
 
 func getStringValueFromEnv(envVar, fallback string) string {

test/e2e/must-gather/must_gather_test.go

@@ -31,9 +31,11 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/yaml"
 
+	nropv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1"
 	nodegroupv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1/helper/nodegroup"
 	"github.com/openshift-kni/numaresources-operator/internal/wait"
 	e2eclient "github.com/openshift-kni/numaresources-operator/test/utils/clients"
+	"github.com/openshift-kni/numaresources-operator/test/utils/objects"
 
 	mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 )
@@ -103,13 +105,15 @@ var _ = Describe("[must-gather] NRO data collected", func() {
 				Expect(err).ToNot(HaveOccurred())
 
 				By("Looking for resources instances")
-				nropInstanceFileName := fmt.Sprintf("%s.yaml", filepath.Join("cluster-scoped-resources/nodetopology.openshift.io/numaresourcesoperators", deployment.NroObj.Name))
+				nropInstanceFileName := fmt.Sprintf("%s.yaml", filepath.Join("cluster-scoped-resources/nodetopology.openshift.io/numaresourcesoperators", objects.NROObjectKey().Name))
 				nroschedInstanceFileName := fmt.Sprintf("%s.yaml", filepath.Join("cluster-scoped-resources/nodetopology.openshift.io/numaresourcesschedulers", deployment.NroSchedObj.Name))
 
 				collectedMCPs, err := getMachineConfigPools(filepath.Join(mgContentFolder, "cluster-scoped-resources/machineconfiguration.openshift.io/machineconfigpools"))
 				Expect(err).ToNot(HaveOccurred())
 
-				ngMCPs, err := nodegroupv1.FindMachineConfigPools(&collectedMCPs, deployment.NroObj.Spec.NodeGroups)
+				nro := &nropv1.NUMAResourcesOperator{}
+				Expect(e2eclient.Client.Get(context.TODO(), objects.NROObjectKey(), nro)).To(Succeed())
+				ngMCPs, err := nodegroupv1.FindMachineConfigPools(&collectedMCPs, nro.Spec.NodeGroups)
 				Expect(err).ToNot(HaveOccurred())
 
 				nglabels := collectMachineConfigPoolsNodeSelector(ngMCPs)
@@ -124,7 +128,7 @@ var _ = Describe("[must-gather] NRO data collected", func() {
 				Expect(err).ToNot(HaveOccurred())
 
 				By("Looking for namespace in NUMAResourcesOperator")
-				updatedNRO, err := wait.With(e2eclient.Client).Interval(5*time.Second).Timeout(2*time.Minute).ForDaemonsetInNUMAResourcesOperatorStatus(ctx, deployment.NroObj)
+				updatedNRO, err := wait.With(e2eclient.Client).Interval(5*time.Second).Timeout(2*time.Minute).ForDaemonsetInNUMAResourcesOperatorStatus(ctx, nro)
 				Expect(err).ToNot(HaveOccurred())
 				namespace := updatedNRO.Status.DaemonSets[0].Namespace
 

test/e2e/uninstall/uninstall_test.go

@@ -56,7 +56,7 @@ var _ = Describe("[Uninstall] clusterCleanup", Serial, func() {
 		It("should delete all components after NRO deletion", func() {
 			By("deleting the NRO object")
 			// since we are getting an existing object, we don't need the real labels here
-			nroObj := objects.TestNRO(objects.EmptyMatchLabels())
+			nroObj := objects.TestNRO(objects.NROWithMCPSelector(objects.EmptyMatchLabels()))
 			By("deleting the KC object")
 			kcObj, err := objects.TestKC(objects.EmptyMatchLabels())
 			Expect(err).To(Not(HaveOccurred()))

test/utils/clients/clients.go

@@ -17,6 +17,7 @@
 package clients
 
 import (
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/klog/v2"
@@ -26,8 +27,8 @@ import (
 
 	nrtv1alpha2 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha2"
 	perfprof "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
+	hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
-	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
 	nropv1 "github.com/openshift-kni/numaresources-operator/api/numaresourcesoperator/v1"
 	"github.com/openshift-kni/numaresources-operator/test/utils/hypershift"
@@ -65,6 +66,9 @@ func init() {
 	if err := perfprof.AddToScheme(scheme.Scheme); err != nil {
 		klog.Exit(err.Error())
 	}
+	if err := hypershiftv1beta1.AddToScheme(scheme.Scheme); err != nil {
+		klog.Exit(err.Error())
+	}
 
 	var err error
 	Client, err = New()