Convert MNIST PyTorch and MNIST Ray test cases to go

tests/go.mod

@@ -3,7 +3,8 @@ module github.com/opendatahub-io/distributed-workloads/tests/new-tests
 require (
 	github.com/onsi/gomega v1.27.10
 	github.com/openshift/api v0.0.0-20230718161610-2a3e8b481cec
-	github.com/project-codeflare/codeflare-operator v0.2.2
+	github.com/project-codeflare/codeflare-operator v0.2.3-0.20230906080827-4ae9e8f9e640
+	github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c
 	github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576
 	k8s.io/api v0.27.2
 	k8s.io/apimachinery v0.27.2
@@ -30,7 +31,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a // indirect
-	github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	golang.org/x/net v0.12.0 // indirect

tests/go.sum

@@ -107,10 +107,10 @@ github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a h1:ZKewwwEIURD
 github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a/go.mod h1:EjhPQjEm8HM3GThz5ywNGLEec1P1IjTn08kwzdvupvA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/project-codeflare/codeflare-operator v0.2.2 h1:7Vf1Hv+q0muzEGbTzPBdKjXOd903NbMu1bsdL02PCr0=
-github.com/project-codeflare/codeflare-operator v0.2.2/go.mod h1:2oYGL1iF92OzEhw+FRd7S4kFAEF9dZZnDCJLsuYZ4r0=
-github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 h1:4J3vW+vuCBHyocd3huZy3FJhOw5acaBHFLYwutkTE/E=
-github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40=
+github.com/project-codeflare/codeflare-operator v0.2.3-0.20230906080827-4ae9e8f9e640 h1:D9oeAMirZcWTNzO7G8IDXy2IEKLnH8EBXjgiaufzGx8=
+github.com/project-codeflare/codeflare-operator v0.2.3-0.20230906080827-4ae9e8f9e640/go.mod h1:q+Y/3BS/N3MDzljiD7XRaUae3yZvzzXvhegMJjAaasE=
+github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c h1:LjCgRcjtkd9byePczvxGBLjdM22M2BCTB76FoEk2fkU=
+github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576 h1:NYt+ipGo+jMLYCFI5S04DzVFSXn8WCR0m6m3bFXPk+4=
 github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576/go.mod h1:7PBzsR4L0b/f5tTxq1XGnzvudbrMyPadiLiNiP9pOl0=

tests/integration/mcad_test.go

@@ -0,0 +1,362 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bytes"
+	"html/template"
+	"testing"
+
+	. "github.com/onsi/gomega"
+	support "github.com/project-codeflare/codeflare-operator/test/support"
+	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+	rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
+
+	authenticationv1 "k8s.io/api/authentication/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/yaml"
+)
+
+var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"}
+var ingressConfigResource = schema.GroupVersionResource{Group: "config.openshift.io", Version: "v1", Resource: "ingresses"}
+var infrastructureConfigResource = schema.GroupVersionResource{Group: "config.openshift.io", Version: "v1", Resource: "infrastructures"}
+
+type NotebookProps struct {
+	IngressDomain           string
+	OpenShiftApiUrl         string
+	KubernetesBearerToken   string
+	Namespace               string
+	OpenDataHubNamespace    string
+	CodeFlareImageStreamTag string
+	JobType                 string
+	NotebookPVC             string
+}
+
+func TestMnistPyTorchMCAD(t *testing.T) {
+	test := support.With(t)
+
+	// Create a namespace
+	namespace := test.NewTestNamespace()
+
+	// Test configuration
+	config := &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "notebooks-mcad",
+		},
+		BinaryData: map[string][]byte{
+			// MNIST MCAD Notebook
+			"mnist_mcad_mini.ipynb": ReadFile(test, "resources/mnist_mcad_mini.ipynb"),
+		},
+		Immutable: support.Ptr(true),
+	}
+	config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name)
+
+	// Create PVC for Notebook
+	notebookPVC := &corev1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "jupyterhub-nb-kube-3aadmin-pvc",
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			AccessModes: []corev1.PersistentVolumeAccessMode{
+				corev1.ReadWriteOnce,
+			},
+			Resources: corev1.ResourceRequirements{
+				Requests: corev1.ResourceList{
+					corev1.ResourceStorage: resource.MustParse("10Gi"),
+				},
+			},
+		},
+	}
+	notebookPVC, err = test.Client().Core().CoreV1().PersistentVolumeClaims(namespace.Name).Create(test.Ctx(), notebookPVC, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.T().Logf("Created PersistentVolumeClaim %s/%s successfully", notebookPVC.Namespace, notebookPVC.Name)
+
+	// Create RBAC, retrieve token for user with limited rights
+	policyRules := []rbacv1.PolicyRule{
+		{
+			Verbs: []string{"get", "create", "delete", "list", "patch", "update"},
+			// APIGroups: []string{mcadv1beta1.GroupName},
+			APIGroups: []string{"mcad.ibm.com"},
+			Resources: []string{"appwrappers"},
+		},
+		// Needed for job.logs()
+		{
+			Verbs:     []string{"get"},
+			APIGroups: []string{""},
+			Resources: []string{"pods/log"},
+		},
+	}
+	token := createTestMnistPyTorchMCADRBAC(test, namespace, policyRules)
+
+	// Read the Notebook CR from resources and perform replacements for custom values using go template
+	notebookProps := NotebookProps{
+		IngressDomain:           getIngressDomain(test),
+		OpenShiftApiUrl:         getOpenShiftApiUrl(test),
+		KubernetesBearerToken:   token,
+		Namespace:               namespace.Name,
+		OpenDataHubNamespace:    GetOpenDataHubNamespace(),
+		CodeFlareImageStreamTag: getCodeFlareImageStreamTag(test),
+		JobType:                 "mcad",
+		NotebookPVC:             "jupyterhub-nb-kube-3aadmin-pvc",
+	}
+	notebookTemplate := string(ReadFile(test, "resources/custom-nb-small.yaml"))
+	parsedNotebookTemplate, err := template.New("notebook").Parse(notebookTemplate)
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Filter template and store results to the buffer
+	notebookBuffer := new(bytes.Buffer)
+	err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps)
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Create Notebook CR
+	notebookCR := &unstructured.Unstructured{}
+	err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR)
+	test.Expect(err).NotTo(HaveOccurred())
+	_, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Make sure the AppWrapper is created and running
+	test.Eventually(support.AppWrappers(test, namespace), support.TestTimeoutLong).
+		Should(
+			And(
+				HaveLen(1),
+				ContainElement(WithTransform(support.AppWrapperName, HavePrefix("mnistjob"))),
+				ContainElement(WithTransform(support.AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))),
+			),
+		)
+
+	// Make sure the AppWrapper finishes and is deleted
+	test.Eventually(support.AppWrappers(test, namespace), support.TestTimeoutLong).
+		Should(HaveLen(0))
+}
+
+func TestMCADRay(t *testing.T) {
+	test := support.With(t)
+
+	// Create a namespace
+	namespace := test.NewTestNamespace()
+
+	// Test configuration
+	config := &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "notebooks-ray",
+		},
+		BinaryData: map[string][]byte{
+			// MNIST MCAD Notebook
+			"mnist_ray_mini.ipynb": ReadFile(test, "resources/mnist_ray_mini.ipynb"),
+			"mnist.py":             ReadFile(test, "resources/mnist.py"),
+			"requirements.txt":     ReadFile(test, "resources/requirements.txt"),
+		},
+		Immutable: support.Ptr(true),
+	}
+	config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name)
+
+	// Create PVC for Notebook
+	notebookPVC := &corev1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "jupyterhub-nb-kube-3aadmin-pvc",
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			AccessModes: []corev1.PersistentVolumeAccessMode{
+				corev1.ReadWriteOnce,
+			},
+			Resources: corev1.ResourceRequirements{
+				Requests: corev1.ResourceList{
+					corev1.ResourceStorage: resource.MustParse("10Gi"),
+				},
+			},
+		},
+	}
+	notebookPVC, err = test.Client().Core().CoreV1().PersistentVolumeClaims(namespace.Name).Create(test.Ctx(), notebookPVC, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.T().Logf("Created PersistentVolumeClaim %s/%s successfully", notebookPVC.Namespace, notebookPVC.Name)
+
+	// Create RBAC, retrieve token for user with limited rights
+	policyRules := []rbacv1.PolicyRule{
+		{
+			Verbs: []string{"get", "create", "delete", "list", "patch", "update"},
+			// APIGroups: []string{mcadv1beta1.GroupName},
+			APIGroups: []string{"mcad.ibm.com"},
+			Resources: []string{"appwrappers"},
+		},
+		{
+			Verbs:     []string{"get", "list"},
+			APIGroups: []string{rayv1alpha1.GroupVersion.Group},
+			Resources: []string{"rayclusters", "rayclusters/status"},
+		},
+		{
+			Verbs:     []string{"get", "list"},
+			APIGroups: []string{"route.openshift.io"},
+			Resources: []string{"routes"},
+		},
+	}
+	token := createTestMnistPyTorchMCADRBAC(test, namespace, policyRules)
+
+	// Read the Notebook CR from resources and perform replacements for custom values using go template
+	notebookProps := NotebookProps{
+		IngressDomain:           getIngressDomain(test),
+		OpenShiftApiUrl:         getOpenShiftApiUrl(test),
+		KubernetesBearerToken:   token,
+		Namespace:               namespace.Name,
+		OpenDataHubNamespace:    GetOpenDataHubNamespace(),
+		CodeFlareImageStreamTag: getCodeFlareImageStreamTag(test),
+		JobType:                 "ray",
+		NotebookPVC:             "jupyterhub-nb-kube-3aadmin-pvc",
+	}
+	notebookTemplate := string(ReadFile(test, "resources/custom-nb-small.yaml"))
+	parsedNotebookTemplate, err := template.New("notebook").Parse(notebookTemplate)
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Filter template and store results to the buffer
+	notebookBuffer := new(bytes.Buffer)
+	err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps)
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Create Notebook CR
+	notebookCR := &unstructured.Unstructured{}
+	err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR)
+	test.Expect(err).NotTo(HaveOccurred())
+	_, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Make sure the AppWrapper is created and running
+	test.Eventually(support.AppWrappers(test, namespace), support.TestTimeoutLong).
+		Should(
+			And(
+				HaveLen(1),
+				ContainElement(WithTransform(support.AppWrapperName, HavePrefix("mnistjob"))),
+				ContainElement(WithTransform(support.AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))),
+			),
+		)
+
+	// Make sure the AppWrapper finishes and is deleted
+	test.Eventually(support.AppWrappers(test, namespace), support.TestTimeoutLong).
+		Should(HaveLen(0))
+}
+
+func getIngressDomain(test support.Test) string {
+	cluster, err := test.Client().Dynamic().Resource(ingressConfigResource).Get(test.Ctx(), "cluster", metav1.GetOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	ingressDomain := cluster.UnstructuredContent()["spec"].(map[string]interface{})["domain"].(string)
+	test.T().Logf("Ingress domain: %s", ingressDomain)
+	return ingressDomain
+}
+
+func getOpenShiftApiUrl(test support.Test) string {
+	cluster, err := test.Client().Dynamic().Resource(infrastructureConfigResource).Get(test.Ctx(), "cluster", metav1.GetOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	openShiftApiUrl := cluster.UnstructuredContent()["status"].(map[string]interface{})["apiServerURL"].(string)
+	test.T().Logf("OpenShift API URL: %s", openShiftApiUrl)
+	return openShiftApiUrl
+}
+
+func getCodeFlareImageStreamTag(test support.Test) string {
+	cfis, err := test.Client().Image().ImageV1().ImageStreams(GetOpenDataHubNamespace()).Get(test.Ctx(), "codeflare-notebook", metav1.GetOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.Expect(cfis.Spec.Tags).To(HaveLen(1))
+	return cfis.Spec.Tags[0].Name
+}
+
+func createTestMnistPyTorchMCADRBAC(test support.Test, namespace *corev1.Namespace, policyRules []rbacv1.PolicyRule) (token string) {
+	serviceAccount := &corev1.ServiceAccount{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ServiceAccount",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "odh-dw-test-user",
+			Namespace: namespace.Name,
+		},
+	}
+	serviceAccount, err := test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	role := &rbacv1.Role{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: rbacv1.SchemeGroupVersion.String(),
+			Kind:       "Role",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "odh-dw-test-role",
+			Namespace: namespace.Name,
+		},
+		Rules: policyRules,
+	}
+	role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	roleBinding := &rbacv1.RoleBinding{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: rbacv1.SchemeGroupVersion.String(),
+			Kind:       "RoleBinding",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "odh-dw-test",
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: rbacv1.SchemeGroupVersion.Group,
+			Kind:     "Role",
+			Name:     role.Name,
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				APIGroup:  corev1.SchemeGroupVersion.Group,
+				Name:      serviceAccount.Name,
+				Namespace: serviceAccount.Namespace,
+			},
+		},
+	}
+	_, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	treq := &authenticationv1.TokenRequest{
+		Spec: authenticationv1.TokenRequestSpec{
+			ExpirationSeconds: support.Ptr(int64(3600)),
+		},
+	}
+	treq, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).CreateToken(test.Ctx(), "odh-dw-test-user", treq, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	return treq.Status.Token
+}