diff --git a/tests/go.mod b/tests/go.mod index 567a106f..ba6e6ef9 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -1,9 +1,10 @@ -module github.com/opendatahub-io/distributed-workloads/tests/new-tests +module github.com/opendatahub-io/distributed-workloads/tests require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230718161610-2a3e8b481cec - github.com/project-codeflare/codeflare-operator v0.2.3 + github.com/project-codeflare/codeflare-operator v0.2.4-0.20230913142530-526bb53289e1 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1 github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.2 @@ -30,7 +31,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a // indirect - github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 // indirect golang.org/x/net v0.12.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index eb05716f..421cd086 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -107,8 +107,8 @@ github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a h1:ZKewwwEIURD github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a/go.mod h1:EjhPQjEm8HM3GThz5ywNGLEec1P1IjTn08kwzdvupvA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-operator v0.2.3 h1:aVsJD519hBjFoftSFlVjEzySM7JjoKgRGhxLwy88YKE= -github.com/project-codeflare/codeflare-operator v0.2.3/go.mod h1:6J91NMtSthXp/gFTl1CDlyJo+rhBPQ+jc5OagdEqaVk= +github.com/project-codeflare/codeflare-operator v0.2.4-0.20230913142530-526bb53289e1 h1:1iBWbUlDja0qpMnyH+u8uL0qEjlxMfldFxygFgyViqU= +github.com/project-codeflare/codeflare-operator v0.2.4-0.20230913142530-526bb53289e1/go.mod h1:6J91NMtSthXp/gFTl1CDlyJo+rhBPQ+jc5OagdEqaVk= github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1 h1:ZNQ/JPdjS6CtaAzt6SNqaoWcpwS1PyVdgZlmIYikPLI= github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/tests/integration/mcad_ray_test.go b/tests/integration/mcad_ray_test.go new file mode 100644 index 00000000..fe857242 --- /dev/null +++ b/tests/integration/mcad_ray_test.go @@ -0,0 +1,100 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "testing" + + . "github.com/onsi/gomega" + cfosupport "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + support "github.com/opendatahub-io/distributed-workloads/tests/integration/support" +) + +func TestMCADRay(t *testing.T) { + test := cfosupport.With(t) + + test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/issues/190") + + // Create a namespace + namespace := test.NewTestNamespace() + + // Test configuration + jupyterNotebookConfigMapFileName := "mnist_ray_mini.ipynb" + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "notebooks-ray", + }, + BinaryData: map[string][]byte{ + // MNIST MCAD Notebook + jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"), + "mnist.py": ReadFile(test, "resources/mnist.py"), + "requirements.txt": ReadFile(test, "resources/requirements.txt"), + }, + Immutable: cfosupport.Ptr(true), + } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + } + token := support.CreateTestRBAC(test, namespace, policyRules) + + // Create Notebook CR + support.CreateNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) + + // Make sure the AppWrapper is created and running + test.Eventually(cfosupport.AppWrappers(test, namespace), cfosupport.TestTimeoutLong). + Should( + And( + HaveLen(1), + ContainElement(WithTransform(cfosupport.AppWrapperName, HavePrefix("mnistjob"))), + ContainElement(WithTransform(cfosupport.AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), + ), + ) + + // Make sure the AppWrapper finishes and is deleted + test.Eventually(cfosupport.AppWrappers(test, namespace), cfosupport.TestTimeoutLong). + Should(HaveLen(0)) +} diff --git a/tests/integration/pytorch_mcad_test.go b/tests/integration/pytorch_mcad_test.go new file mode 100644 index 00000000..df1bbdd3 --- /dev/null +++ b/tests/integration/pytorch_mcad_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "testing" + + . "github.com/onsi/gomega" + cfosupport "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + support "github.com/opendatahub-io/distributed-workloads/tests/integration/support" +) + +func TestMnistPyTorchMCAD(t *testing.T) { + test := cfosupport.With(t) + + // Create a namespace + namespace := test.NewTestNamespace() + + // Test configuration + jupyterNotebookConfigMapFileName := "mnist_mcad_mini.ipynb" + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "notebooks-mcad", + }, + BinaryData: map[string][]byte{ + // MNIST MCAD Notebook + jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_mcad_mini.ipynb"), + }, + Immutable: cfosupport.Ptr(true), + } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + // Needed for job.logs() + { + Verbs: []string{"get"}, + APIGroups: []string{""}, + Resources: []string{"pods/log"}, + }, + } + token := support.CreateTestRBAC(test, namespace, policyRules) + + // Create Notebook CR + support.CreateNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) + + // Make sure the AppWrapper is created and running + test.Eventually(cfosupport.AppWrappers(test, namespace), cfosupport.TestTimeoutLong). + Should( + And( + HaveLen(1), + ContainElement(WithTransform(cfosupport.AppWrapperName, HavePrefix("mnistjob"))), + ContainElement(WithTransform(cfosupport.AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), + ), + ) + + // Make sure the AppWrapper finishes and is deleted + test.Eventually(cfosupport.AppWrappers(test, namespace), cfosupport.TestTimeoutLong). + Should(HaveLen(0)) +} diff --git a/tests/integration/ray_test.go b/tests/integration/ray_test.go index 365b0b39..80a64a74 100644 --- a/tests/integration/ray_test.go +++ b/tests/integration/ray_test.go @@ -38,6 +38,10 @@ func TestRayCluster(t *testing.T) { test := support.With(t) test.T().Parallel() + // This test is unstable. It seems that RayJob CR sometimes trigger 2 jobs in Ray, causing confusion in KubeRay operator. + // Needs to be checked with newer KubeRay version. If still unstable then it needs to be reported. + test.T().Skip("Requires https://github.com/opendatahub-io/distributed-workloads/issues/65") + // Create a namespace namespace := test.NewTestNamespace() diff --git a/tests/integration/resources/mnist.py b/tests/integration/resources/mnist.py index 244c84d2..d6a21194 100644 --- a/tests/integration/resources/mnist.py +++ b/tests/integration/resources/mnist.py @@ -149,7 +149,7 @@ def test_dataloader(self): trainer = Trainer( accelerator="auto", # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs - max_epochs=5, + max_epochs=2, callbacks=[TQDMProgressBar(refresh_rate=20)], num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), diff --git a/tests/integration/resources/mnist_mcad_mini.ipynb b/tests/integration/resources/mnist_mcad_mini.ipynb new file mode 100644 index 00000000..0b53324a --- /dev/null +++ b/tests/integration/resources/mnist_mcad_mini.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Import pieces from codeflare-sdk\n", + "from codeflare_sdk.job.jobs import DDPJobDefinition\n", + "from time import sleep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47ca5c15", + "metadata": { + "tags": ["parameters"] + }, + "outputs": [], + "source": [ + "#parameters\n", + "namespace = \"default\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26b21373", + "metadata": {}, + "outputs": [], + "source": [ + "job = DDPJobDefinition(name=\"mnistjob\", script=\"mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/mnist-job-test:v0.0.1\").submit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d24e9f95", + "metadata": {}, + "outputs": [], + "source": [ + "finished = False\n", + "while not finished:\n", + " sleep(1)\n", + " try:\n", + " finished = (\"Epoch 4: 100%\" in job.logs())\n", + " except:\n", + " finished = False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f078b7cd", + "metadata": {}, + "outputs": [], + "source": [ + "job.cancel()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/integration/resources/mnist_ray_mini.ipynb b/tests/integration/resources/mnist_ray_mini.ipynb new file mode 100644 index 00000000..39d7d84d --- /dev/null +++ b/tests/integration/resources/mnist_ray_mini.ipynb @@ -0,0 +1,162 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Import pieces from codeflare-sdk\n", + "from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration\n", + "from codeflare_sdk.job.jobs import DDPJobDefinition\n", + "from time import sleep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30888aed", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#parameters\n", + "namespace = \"default\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f4bc870-091f-4e11-9642-cba145710159", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Create our cluster and submit appwrapper\n", + "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', num_workers=1, min_cpus=1, max_cpus=1, min_memory=4, max_memory=4, num_gpus=0, instascale=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Bring up the cluster\n", + "cluster.up()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a99d5aff", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cluster.wait_ready()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df71c1ed", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cluster.status()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cluster.details()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47ca5c15", + "metadata": {}, + "outputs": [], + "source": [ + "job = DDPJobDefinition(name=\"mnisttest\", script=\"mnist.py\", workspace=\"file:///opt/app-root/notebooks/..data\", scheduler_args={\"requirements\": \"/opt/app-root/notebooks/requirements.txt\"}).submit(cluster)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f63a178a", + "metadata": {}, + "outputs": [], + "source": [ + "finished = False\n", + "while not finished:\n", + " sleep(1)\n", + " status = job.status()\n", + " finished = (str(status.state) == \"SUCCEEDED\")" + ] + }, + { + "cell_type": "markdown", + "id": "885ad96b", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b099777", + "metadata": {}, + "outputs": [], + "source": [ + "cluster.down()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/integration/resources/requirements.txt b/tests/integration/resources/requirements.txt new file mode 100644 index 00000000..073e4247 --- /dev/null +++ b/tests/integration/resources/requirements.txt @@ -0,0 +1,4 @@ +pytorch_lightning==1.5.10 +ray_lightning +torchmetrics==0.9.1 +torchvision==0.12.0 \ No newline at end of file diff --git a/tests/integration/sanity_test.go b/tests/integration/sanity_test.go index 8c7f8e0d..4d63e250 100644 --- a/tests/integration/sanity_test.go +++ b/tests/integration/sanity_test.go @@ -20,17 +20,19 @@ import ( "testing" . "github.com/onsi/gomega" - support "github.com/project-codeflare/codeflare-operator/test/support" + cfosupport "github.com/project-codeflare/codeflare-operator/test/support" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + support "github.com/opendatahub-io/distributed-workloads/tests/integration/support" ) func TestMcadReadyStatus(t *testing.T) { - test := support.With(t) + test := cfosupport.With(t) - mcad, err := test.Client().CodeFlare().CodeflareV1alpha1().MCADs(GetOpenDataHubNamespace()).Get(test.Ctx(), "mcad", metav1.GetOptions{}) + mcad, err := test.Client().CodeFlare().CodeflareV1alpha1().MCADs(support.GetOpenDataHubNamespace()).Get(test.Ctx(), "mcad", metav1.GetOptions{}) test.Expect(err).NotTo(HaveOccurred()) // Assert the MCAD status is 'ready' @@ -38,9 +40,9 @@ func TestMcadReadyStatus(t *testing.T) { } func TestInstaScaleReadyStatus(t *testing.T) { - test := support.With(t) + test := cfosupport.With(t) - instascale, err := test.Client().CodeFlare().CodeflareV1alpha1().InstaScales(GetOpenDataHubNamespace()).Get(test.Ctx(), "instascale", metav1.GetOptions{}) + instascale, err := test.Client().CodeFlare().CodeflareV1alpha1().InstaScales(support.GetOpenDataHubNamespace()).Get(test.Ctx(), "instascale", metav1.GetOptions{}) test.Expect(err).NotTo(HaveOccurred()) // Assert the InstaScale status is 'ready' @@ -48,11 +50,11 @@ func TestInstaScaleReadyStatus(t *testing.T) { } func TestKubeRayRunning(t *testing.T) { - test := support.With(t) + test := cfosupport.With(t) - kuberay, err := test.Client().Core().AppsV1().Deployments(GetOpenDataHubNamespace()).Get(test.Ctx(), "kuberay-operator", metav1.GetOptions{}) + kuberay, err := test.Client().Core().AppsV1().Deployments(support.GetOpenDataHubNamespace()).Get(test.Ctx(), "kuberay-operator", metav1.GetOptions{}) test.Expect(err).NotTo(HaveOccurred()) // Assert the KubeRay Deployment is running - test.Expect(kuberay).To(WithTransform(support.ConditionStatus(appsv1.DeploymentAvailable), Equal(corev1.ConditionTrue))) + test.Expect(kuberay).To(WithTransform(cfosupport.ConditionStatus(appsv1.DeploymentAvailable), Equal(corev1.ConditionTrue))) } diff --git a/tests/integration/config.go b/tests/integration/support/config.go similarity index 97% rename from tests/integration/config.go rename to tests/integration/support/config.go index 56aa00af..13ee57a5 100644 --- a/tests/integration/config.go +++ b/tests/integration/support/config.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration +package support import ( "os" diff --git a/tests/integration/support/net.go b/tests/integration/support/net.go new file mode 100644 index 00000000..f66c42fa --- /dev/null +++ b/tests/integration/support/net.go @@ -0,0 +1,57 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + gomega "github.com/onsi/gomega" + cfosupport "github.com/project-codeflare/codeflare-operator/test/support" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ingressConfigResource = schema.GroupVersionResource{Group: "config.openshift.io", Version: "v1", Resource: "ingresses"} +var infrastructureConfigResource = schema.GroupVersionResource{Group: "config.openshift.io", Version: "v1", Resource: "infrastructures"} + +func GetIngressDomain(test cfosupport.Test) string { + test.T().Helper() + + cluster, err := test.Client().Dynamic().Resource(ingressConfigResource).Get(test.Ctx(), "cluster", metav1.GetOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + ingressDomain, found, err := unstructured.NestedString(cluster.UnstructuredContent(), "spec", "domain") + test.Expect(err).NotTo(gomega.HaveOccurred()) + test.Expect(found).To(gomega.BeTrue()) + + test.T().Logf("Ingress domain: %s", ingressDomain) + return ingressDomain +} + +func GetOpenShiftApiUrl(test cfosupport.Test) string { + test.T().Helper() + + cluster, err := test.Client().Dynamic().Resource(infrastructureConfigResource).Get(test.Ctx(), "cluster", metav1.GetOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + openShiftApiUrl, found, err := unstructured.NestedString(cluster.UnstructuredContent(), "status", "apiServerURL") + test.Expect(err).NotTo(gomega.HaveOccurred()) + test.Expect(found).To(gomega.BeTrue()) + + test.T().Logf("OpenShift API URL: %s", openShiftApiUrl) + return openShiftApiUrl +} diff --git a/tests/integration/support/notebook.go b/tests/integration/support/notebook.go new file mode 100644 index 00000000..58721265 --- /dev/null +++ b/tests/integration/support/notebook.go @@ -0,0 +1,106 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "bytes" + "embed" + "html/template" + + . "github.com/onsi/gomega" + gomega "github.com/onsi/gomega" + cfosupport "github.com/project-codeflare/codeflare-operator/test/support" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" +) + +//go:embed resources/* +var files embed.FS + +var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"} + +type NotebookProps struct { + IngressDomain string + OpenShiftApiUrl string + KubernetesBearerToken string + Namespace string + OpenDataHubNamespace string + CodeFlareImageStreamTag string + NotebookConfigMapName string + NotebookConfigMapFileName string + NotebookPVC string +} + +func CreateNotebook(test cfosupport.Test, namespace *corev1.Namespace, notebookToken, jupyterNotebookConfigMapName, jupyterNotebookConfigMapFileName string) { + // Create PVC for Notebook + notebookPVC := &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "PersistentVolumeClaim", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "jupyterhub-nb-kube-3aadmin-pvc", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + } + notebookPVC, err := test.Client().Core().CoreV1().PersistentVolumeClaims(namespace.Name).Create(test.Ctx(), notebookPVC, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created PersistentVolumeClaim %s/%s successfully", notebookPVC.Namespace, notebookPVC.Name) + + // Read the Notebook CR from resources and perform replacements for custom values using go template + notebookProps := NotebookProps{ + IngressDomain: GetIngressDomain(test), + OpenShiftApiUrl: GetOpenShiftApiUrl(test), + KubernetesBearerToken: notebookToken, + Namespace: namespace.Name, + OpenDataHubNamespace: GetOpenDataHubNamespace(), + CodeFlareImageStreamTag: GetODHCodeFlareImageStreamTag(test), + NotebookConfigMapName: jupyterNotebookConfigMapName, + NotebookConfigMapFileName: jupyterNotebookConfigMapFileName, + NotebookPVC: notebookPVC.Name, + } + notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml") + test.Expect(err).NotTo(gomega.HaveOccurred()) + parsedNotebookTemplate, err := template.New("notebook").Parse(string(notebookTemplate)) + test.Expect(err).NotTo(HaveOccurred()) + + // Filter template and store results to the buffer + notebookBuffer := new(bytes.Buffer) + err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps) + test.Expect(err).NotTo(HaveOccurred()) + + // Create Notebook CR + notebookCR := &unstructured.Unstructured{} + err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR) + test.Expect(err).NotTo(HaveOccurred()) + _, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) +} diff --git a/tests/integration/support/odh.go b/tests/integration/support/odh.go new file mode 100644 index 00000000..6bb356c3 --- /dev/null +++ b/tests/integration/support/odh.go @@ -0,0 +1,33 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + gomega "github.com/onsi/gomega" + cfosupport "github.com/project-codeflare/codeflare-operator/test/support" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GetODHCodeFlareImageStreamTag(test cfosupport.Test) string { + test.T().Helper() + + cfis, err := test.Client().Image().ImageV1().ImageStreams(GetOpenDataHubNamespace()).Get(test.Ctx(), "codeflare-notebook", metav1.GetOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) + test.Expect(cfis.Spec.Tags).To(gomega.HaveLen(1)) + return cfis.Spec.Tags[0].Name +} diff --git a/tests/integration/support/rbac.go b/tests/integration/support/rbac.go new file mode 100644 index 00000000..80a45647 --- /dev/null +++ b/tests/integration/support/rbac.go @@ -0,0 +1,92 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + gomega "github.com/onsi/gomega" + cfosupport "github.com/project-codeflare/codeflare-operator/test/support" + + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func CreateTestRBAC(test cfosupport.Test, namespace *corev1.Namespace, policyRules []rbacv1.PolicyRule) (token string) { + test.T().Helper() + + serviceAccount := &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "odh-dw-test-user", + Namespace: namespace.Name, + }, + } + serviceAccount, err := test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + role := &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "Role", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "odh-dw-test-role", + Namespace: namespace.Name, + }, + Rules: policyRules, + } + role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + roleBinding := &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "odh-dw-test", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: corev1.SchemeGroupVersion.Group, + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + }, + }, + } + _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + treq := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: cfosupport.Ptr(int64(3600)), + }, + } + treq, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).CreateToken(test.Ctx(), "odh-dw-test-user", treq, metav1.CreateOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) + return treq.Status.Token +} diff --git a/tests/integration/support/resources/custom-nb-small.yaml b/tests/integration/support/resources/custom-nb-small.yaml new file mode 100644 index 00000000..1afa0fb3 --- /dev/null +++ b/tests/integration/support/resources/custom-nb-small.yaml @@ -0,0 +1,165 @@ +# This template maybe used to spin up a custom notebook image +# i.e.: sed s/{{ .IngressDomain}}/$(oc get ingresses.config/cluster -o jsonpath={.spec.domain})/g tests/resources/custom-nb.template | oc apply -f - +# resources generated: +# pod/jupyter-nb-kube-3aadmin-0 +# service/jupyter-nb-kube-3aadmin +# route.route.openshift.io/jupyter-nb-kube-3aadmin (jupyter-nb-kube-3aadmin-opendatahub.apps.tedbig412.cp.fyre.ibm.com) +# service/jupyter-nb-kube-3aadmin-tls +apiVersion: kubeflow.org/v1 +kind: Notebook +metadata: + annotations: + notebooks.opendatahub.io/inject-oauth: "true" + notebooks.opendatahub.io/last-image-selection: codeflare-notebook:latest + notebooks.opendatahub.io/last-size-selection: Small + notebooks.opendatahub.io/oauth-logout-url: https://odh-dashboard-{{ .OpenDataHubNamespace}}.{{ .IngressDomain}}/notebookController/kube-3aadmin/home + opendatahub.io/link: https://jupyter-nb-kube-3aadmin-{{ .Namespace}}.{{ .IngressDomain}}/notebook/{{ .Namespace}}/jupyter-nb-kube-3aadmin + opendatahub.io/username: kube:admin + generation: 1 + labels: + app: jupyter-nb-kube-3aadmin + opendatahub.io/dashboard: "true" + opendatahub.io/odh-managed: "true" + opendatahub.io/user: kube-3aadmin + name: jupyter-nb-kube-3aadmin + namespace: {{ .Namespace}} +spec: + template: + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: nvidia.com/gpu.present + operator: NotIn + values: + - "true" + weight: 1 + containers: + - env: + - name: NOTEBOOK_ARGS + value: |- + --ServerApp.port=8888 + --ServerApp.token='' + --ServerApp.password='' + --ServerApp.base_url=/notebook/{{ .Namespace}}/jupyter-nb-kube-3aadmin + --ServerApp.quit_button=False + --ServerApp.tornado_settings={"user":"kube-3aadmin","hub_host":"https://odh-dashboard-{{ .OpenDataHubNamespace}}.{{ .IngressDomain}}","hub_prefix":"/notebookController/kube-3aadmin"} + - name: JUPYTER_IMAGE + value: image-registry.openshift-image-registry.svc:5000/{{ .OpenDataHubNamespace}}/codeflare-notebook:{{ .CodeFlareImageStreamTag}} + - name: JUPYTER_NOTEBOOK_PORT + value: "8888" + - name: OCP_SERVER + value: {{ .OpenShiftApiUrl}} + - name: OCP_TOKEN + value: {{ .KubernetesBearerToken}} + image: image-registry.openshift-image-registry.svc:5000/{{ .OpenDataHubNamespace}}/codeflare-notebook:{{ .CodeFlareImageStreamTag}} + command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{ .NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{ .Namespace}} && sleep infinity"] + # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ] + imagePullPolicy: Always + # livenessProbe: + # failureThreshold: 3 + # httpGet: + # path: /notebook/{{ .Namespace}}/jupyter-nb-kube-3aadmin/api + # port: notebook-port + # scheme: HTTP + # initialDelaySeconds: 10 + # periodSeconds: 5 + # successThreshold: 1 + # timeoutSeconds: 1 + name: jupyter-nb-kube-3aadmin + ports: + - containerPort: 8888 + name: notebook-port + protocol: TCP + resources: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 3Gi + volumeMounts: + - mountPath: /opt/app-root/src + name: jupyterhub-nb-kube-3aadmin-pvc + - mountPath: /opt/app-root/notebooks + name: {{ .NotebookConfigMapName}} + workingDir: /opt/app-root/src + - args: + - --provider=openshift + - --https-address=:8443 + - --http-address= + - --openshift-service-account=jupyter-nb-kube-3aadmin + - --cookie-secret-file=/etc/oauth/config/cookie_secret + - --cookie-expire=24h0m0s + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:8888 + - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - --skip-auth-regex=^(?:/notebook/$(NAMESPACE)/jupyter-nb-kube-3aadmin)?/api$ + - --email-domain=* + - --skip-provider-button + - --openshift-sar={"verb":"get","resource":"notebooks","resourceAPIGroup":"kubeflow.org","resourceName":"jupyter-nb-kube-3aadmin","namespace":"$(NAMESPACE)"} + - --logout-url=https://odh-dashboard-{{ .OpenDataHubNamespace}}.{{ .IngressDomain}}/notebookController/kube-3aadmin/home + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.redhat.io/openshift4/ose-oauth-proxy:v4.10 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /oauth/healthz + port: oauth-proxy + scheme: HTTPS + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: oauth-proxy + ports: + - containerPort: 8443 + name: oauth-proxy + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /oauth/healthz + port: oauth-proxy + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 100m + memory: 64Mi + volumeMounts: + - mountPath: /etc/oauth/config + name: oauth-config + - mountPath: /etc/tls/private + name: tls-certificates + enableServiceLinks: false + serviceAccountName: jupyter-nb-kube-3aadmin + volumes: + - name: jupyterhub-nb-kube-3aadmin-pvc + persistentVolumeClaim: + claimName: {{ .NotebookPVC}} + - name: oauth-config + secret: + defaultMode: 420 + secretName: jupyter-nb-kube-3aadmin-oauth-config + - name: tls-certificates + secret: + defaultMode: 420 + secretName: jupyter-nb-kube-3aadmin-tls + - name: {{ .NotebookConfigMapName}} + configMap: + name: {{ .NotebookConfigMapName}}