WIP - Convert e2e test to Python test + unpriviliged user test case

tests/e2e/config

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJek1USXhNakV3TWpneE4xb1hEVE16TVRJd09URXdNamd4TjFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTGFPCi9vY1dITXVnS1BOMEwxcjM5b1Nqa0Z3TXlSMkpQTXlmeHFpdHZaclo4TUVsNEN5ekk0emVaakp5V1RTaG1ZTTEKT08yN1BqeE11QTZSQUtMekE3QVR3Z05wSUNQYm1lL1pYNTFaaks1SWFETG83SHUwZXc4Z2ZGQW9qSWs4eG4rOApENS9kaHlBeTRlbVhUUXpSbGd1MENnZjE3ZEl3Sm9YU0dBaU9nSytYcEtUQ3p4bGNad1RvdWNGUjNYa0YyeUhjCnliaTdJV3M5RDl4V3B6WXB3Z1RXemh2SDdNcU1UdVppRU93L2lQOEJxZVhTZVhiN21NV3MyNGh5aTVYczQwS2cKUmtzRFFCWUFhUCsrRzZmOEptRVo0SGQxUTl6a01ZQThGSXp1L29ZVEdoKzZ0THRhS3FYaTRNOHc5WWM4OEZLNwp1ZWlQQlhXbitONmFUM3dWcGo4Q0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZFanE4ampuOTA3c1A1UkNXZFc0bkQ1MXAya2xNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBS1VINVVsU2UzVE1hdHQrWXZPWgpHZkcwSzZDd0hBQUlGVVhFQVdUa3Fyb0pKVjhCT3NkVEh4SGhJOFZtWlkrMEJoU3pCckZsek13NmVLNHhtR3V6CnRucFVMQnUvZXVjQXNhNWd2ZU8wWjZQamY4VER2MXhJcmdDS044RlgydWx6TEZjdEJrdXB1MXByM2g1MVd1eGEKL3NGNS9vU0dMR0VtcEZ2bGZFNFdub1FwejAyeEFGSE1QS0poRVpGQ3dndmJ6dWNUVDZPYWRuQ2RQdVMrcVRnVgo4TlVlMVgyTkE5RHFDZlllZ2VpWm5PTXlFY2o3WVpiL1BGZ09QdWtFYU1NOHh1YXZpbm9tenlLVHFKd1VJcG5iCkhBNzFKK3IyWFdBellPN1gyRkRWNnhyanlJdFVSWithcFJFREEwL29maTB5andzRjBtV01oMVJSY0tCQ1ZJUzMKWHBJPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+    server: https://127.0.0.1:34211
+  name: kind-kind
+contexts:
+- context:
+    cluster: kind-kind
+    user: sdk-user
+  name: sdk-user
+users:
+- name: sdk-user
+  user:
+    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN3VENDQWFrQ0ZCT0FEQ1l4OUxDczZDNVdPZzFQRWRuQlc5YkpNQTBHQ1NxR1NJYjNEUUVCQ3dVQU1CVXgKRXpBUkJnTlZCQU1UQ210MVltVnlibVYwWlhNd0hoY05Nak14TWpFeU1UQXlPRFEwV2hjTk1qUXhNakEyTVRBeQpPRFEwV2pBbE1SRXdEd1lEVlFRRERBaHpaR3N0ZFhObGNqRVFNQTRHQTFVRUNnd0hkR1Z1WVc1ME1UQ0NBU0l3CkRRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFPMkw0UnJuMlB2TEQweGZSL2g3MDBhNnB3aG0KSmtrTGttWHFYY3ZISmdiS1VkZG5QcU5rTzVsN2w4STR6MjMydWZiYllCbFZHL1ljbTd3OGNlK2RFbmpoTHAxeQozRWU0RUdsQ1JETHU5U080SUZUSTUwRHk4WUxhUTU5TitteXQ5UDBENW14RmEremREYUQ1Q2E2NTFtd3B1R2g3CkNDbnZhaFNnYmt5MytlQ0FiOU1FdGFwZi82bjlvT045dHNVZTNOQlo1aUZ2cTV4bER6Q0k2dWlUKzJTVytCWWIKM3dXbkhsTkoxTVZuVHVSbUVJc3pPUXltREZkQVcyVDJjNkl2dnJKZndCNzVOd1Q5c2poUGY4SHV0M0tIUGErVQpQazVYRVErdVlYcjFiMlVQZUdrT3VtdkZnN1JQRHZMK1JFL0t0QXpDOGNaYVNycE5sWUk2TVZMVVV5OENBd0VBCkFUQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFNeGJWdGx5UkpGNVN6VmtiMmFnNlNHRTJoeWlWUVVRT1VlNzQKd3UyNWQ0RW9RWWJBNHU2OXVXOXJJUkE4ZmlLN1VaSU1KbFhvUU1UYmlQblg0MWdmRVY2eUhqMzFkSjZmQUhRYQphVG5tbUt5N2lUQjNWUzBKc0dobURDbUdNblZWWHNnSy9GS08yeER2M2ZkYzkyODI1SGpRM2hwd28reFJwTjYvCkliaHlHdUtHaXB4VzJsUjRydlYvTGd3UHhvcHZ2YkNScmxQQUNVeHd2bU04aHo2WVlNakdndGZPRzRtcnhPRDcKWGY2RXJBV3ZRQy9BaVBlVmhXMm5WWWRNNHlUSVdoNEorcXY0SGd0NHBDQ0FVMzNkMnJvR0dIY29VWTNkY1pteApQeTdlaExIbHFWLzY1REpycnU4cS9YSzZWYnIyTjQyUzVQOHlINUlxeGVsSytkeDlpUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    client-key-data: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRRHRpK0VhNTlqN3l3OU0KWDBmNGU5Tkd1cWNJWmlaSkM1Smw2bDNMeHlZR3lsSFhaejZqWkR1WmU1ZkNPTTl0OXJuMjIyQVpWUnYySEp1OApQSEh2blJKNDRTNmRjdHhIdUJCcFFrUXk3dlVqdUNCVXlPZEE4dkdDMmtPZlRmcHNyZlQ5QStac1JXdnMzUTJnCitRbXV1ZFpzS2Job2V3Z3A3Mm9Vb0c1TXQvbmdnRy9UQkxXcVgvK3AvYURqZmJiRkh0elFXZVloYjZ1Y1pROHcKaU9yb2svdGtsdmdXRzk4RnB4NVRTZFRGWjA3a1poQ0xNemtNcGd4WFFGdGs5bk9pTDc2eVg4QWUrVGNFL2JJNApUMy9CN3JkeWh6MnZsRDVPVnhFUHJtRjY5VzlsRDNocERycHJ4WU8wVHc3eS9rUlB5clFNd3ZIR1drcTZUWldDCk9qRlMxRk12QWdNQkFBRUNnZ0VBSTN3bnpsc2dBTTBlcGV2OTFsSzVCdkhQRGVRa2gvamdqN1RvK2czdjBrZmEKM2ZDZ1hNd1dVeUV4VkdRa0tHc1k4SlZvejUvMUkxaTJzdkhjbTB5OWU2MDN4M0ZuRXBlZWEvcm9NU3VkRnc3awpWWVlGZGFCVkQ1VUhVUDdYRC9FVWpjOU5WcE13UWh2cmMwYUFlY2R0RG85VTZ0YW1LNDJHTFRxaC8zZmtMZkw0CkVQMWVuNDc5ZjVjcUVkNzdpOUNhRE1RSGVYMEtScXdmWldhSTNQellrYmQ3M3ZMZVVJQ0FXMzQzUEJVdUdXci8KV1hMekRsbVVkd3hjbG1OYTlGR1RRdnNxcGFFeFR4b0duNVZ6NGpWN2xFbGs5aW1aeUxJN2FRL2FhN3BaMXZiTQorVUk3TDB5YnR6K0ZqRjZLSEIzNWdBMkt1MjQvcVFycGN6TVdJQkpPOFFLQmdRRDdTdTl3a2Z0VzY2ZklFb2xsCnV1VzlvNStLd1pjYUhvOGo5Y0ZLZzRmcC95cXpoWnRtUlBHTW1CWExvUlZmcGJFUTBzZjJpOWlwejBsUklaUTIKYWF3a3hKMEhyU3NBVlB5bnFrbXpSNVMyVSsySVd1eDM0TC9VRU9zRC9LZUUzU24vZzlTTmpZTEcrNTdMZGxjSwpueVZjU1BJaEl0TlJlNUViNmJpUk1EbjZoUUtCZ1FEeC93WXFjNEhDKzJjQ2VJcnpaN3FDM3lybGdiakc4cWpXCkFreXFtdmxvdURXdDRjRUc2bmxuWHF3eXRoQ0JNWnFSZHNiZmhWWHFFbUI3b0ZHNjV1bzRzUlZjaVNKK2J0dWEKVlRtSklHcEdVUEF0VUlKVXVWRjROeHVOdVN3SnhWU0F5bmlvKy9pMVowZXkxVEVmazNZVFFPTEkrbEVUNzRxcwpsV2dmSXlxM0l3S0JnR3lLRWwvSi9naXVJcnN0SG9GOU40d3dwMUdVaW9KeW5wc0dwQ1ZlS0k5dWNuQTJEa2dmCkVVSUwwcVl3Zm4zZ29GbEc0YTNnKzRWbERpTG40UStibHdvT2psRHBnQUJWdFFkcWF3anZxeEVSc1RCTExZWWQKNGwxanJVNzhpeEs5UUUyb0VGL1B0cVBodk5YZTJIdXkvNzBibU5HdExCOHV3eCtPVlBVSklwSE5Bb0dCQUpHSgpaRFhubmFTYitZbDg0V1FkZ0FmeEd5Vkg3TTZKWll5L2VVZ3BSOUg0NXgrWjQ3SzdGU1JieFlnQ0FzOFArL3Q3CnlZTG45NUY4VjlaQnhxVjI5bW45NWZEdThIWEZTZ0Q1UEU4QjFhaFFTUUdYcDZvNGdZeWc1OHRHRC93WVZ3ZlYKdk5jMElwRkdlZEpOY091aWphSnFwWGxsUVptUnVINnVwQjRGMGt5dEFvR0JBTzh5ZVVOVmhQNWhjandQVmNFawpYV0FSS3ErNnhBM3d3R0ZLV25CY1JSd2xOcFNjMUkrWWo0dFZaLzN5R3VxVndCL2JqWm1UemlQOG5HK3FGL3NoCm1zTzFsN0JZVlcrMDFpQ1k5c0M1S3FBR0lyTG5yUkd6LzdSZUhvWDdIVFVTM29PQW9YSmtkSWZPRkJVNVdDVGkKMzg3dHFTTDBjQkN6bHlEU2dGZUJFZWVyCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K

tests/e2e/install-codeflare-sdk.sh

@@ -1,17 +1,9 @@
-#!/bin/bash
-
-cd ..
+# !/bin/bash
 
 # Install Poetry and configure virtualenvs
 pip install poetry
 poetry config virtualenvs.create false
 
-cd codeflare-sdk
-
 # Lock dependencies and install them
 poetry lock --no-update
 poetry install --with test,docs
-
-# Return to the workdir
-cd ..
-cd workdir

tests/e2e/kind.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+# Copyright 2022 IBM, Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+: "${INGRESS_NGINX_VERSION:=controller-v1.6.4}"
+
+echo "Creating KinD cluster"
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+EOF
+
+echo "Deploying Ingress controller into KinD cluster"
+curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${INGRESS_NGINX_VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n        - --status-update-interval=10/g" | kubectl apply -f -
+kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true"
+kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all
+
+## Create a user with limited permissions to test the SDK
+# Create a CA and a user certificate and key
+docker cp kind-control-plane:/etc/kubernetes/pki/ca.crt .
+docker cp kind-control-plane:/etc/kubernetes/pki/ca.key .
+openssl genrsa -out sdk-user.key 2048
+openssl req -new -key sdk-user.key -out sdk-user.csr -subj /CN=sdk-user/O=tenant1
+openssl x509 -req -in sdk-user.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out sdk-user.crt -days 360
+base64 -w 0 < ca.crt > ca.crt.base64
+base64 -w 0 < sdk-user.crt > sdk-user.crt.base64
+base64 -w 0 < sdk-user.key > sdk-user.key.base64
+SERVER_ADDRESS=$(kubectl cluster-info | grep -o "https://127.0.0.1:[0-9]*" | head -n 1)
+
+# Replace the placeholders in the user config file with the actual values
+sed -i 's|certificate-authority-data:.*|certificate-authority-data: '$(cat ca.crt.base64)'|g' ./tests/e2e/config
+sed -i 's|client-certificate-data:.*|client-certificate-data: '$(cat sdk-user.crt.base64)'|g' ./tests/e2e/config
+sed -i 's|client-key-data:.*|client-key-data: '$(cat sdk-user.key.base64)'|g' ./tests/e2e/config
+sed -i 's|server:.*|server: '$(echo $SERVER_ADDRESS)'|g' ./tests/e2e/config
+
+# Apply to the user limited RBAC permissions
+cat <<EOF | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: tenant-user
+rules:
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["mcadv1beta1.groupname"]
+  resources: ["appwrappers"]
+  verbs: ["get", "create", "delete", "list", "patch", "update"]
+- apiGroups: ["rayv1.groupversion.group"]
+  resources: ["rayclusters", "rayclusters/status"]
+  verbs: ["get", "list"]
+- apiGroups: ["route.openshift.io"]
+  resources: ["routes"]
+  verbs: ["get", "list"]
+- apiGroups: ["networking.k8s.io"]
+  resources: ["ingresses"]
+  verbs: ["get", "list"]
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: tenant-user
+subjects:
+- kind: User
+  name: sdk-user
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: Role
+  name: tenant-user
+  apiGroup: rbac.authorization.k8s.io
+EOF
+
+# Temporary ClusterRoles
+cat <<EOF | kubectl apply -f -
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cr-tenant-user
+rules:
+- apiGroups: ["config.openshift.io"]
+  resources: ["ingresses"]
+  verbs: ["get", "list"]
+  resourceNames: ["cluster"]
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cr-tenant-user
+subjects:
+- kind: User
+  name: sdk-user
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: cr-tenant-user
+  apiGroup: rbac.authorization.k8s.io
+EOF
+
+# Cleanup csr/crt/keys from local machine
+rm -f ca.crt.base64 sdk-user.crt.base64 sdk-user.key.base64 ca.crt sdk-user.crt sdk-user.key sdk-user.csr ca.key ca.srl kind.csr
+
+# Install CodeFlare SDK
+chmod +x ./tests/e2e/install-codeflare-sdk.sh
+./tests/e2e/install-codeflare-sdk.sh
+
+# Confirming the user can access the cluster
+kubectl get ns default --as sdk-user

tests/e2e/mnist_raycluster_sdk.py

@@ -11,7 +11,7 @@
 from codeflare_sdk.job.jobs import DDPJobDefinition
 
 namespace = sys.argv[1]
-ray_image = os.getenv("RAY_IMAGE")
+ray_image = "quay.io/project-codeflare/ray:latest-py39-cu118"
 host = os.getenv("CLUSTER_HOSTNAME")
 
 ingress_options = {}
@@ -46,7 +46,6 @@
     )
 )
 
-
 cluster.up()
 
 cluster.status()
@@ -62,6 +61,14 @@
     script="mnist.py",
     scheduler_args={"requirements": "requirements.txt"},
 )
+
+# THIS DOESN'T WORK: (CURRENT STATUS: CONNECTION REFUSED)
+host_alias = {
+    "ip": "172.18.0.2",  # Replace with the actual node IP
+    "hostnames": ["kind"],  # Replace with the actual hostname
+}
+jobdef["spec"]["template"]["spec"]["hostAliases"] = [host_alias]
+
 job = jobdef.submit(cluster)
 
 done = False