add dws multiclusters example folder (#949)

* add dws multiclusters example folder

* add terraform support

* update README Terraform command

tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/.gitignore

@@ -0,0 +1 @@
+*.kubeconfig

tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/README.md

@@ -0,0 +1,80 @@
+# Multikueue-dws-integration
+
+This repository provides the files needed to demonstrate how to use MultiKueue with Dynamic Workload Scheduler (DWS) GKE Autopilot.  This setup allows you to run workloads across multiple GKE clusters in different regions, automatically leveraging available GPU resources thanks to DWS.
+
+## Repository Contents
+
+This repository contains the following files:
+
+* `create-clusters.sh`: Script to create the required GKE clusters (one manager and three workers).
+* `tf folder`: contains the terraform script to create the required GKE clusters (one manager and three workers). You can use it instead of the bash script.
+* `deploy-multikueue.sh`: Script to install and configure Kueue and MultiKueue on the clusters.
+* `dws-multi-worker.yaml`: Kueue configuration for the worker clusters, including manager configuration.
+* `job-multi-dws-autopilot.yaml`: Example job definition to be submitted to the MultiKueue setup.
+
+## Setup and Usage
+
+### Create Clusters
+
+```
+terraform -chdir=tf init
+terraform -chdir=tf plan
+terraform -chdir=tf apply -var project_id=<YOUR PROJECT ID>
+```
+
+### Install Kueue
+
+After creating the GKE clusters and updating your kubeconfig files, install the Kueue components:
+
+```
+./deploy-multikueue.sh  
+```
+
+### Validate installation
+
+Verify the Kueue installation and the connection between the manager and worker clusters:
+
+```
+kubectl get clusterqueues dws-cluster-queue -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}CQ - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}"
+kubectl get admissionchecks sample-dws-multikueue -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}AC - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}"
+kubectl get multikueuecluster multikueue-dws-worker-asia -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}MC-ASIA - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}"
+kubectl get multikueuecluster multikueue-dws-worker-us -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}MC-US - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}"
+kubectl get multikueuecluster multikueue-dws-worker-eu -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}MC-EU - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}"
+```
+
+A successful output should look like this:
+
+```
+CQ - Active: True Reason: Ready Message: Can admit new workloads
+AC - Active: True Reason: Active Message: The admission check is active
+MC-ASIA - Active: True Reason: Active Message: Connected
+MC-US - Active: True Reason: Active Message: Connected
+MC-EU - Active: True Reason: Active Message: Connected
+```
+
+### Launch job
+
+Submit your job to the Kueue controller, which will run it on a worker cluster with available resources:
+
+```
+kubectl create -f job-multi-dws-autopilot.yaml
+```
+
+### Get the status of the job
+
+To check the job status and see where it's scheduled:
+
+```
+kubectl get workloads.kueue.x-k8s.io -o jsonpath='{range .items[*]}{.status.admissionChecks}{"\n"}{end}'
+```
+
+In the output message, you can find where the job is scheduled#
+
+### Destroy resources
+
+
+```
+terraform -chdir=tf destroy -var project_id=<YOUR PROJECT ID>
+```
+
+

tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-clusters.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo 'Create GKE Autopilot clusters'
+
+KUEUE_VERSION=v0.8.1
+regions=("europe-west4" "asia-southeast1" "us-east4" "europe-west4")
+kubeconfigs=("manager-europe-west4" "worker-asia-southeast1" "worker-us-east4" "worker-eu-west4")
+PROJECT_ID=$(gcloud config get-value project)
+PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
+PREFIX_MANAGER="man"
+PREFIX_WORKER="w"
+JOBSET_VERSION=v0.6.0
+
+# Loop through the regions
+for i in "${!regions[@]}"; do
+    region="${regions[$i]}"
+    echo "$region"
+    # Construct the cluster name, adding "manager" if it's the first region
+    if [[ $i -eq 0 ]]; then
+        cluster_name="$PREFIX_MANAGER-$region"
+    else
+        cluster_name="$PREFIX_WORKER-$region"
+    fi
+
+    #Create the cluster
+    gcloud container clusters create-auto "$cluster_name" \
+        --project "$PROJECT_ID" \
+        --region "$region" \
+        --release-channel "regular" \
+        --async
+done
+for i in "${!regions[@]}"; do
+    region="${regions[$i]}"
+    if [[ $i -eq 0 ]]; then
+        cluster_name="$PREFIX_MANAGER-$region"
+    else
+        cluster_name="$PREFIX_WORKER-$region"
+    fi
+
+    opId=$(gcloud container operations list --filter "TARGET=https://container.googleapis.com/v1/projects/$PROJECT_NUMBER/locations/$region/clusters/$cluster_name" --format="value(name)")
+    gcloud container operations wait "$opId" --project "$PROJECT_ID" --region "$region"
+    set +e
+    until gcloud -q container clusters get-credentials "$cluster_name" \
+        --project "$PROJECT_ID" \
+        --region "$region"; do
+        echo "GKE Cluster is provisioning. Retrying in 15 seconds..."
+        sleep 15
+    done
+    set -e
+    configname="${kubeconfigs[$i]}"
+    kubectl config rename-context "gke_$PROJECT_ID"_"$region"_"$cluster_name" "$configname"
+done

tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-multikueue-kubeconfig.sh

@@ -0,0 +1,232 @@
+#!/bin/bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBECONFIG_OUT=${1:-kubeconfig}
+MULTIKUEUE_SA=multikueue-sa
+NAMESPACE=kueue-system
+
+# Creating a restricted MultiKueue role, service account and role binding"
+kubectl apply -f - <<EOF
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ${MULTIKUEUE_SA}
+  namespace: ${NAMESPACE}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ${MULTIKUEUE_SA}-role
+rules:
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - jobs/status
+  verbs:
+  - get
+- apiGroups:
+  - jobset.x-k8s.io
+  resources:
+  - jobsets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - jobset.x-k8s.io
+  resources:
+  - jobsets/status
+  verbs:
+  - get
+- apiGroups:
+  - kueue.x-k8s.io
+  resources:
+  - workloads
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - kueue.x-k8s.io
+  resources:
+  - workloads/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - tfjobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - tfjobs/status
+  verbs:
+  - get
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - paddlejobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - paddlejobs/status
+  verbs:
+  - get
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - pytorchjobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - pytorchjobs/status
+  verbs:
+  - get
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - xgboostjobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - xgboostjobs/status
+  verbs:
+  - get
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - mpijobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - mpijobs/status
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ${MULTIKUEUE_SA}-crb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ${MULTIKUEUE_SA}-role
+subjects:
+- kind: ServiceAccount
+  name: ${MULTIKUEUE_SA}
+  namespace: ${NAMESPACE}
+EOF
+
+# Get or create a secret bound to the new service account.
+SA_SECRET_NAME=$(kubectl get -n ${NAMESPACE} sa/${MULTIKUEUE_SA} -o "jsonpath={.secrets[0]..name}")
+if [ -z "$SA_SECRET_NAME" ]; then
+  kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/service-account-token
+metadata:
+  name: ${MULTIKUEUE_SA}
+  namespace: ${NAMESPACE}
+  annotations:
+    kubernetes.io/service-account.name: "${MULTIKUEUE_SA}"
+EOF
+
+  SA_SECRET_NAME=${MULTIKUEUE_SA}
+fi
+
+# Note: service account token is stored base64-encoded in the secret but must
+# be plaintext in kubeconfig.
+SA_TOKEN=$(kubectl get -n ${NAMESPACE} "secrets/${SA_SECRET_NAME}" -o "jsonpath={.data['token']}" | base64 -d)
+CA_CERT=$(kubectl get -n ${NAMESPACE} "secrets/${SA_SECRET_NAME}" -o "jsonpath={.data['ca\.crt']}")
+
+# Extract cluster IP from the current context
+CURRENT_CONTEXT=$(kubectl config current-context)
+CURRENT_CLUSTER=$(kubectl config view -o jsonpath="{.contexts[?(@.name == \"${CURRENT_CONTEXT}\"})].context.cluster}")
+CURRENT_CLUSTER_ADDR=$(kubectl config view -o jsonpath="{.clusters[?(@.name == \"${CURRENT_CLUSTER}\"})].cluster.server}")
+
+# Create the Kubeconfig file
+echo "Writing kubeconfig in ${KUBECONFIG_OUT}"
+cat >"${KUBECONFIG_OUT}" <<EOF
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: ${CA_CERT}
+    server: ${CURRENT_CLUSTER_ADDR}
+  name: ${CURRENT_CLUSTER}
+contexts:
+- context:
+    cluster: ${CURRENT_CLUSTER}
+    user: ${CURRENT_CLUSTER}-${MULTIKUEUE_SA}
+  name: ${CURRENT_CONTEXT}
+current-context: ${CURRENT_CONTEXT}
+kind: Config
+preferences: {}
+users:
+- name: ${CURRENT_CLUSTER}-${MULTIKUEUE_SA}
+  user:
+    token: ${SA_TOKEN}
+EOF