From 31ae008e9c60296b0c21ef10f8d3c9030774d902 Mon Sep 17 00:00:00 2001
From: Craig Condit <ccondit@apache.org>
Date: Fri, 16 Aug 2024 10:48:30 -0500
Subject: [PATCH] [YUNIKORN-2815] Run e2e tests against Kubernetes 1.31

Update the ee2e test matrix to include Kubernetes 1.31. Also, set the
InPlacePodVerticalScaling feature flag when running on Kind v1.27.x or
later to facilitate easier testing of pod resizing.
---
 .github/workflows/pre-commit.yml | 16 +++++++++---
 Makefile                         |  2 +-
 scripts/kind-pod-resize.yaml     | 25 ++++++++++++++++++
 scripts/run-e2e-tests.sh         | 44 +++++++++++++++++++++++++++-----
 4 files changed, 77 insertions(+), 10 deletions(-)
 create mode 100644 scripts/kind-pod-resize.yaml

diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index ab442ed60..066246f87 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -43,8 +43,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        k8s: [v1.30.0, v1.29.4, v1.28.9, v1.27.13, v1.26.15, v1.25.16, v1.24.17]
-        plugin: ['', '--plugin']
+        k8s:
+          [
+            v1.31.0,
+            v1.30.4,
+            v1.29.8,
+            v1.28.13,
+            v1.27.16,
+            v1.26.15,
+            v1.25.16,
+            v1.24.17,
+          ]
+        plugin: ["", "--plugin"]
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
@@ -76,5 +86,5 @@ jobs:
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
-          name: ${{ github.job }} stdout (${{ matrix.k8s }}${{ matrix.plugin == '--plugin' && format(', {0}', matrix.plugin) || matrix.plugin }}) 
+          name: ${{ github.job }} stdout (${{ matrix.k8s }}${{ matrix.plugin == '--plugin' && format(', {0}', matrix.plugin) || matrix.plugin }})
           path: build/e2e
diff --git a/Makefile b/Makefile
index 3c8dd3019..d92b1ab97 100644
--- a/Makefile
+++ b/Makefile
@@ -166,7 +166,7 @@ KUBECTL_BIN=$(KUBECTL_PATH)/kubectl
 export PATH := $(BASE_DIR)/$(KUBECTL_PATH):$(PATH)
 
 # kind
-KIND_VERSION=v0.23.0
+KIND_VERSION=v0.24.0
 KIND_PATH=$(TOOLS_DIR)/kind-$(KIND_VERSION)
 KIND_BIN=$(KIND_PATH)/kind
 export PATH := $(BASE_DIR)/$(KIND_PATH):$(PATH)
diff --git a/scripts/kind-pod-resize.yaml b/scripts/kind-pod-resize.yaml
new file mode 100644
index 000000000..ae514fabf
--- /dev/null
+++ b/scripts/kind-pod-resize.yaml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+featureGates:
+  "InPlacePodVerticalScaling": true
+nodes:
+  - role: control-plane
+  - role: worker
+  - role: worker
diff --git a/scripts/run-e2e-tests.sh b/scripts/run-e2e-tests.sh
index a9ea909d4..53dfc8da8 100755
--- a/scripts/run-e2e-tests.sh
+++ b/scripts/run-e2e-tests.sh
@@ -23,9 +23,39 @@ KUBECTL_VERSION=$(make -s print_kubectl_version)
 HELM=$TOOLS_DIRECTORY/helm-$HELM_VERSION/helm
 KIND=$TOOLS_DIRECTORY/kind-$KIND_VERSION/kind
 KUBECTL=$TOOLS_DIRECTORY/kubectl-$KUBECTL_VERSION/kubectl
+KIND_CONFIG=./scripts/kind.yaml
 GO="${GO:-go}"
 export GO
 
+# return 0 if arg1 <= arg2
+function verlte() {
+	if [ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]; then
+		return 0
+	else 
+		return 1
+	fi
+}
+
+# return 0 if arg1 < arg2
+function verlt() {
+  if [ "$1" = "$2" ]; then
+		return 1
+	fi
+	verlte "$1" "$2"
+}
+
+function update_kind_config() {
+  # use a different kind config for different cluster versions
+  version=$(echo "$1" | sed 's/.*://' | sed 's/^v//')
+  if verlt "${version}" "1.27"; then
+    # 1.26 or earlier
+    KIND_CONFIG=./scripts/kind.yaml
+  else
+    # 1.27 or later; enable InPlacePodVerticalScaling feature flag
+    KIND_CONFIG=./scripts/kind-pod-resize.yaml
+  fi
+}
+
 function check_cmd() {
   CMD=$1
   if ! command -v "${CMD}" &> /dev/null
@@ -164,13 +194,14 @@ Examples:
   ${NAME} -a test -n yk8s -v kindest/node:v1.24.17
   ${NAME} -a test -n yk8s -v kindest/node:v1.25.16
   ${NAME} -a test -n yk8s -v kindest/node:v1.26.15
-  ${NAME} -a test -n yk8s -v kindest/node:v1.27.13
-  ${NAME} -a test -n yk8s -v kindest/node:v1.28.9
-  ${NAME} -a test -n yk8s -v kindest/node:v1.29.4
-  ${NAME} -a test -n yk8s -v kindest/node:v1.30.0
+  ${NAME} -a test -n yk8s -v kindest/node:v1.27.16
+  ${NAME} -a test -n yk8s -v kindest/node:v1.28.13
+  ${NAME} -a test -n yk8s -v kindest/node:v1.29.8
+  ${NAME} -a test -n yk8s -v kindest/node:v1.30.4
+  ${NAME} -a test -n yk8s -v kindest/node:v1.31.0
 
   Use a local helm chart path:
-    ${NAME} -a test -n yk8s -v kindest/node:v1.30.0 -p ../yunikorn-release/helm-charts/yunikorn
+    ${NAME} -a test -n yk8s -v kindest/node:v1.31.0 -p ../yunikorn-release/helm-charts/yunikorn
 EOF
 }
 
@@ -181,7 +212,6 @@ eval "$(make arch)"
 OS=$(uname -s | tr '[:upper:]' '[:lower:]')
 check_os
 
-KIND_CONFIG=./scripts/kind.yaml
 CHART_PATH="./build/yunikorn-release/helm-charts/yunikorn"
 GIT_CLONE=true
 SCHEDULER_IMAGE="scheduler-${DOCKER_ARCH}-latest"
@@ -228,6 +258,8 @@ case ${key} in
 esac
 done
 
+update_kind_config "${CLUSTER_VERSION}"
+
 echo "e2e test run details"
 echo "  action             : ${ACTION}"
 echo "  kind cluster name  : ${CLUSTER_NAME}"