Windows test

Signed-off-by: Connor Catlett <[email protected]>

Makefile

@@ -91,7 +91,7 @@ test/coverage:
 #	go test -v -race ./tests/sanity/...
 
 .PHONY: tools
-tools: bin/aws bin/ct bin/eksctl bin/ginkgo bin/golangci-lint bin/helm bin/kops bin/kubetest2 bin/mockgen bin/shfmt
+tools: bin/aws bin/ct bin/eksctl bin/ginkgo bin/golangci-lint bin/gomplate bin/helm bin/kops bin/kubetest2 bin/mockgen bin/shfmt
 
 .PHONY: update
 update: update/gofmt update/kustomize update/mockgen update/gomod update/shfmt
@@ -105,7 +105,7 @@ verify: verify/govet verify/golangci-lint verify/update
 all-push: all-image-registry push-manifest
 
 .PHONY: cluster/create
-cluster/create: bin/kops bin/eksctl bin/aws
+cluster/create: bin/kops bin/eksctl bin/aws bin/gomplate
 	./hack/e2e/create-cluster.sh
 
 .PHONY: cluster/kubeconfig

hack/e2e/config.sh

@@ -58,6 +58,3 @@ TEST_PATH=${TEST_PATH:-"./tests/e2e-kubernetes/..."}
 GINKGO_FOCUS=${GINKGO_FOCUS:-"External.Storage"}
 GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]|\[Serial\]"}
 GINKGO_PARALLEL=${GINKGO_PARALLEL:-25}
-
-# TODO: Left in for now, but look into if this is still necessary and remove if not
-EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-"Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV"}

hack/e2e/create-cluster.sh

@@ -59,16 +59,21 @@ elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
   eksctl_create_cluster \
     "$CLUSTER_NAME" \
     "${BIN}/eksctl" \
+    "${BIN}/gomplate" \
+    "$AWS_REGION" \
     "$ZONES" \
     "$INSTANCE_TYPE" \
     "$K8S_VERSION_EKSCTL" \
     "$CLUSTER_FILE" \
     "$KUBECONFIG" \
-    "${BASE_DIR}/eksctl/patch.yaml" \
-    "$EKSCTL_ADMIN_ROLE" \
     "$WINDOWS" \
-    "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml"
+    "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml" \
+    "${BASE_DIR}/eksctl/cluster.yaml"
 else
   echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2
   exit 1
 fi
+
+if [[ "$WINDOWS" == true ]]; then
+  kubectl apply --kubeconfig "${KUBECONFIG}" -f "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml"
+fi

hack/e2e/eksctl/cluster.yaml

@@ -0,0 +1,33 @@
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+metadata:
+  name: {{ .Env.CLUSTER_NAME }}
+  region: {{ .Env.REGION }}
+  version: "{{ .Env.K8S_VERSION }}"
+availabilityZones: [{{ .Env.ZONES }}]
+iam:
+  vpcResourceControllerPolicy: true
+  withOIDC: true
+  serviceAccounts:
+    - metadata:
+        name: ebs-csi-controller-sa
+        namespace: kube-system
+      wellKnownPolicies:
+        ebsCSIController: true
+managedNodeGroups:
+  - name: ng-linux
+    amiFamily: AmazonLinux2
+    desiredCapacity: 3
+    disablePodIMDS: true
+    instanceTypes: [{{ .Env.INSTANCE_TYPE }}]
+    ssh:
+      allow: false
+{{- if eq .Env.WINDOWS "true" }}
+  - name: ng-windows
+    amiFamily: WindowsServer2022CoreContainer
+    desiredCapacity: 3
+    disablePodIMDS: true
+    instanceTypes: [m5.2xlarge]
+    ssh:
+      allow: false
+{{- end }}

hack/e2e/eksctl/eksctl.sh

@@ -22,68 +22,39 @@ set -euo pipefail
 function eksctl_create_cluster() {
   CLUSTER_NAME=${1}
   EKSCTL_BIN=${2}
-  ZONES=${3}
-  INSTANCE_TYPE=${4}
-  K8S_VERSION=${5}
-  CLUSTER_FILE=${6}
-  KUBECONFIG=${7}
-  EKSCTL_PATCH_FILE=${8}
-  EKSCTL_ADMIN_ROLE=${9}
+  GOMPLATE_BIN=${3}
+  REGION=${4}
+  ZONES=${5}
+  INSTANCE_TYPE=${6}
+  K8S_VERSION=${7}
+  CLUSTER_FILE=${8}
+  KUBECONFIG=${9}
   WINDOWS=${10}
   VPC_CONFIGMAP_FILE=${11}
+  TEMPLATE_FILE=${12}
 
   CLUSTER_NAME="${CLUSTER_NAME//./-}"
 
+  loudecho "Templating $CLUSTER_NAME to $CLUSTER_FILE"
+  CLUSTER_NAME="${CLUSTER_NAME}" \
+  REGION="${REGION}" \
+  K8S_VERSION="${K8S_VERSION}" \
+  ZONES="${ZONES}" \
+  INSTANCE_TYPE="${INSTANCE_TYPE}" \
+  ${GOMPLATE_BIN} -f "${TEMPLATE_FILE}" -o "${CLUSTER_FILE}"
+
   if eksctl_cluster_exists "${CLUSTER_NAME}" "${EKSCTL_BIN}"; then
     loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE"
     ${EKSCTL_BIN} upgrade cluster -f "${CLUSTER_FILE}"
   else
-    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
-    ${EKSCTL_BIN} create cluster \
-      --managed \
-      --ssh-access=false \
-      --zones "${ZONES}" \
-      --nodes=3 \
-      --instance-types="${INSTANCE_TYPE}" \
-      --version="${K8S_VERSION}" \
-      --disable-pod-imds \
-      --dry-run \
-      "${CLUSTER_NAME}" >"${CLUSTER_FILE}"
-
-    if test -f "$EKSCTL_PATCH_FILE"; then
-      eksctl_patch_cluster_file "$CLUSTER_FILE" "$EKSCTL_PATCH_FILE"
-    fi
-
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
     ${EKSCTL_BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
   fi
 
-  loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}"
-  loudecho "Getting cluster ${CLUSTER_NAME}"
-  ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}"
-
-  if [[ -n "$EKSCTL_ADMIN_ROLE" ]]; then
-    AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-    ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}"
-    loudecho "Granting ${ADMIN_ARN} admin access to the cluster"
-    ${EKSCTL_BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin
-  fi
-
   if [[ "$WINDOWS" == true ]]; then
-    ${EKSCTL_BIN} create nodegroup \
-      --managed=true \
-      --ssh-access=false \
-      --cluster="${CLUSTER_NAME}" \
-      --node-ami-family=WindowsServer2022CoreContainer \
-      --instance-types=m5.2xlarge \
-      -n ng-windows \
-      -m 3 \
-      -M 3
-
-    kubectl apply --kubeconfig "${KUBECONFIG}" -f "$VPC_CONFIGMAP_FILE"
+    loudecho "Applying VPC ConfigMap (Windows only)"
+    kubectl apply --kubeconfig "${KUBECONFIG}" -f "${VPC_CONFIGMAP_FILE}"
   fi
-
-  return $?
 }
 
 function eksctl_cluster_exists() {
@@ -108,23 +79,3 @@ function eksctl_delete_cluster() {
   loudecho "Deleting cluster ${CLUSTER_NAME}"
   ${EKSCTL_BIN} delete cluster "${CLUSTER_NAME}"
 }
-
-function eksctl_patch_cluster_file() {
-  CLUSTER_FILE=${1}      # input must be yaml
-  EKSCTL_PATCH_FILE=${2} # input must be yaml
-
-  loudecho "Patching cluster $CLUSTER_NAME with $EKSCTL_PATCH_FILE"
-
-  # Temporary intermediate files for patching
-  CLUSTER_FILE_0=$CLUSTER_FILE.0
-  CLUSTER_FILE_1=$CLUSTER_FILE.1
-
-  cp "$CLUSTER_FILE" "$CLUSTER_FILE_0"
-
-  # Patch only the Cluster
-  kubectl patch --kubeconfig "/dev/null" -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml >"$CLUSTER_FILE_1"
-  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
-
-  # Done patching, overwrite original CLUSTER_FILE
-  mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml
-}

hack/tools/install.sh

@@ -26,6 +26,8 @@ EKSCTL_VERSION="v0.175.0"
 GINKGO_VERSION="v2.17.1"
 # https://github.com/golangci/golangci-lint
 GOLANGCI_LINT_VERSION="v1.57.2"
+# https://github.com/hairyhenderson/gomplate
+GOMPLATE_VERSION="v3.11.7"
 # https://github.com/helm/helm
 HELM_VERSION="v3.14.4"
 # https://github.com/kubernetes/kops
@@ -124,6 +126,14 @@ function install_golangci-lint() {
   install_tar_binary "${INSTALL_PATH}" "https://github.com/golangci/golangci-lint/releases/download/${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}.tar.gz" "golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}/golangci-lint"
 }
 
+function install_gomplate() {
+  INSTALL_PATH="${1}"
+
+  # gomplate includes library from no longer existing domain inet.af, and thus cannot be installed via go install
+  # install the released binary from GitHub releases instead
+  install_binary "${INSTALL_PATH}" "https://github.com/hairyhenderson/gomplate/releases/download/${GOMPLATE_VERSION}/gomplate_${OS}-${ARCH}" "gomplate"
+}
+
 function install_helm() {
   INSTALL_PATH="${1}"