diff --git a/Makefile b/Makefile index 7f7c72b86b..a29715a104 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ test/coverage: # go test -v -race ./tests/sanity/... .PHONY: tools -tools: bin/aws bin/ct bin/eksctl bin/ginkgo bin/golangci-lint bin/helm bin/kops bin/kubetest2 bin/mockgen bin/shfmt +tools: bin/aws bin/ct bin/eksctl bin/ginkgo bin/golangci-lint bin/gomplate bin/helm bin/kops bin/kubetest2 bin/mockgen bin/shfmt .PHONY: update update: update/gofmt update/kustomize update/mockgen update/gomod update/shfmt @@ -105,7 +105,7 @@ verify: verify/govet verify/golangci-lint verify/update all-push: all-image-registry push-manifest .PHONY: cluster/create -cluster/create: bin/kops bin/eksctl bin/aws +cluster/create: bin/kops bin/eksctl bin/aws bin/gomplate ./hack/e2e/create-cluster.sh .PHONY: cluster/kubeconfig diff --git a/hack/e2e/config.sh b/hack/e2e/config.sh index 0d3c27d373..8ee646cc00 100644 --- a/hack/e2e/config.sh +++ b/hack/e2e/config.sh @@ -58,6 +58,3 @@ TEST_PATH=${TEST_PATH:-"./tests/e2e-kubernetes/..."} GINKGO_FOCUS=${GINKGO_FOCUS:-"External.Storage"} GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]|\[Serial\]"} GINKGO_PARALLEL=${GINKGO_PARALLEL:-25} - -# TODO: Left in for now, but look into if this is still necessary and remove if not -EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-"Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV"} diff --git a/hack/e2e/create-cluster.sh b/hack/e2e/create-cluster.sh index 7171111419..d21cabdf75 100755 --- a/hack/e2e/create-cluster.sh +++ b/hack/e2e/create-cluster.sh @@ -59,16 +59,21 @@ elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then eksctl_create_cluster \ "$CLUSTER_NAME" \ "${BIN}/eksctl" \ + "${BIN}/gomplate" \ + "$AWS_REGION" \ "$ZONES" \ "$INSTANCE_TYPE" \ "$K8S_VERSION_EKSCTL" \ "$CLUSTER_FILE" \ "$KUBECONFIG" \ - "${BASE_DIR}/eksctl/patch.yaml" \ - "$EKSCTL_ADMIN_ROLE" \ "$WINDOWS" \ - "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml" + "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml" \ + "${BASE_DIR}/eksctl/cluster.yaml" else echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2 exit 1 fi + +if [[ "$WINDOWS" == true ]]; then + kubectl apply --kubeconfig "${KUBECONFIG}" -f "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml" +fi diff --git a/hack/e2e/eksctl/cluster.yaml b/hack/e2e/eksctl/cluster.yaml new file mode 100644 index 0000000000..9fce5f77bf --- /dev/null +++ b/hack/e2e/eksctl/cluster.yaml @@ -0,0 +1,33 @@ +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: {{ .Env.CLUSTER_NAME }} + region: {{ .Env.REGION }} + version: "{{ .Env.K8S_VERSION }}" +availabilityZones: [{{ .Env.ZONES }}] +iam: + vpcResourceControllerPolicy: true + withOIDC: true + serviceAccounts: + - metadata: + name: ebs-csi-controller-sa + namespace: kube-system + wellKnownPolicies: + ebsCSIController: true +managedNodeGroups: + - name: ng-linux + amiFamily: AmazonLinux2 + desiredCapacity: 3 + disablePodIMDS: true + instanceTypes: [{{ .Env.INSTANCE_TYPE }}] + ssh: + allow: false +{{- if eq .Env.WINDOWS "true" }} + - name: ng-windows + amiFamily: WindowsServer2022CoreContainer + desiredCapacity: 3 + disablePodIMDS: true + instanceTypes: [m5.2xlarge] + ssh: + allow: false +{{- end }} diff --git a/hack/e2e/eksctl/eksctl.sh b/hack/e2e/eksctl/eksctl.sh index 4b2ccaf68e..4186c0a70a 100644 --- a/hack/e2e/eksctl/eksctl.sh +++ b/hack/e2e/eksctl/eksctl.sh @@ -22,68 +22,39 @@ set -euo pipefail function eksctl_create_cluster() { CLUSTER_NAME=${1} EKSCTL_BIN=${2} - ZONES=${3} - INSTANCE_TYPE=${4} - K8S_VERSION=${5} - CLUSTER_FILE=${6} - KUBECONFIG=${7} - EKSCTL_PATCH_FILE=${8} - EKSCTL_ADMIN_ROLE=${9} + GOMPLATE_BIN=${3} + REGION=${4} + ZONES=${5} + INSTANCE_TYPE=${6} + K8S_VERSION=${7} + CLUSTER_FILE=${8} + KUBECONFIG=${9} WINDOWS=${10} VPC_CONFIGMAP_FILE=${11} + TEMPLATE_FILE=${12} CLUSTER_NAME="${CLUSTER_NAME//./-}" + loudecho "Templating $CLUSTER_NAME to $CLUSTER_FILE" + CLUSTER_NAME="${CLUSTER_NAME}" \ + REGION="${REGION}" \ + K8S_VERSION="${K8S_VERSION}" \ + ZONES="${ZONES}" \ + INSTANCE_TYPE="${INSTANCE_TYPE}" \ + ${GOMPLATE_BIN} -f "${TEMPLATE_FILE}" -o "${CLUSTER_FILE}" + if eksctl_cluster_exists "${CLUSTER_NAME}" "${EKSCTL_BIN}"; then loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE" ${EKSCTL_BIN} upgrade cluster -f "${CLUSTER_FILE}" else - loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)" - ${EKSCTL_BIN} create cluster \ - --managed \ - --ssh-access=false \ - --zones "${ZONES}" \ - --nodes=3 \ - --instance-types="${INSTANCE_TYPE}" \ - --version="${K8S_VERSION}" \ - --disable-pod-imds \ - --dry-run \ - "${CLUSTER_NAME}" >"${CLUSTER_FILE}" - - if test -f "$EKSCTL_PATCH_FILE"; then - eksctl_patch_cluster_file "$CLUSTER_FILE" "$EKSCTL_PATCH_FILE" - fi - loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE" ${EKSCTL_BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}" fi - loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}" - loudecho "Getting cluster ${CLUSTER_NAME}" - ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}" - - if [[ -n "$EKSCTL_ADMIN_ROLE" ]]; then - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}" - loudecho "Granting ${ADMIN_ARN} admin access to the cluster" - ${EKSCTL_BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin - fi - if [[ "$WINDOWS" == true ]]; then - ${EKSCTL_BIN} create nodegroup \ - --managed=true \ - --ssh-access=false \ - --cluster="${CLUSTER_NAME}" \ - --node-ami-family=WindowsServer2022CoreContainer \ - --instance-types=m5.2xlarge \ - -n ng-windows \ - -m 3 \ - -M 3 - - kubectl apply --kubeconfig "${KUBECONFIG}" -f "$VPC_CONFIGMAP_FILE" + loudecho "Applying VPC ConfigMap (Windows only)" + kubectl apply --kubeconfig "${KUBECONFIG}" -f "${VPC_CONFIGMAP_FILE}" fi - - return $? } function eksctl_cluster_exists() { @@ -108,23 +79,3 @@ function eksctl_delete_cluster() { loudecho "Deleting cluster ${CLUSTER_NAME}" ${EKSCTL_BIN} delete cluster "${CLUSTER_NAME}" } - -function eksctl_patch_cluster_file() { - CLUSTER_FILE=${1} # input must be yaml - EKSCTL_PATCH_FILE=${2} # input must be yaml - - loudecho "Patching cluster $CLUSTER_NAME with $EKSCTL_PATCH_FILE" - - # Temporary intermediate files for patching - CLUSTER_FILE_0=$CLUSTER_FILE.0 - CLUSTER_FILE_1=$CLUSTER_FILE.1 - - cp "$CLUSTER_FILE" "$CLUSTER_FILE_0" - - # Patch only the Cluster - kubectl patch --kubeconfig "/dev/null" -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml >"$CLUSTER_FILE_1" - mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0" - - # Done patching, overwrite original CLUSTER_FILE - mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml -} diff --git a/hack/tools/install.sh b/hack/tools/install.sh index 47c88cdede..8107011931 100755 --- a/hack/tools/install.sh +++ b/hack/tools/install.sh @@ -26,6 +26,8 @@ EKSCTL_VERSION="v0.175.0" GINKGO_VERSION="v2.17.1" # https://github.com/golangci/golangci-lint GOLANGCI_LINT_VERSION="v1.57.2" +# https://github.com/hairyhenderson/gomplate +GOMPLATE_VERSION="v3.11.7" # https://github.com/helm/helm HELM_VERSION="v3.14.4" # https://github.com/kubernetes/kops @@ -124,6 +126,14 @@ function install_golangci-lint() { install_tar_binary "${INSTALL_PATH}" "https://github.com/golangci/golangci-lint/releases/download/${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}.tar.gz" "golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}/golangci-lint" } +function install_gomplate() { + INSTALL_PATH="${1}" + + # gomplate includes library from no longer existing domain inet.af, and thus cannot be installed via go install + # install the released binary from GitHub releases instead + install_binary "${INSTALL_PATH}" "https://github.com/hairyhenderson/gomplate/releases/download/${GOMPLATE_VERSION}/gomplate_${OS}-${ARCH}" "gomplate" +} + function install_helm() { INSTALL_PATH="${1}"