diff --git a/Makefile b/Makefile index 5b25f24dc11..b8764a9cac9 100644 --- a/Makefile +++ b/Makefile @@ -190,6 +190,7 @@ E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml SKIP_CLEANUP ?= false AZWI_SKIP_CLEANUP ?= false SKIP_LOG_COLLECTION ?= false +MGMT_CLUSTER_TYPE ?= kind # @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster # to be created with extra mounts for key pairs which is not yet supported # by existing e2e framework. A mgmt cluster(kind) is created as part of e2e suite @@ -717,7 +718,13 @@ test-cover: test ## Run tests with code coverage and generate reports. .PHONY: kind-create-bootstrap kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster. - KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh + if [ "$(MGMT_CLUSTER_TYPE)" == "aks" ]; then \ + MGMT_CLUSTER_NAME=capz-e2e \ + AKS_RESOURCE_GROUP=capz-e2e \ + ./scripts/aks-as-mgmt.sh; \ + else \ + KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh; \ + fi .PHONY: cleanup-workload-identity cleanup-workload-identity: ## Cleanup CI workload-identity infra diff --git a/scripts/aks-as-mgmt.sh b/scripts/aks-as-mgmt.sh index 1fffb9a08bc..c9bab872eca 100755 --- a/scripts/aks-as-mgmt.sh +++ b/scripts/aks-as-mgmt.sh @@ -217,6 +217,7 @@ kustomize_substitutions: ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE}" REGISTRY: "${REGISTRY}" APISERVER_LB_DNS_SUFFIX: "${APISERVER_LB_DNS_SUFFIX}" + AZURE_LOCATION: "${AZURE_LOCATION}" allowed_contexts: - "$MGMT_CLUSTER_NAME" - "kind-capz" diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 04498f7a1e5..aebb156b614 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -25,9 +25,6 @@ set -o pipefail # Install kubectl REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl" -KIND="${REPO_ROOT}/hack/tools/bin/kind" -make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}" # shellcheck source=hack/ensure-go.sh source "${REPO_ROOT}/hack/ensure-go.sh" diff --git a/scripts/peer-vnets.sh b/scripts/peer-vnets.sh new file mode 100644 index 00000000000..9cf75514bcf --- /dev/null +++ b/scripts/peer-vnets.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit # exit immediately if a command exits with a non-zero status. +set -o nounset # exit when script tries to use undeclared variables. +set -o pipefail # make the pipeline fail if any command in it fails. + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. + +source_tilt_settings() { + if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 + fi + + TILT_SETTINGS_FILE="$1" + + # Check that the file exists + if [ ! -f "$TILT_SETTINGS_FILE" ]; then + echo "File not found: $TILT_SETTINGS_FILE" + exit 1 + fi + + echo "Reading variables from $TILT_SETTINGS_FILE under 'kustomize_substition'..." + + # Get the list of keys under kustomize_substition + VAR_KEYS=$(yq e '.kustomize_substition | keys | .[]' "$TILT_SETTINGS_FILE" 2>/dev/null || true) + + # If there's no such key or it's empty, VAR_KEYS will be empty + if [ -z "$VAR_KEYS" ]; then + echo "No variables found under 'kustomize_substition'." + else + for key in $VAR_KEYS; do + # Read the value of each key + value=$(yq e ".kustomize_substition[\"$key\"]" "$TILT_SETTINGS_FILE") + # Export the key/value pair + export "$key=$value" + echo "Exported $key=$value" + done + fi + + echo "All variables exported" +} + + +peer_vnets() { + # ------------------------------------------------------------------------------ + # Peer Vnets + # ------------------------------------------------------------------------------ + + echo "--------Peering VNETs--------" + az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180 + export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv) + echo " 1/8 ${AKS_MGMT_VNET_NAME} found " + + # wait for workload VNet to be created + az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180 + export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv) + echo " 2/8 ${CLUSTER_NAME}-vnet found" + + # peer mgmt vnet + az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none + az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors --output none + echo " 3/8 mgmt-to-${CLUSTER_NAME} peering created in ${AKS_MGMT_VNET_NAME}" + + # peer workload vnet + az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none + az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors --output none + echo " 4/8 ${CLUSTER_NAME}-to-mgmt peering created in ${CLUSTER_NAME}-vnet" + + # create private DNS zone + az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors --output none + az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors --output none + echo " 5/8 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created in ${CLUSTER_NAME}" + + # link private DNS Zone to workload vnet + az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors --output none + az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors --output none + echo " 6/8 workload cluster vnet ${CLUSTER_NAME}-vnet linked with private DNS zone" + + # link private DNS Zone to mgmt vnet + az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors --output none + az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors --output none + echo " 7/8 management cluster vnet ${AKS_MGMT_VNET_NAME} linked with private DNS zone" + + # create private DNS zone record + az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --record-set-name \"@\" --ipv4-address ${AZURE_INTERNAL_LB_PRIVATE_IP} --only-show-errors --output none + echo " 8/8 \"@\" private DNS zone record created to point ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com to ${AZURE_INTERNAL_LB_PRIVATE_IP}" +} diff --git a/test/e2e/azure_apiserver_ilb.go b/test/e2e/azure_apiserver_ilb.go index 710b8224a5b..62b64c4381f 100644 --- a/test/e2e/azure_apiserver_ilb.go +++ b/test/e2e/azure_apiserver_ilb.go @@ -462,3 +462,7 @@ func AzureAPIServerILBSpec(ctx context.Context, inputGetter func() AzureAPIServe err = wait.ExponentialBackoffWithContext(ctx, backoff, retryDSFn) Expect(err).NotTo(HaveOccurred()) } + +func PeerVnets(ctx context.Context, inputGetter func() AzureAPIServerILBSpecInput) { + +} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 1e62b568ff4..96fc9b3c5ab 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -1180,6 +1180,11 @@ var _ = Describe("Workload cluster creation", func() { } }) }), + withPreWaitForCluster(func() { + // TODO: only invoke this in local runs + // Peer VNets of the mgmt cluster and workload cluster + + }), ), result) By("Probing workload cluster with APIServerILB feature gate", func() { diff --git a/test/e2e/common.go b/test/e2e/common.go index 466e5314228..943e65ed3f6 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -443,3 +443,9 @@ func withAzureCNIv1Manifest(manifestPath string) func(*clusterctl.ApplyClusterTe input.CNIManifestPath = manifestPath } } + +func withPreWaitForCluster(preWaitForCluster func()) func(*clusterctl.ApplyClusterTemplateAndWaitInput) { + return func(input *clusterctl.ApplyClusterTemplateAndWaitInput) { + input.PreWaitForCluster = preWaitForCluster + } +} diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index b8dafc2b604..73a7ed8f671 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -226,9 +226,9 @@ variables: CONFORMANCE_IMAGE: "${CONFORMANCE_IMAGE:-}" CONFORMANCE_NODES: "${CONFORMANCE_NODES:-1}" IP_FAMILY: "IPv4" - CLUSTER_IDENTITY_NAME: "cluster-identity-ci" - ASO_CREDENTIAL_SECRET_NAME: "aso-credentials" - ASO_CREDENTIAL_SECRET_MODE: workloadidentity + CLUSTER_IDENTITY_NAME: "${CLUSTER_IDENTITY_NAME:-cluster-identity-ci}" + ASO_CREDENTIAL_SECRET_NAME: "${ASO_CREDENTIAL_SECRET_NAME:-aso-credentials}" + ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE:-workloadidentity} NODE_DRAIN_TIMEOUT: "60s" CI_VERSION: "" KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" @@ -240,8 +240,8 @@ variables: LATEST_PROVIDER_UPGRADE_VERSION: "v1.17.1" OLD_CAAPH_UPGRADE_VERSION: "v0.1.0-alpha.10" LATEST_CAAPH_UPGRADE_VERSION: "v0.2.5" - CI_RG: capz-ci - USER_IDENTITY: cloud-provider-user-identity + CI_RG: "${CI_RG:-capz-ci}" + USER_IDENTITY: "${USER_IDENTITY:-cloud-provider-user-identity}" EXP_APISERVER_ILB: "true" intervals: