wip of running e2e tests locally

Makefile

@@ -190,6 +190,7 @@ E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
 SKIP_CLEANUP ?= false
 AZWI_SKIP_CLEANUP ?= false
 SKIP_LOG_COLLECTION ?= false
+MGMT_CLUSTER_TYPE ?= kind
 # @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster
 # to be created with extra mounts for key pairs which is not yet supported
 # by existing e2e framework. A mgmt cluster(kind) is created as part of e2e suite
@@ -717,7 +718,13 @@ test-cover: test ## Run tests with code coverage and generate reports.
 
 .PHONY: kind-create-bootstrap
 kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
-	KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh
+	if [ "$(MGMT_CLUSTER_TYPE)" == "aks" ]; then \
+		MGMT_CLUSTER_NAME=capz-e2e \
+		AKS_RESOURCE_GROUP=capz-e2e \
+		./scripts/aks-as-mgmt.sh; \
+	else \
+		KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh; \
+	fi
 
 .PHONY: cleanup-workload-identity
 cleanup-workload-identity: ## Cleanup CI workload-identity infra

scripts/aks-as-mgmt.sh

@@ -217,6 +217,7 @@ kustomize_substitutions:
   ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE}"
   REGISTRY: "${REGISTRY}"
   APISERVER_LB_DNS_SUFFIX: "${APISERVER_LB_DNS_SUFFIX}"
+  AZURE_LOCATION: "${AZURE_LOCATION}"
 allowed_contexts:
   - "$MGMT_CLUSTER_NAME"
   - "kind-capz"

scripts/ci-e2e.sh

@@ -25,9 +25,6 @@ set -o pipefail
 
 # Install kubectl
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
-KIND="${REPO_ROOT}/hack/tools/bin/kind"
-make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}"
 
 # shellcheck source=hack/ensure-go.sh
 source "${REPO_ROOT}/hack/ensure-go.sh"

scripts/peer-vnets.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit # exit immediately if a command exits with a non-zero status.
+set -o nounset # exit when script tries to use undeclared variables.
+set -o pipefail # make the pipeline fail if any command in it fails.
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+
+source_tilt_settings() {
+  if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <tilt-settings.yaml>"
+    exit 1
+  fi
+
+  TILT_SETTINGS_FILE="$1"
+
+  # Check that the file exists
+  if [ ! -f "$TILT_SETTINGS_FILE" ]; then
+    echo "File not found: $TILT_SETTINGS_FILE"
+    exit 1
+  fi
+
+  echo "Reading variables from $TILT_SETTINGS_FILE under 'kustomize_substition'..."
+
+  # Get the list of keys under kustomize_substition
+  VAR_KEYS=$(yq e '.kustomize_substition | keys | .[]' "$TILT_SETTINGS_FILE" 2>/dev/null || true)
+
+  # If there's no such key or it's empty, VAR_KEYS will be empty
+  if [ -z "$VAR_KEYS" ]; then
+    echo "No variables found under 'kustomize_substition'."
+  else
+    for key in $VAR_KEYS; do
+      # Read the value of each key
+      value=$(yq e ".kustomize_substition[\"$key\"]" "$TILT_SETTINGS_FILE")
+      # Export the key/value pair
+      export "$key=$value"
+      echo "Exported $key=$value"
+    done
+  fi
+
+  echo "All variables exported"
+}
+
+
+peer_vnets() {
+  # ------------------------------------------------------------------------------
+  # Peer Vnets
+  # ------------------------------------------------------------------------------
+
+  echo "--------Peering VNETs--------"
+  az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180
+  export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv)
+  echo " 1/8 ${AKS_MGMT_VNET_NAME} found "
+
+  # wait for workload VNet to be created
+  az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180
+  export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv)
+  echo " 2/8 ${CLUSTER_NAME}-vnet found"
+
+  # peer mgmt vnet
+  az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none
+  az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors --output none
+  echo " 3/8 mgmt-to-${CLUSTER_NAME} peering created in ${AKS_MGMT_VNET_NAME}"
+
+  # peer workload vnet
+  az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none
+  az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors --output none
+  echo " 4/8 ${CLUSTER_NAME}-to-mgmt peering created in ${CLUSTER_NAME}-vnet"
+
+  # create private DNS zone
+  az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors --output none
+  az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors --output none
+  echo " 5/8 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created in ${CLUSTER_NAME}"
+
+  # link private DNS Zone to workload vnet
+  az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors --output none
+  az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors --output none
+  echo " 6/8 workload cluster vnet ${CLUSTER_NAME}-vnet linked with private DNS zone"
+
+  # link private DNS Zone to mgmt vnet
+  az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors --output none
+  az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors --output none
+  echo " 7/8 management cluster vnet ${AKS_MGMT_VNET_NAME} linked with private DNS zone"
+
+  # create private DNS zone record
+  az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --record-set-name \"@\" --ipv4-address ${AZURE_INTERNAL_LB_PRIVATE_IP} --only-show-errors --output none
+  echo " 8/8 \"@\" private DNS zone record created to point ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com to ${AZURE_INTERNAL_LB_PRIVATE_IP}"
+}

test/e2e/azure_apiserver_ilb.go

@@ -462,3 +462,7 @@ func AzureAPIServerILBSpec(ctx context.Context, inputGetter func() AzureAPIServe
 	err = wait.ExponentialBackoffWithContext(ctx, backoff, retryDSFn)
 	Expect(err).NotTo(HaveOccurred())
 }
+
+func PeerVnets(ctx context.Context, inputGetter func() AzureAPIServerILBSpecInput) {
+
+}

test/e2e/azure_test.go

@@ -1180,6 +1180,11 @@ var _ = Describe("Workload cluster creation", func() {
 						}
 					})
 				}),
+				withPreWaitForCluster(func() {
+					// TODO: only invoke this in local runs
+					// Peer VNets of the mgmt cluster and workload cluster
+
+				}),
 			), result)
 
 			By("Probing workload cluster with APIServerILB feature gate", func() {

test/e2e/common.go

@@ -443,3 +443,9 @@ func withAzureCNIv1Manifest(manifestPath string) func(*clusterctl.ApplyClusterTe
 		input.CNIManifestPath = manifestPath
 	}
 }
+
+func withPreWaitForCluster(preWaitForCluster func()) func(*clusterctl.ApplyClusterTemplateAndWaitInput) {
+	return func(input *clusterctl.ApplyClusterTemplateAndWaitInput) {
+		input.PreWaitForCluster = preWaitForCluster
+	}
+}

test/e2e/config/azure-dev.yaml

@@ -226,9 +226,9 @@ variables:
   CONFORMANCE_IMAGE: "${CONFORMANCE_IMAGE:-}"
   CONFORMANCE_NODES: "${CONFORMANCE_NODES:-1}"
   IP_FAMILY: "IPv4"
-  CLUSTER_IDENTITY_NAME: "cluster-identity-ci"
-  ASO_CREDENTIAL_SECRET_NAME: "aso-credentials"
-  ASO_CREDENTIAL_SECRET_MODE: workloadidentity
+  CLUSTER_IDENTITY_NAME: "${CLUSTER_IDENTITY_NAME:-cluster-identity-ci}"
+  ASO_CREDENTIAL_SECRET_NAME: "${ASO_CREDENTIAL_SECRET_NAME:-aso-credentials}"
+  ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE:-workloadidentity}
   NODE_DRAIN_TIMEOUT: "60s"
   CI_VERSION: ""
   KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml"
@@ -240,8 +240,8 @@ variables:
   LATEST_PROVIDER_UPGRADE_VERSION: "v1.17.1"
   OLD_CAAPH_UPGRADE_VERSION: "v0.1.0-alpha.10"
   LATEST_CAAPH_UPGRADE_VERSION: "v0.2.5"
-  CI_RG: capz-ci
-  USER_IDENTITY: cloud-provider-user-identity
+  CI_RG: "${CI_RG:-capz-ci}"
+  USER_IDENTITY: "${USER_IDENTITY:-cloud-provider-user-identity}"
   EXP_APISERVER_ILB: "true"
 
 intervals: