From bc41421de9e88cf9d02f5362313d6fe28dd418f1 Mon Sep 17 00:00:00 2001
From: Francesco Torta <62566275+fra98@users.noreply.github.com>
Date: Fri, 26 Jul 2024 10:37:58 +0200
Subject: [PATCH] Added AKS provider

---
 test/e2e/cruise/network/network_test.go       |  24 ++++
 test/e2e/pipeline/infra/aks/clean.sh          |  73 ++++++++++
 .../pipeline/infra/aks/pre-requirements.sh    |  57 ++++++++
 test/e2e/pipeline/infra/aks/setup.sh          | 136 ++++++++++++++++++
 test/e2e/pipeline/installer/liqoctl/setup.sh  |   7 +
 5 files changed, 297 insertions(+)
 create mode 100755 test/e2e/pipeline/infra/aks/clean.sh
 create mode 100755 test/e2e/pipeline/infra/aks/pre-requirements.sh
 create mode 100755 test/e2e/pipeline/infra/aks/setup.sh

diff --git a/test/e2e/cruise/network/network_test.go b/test/e2e/cruise/network/network_test.go
index c2caf3e622..82e3f5e9a9 100644
--- a/test/e2e/cruise/network/network_test.go
+++ b/test/e2e/cruise/network/network_test.go
@@ -99,6 +99,12 @@ var _ = BeforeSuite(func() {
 		ovverideArgsClusterAPI(&args)
 	case "kind":
 		overrideArgsKind(&args)
+	case "eks":
+		overrideArgsEKS(&args)
+	case "gke":
+		overrideArgsGKE(&args)
+	case "aks":
+		overrideArgsAKS(&args)
 	}
 })
 
@@ -211,6 +217,24 @@ func overrideArgsKind(args *networkTestsArgs) {
 	args.loadBalancer = false
 }
 
+func overrideArgsEKS(args *networkTestsArgs) {
+	args.failfast = false
+	args.loadBalancer = true
+	args.nodePortExt = false // nodeport are not exposed
+}
+
+func overrideArgsGKE(args *networkTestsArgs) {
+	args.failfast = false
+	args.loadBalancer = true
+	args.nodePortExt = false // nodeport are not exposed by default // TODO: modify GKE plugin to open nodeport firewall
+}
+
+func overrideArgsAKS(args *networkTestsArgs) {
+	args.failfast = false
+	args.loadBalancer = true
+	args.nodePortExt = false // nodeport are not exposed
+}
+
 func RestartPods(cl client.Client) {
 	podList := &corev1.PodList{}
 	Expect(
diff --git a/test/e2e/pipeline/infra/aks/clean.sh b/test/e2e/pipeline/infra/aks/clean.sh
new file mode 100755
index 0000000000..d6cf7f3ada
--- /dev/null
+++ b/test/e2e/pipeline/infra/aks/clean.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# This scripts expects the following variables to be set:
+# CLUSTER_NUMBER        -> the number of liqo clusters
+# K8S_VERSION           -> the Kubernetes version
+# CNI                   -> the CNI plugin used
+# TMPDIR                -> the directory where the test-related files are stored
+# BINDIR                -> the directory where the test-related binaries are stored
+# TEMPLATE_DIR          -> the directory where to read the cluster templates
+# NAMESPACE             -> the namespace where liqo is running
+# KUBECONFIGDIR         -> the directory where the kubeconfigs are stored
+# LIQO_VERSION          -> the liqo version to test
+# INFRA                 -> the Kubernetes provider for the infrastructure
+# LIQOCTL               -> the path where liqoctl is stored
+# KUBECTL               -> the path where kubectl is stored
+# POD_CIDR_OVERLAPPING  -> the pod CIDR of the clusters is overlapping
+# CLUSTER_TEMPLATE_FILE -> the file where the cluster template is stored
+
+set -e           # Fail in case of error
+set -o nounset   # Fail if undefined variables are used
+set -o pipefail  # Fail if one of the piped commands fails
+
+error() {
+   local sourcefile=$1
+   local lineno=$2
+   echo "An error occurred at $sourcefile:$lineno."
+}
+trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR
+
+CLUSTER_NAME=cluster
+RUNNER_NAME=${RUNNER_NAME:-"test"}
+CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}"
+
+PIDS=()
+
+# Cleaning all remaining clusters
+for i in $(seq 1 "${CLUSTER_NUMBER}")
+do
+    AKS_RESOURCE_GROUP="liqo${i}"
+    RUNNER_NAME=${RUNNER_NAME:-"test"}
+    AKS_CLUSTER_NAME="${RUNNER_NAME}-cluster${i}"
+
+    # if the cluster exists, delete it
+    if az aks show --resource-group "${AKS_RESOURCE_GROUP}" --name "${AKS_CLUSTER_NAME}" &> /dev/null; then
+        echo "Deleting cluster ${CLUSTER_NAME}${i}"
+        az aks delete --resource-group "${AKS_RESOURCE_GROUP}" --name "${AKS_CLUSTER_NAME}" --yes &
+        PIDS+=($!)
+    else
+        echo "Cluster ${CLUSTER_NAME}${i} does not exist"
+    fi
+done
+
+for PID in "${PIDS[@]}"; do
+    wait "${PID}"
+done
+
+# Cleaning the resource group
+for i in $(seq 1 "${CLUSTER_NUMBER}")
+do
+    AKS_RESOURCE_GROUP="liqo${i}"
+    rg_exists=$(az group exists --name "${AKS_RESOURCE_GROUP}")
+    if [[ $rg_exists == "true" ]]; then
+        echo "Deleting resource group ${AKS_RESOURCE_GROUP}"
+        az group delete --name "${AKS_RESOURCE_GROUP}" --yes &
+        PIDS+=($!)
+    else
+        echo "Resource group ${AKS_RESOURCE_GROUP} does not exist. Nothing to delete"
+    fi
+done
+
+for PID in "${PIDS[@]}"; do
+    wait "${PID}"
+done
diff --git a/test/e2e/pipeline/infra/aks/pre-requirements.sh b/test/e2e/pipeline/infra/aks/pre-requirements.sh
new file mode 100755
index 0000000000..a958749900
--- /dev/null
+++ b/test/e2e/pipeline/infra/aks/pre-requirements.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#shellcheck disable=SC1091
+
+# This scripts expects the following variables to be set:
+# CLUSTER_NUMBER        -> the number of liqo clusters
+# K8S_VERSION           -> the Kubernetes version
+# CNI                   -> the CNI plugin used
+# TMPDIR                -> the directory where the test-related files are stored
+# BINDIR                -> the directory where the test-related binaries are stored
+# TEMPLATE_DIR          -> the directory where to read the cluster templates
+# NAMESPACE             -> the namespace where liqo is running
+# KUBECONFIGDIR         -> the directory where the kubeconfigs are stored
+# LIQO_VERSION          -> the liqo version to test
+# INFRA                 -> the Kubernetes provider for the infrastructure
+# LIQOCTL               -> the path where liqoctl is stored
+# KUBECTL               -> the path where kubectl is stored
+# HELM                  -> the path where helm is stored
+# POD_CIDR_OVERLAPPING  -> the pod CIDR of the clusters is overlapping
+# CLUSTER_TEMPLATE_FILE -> the file where the cluster template is stored
+
+set -e           # Fail in case of error
+set -o nounset   # Fail if undefined variables are used
+set -o pipefail  # Fail if one of the piped commands fails
+
+error() {
+   local sourcefile=$1
+   local lineno=$2
+   echo "An error occurred at $sourcefile:$lineno."
+}
+trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR
+
+FILEPATH=$(realpath "$0")
+WORKDIR=$(dirname "$FILEPATH")
+
+source "$WORKDIR/../../utils.sh"
+
+setup_arch_and_os
+
+install_kubectl "${OS}" "${ARCH}" "${K8S_VERSION}"
+
+install_helm "${OS}" "${ARCH}"
+
+# Install AZ cli
+if ! command -v az &> /dev/null
+then
+    echo "Azure CLI could not be found. Downloading and installing..."
+    if [[ "${OS}" == "linux" ]]
+    then
+        curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+    elif [[ "${OS}" == "darwin" ]]
+    then
+        brew update && brew install azure-cli
+    fi
+fi
+
+echo "Azure CLI version:"
+az --version
diff --git a/test/e2e/pipeline/infra/aks/setup.sh b/test/e2e/pipeline/infra/aks/setup.sh
new file mode 100755
index 0000000000..473f995c0f
--- /dev/null
+++ b/test/e2e/pipeline/infra/aks/setup.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+#shellcheck disable=SC1091
+
+# This scripts expects the following variables to be set:
+# CLUSTER_NUMBER        -> the number of liqo clusters
+# K8S_VERSION           -> the Kubernetes version
+# CNI                   -> the CNI plugin used
+# TMPDIR                -> the directory where the test-related files are stored
+# BINDIR                -> the directory where the test-related binaries are stored
+# TEMPLATE_DIR          -> the directory where to read the cluster templates
+# NAMESPACE             -> the namespace where liqo is running
+# KUBECONFIGDIR         -> the directory where the kubeconfigs are stored
+# LIQO_VERSION          -> the liqo version to test
+# INFRA                 -> the Kubernetes provider for the infrastructure
+# LIQOCTL               -> the path where liqoctl is stored
+# KUBECTL               -> the path where kubectl is stored
+# HELM                  -> the path where helm is stored
+# POD_CIDR_OVERLAPPING  -> the pod CIDR of the clusters is overlapping
+# CLUSTER_TEMPLATE_FILE -> the file where the cluster template is stored
+
+set -e           # Fail in case of error
+set -o nounset   # Fail if undefined variables are used
+set -o pipefail  # Fail if one of the piped commands fails
+
+error() {
+   local sourcefile=$1
+   local lineno=$2
+   echo "An error occurred at $sourcefile:$lineno."
+}
+trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR
+
+FILEPATH=$(realpath "$0")
+WORKDIR=$(dirname "$FILEPATH")
+
+source "$WORKDIR/../../utils.sh"
+
+NUM_NODES="2"
+VM_TYPE="Standard_B2s"
+REGIONS=("italynorth" "francecentral" "germanywestcentral" "switzerlandnorth")
+
+POD_CIDR_OVERLAPPING=${POD_CIDR_OVERLAPPING:-"true"}
+CNI=${CNI:-"azure"} # "azure", "kubenet", "none"
+
+if [[ "${CNI}" == "azure" ]]; then
+    POD_CIDR_OVERLAPPING="true"
+fi
+
+function create_resource_group() {
+    local aks_resource_group=$1
+    local region=$2
+
+    rg_exists=$(az group exists --name "$aks_resource_group")
+    if  [[ $rg_exists == "false" ]]; then
+        echo "Creating resource group $aks_resource_group in region $region"
+        az group create \
+            --name "$aks_resource_group" \
+            --location "$region"
+    fi
+}
+
+function aks_create_cluster() {
+    local aks_resource_group=$1
+    local aks_resource_name=$2
+    local kubeconfig=$3
+    local pod_cidr=$4
+
+    args=()
+    args+=("--resource-group $aks_resource_group")
+    args+=("--name $aks_resource_name")
+    args+=("--node-count $NUM_NODES")
+    args+=("--node-vm-size $VM_TYPE")
+    args+=("--kubernetes-version $K8S_VERSION")
+    args+=("--tier free")
+    args+=("--generate-ssh-keys")
+
+    args+=("--network-plugin $CNI")
+    if [[ "${CNI}" == "kubenet" ]]; then
+        args+=("--pod-cidr $pod_cidr")
+    fi
+    
+    ARGS="${args[*]}"
+    eval "az aks create $ARGS"
+
+    az aks get-credentials \
+        --resource-group "$aks_resource_group" \
+        --name "$aks_resource_name" \
+        --file "$kubeconfig"
+}
+
+# Create the clusters
+PIDS=()
+
+for i in $(seq 1 "${CLUSTER_NUMBER}");
+do
+    AKS_RESOURCE_GROUP="liqo${i}"
+    RUNNER_NAME=${RUNNER_NAME:-"test"}
+    AKS_CLUSTER_NAME="${RUNNER_NAME}-cluster${i}"
+    REGION=${REGIONS[$i-1]}
+    KUBECONFIG="${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
+    
+    create_resource_group "${AKS_RESOURCE_GROUP}" "${REGION}"
+
+    # The PodCIDR can be set only for kubenet. On AzureCNI it is fixed, so only pod cidr overlapping is possible.
+    POD_CIDR=""
+    if [[ ${CNI} == "kubenet" ]]; then
+        if [[ ${POD_CIDR_OVERLAPPING} == "true" ]]; then
+            POD_CIDR="10.50.0.0/16"
+        else
+            POD_CIDR="10.$((i * 10)).0.0/16"
+        fi
+    fi
+
+	aks_create_cluster "${AKS_RESOURCE_GROUP}" "${AKS_CLUSTER_NAME}" "${KUBECONFIG}" "${POD_CIDR}" &
+    PIDS+=($!)
+done
+
+for PID in "${PIDS[@]}"; do
+    wait "${PID}"
+done
+
+# Install needed utilities
+PIDS=()
+
+for i in $(seq 1 "${CLUSTER_NUMBER}");
+do
+    KUBECONFIG="${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
+    
+    # Install kyverno for network tests
+    install_kyverno "$KUBECONFIG" &
+
+    PIDS+=($!)
+done
+
+for PID in "${PIDS[@]}"; do
+    wait "${PID}"
+done
diff --git a/test/e2e/pipeline/installer/liqoctl/setup.sh b/test/e2e/pipeline/installer/liqoctl/setup.sh
index 1a19b58352..e9804299ea 100755
--- a/test/e2e/pipeline/installer/liqoctl/setup.sh
+++ b/test/e2e/pipeline/installer/liqoctl/setup.sh
@@ -16,6 +16,7 @@
 # HELM                  -> the path where helm is stored
 # POD_CIDR_OVERLAPPING  -> the pod CIDR of the clusters is overlapping
 # CLUSTER_TEMPLATE_FILE -> the file where the cluster template is stored
+# AZ_SUBSCRIPTION_ID    -> the ID of the Azure subscription to use (only for AKS)
 
 set -e           # Fail in case of error
 set -o nounset   # Fail if undefined variables are used
@@ -90,6 +91,12 @@ do
     fi
     set -u
   fi
+  if [[ "${INFRA}" == "aks" ]]; then
+    AKS_RESOURCE_GROUP="liqo${i}"
+    RUNNER_NAME=${RUNNER_NAME:-"test"}
+    AKS_CLUSTER_NAME="${RUNNER_NAME}-cluster${i}"
+    COMMON_ARGS=("${COMMON_ARGS[@]}" --subscription-id "${AZ_SUBSCRIPTION_ID}" --resource-group-name "${AKS_RESOURCE_GROUP}" --resource-name "${AKS_CLUSTER_NAME}")
+  fi
   if [[ "${INFRA}" == "cluster-api" ]]; then
     LIQO_PROVIDER="kubeadm"
     COMMON_ARGS=("${COMMON_ARGS[@]}")