AKS Backup scenario in Terraform

.gitignore

@@ -48,20 +48,20 @@ override.tf.json
 
 # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
 # example: *tfplan*
-*.tfvars
+# *.tfvars
 *.hcl
 *.vs
 
 !.vscode/cspell.json #workspace spelling should be source controlled
 
 # Ignore any .tfvars files that are generated automatically for each Terraform run
-**/*.tfvars
+# **/*.tfvars
 **/*.tfvars.json
 
 *tfplan.json
 **/tfplan.json
-*tfplan
-**/tfplan
+# *tfplan
+# **/tfplan
 
 **/.terraform
 **/*.tfstate

Scenarios/AKS-Backup/Readme.md

@@ -0,0 +1,29 @@
+# Scenario: creating a backup of an AKS cluster
+
+## Introduction
+
+This scenario shows how to create a backup of an existing AKS cluster and then use it to restore to a new cluster.
+
+## Steps to follow
+
+1. Deploy the AKS Landing Zone by following the steps in folder `Scenarios/AKS-Landing-Zone`.
+2. Provide the required input parameters in the `Scenarios/AKS-Backup/terraform.tfvars` file.
+3. Run the current Terraform templates to create a backup of the AKS cluster and a new cluster for restore:
+
+```sh
+cd Scenarios/AKS-Backup
+terraform init
+terraform apply
+```
+
+4. Deploy a kubernetes deployment and persistent volumes to the original AKS cluster.
+
+```sh
+kubectl apply -f ./kubernetes/deploy_disk_lrs.yaml
+kubectl apply -f ./kubernetes/deploy_disk_zrs_sc.yaml
+```
+
+5. Trigger the backup of the AKS cluster on the azure portal through Vault backup.
+> You might need to click on the 'fix errors' button to resolve the backup misconfigurations.
+6. Watch for the resources created in the backup resource group (disk snapshots and blobs in the backup storage account).
+7. Trigger the restore operation to the new AKS cluster using the backup created in the previous step.

Scenarios/AKS-Backup/aks-1.tf

@@ -0,0 +1,15 @@
+data "azurerm_kubernetes_cluster" "aks-1" {
+  name                = var.aksName
+  resource_group_name = var.rgLzName
+}
+
+data "azurerm_user_assigned_identity" "aks1_identity" {
+  name                = split("/", data.azurerm_kubernetes_cluster.aks-1.identity.0.identity_ids[0])[8]
+  resource_group_name = var.rgLzName
+}
+
+resource "azurerm_role_assignment" "cluster_msi_contributor_on_snap_rg" {
+  scope                = azurerm_resource_group.rg-backup.id
+  role_definition_name = "Contributor"
+  principal_id         = data.azurerm_user_assigned_identity.aks1_identity.principal_id
+}

Scenarios/AKS-Backup/aks-2.tf

@@ -0,0 +1,73 @@
+resource "azurerm_kubernetes_cluster" "aks-2" {
+  name                = "aks-cluster"
+  location            = azurerm_resource_group.rg-2.location
+  resource_group_name = azurerm_resource_group.rg-2.name
+  dns_prefix          = "aks"
+  kubernetes_version  = "1.30.5"
+
+  network_profile {
+    network_plugin      = "azure"
+    network_plugin_mode = "overlay"
+    # subnet_id           = azurerm_subnet.snet-aks-2.id
+  }
+
+  default_node_pool {
+    name                        = "systempool"
+    temporary_name_for_rotation = "syspool"
+    node_count                  = 3
+    vm_size                     = "standard_b2als_v2"
+    zones                       = [1, 2, 3]
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  lifecycle {
+    ignore_changes = [
+      default_node_pool.0.upgrade_settings
+    ]
+  }
+}
+
+
+
+resource "azurerm_role_assignment" "cluster_2_msi_contributor_on_snap_rg" {
+  scope                = azurerm_resource_group.rg-backup.id
+  role_definition_name = "Contributor"
+  principal_id         = azurerm_kubernetes_cluster.aks-2.identity.0.principal_id
+}
+
+resource "azurerm_kubernetes_cluster_extension" "extension-2" {
+  name              = "backup-extension"
+  cluster_id        = azurerm_kubernetes_cluster.aks-2.id
+  extension_type    = "Microsoft.DataProtection.Kubernetes"
+  release_train     = "stable"
+  release_namespace = "dataprotection-microsoft"
+  configuration_settings = {
+    "configuration.backupStorageLocation.bucket"                = azurerm_storage_container.container.name
+    "configuration.backupStorageLocation.config.storageAccount" = azurerm_storage_account.storage.name
+    "configuration.backupStorageLocation.config.resourceGroup"  = azurerm_storage_account.storage.resource_group_name
+    "configuration.backupStorageLocation.config.subscriptionId" = data.azurerm_client_config.current.subscription_id
+    "credentials.tenantId"                                      = data.azurerm_client_config.current.tenant_id
+  }
+}
+
+resource "azurerm_role_assignment" "extension_2_storage_account_contributor" {
+  scope                = azurerm_storage_account.storage.id
+  role_definition_name = "Storage Account Contributor"
+  principal_id         = azurerm_kubernetes_cluster_extension.extension-2.aks_assigned_identity[0].principal_id
+}
+
+resource "azurerm_kubernetes_cluster_trusted_access_role_binding" "aks_2_trusted_access" {
+  kubernetes_cluster_id = azurerm_kubernetes_cluster.aks-2.id
+  name                  = "trusted-access"
+  roles                 = ["Microsoft.DataProtection/backupVaults/backup-operator"]
+  source_resource_id    = azurerm_data_protection_backup_vault.backup-vault.id
+}
+
+resource "azurerm_role_assignment" "vault_msi_read_on_cluster_2" {
+  scope                = azurerm_kubernetes_cluster.aks-2.id
+  role_definition_name = "Reader"
+  principal_id         = azurerm_data_protection_backup_vault.backup-vault.identity[0].principal_id
+}

Scenarios/AKS-Backup/aks-backup-extension.tf

@@ -0,0 +1,22 @@
+resource "azurerm_kubernetes_cluster_extension" "extension-1" {
+  name              = "backup-extension"
+  cluster_id        = data.azurerm_kubernetes_cluster.aks-1.id
+  extension_type    = "Microsoft.DataProtection.Kubernetes"
+  release_train     = "stable"
+  release_namespace = "dataprotection-microsoft"
+  configuration_settings = {
+    "configuration.backupStorageLocation.bucket"                = azurerm_storage_container.container.name
+    "configuration.backupStorageLocation.config.storageAccount" = azurerm_storage_account.storage.name
+    "configuration.backupStorageLocation.config.resourceGroup"  = azurerm_storage_account.storage.resource_group_name
+    "configuration.backupStorageLocation.config.subscriptionId" = data.azurerm_client_config.current.subscription_id
+    "credentials.tenantId"                                      = data.azurerm_client_config.current.tenant_id
+  }
+}
+
+resource "azurerm_role_assignment" "extension_1_storage_account_contributor" {
+  scope                = azurerm_storage_account.storage.id
+  role_definition_name = "Storage Account Contributor"
+  principal_id         = azurerm_kubernetes_cluster_extension.extension-1.aks_assigned_identity[0].principal_id
+}
+
+data "azurerm_client_config" "current" {}

Scenarios/AKS-Backup/backup-instance.tf

@@ -0,0 +1,28 @@
+resource "azurerm_data_protection_backup_instance_kubernetes_cluster" "backup_instance_aks" {
+  name                         = "backup-instance-aks"
+  location                     = azurerm_resource_group.rg-backup.location
+  vault_id                     = azurerm_data_protection_backup_vault.backup-vault.id
+  kubernetes_cluster_id        = data.azurerm_kubernetes_cluster.aks-1.id
+  snapshot_resource_group_name = azurerm_resource_group.rg-backup.name
+  backup_policy_id             = azurerm_data_protection_backup_policy_kubernetes_cluster.backup_policy_aks.id
+
+  backup_datasource_parameters {
+    # excluded_namespaces              = ["ns1"]
+    # excluded_resource_types          = ["exvolumesnapshotcontents.snapshot.storage.k8s.io"]
+    cluster_scoped_resources_enabled = true
+    # included_namespaces              = ["*"] # ["test-included-namespaces"]
+    # included_resource_types          = ["*"] # ["involumesnapshotcontents.snapshot.storage.k8s.io"]
+    # label_selectors                  = ["*"] # ["kubernetes.io/metadata.name:test"]
+    volume_snapshot_enabled = true
+  }
+
+  depends_on = [
+    azurerm_role_assignment.extension_1_storage_account_contributor,
+    azurerm_role_assignment.vault_msi_read_on_cluster,
+    azurerm_role_assignment.vault_msi_read_on_snap_rg,
+    azurerm_role_assignment.cluster_msi_contributor_on_snap_rg,
+    azurerm_role_assignment.vault_msi_snapshot_contributor_on_snap_rg,
+    azurerm_role_assignment.vault_msi_data_operator_on_snap_rg,
+    azurerm_role_assignment.vault_msi_data_contributor_on_storage,
+  ]
+}

Scenarios/AKS-Backup/backup-policy.tf

@@ -0,0 +1,31 @@
+resource "azurerm_data_protection_backup_policy_kubernetes_cluster" "backup_policy_aks" {
+  name                = "backup-policy-aks"
+  resource_group_name = azurerm_data_protection_backup_vault.backup-vault.resource_group_name
+  vault_name          = azurerm_data_protection_backup_vault.backup-vault.name
+
+  backup_repeating_time_intervals = ["R/2023-05-23T02:30:00+00:00/P1W"]
+
+  retention_rule {
+    name     = "Daily"
+    priority = 25
+
+    life_cycle {
+      duration        = "P84D"
+      data_store_type = "OperationalStore"
+    }
+
+    criteria {
+      days_of_week           = ["Thursday"]
+      months_of_year         = ["November"]
+      weeks_of_month         = ["First"]
+      scheduled_backup_times = ["2023-05-23T02:30:00Z"]
+    }
+  }
+
+  default_retention_rule {
+    life_cycle {
+      duration        = "P14D"
+      data_store_type = "OperationalStore"
+    }
+  }
+}

Scenarios/AKS-Backup/backup-vault.tf

@@ -0,0 +1,44 @@
+resource "azurerm_data_protection_backup_vault" "backup-vault" {
+  name                = "backup-vault"
+  resource_group_name = azurerm_resource_group.rg-backup.name
+  location            = azurerm_resource_group.rg-backup.location
+  datastore_type      = "VaultStore"
+  redundancy          = "LocallyRedundant" # `GeoRedundant`
+  # cross_region_restore_enabled = "false" # can only be specified when `redundancy` is specified for `GeoRedundant`
+  soft_delete                = "Off"
+  retention_duration_in_days = 14
+
+  identity {
+    type = "SystemAssigned"
+  }
+}
+
+resource "azurerm_role_assignment" "vault_msi_read_on_cluster" {
+  scope                = data.azurerm_kubernetes_cluster.aks-1.id # var.aksClusterId
+  role_definition_name = "Reader"
+  principal_id         = azurerm_data_protection_backup_vault.backup-vault.identity[0].principal_id
+}
+
+resource "azurerm_role_assignment" "vault_msi_read_on_snap_rg" {
+  scope                = azurerm_resource_group.rg-backup.id
+  role_definition_name = "Reader"
+  principal_id         = azurerm_data_protection_backup_vault.backup-vault.identity[0].principal_id
+}
+
+resource "azurerm_role_assignment" "vault_msi_snapshot_contributor_on_snap_rg" {
+  scope                = azurerm_resource_group.rg-backup.id
+  role_definition_name = "Disk Snapshot Contributor"
+  principal_id         = azurerm_data_protection_backup_vault.backup-vault.identity[0].principal_id
+}
+
+resource "azurerm_role_assignment" "vault_msi_data_operator_on_snap_rg" {
+  scope                = azurerm_resource_group.rg-backup.id
+  role_definition_name = "Data Operator for Managed Disks"
+  principal_id         = azurerm_data_protection_backup_vault.backup-vault.identity[0].principal_id
+}
+
+resource "azurerm_role_assignment" "vault_msi_data_contributor_on_storage" {
+  scope                = azurerm_storage_account.storage.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = azurerm_data_protection_backup_vault.backup-vault.identity[0].principal_id
+}

Scenarios/AKS-Backup/input.tf

@@ -0,0 +1,17 @@
+variable "location1" {
+  default = "swedencentral"
+}
+
+variable "location2" {
+  default = "swedencentral"
+}
+
+variable "aksName" {
+  type    = string
+  default = ""
+}
+
+variable "rgLzName" {
+  type    = string
+  default = "AksTerra-AVM-LZ-RG"
+}

Scenarios/AKS-Backup/kubernetes/deploy_disk_lrs.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-lrs
+spec:
+  selector:
+    matchLabels:
+      app: nginx-lrs
+  template:
+    metadata:
+      labels:
+        app: nginx-lrs
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        command:
+        - "/bin/sh"
+        - "-c"
+        - while true; do echo $(date) >> /mnt/azuredisk/outfile; sleep 60; done
+        volumeMounts:
+          - name: azuredisk-lrs
+            mountPath: "/mnt/azuredisk"
+      volumes:
+        - name: azuredisk-lrs
+          persistentVolumeClaim:
+            claimName: azure-managed-disk-lrs
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: azure-managed-disk-lrs
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: managed-csi
+  resources:
+    requests:
+      storage: 5Gi

Scenarios/AKS-Backup/kubernetes/deploy_disk_zrs_sc.yaml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-zrs
+spec:
+  selector:
+    matchLabels:
+      app: nginx-zrs
+  template:
+    metadata:
+      labels:
+        app: nginx-zrs
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        command:
+        - "/bin/sh"
+        - "-c"
+        - while true; do echo $(date) >> /mnt/azuredisk/outfile; sleep 60; done
+        volumeMounts:
+          - name: azuredisk-zrs
+            mountPath: "/mnt/azuredisk"
+      volumes:
+        - name: azuredisk-zrs
+          persistentVolumeClaim:
+            claimName: azure-managed-disk-zrs
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: managed-csi-zrs
+parameters:
+  skuname: StandardSSD_ZRS
+provisioner: disk.csi.azure.com
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: azure-managed-disk-zrs
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: managed-csi-zrs
+  resources:
+    requests:
+      storage: 5Gi