Merge pull request #47 from ai-cfia/45-as-a-devops-i-would-like-a-vau…

…lt-integration-for-azure-aks-cluster

Issue #45: Adds vault module

apply-terraform.yml

@@ -19,11 +19,35 @@ stages:
       - job: TerraformInitPlan
         displayName: "Terraform Init and Plan"
         steps:
+          - task: AzureCLI@2
+            displayName: "Azure CLI Login"
+            inputs:
+              azureSubscription: $(serviceConnectionName)
+              scriptType: "bash"
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                echo "Successfully logged in with Azure CLI"
+
+          - task: AzureCLI@2
+            displayName: "Get AKS Credentials"
+            inputs:
+              azureSubscription: $(serviceConnectionName)
+              scriptType: "bash"
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                az aks get-credentials \
+                  --name $(aks-name) \
+                  --resource-group $(aks-resource-group) \
+                  --admin \
+                  --file ~/.kube/config
+                echo "AKS credentials configured for kubectl"
+
           - task: DownloadSecureFile@1
             name: DownloadSecureVars
-            displayName: 'Download staging.tfvars'
+            displayName: "Download staging.tfvars"
             inputs:
-              secureFile: 'staging.tfvars'
+              secureFile: "staging.tfvars"
+
           - task: TerraformTaskV2@2
             displayName: Terra Init
             inputs:
@@ -35,12 +59,13 @@ stages:
               backendAzureRmStorageAccountName: "$(storageAccountName)"
               backendAzureRmContainerName: "$(containerName)"
               backendAzureRmKey: "$(stateKey)"
+
           - task: TerraformTaskV2@2
             displayName: Terra Plan
             inputs:
               provider: "azurerm"
               command: "plan"
-              commandOptions: '-var-file=$(DownloadSecureVars.SecureFilePath)'
+              commandOptions: "-var-file=$(DownloadSecureVars.SecureFilePath)"
               workingDirectory: "$(System.DefaultWorkingDirectory)/terraform/staging"
               environmentServiceNameAzureRM: "$(serviceConnectionName)"
 
@@ -59,11 +84,36 @@ stages:
             deploy:
               steps:
                 - checkout: self
+
+                - task: AzureCLI@2
+                  displayName: "Azure CLI Login"
+                  inputs:
+                    azureSubscription: $(serviceConnectionName)
+                    scriptType: "bash"
+                    scriptLocation: "inlineScript"
+                    inlineScript: |
+                      echo "Successfully logged in with Azure CLI"
+
+                - task: AzureCLI@2
+                  displayName: "Get AKS Credentials"
+                  inputs:
+                    azureSubscription: $(serviceConnectionName)
+                    scriptType: "bash"
+                    scriptLocation: "inlineScript"
+                    inlineScript: |
+                      az aks get-credentials \
+                        --name $(aks-name) \
+                        --resource-group $(aks-resource-group) \
+                        --admin \
+                        --file ~/.kube/config
+                      echo "AKS credentials configured for kubectl"
+
                 - task: DownloadSecureFile@1
                   name: DownloadSecureVars
-                  displayName: 'Download staging.tfvars'
+                  displayName: "Download staging.tfvars"
                   inputs:
-                    secureFile: 'staging.tfvars'
+                    secureFile: "staging.tfvars"
+
                 - task: TerraformTaskV2@2
                   displayName: Terra Init
                   inputs:
@@ -80,6 +130,6 @@ stages:
                   inputs:
                     provider: "azurerm"
                     command: "apply"
-                    commandOptions: '-var-file=$(DownloadSecureVars.SecureFilePath)'
+                    commandOptions: "-var-file=$(DownloadSecureVars.SecureFilePath)"
                     workingDirectory: "$(System.DefaultWorkingDirectory)/terraform/staging"
                     environmentServiceNameAzureRM: "$(serviceConnectionName)"

kubernetes/system/vault/vault.yml

@@ -826,15 +826,23 @@ server:
           }
         }
 
-        seal "gcpckms" {
-          project     = "spartan-rhino-408115"
-          region      = "northamerica-northeast1"
-          key_ring    = "vault-keyring-gcp"
-          crypto_key  = "vault-cryptokey-gcp"
+
+        seal "azurekeyvault" {
+          tenant_id      = "YOUR-AZURE-TENANT-ID"
+          vault_name     = "staging-vault
+          key_name       = "staging-vault-unseal"
+          resource       = "https://managedhsm.azure.net"
         }
 
         service_registration "kubernetes" {}
 
+    # unseal for gcp kms setup
+    # seal "gcpckms" {
+    #       project     = "spartan-rhino-408115"
+    #       region      = "northamerica-northeast1"
+    #       key_ring    = "vault-keyring-gcp"
+    #       crypto_key  = "vault-cryptokey-gcp"
+    # }
     # A disruption budget limits the number of pods of a replicated application
     # that are down simultaneously from voluntary disruptions
     disruptionBudget:

terraform/modules/azure-kubernetes-cluster/main.tf

@@ -35,6 +35,12 @@ resource "azurerm_kubernetes_cluster" "k8s" {
     pod_cidr       = var.pod_cidr
   }
 
+  lifecycle {
+    ignore_changes = [
+      default_node_pool
+    ]
+  }
+
   tags = var.tags
 
   sku_tier = var.sku_tier

terraform/modules/azure-kubernetes-cluster/output.tf

@@ -2,3 +2,13 @@ output "cluster_name" {
   description = "Cluster name to be used in the context of kubectl"
   value       = azurerm_kubernetes_cluster.k8s.name
 }
+
+output "cluster_ca_certificate" {
+  description = "Cluster certificate"
+  value       = azurerm_kubernetes_cluster.k8s.kube_config[0].cluster_ca_certificate
+}
+
+output "cluster_principal_id" {
+  description = "AKS cluster principal id"
+  value       = azurerm_kubernetes_cluster.k8s.identity[0].principal_id
+}

terraform/modules/azure-kubernetes-cluster/versions.tf → terraform/modules/azure-kubernetes-cluster/providers.tf

@@ -11,10 +11,6 @@ terraform {
       source  = "hashicorp/local"
       version = "2.4.0"
     }
-    tls = {
-      source  = "hashicorp/tls"
-      version = "4.0.4"
-    }
     azuread = {
       source  = "hashicorp/azuread"
       version = "2.47.0"

terraform/modules/azure-kubernetes-cluster/resources.tf

@@ -9,8 +9,3 @@ data "azurerm_subnet" "subnet" {
   virtual_network_name = var.network_vnet
   resource_group_name  = var.network_resource_group
 }
-
-
-resource "tls_private_key" "pair" {
-  algorithm = "RSA"
-}

terraform/modules/vault/data.tf

@@ -0,0 +1,13 @@
+# This requires that jq and openssl are installed in the runtime environment
+# It creates a certificate signing request (CSR) based on the vault-csr.conf file
+# The 2 jq at the beginning and end of the pipes are used to read the input and wrap the result in json
+# since this is how terraform "external" passes data.
+data "external" "k8s_cert_request" {
+  program = [
+    "bash", "-c",
+    "curl -o jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && chmod +x jq && jq -rc '.key' | openssl req -new -noenc -config ${path.module}/vault-csr.conf -key /dev/stdin | jq -rRncs '{\"request\": inputs}'"
+  ]
+  query = {
+    "key" = tls_private_key.pair.private_key_pem
+  }
+}

terraform/modules/vault/main.tf

@@ -0,0 +1,103 @@
+data "azurerm_client_config" "current" {}
+
+# Needed to provide unique vault name
+resource "random_string" "azurerm_key_vault_name" {
+  length  = 5
+  numeric = true
+  special = false
+  upper   = false
+}
+
+locals {
+  suffix = random_string.azurerm_key_vault_name.result
+}
+
+# Create Azure Key Vault
+resource "azurerm_key_vault" "vault" {
+  name                       = "${var.prefix}-vault-${local.suffix}"
+  location                   = var.location
+  resource_group_name        = var.resource_group
+  tenant_id                  = data.azurerm_client_config.current.tenant_id
+  sku_name                   = "standard"
+  soft_delete_retention_days = 7
+  purge_protection_enabled   = false
+
+  access_policy {
+    tenant_id = data.azurerm_client_config.current.tenant_id
+    object_id = data.azurerm_client_config.current.object_id
+
+    key_permissions    = var.key_permissions
+    secret_permissions = var.secret_permissions
+  }
+}
+
+resource "azurerm_key_vault_key" "vault_unseal" {
+  name         = "${var.prefix}-vault-unseal-${local.suffix}"
+  key_vault_id = azurerm_key_vault.vault.id
+  key_type     = var.key_type
+  key_size     = var.key_size
+  key_opts     = var.key_opts
+
+  # Define the rotation policy
+  rotation_policy {
+    automatic {
+      time_before_expiry = "P7D"
+    }
+
+    expire_after         = "P28D"
+    notify_before_expiry = "P7D"
+  }
+}
+
+# Set an access policy for the service principal to the Key Vault
+resource "azurerm_key_vault_access_policy" "vault" {
+  key_vault_id = azurerm_key_vault.vault.id
+  tenant_id    = data.azurerm_client_config.current.tenant_id
+  object_id    = var.cluster_principal_id
+
+  key_permissions    = var.aks_key_permissions
+  secret_permissions = var.aks_secret_permissions
+}
+
+# We make ask Kubernetes to sign the certificate
+# https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/
+resource "kubernetes_certificate_signing_request_v1" "vault_kube_cert_req" {
+  metadata {
+    name = "vault.svc"
+  }
+  spec {
+    request     = data.external.k8s_cert_request.result["request"]
+    signer_name = "kubernetes.io/kubelet-serving"
+    usages      = ["digital signature", "key encipherment", "server auth"]
+  }
+  auto_approve = true
+  lifecycle {
+    ignore_changes       = [spec[0].request]
+    replace_triggered_by = [tls_private_key.pair]
+  }
+}
+
+# Makes sure the vault namespace is created before adding secrets
+resource "kubernetes_namespace" "vault_ns" {
+  metadata {
+    name = "vault"
+    labels = {
+      "pod-security.kubernetes.io/enforce" = "privileged"
+    }
+  }
+}
+
+resource "kubernetes_secret" "vault_tls" {
+  metadata {
+    name      = "vault-tls"
+    namespace = kubernetes_namespace.vault_ns.metadata[0].name
+  }
+
+  data = {
+    "vault.crt" = kubernetes_certificate_signing_request_v1.vault_kube_cert_req.certificate
+    "vault.key" = tls_private_key.pair.private_key_pem
+    "vault.ca"  = var.ca_cluster
+  }
+
+  type = "kubernetes.io/generic"
+}

terraform/modules/vault/providers.tf

@@ -0,0 +1,35 @@
+terraform {
+
+  required_version = ">= 1.7.2"
+
+  required_providers {
+    local = {
+      source  = "hashicorp/local"
+      version = "2.4.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = "4.0.4"
+    }
+    external = {
+      source  = "hashicorp/external"
+      version = "2.3.1"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.24.0"
+    }
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~> 3.25"
+    }
+    azuread = {
+      source  = "hashicorp/azuread"
+      version = "2.47.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~>3.0"
+    }
+  }
+}

terraform/modules/vault/resources.tf

@@ -0,0 +1,4 @@
+resource "tls_private_key" "pair" {
+  algorithm = "RSA"
+  rsa_bits  = 2048
+}