diff --git a/apply-terraform.yml b/apply-terraform.yml index 954f5cdf..c979c446 100644 --- a/apply-terraform.yml +++ b/apply-terraform.yml @@ -19,11 +19,35 @@ stages: - job: TerraformInitPlan displayName: "Terraform Init and Plan" steps: + - task: AzureCLI@2 + displayName: "Azure CLI Login" + inputs: + azureSubscription: $(serviceConnectionName) + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + echo "Successfully logged in with Azure CLI" + + - task: AzureCLI@2 + displayName: "Get AKS Credentials" + inputs: + azureSubscription: $(serviceConnectionName) + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + az aks get-credentials \ + --name $(aks-name) \ + --resource-group $(aks-resource-group) \ + --admin \ + --file ~/.kube/config + echo "AKS credentials configured for kubectl" + - task: DownloadSecureFile@1 name: DownloadSecureVars - displayName: 'Download staging.tfvars' + displayName: "Download staging.tfvars" inputs: - secureFile: 'staging.tfvars' + secureFile: "staging.tfvars" + - task: TerraformTaskV2@2 displayName: Terra Init inputs: @@ -35,12 +59,13 @@ stages: backendAzureRmStorageAccountName: "$(storageAccountName)" backendAzureRmContainerName: "$(containerName)" backendAzureRmKey: "$(stateKey)" + - task: TerraformTaskV2@2 displayName: Terra Plan inputs: provider: "azurerm" command: "plan" - commandOptions: '-var-file=$(DownloadSecureVars.SecureFilePath)' + commandOptions: "-var-file=$(DownloadSecureVars.SecureFilePath)" workingDirectory: "$(System.DefaultWorkingDirectory)/terraform/staging" environmentServiceNameAzureRM: "$(serviceConnectionName)" @@ -59,11 +84,36 @@ stages: deploy: steps: - checkout: self + + - task: AzureCLI@2 + displayName: "Azure CLI Login" + inputs: + azureSubscription: $(serviceConnectionName) + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + echo "Successfully logged in with Azure CLI" + + - task: AzureCLI@2 + displayName: "Get AKS Credentials" + inputs: + azureSubscription: $(serviceConnectionName) + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + az aks get-credentials \ + --name $(aks-name) \ + --resource-group $(aks-resource-group) \ + --admin \ + --file ~/.kube/config + echo "AKS credentials configured for kubectl" + - task: DownloadSecureFile@1 name: DownloadSecureVars - displayName: 'Download staging.tfvars' + displayName: "Download staging.tfvars" inputs: - secureFile: 'staging.tfvars' + secureFile: "staging.tfvars" + - task: TerraformTaskV2@2 displayName: Terra Init inputs: @@ -80,6 +130,6 @@ stages: inputs: provider: "azurerm" command: "apply" - commandOptions: '-var-file=$(DownloadSecureVars.SecureFilePath)' + commandOptions: "-var-file=$(DownloadSecureVars.SecureFilePath)" workingDirectory: "$(System.DefaultWorkingDirectory)/terraform/staging" environmentServiceNameAzureRM: "$(serviceConnectionName)" diff --git a/kubernetes/system/vault/vault.yml b/kubernetes/system/vault/vault.yml index ca2a2c4d..380d58fa 100644 --- a/kubernetes/system/vault/vault.yml +++ b/kubernetes/system/vault/vault.yml @@ -826,15 +826,23 @@ server: } } - seal "gcpckms" { - project = "spartan-rhino-408115" - region = "northamerica-northeast1" - key_ring = "vault-keyring-gcp" - crypto_key = "vault-cryptokey-gcp" + + seal "azurekeyvault" { + tenant_id = "YOUR-AZURE-TENANT-ID" + vault_name = "staging-vault + key_name = "staging-vault-unseal" + resource = "https://managedhsm.azure.net" } service_registration "kubernetes" {} + # unseal for gcp kms setup + # seal "gcpckms" { + # project = "spartan-rhino-408115" + # region = "northamerica-northeast1" + # key_ring = "vault-keyring-gcp" + # crypto_key = "vault-cryptokey-gcp" + # } # A disruption budget limits the number of pods of a replicated application # that are down simultaneously from voluntary disruptions disruptionBudget: diff --git a/terraform/modules/azure-cluster-network/versions.tf b/terraform/modules/azure-cluster-network/providers.tf similarity index 100% rename from terraform/modules/azure-cluster-network/versions.tf rename to terraform/modules/azure-cluster-network/providers.tf diff --git a/terraform/modules/azure-kubernetes-cluster/main.tf b/terraform/modules/azure-kubernetes-cluster/main.tf index 0c70dd40..972ea865 100644 --- a/terraform/modules/azure-kubernetes-cluster/main.tf +++ b/terraform/modules/azure-kubernetes-cluster/main.tf @@ -35,6 +35,12 @@ resource "azurerm_kubernetes_cluster" "k8s" { pod_cidr = var.pod_cidr } + lifecycle { + ignore_changes = [ + default_node_pool + ] + } + tags = var.tags sku_tier = var.sku_tier diff --git a/terraform/modules/azure-kubernetes-cluster/output.tf b/terraform/modules/azure-kubernetes-cluster/output.tf index 3cebd6d8..9dd4f4bf 100644 --- a/terraform/modules/azure-kubernetes-cluster/output.tf +++ b/terraform/modules/azure-kubernetes-cluster/output.tf @@ -2,3 +2,13 @@ output "cluster_name" { description = "Cluster name to be used in the context of kubectl" value = azurerm_kubernetes_cluster.k8s.name } + +output "cluster_ca_certificate" { + description = "Cluster certificate" + value = azurerm_kubernetes_cluster.k8s.kube_config[0].cluster_ca_certificate +} + +output "cluster_principal_id" { + description = "AKS cluster principal id" + value = azurerm_kubernetes_cluster.k8s.identity[0].principal_id +} diff --git a/terraform/modules/azure-kubernetes-cluster/versions.tf b/terraform/modules/azure-kubernetes-cluster/providers.tf similarity index 81% rename from terraform/modules/azure-kubernetes-cluster/versions.tf rename to terraform/modules/azure-kubernetes-cluster/providers.tf index 46c15132..d326b78b 100644 --- a/terraform/modules/azure-kubernetes-cluster/versions.tf +++ b/terraform/modules/azure-kubernetes-cluster/providers.tf @@ -11,10 +11,6 @@ terraform { source = "hashicorp/local" version = "2.4.0" } - tls = { - source = "hashicorp/tls" - version = "4.0.4" - } azuread = { source = "hashicorp/azuread" version = "2.47.0" diff --git a/terraform/modules/azure-kubernetes-cluster/resources.tf b/terraform/modules/azure-kubernetes-cluster/resources.tf index 9379497f..ab9546da 100644 --- a/terraform/modules/azure-kubernetes-cluster/resources.tf +++ b/terraform/modules/azure-kubernetes-cluster/resources.tf @@ -9,8 +9,3 @@ data "azurerm_subnet" "subnet" { virtual_network_name = var.network_vnet resource_group_name = var.network_resource_group } - - -resource "tls_private_key" "pair" { - algorithm = "RSA" -} diff --git a/terraform/modules/gcp-kubernetes-cluster/provider.tf b/terraform/modules/gcp-kubernetes-cluster/providers.tf similarity index 100% rename from terraform/modules/gcp-kubernetes-cluster/provider.tf rename to terraform/modules/gcp-kubernetes-cluster/providers.tf diff --git a/terraform/modules/vault/data.tf b/terraform/modules/vault/data.tf new file mode 100644 index 00000000..f59eb0e3 --- /dev/null +++ b/terraform/modules/vault/data.tf @@ -0,0 +1,13 @@ +# This requires that jq and openssl are installed in the runtime environment +# It creates a certificate signing request (CSR) based on the vault-csr.conf file +# The 2 jq at the beginning and end of the pipes are used to read the input and wrap the result in json +# since this is how terraform "external" passes data. +data "external" "k8s_cert_request" { + program = [ + "bash", "-c", + "curl -o jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && chmod +x jq && jq -rc '.key' | openssl req -new -noenc -config ${path.module}/vault-csr.conf -key /dev/stdin | jq -rRncs '{\"request\": inputs}'" + ] + query = { + "key" = tls_private_key.pair.private_key_pem + } +} diff --git a/terraform/modules/vault/main.tf b/terraform/modules/vault/main.tf new file mode 100644 index 00000000..10ad0440 --- /dev/null +++ b/terraform/modules/vault/main.tf @@ -0,0 +1,103 @@ +data "azurerm_client_config" "current" {} + +# Needed to provide unique vault name +resource "random_string" "azurerm_key_vault_name" { + length = 5 + numeric = true + special = false + upper = false +} + +locals { + suffix = random_string.azurerm_key_vault_name.result +} + +# Create Azure Key Vault +resource "azurerm_key_vault" "vault" { + name = "${var.prefix}-vault-${local.suffix}" + location = var.location + resource_group_name = var.resource_group + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = "standard" + soft_delete_retention_days = 7 + purge_protection_enabled = false + + access_policy { + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azurerm_client_config.current.object_id + + key_permissions = var.key_permissions + secret_permissions = var.secret_permissions + } +} + +resource "azurerm_key_vault_key" "vault_unseal" { + name = "${var.prefix}-vault-unseal-${local.suffix}" + key_vault_id = azurerm_key_vault.vault.id + key_type = var.key_type + key_size = var.key_size + key_opts = var.key_opts + + # Define the rotation policy + rotation_policy { + automatic { + time_before_expiry = "P7D" + } + + expire_after = "P28D" + notify_before_expiry = "P7D" + } +} + +# Set an access policy for the service principal to the Key Vault +resource "azurerm_key_vault_access_policy" "vault" { + key_vault_id = azurerm_key_vault.vault.id + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = var.cluster_principal_id + + key_permissions = var.aks_key_permissions + secret_permissions = var.aks_secret_permissions +} + +# We make ask Kubernetes to sign the certificate +# https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/ +resource "kubernetes_certificate_signing_request_v1" "vault_kube_cert_req" { + metadata { + name = "vault.svc" + } + spec { + request = data.external.k8s_cert_request.result["request"] + signer_name = "kubernetes.io/kubelet-serving" + usages = ["digital signature", "key encipherment", "server auth"] + } + auto_approve = true + lifecycle { + ignore_changes = [spec[0].request] + replace_triggered_by = [tls_private_key.pair] + } +} + +# Makes sure the vault namespace is created before adding secrets +resource "kubernetes_namespace" "vault_ns" { + metadata { + name = "vault" + labels = { + "pod-security.kubernetes.io/enforce" = "privileged" + } + } +} + +resource "kubernetes_secret" "vault_tls" { + metadata { + name = "vault-tls" + namespace = kubernetes_namespace.vault_ns.metadata[0].name + } + + data = { + "vault.crt" = kubernetes_certificate_signing_request_v1.vault_kube_cert_req.certificate + "vault.key" = tls_private_key.pair.private_key_pem + "vault.ca" = var.ca_cluster + } + + type = "kubernetes.io/generic" +} diff --git a/terraform/modules/vault/providers.tf b/terraform/modules/vault/providers.tf new file mode 100644 index 00000000..e0f0d73a --- /dev/null +++ b/terraform/modules/vault/providers.tf @@ -0,0 +1,35 @@ +terraform { + + required_version = ">= 1.7.2" + + required_providers { + local = { + source = "hashicorp/local" + version = "2.4.0" + } + tls = { + source = "hashicorp/tls" + version = "4.0.4" + } + external = { + source = "hashicorp/external" + version = "2.3.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.24.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.25" + } + azuread = { + source = "hashicorp/azuread" + version = "2.47.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} diff --git a/terraform/modules/vault/resources.tf b/terraform/modules/vault/resources.tf new file mode 100644 index 00000000..f91a4263 --- /dev/null +++ b/terraform/modules/vault/resources.tf @@ -0,0 +1,4 @@ +resource "tls_private_key" "pair" { + algorithm = "RSA" + rsa_bits = 2048 +} diff --git a/terraform/modules/vault/variables.tf b/terraform/modules/vault/variables.tf new file mode 100644 index 00000000..d040bd44 --- /dev/null +++ b/terraform/modules/vault/variables.tf @@ -0,0 +1,70 @@ +variable "prefix" { + description = "(Required) Base name used by resources (cluster name, main service and others)." + type = string +} + +variable "location" { + description = "(Required) Specifies the supported Azure location where the resource exists. Changing this forces a new resource to be created." + type = string +} + +variable "ca_cluster" { + description = "CA certificate of the AKS cluster" + type = string +} + +variable "cluster_principal_id" { + description = "AKS cluster principal id" + type = string +} + +variable "resource_group" { + description = "(Required) Specifies the Resource Group where the Managed Kubernetes Cluster should exist. Changing this forces a new resource to be created." + type = string +} + +variable "key_permissions" { + type = list(string) + description = "List of key permissions." + default = ["List", "Create", "Delete", "Get", "Purge", "Recover", "Update", "GetRotationPolicy", "SetRotationPolicy"] +} + +variable "secret_permissions" { + type = list(string) + description = "List of secret permissions." + default = ["Set"] +} + +variable "key_type" { + description = "The JsonWebKeyType of the key to be created." + default = "RSA" + type = string + validation { + condition = contains(["EC", "EC-HSM", "RSA", "RSA-HSM"], var.key_type) + error_message = "The key_type must be one of the following: EC, EC-HSM, RSA, RSA-HSM." + } +} + +variable "key_opts" { + type = list(string) + description = "The permitted JSON web key operations of the key to be created." + default = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey"] +} + +variable "key_size" { + type = number + description = "The size in bits of the key to be created." + default = 2048 +} + +variable "aks_key_permissions" { + type = list(string) + description = "List of key permissions for aks." + default = ["Get", "List", "Update", "Create", "Import", "Delete", "Recover", "Backup", "Restore", "Decrypt", "Encrypt", "UnwrapKey", "WrapKey", "Verify", "Sign"] +} + +variable "aks_secret_permissions" { + type = list(string) + description = "List of secret permissions for aks." + default = ["Get"] +} diff --git a/terraform/modules/vault/vault-csr.conf b/terraform/modules/vault/vault-csr.conf new file mode 100644 index 00000000..0e39b79a --- /dev/null +++ b/terraform/modules/vault/vault-csr.conf @@ -0,0 +1,22 @@ +[req] +default_bits = 2048 +prompt = no +encrypt_key = yes +default_md = sha256 +distinguished_name = kubelet_serving +req_extensions = v3_req +[ kubelet_serving ] +O = system:nodes +CN = system:node:*.vault.svc.cluster.local +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment, dataEncipherment +extendedKeyUsage = serverAuth, clientAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = *.vault-internal +DNS.2 = *.vault-internal.vault.svc.cluster.local +DNS.3 = *.vault +DNS.4 = vault.vault.svc.cluster.local +DNS.5 = vault.vault.svc +IP.1 = 127.0.0.1 diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 5beb720a..de4247b9 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -113,3 +113,17 @@ module "aks-cluster-0" { # #... # } + +module "vault" { + source = "../modules/vault" + location = var.location_1 + resource_group = azurerm_resource_group.rg.name + prefix = var.environment + + cluster_principal_id = module.aks-cluster-0.cluster_principal_id + ca_cluster = module.aks-cluster-0.cluster_ca_certificate + + providers = { + kubernetes = kubernetes + } +} diff --git a/terraform/staging/versions.tf b/terraform/staging/providers.tf similarity index 89% rename from terraform/staging/versions.tf rename to terraform/staging/providers.tf index 63dfbe1c..9da27435 100644 --- a/terraform/staging/versions.tf +++ b/terraform/staging/providers.tf @@ -25,3 +25,7 @@ terraform { } } } + +provider "kubernetes" { + config_path = "~/.kube/config" +} diff --git a/terraform/staging/variables.tf b/terraform/staging/variables.tf index b16017b9..054a1207 100644 --- a/terraform/staging/variables.tf +++ b/terraform/staging/variables.tf @@ -14,6 +14,12 @@ variable "resource_group" { type = string } +variable "environment" { + type = string + description = "Name of the deployment environment" + default = "staging" +} + variable "aks_name" { description = "AKS cluster name" type = string