From 404eea36f309245de54648afa0e014e015b9681c Mon Sep 17 00:00:00 2001 From: viniciusdc Date: Mon, 16 Dec 2024 13:09:58 -0300 Subject: [PATCH 1/4] Add Azure AAD object schema --- src/_nebari/stages/infrastructure/__init__.py | 24 +++++++++++++++++++ .../infrastructure/template/azure/main.tf | 1 + .../template/azure/modules/kubernetes/main.tf | 13 ++++++++++ .../azure/modules/kubernetes/variables.tf | 14 +++++++++++ .../template/azure/variables.tf | 14 +++++++++++ 5 files changed, 66 insertions(+) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 243abd1608..4ce5943d09 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -90,6 +90,27 @@ class AzureNodeGroupInputVars(schema.Base): max_nodes: int +class AzureRBAC(schema.Base): + """ + Represents the configuration for Azure Active Directory Role-Based Access Control + (RBAC) integration in a Kubernetes cluster. + + Attributes: + enabled (bool): Indicates whether Azure AD-based Role-Based Access Control is + enabled. + managed (bool): Specifies if the Azure AD integration is managed by Azure. + When set to True, Azure creates and manages the Service Principal used for + integration. + admin_group_object_ids (List[str]): A list of Object IDs of Azure AD groups assigned + administrative roles on the cluster. This property is only applicable when + `managed` is set to True. + """ + + enabled: bool + managed: bool + admin_group_object_ids: List[str] + + class AzureInputVars(schema.Base): name: str environment: str @@ -105,6 +126,7 @@ class AzureInputVars(schema.Base): max_pods: Optional[int] = None network_profile: Optional[Dict[str, str]] = None workload_identity_enabled: bool = False + azure_rbac: Optional[AzureRBAC] = None class AWSAmiTypes(str, enum.Enum): @@ -370,6 +392,7 @@ class AzureProvider(schema.Base): network_profile: Optional[Dict[str, str]] = None max_pods: Optional[int] = None workload_identity_enabled: bool = False + azure_rbac: Optional[AzureRBAC] = None @model_validator(mode="before") @classmethod @@ -809,6 +832,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): network_profile=self.config.azure.network_profile, max_pods=self.config.azure.max_pods, workload_identity_enabled=self.config.azure.workload_identity_enabled, + azure_rbac=self.config.azure.azure_rbac, ).model_dump() elif self.config.provider == schema.ProviderEnum.aws: return AWSInputVars( diff --git a/src/_nebari/stages/infrastructure/template/azure/main.tf b/src/_nebari/stages/infrastructure/template/azure/main.tf index 2d6e2e2afa..41f35f1bf3 100644 --- a/src/_nebari/stages/infrastructure/template/azure/main.tf +++ b/src/_nebari/stages/infrastructure/template/azure/main.tf @@ -43,4 +43,5 @@ module "kubernetes" { vnet_subnet_id = var.vnet_subnet_id private_cluster_enabled = var.private_cluster_enabled workload_identity_enabled = var.workload_identity_enabled + azure_rbac = var.azure_rbac } diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf index f093f048c6..a1760aa1cc 100644 --- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf +++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf @@ -1,3 +1,7 @@ +data "azure_client_config" "current" { + count = var.azure_rbac.enabled ? 1 : 0 +} + # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster resource "azurerm_kubernetes_cluster" "main" { name = var.name @@ -61,6 +65,15 @@ resource "azurerm_kubernetes_cluster" "main" { ] } + dynamic "azure_active_directory_role_based_access_control" { + for_each = var.azure_rbac.enabled ? [var.azure_rbac] : [] + content { + azure_rbac_enabled = var.azure_rbac.azure_rbac_enabled + admin_group_object_ids = var.azure_rbac.admin_group_object_ids + tenant_id = data.azure_client_config.current[0].tenant_id + managed = var.azure_rbac.managed + } + } } # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf index b93a9fae2d..b46e18976c 100644 --- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf +++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf @@ -76,3 +76,17 @@ variable "workload_identity_enabled" { type = bool default = false } + +variable "azure_rbac" { + description = "Azure Active Directory Role-Based Access Control (RBAC) integration in a Kubernetes cluster" + type = object({ + enabled : bool + managed : bool + admin_group_object_ids : list(string) + }) + default = { + enabled : false + managed : false + admin_group_object_ids : [] + } +} diff --git a/src/_nebari/stages/infrastructure/template/azure/variables.tf b/src/_nebari/stages/infrastructure/template/azure/variables.tf index dcef2c97cb..22f149f3c6 100644 --- a/src/_nebari/stages/infrastructure/template/azure/variables.tf +++ b/src/_nebari/stages/infrastructure/template/azure/variables.tf @@ -82,3 +82,17 @@ variable "workload_identity_enabled" { type = bool default = false } + +variable "azure_rbac" { + description = "Azure Active Directory Role-Based Access Control (RBAC) integration in a Kubernetes cluster" + type = object({ + enabled : bool + managed : bool + admin_group_object_ids : list(string) + }) + default = { + enabled : false + managed : false + admin_group_object_ids : [] + } +} From aaadaa70ece1517707a55ecfa8cd458a3cb2e6d6 Mon Sep 17 00:00:00 2001 From: viniciusdc Date: Mon, 16 Dec 2024 13:10:58 -0300 Subject: [PATCH 2/4] update kubernetes output creds based on azure rbac --- .../azure/modules/kubernetes/outputs.tf | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf index e96187bcd6..0f9a826e03 100644 --- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf +++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf @@ -1,21 +1,20 @@ output "credentials" { - description = "Credentials required for connecting to kubernetes cluster" + description = "Credentials required for connecting to Kubernetes cluster" sensitive = true value = { - # see bottom of https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster - endpoint = azurerm_kubernetes_cluster.main.kube_config.0.host - username = azurerm_kubernetes_cluster.main.kube_config.0.username - password = azurerm_kubernetes_cluster.main.kube_config.0.password - client_certificate = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.client_certificate) - client_key = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.client_key) - cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate) + endpoint = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.host : azurerm_kubernetes_cluster.main.kube_config.0.host + username = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.username : azurerm_kubernetes_cluster.main.kube_config.0.username + password = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.password : azurerm_kubernetes_cluster.main.kube_config.0.password + client_certificate = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.client_certificate : azurerm_kubernetes_cluster.main.kube_config.0.client_certificate + client_key = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.client_key : azurerm_kubernetes_cluster.main.kube_config.0.client_key + cluster_ca_certificate = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.cluster_ca_certificate : azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate } } output "kubeconfig" { description = "Kubernetes connection kubeconfig" sensitive = true - value = azurerm_kubernetes_cluster.main.kube_config_raw + value = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config_raw : azurerm_kubernetes_cluster.main.kube_config_raw } output "cluster_oidc_issuer_url" { From f4e67a8bfaa47d3e59eaa862f3973e57dde59566 Mon Sep 17 00:00:00 2001 From: vinicius douglas cerutti Date: Mon, 16 Dec 2024 23:39:10 -0300 Subject: [PATCH 3/4] clean k8s credential handling and fix variable names --- src/_nebari/stages/infrastructure/__init__.py | 2 +- .../azure/modules/credentials/outputs.tf | 12 ++++++++++++ .../azure/modules/credentials/variables.tf | 16 ++++++++++++++++ .../template/azure/modules/kubernetes/main.tf | 8 ++++---- .../azure/modules/kubernetes/outputs.tf | 18 +++++++++--------- .../azure/modules/kubernetes/variables.tf | 4 ++-- .../infrastructure/template/azure/variables.tf | 4 ++-- 7 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 src/_nebari/stages/infrastructure/template/azure/modules/credentials/outputs.tf create mode 100644 src/_nebari/stages/infrastructure/template/azure/modules/credentials/variables.tf diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 4ce5943d09..122e62b119 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -107,7 +107,7 @@ class AzureRBAC(schema.Base): """ enabled: bool - managed: bool + managed_identity: bool admin_group_object_ids: List[str] diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/credentials/outputs.tf b/src/_nebari/stages/infrastructure/template/azure/modules/credentials/outputs.tf new file mode 100644 index 0000000000..0ceac66d76 --- /dev/null +++ b/src/_nebari/stages/infrastructure/template/azure/modules/credentials/outputs.tf @@ -0,0 +1,12 @@ +output "credentials" { + description = "Credentials required for connecting to Kubernetes cluster" + sensitive = true + value = { + endpoint = var.azure_rbac_enabled ? var.kube_admin_config.host : var.kube_config.host + username = var.azure_rbac_enabled ? var.kube_admin_config.username : var.kube_config.username + password = var.azure_rbac_enabled ? var.kube_admin_config.password : var.kube_config.password + client_certificate = var.azure_rbac_enabled ? base64decode(var.kube_admin_config.client_certificate) : base64decode(var.kube_config.client_certificate) + client_key = var.azure_rbac_enabled ? base64decode(var.kube_admin_config.client_key) : base64decode(var.kube_config.client_key) + cluster_ca_certificate = var.azure_rbac_enabled ? base64decode(var.kube_admin_config.cluster_ca_certificate) : base64decode(var.kube_config.cluster_ca_certificate) + } +} diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/credentials/variables.tf b/src/_nebari/stages/infrastructure/template/azure/modules/credentials/variables.tf new file mode 100644 index 0000000000..cf1916cb7e --- /dev/null +++ b/src/_nebari/stages/infrastructure/template/azure/modules/credentials/variables.tf @@ -0,0 +1,16 @@ +variable "azure_rbac_enabled" { + description = "Flag to enable Azure RBAC" + type = bool +} + +variable "kube_admin_config" { + description = "Kube admin config for RBAC" + type = any + sensitive = true +} + +variable "kube_config" { + description = "Kube config for standard access" + type = any + sensitive = true +} diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf index a1760aa1cc..2bae2dff23 100644 --- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf +++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf @@ -1,4 +1,4 @@ -data "azure_client_config" "current" { +data "azurerm_client_config" "current" { count = var.azure_rbac.enabled ? 1 : 0 } @@ -68,10 +68,10 @@ resource "azurerm_kubernetes_cluster" "main" { dynamic "azure_active_directory_role_based_access_control" { for_each = var.azure_rbac.enabled ? [var.azure_rbac] : [] content { - azure_rbac_enabled = var.azure_rbac.azure_rbac_enabled + azure_rbac_enabled = var.azure_rbac.enabled admin_group_object_ids = var.azure_rbac.admin_group_object_ids - tenant_id = data.azure_client_config.current[0].tenant_id - managed = var.azure_rbac.managed + tenant_id = data.azurerm_client_config.current[0].tenant_id + managed = var.azure_rbac.managed_identity } } } diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf index 0f9a826e03..463b1518c0 100644 --- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf +++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf @@ -1,20 +1,20 @@ +module "k8s_credentials" { + source = "../credentials" + azure_rbac_enabled = var.azure_rbac.enabled + kube_admin_config = azurerm_kubernetes_cluster.main.kube_admin_config[0] + kube_config = azurerm_kubernetes_cluster.main.kube_config[0] +} + output "credentials" { description = "Credentials required for connecting to Kubernetes cluster" sensitive = true - value = { - endpoint = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.host : azurerm_kubernetes_cluster.main.kube_config.0.host - username = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.username : azurerm_kubernetes_cluster.main.kube_config.0.username - password = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.password : azurerm_kubernetes_cluster.main.kube_config.0.password - client_certificate = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.client_certificate : azurerm_kubernetes_cluster.main.kube_config.0.client_certificate - client_key = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.client_key : azurerm_kubernetes_cluster.main.kube_config.0.client_key - cluster_ca_certificate = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config.0.cluster_ca_certificate : azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate - } + value = module.k8s_credentials.credentials } output "kubeconfig" { description = "Kubernetes connection kubeconfig" sensitive = true - value = var.azure_rbac.enable ? azurerm_kubernetes_cluster.main.kube_admin_config_raw : azurerm_kubernetes_cluster.main.kube_config_raw + value = var.azure_rbac.enabled ? azurerm_kubernetes_cluster.main.kube_admin_config_raw : azurerm_kubernetes_cluster.main.kube_config_raw } output "cluster_oidc_issuer_url" { diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf index b46e18976c..74be51c062 100644 --- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf +++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf @@ -81,12 +81,12 @@ variable "azure_rbac" { description = "Azure Active Directory Role-Based Access Control (RBAC) integration in a Kubernetes cluster" type = object({ enabled : bool - managed : bool + managed_identity : bool admin_group_object_ids : list(string) }) default = { enabled : false - managed : false + managed_identity : false admin_group_object_ids : [] } } diff --git a/src/_nebari/stages/infrastructure/template/azure/variables.tf b/src/_nebari/stages/infrastructure/template/azure/variables.tf index 22f149f3c6..5f06abf814 100644 --- a/src/_nebari/stages/infrastructure/template/azure/variables.tf +++ b/src/_nebari/stages/infrastructure/template/azure/variables.tf @@ -87,12 +87,12 @@ variable "azure_rbac" { description = "Azure Active Directory Role-Based Access Control (RBAC) integration in a Kubernetes cluster" type = object({ enabled : bool - managed : bool + managed_identity : bool admin_group_object_ids : list(string) }) default = { enabled : false - managed : false + managed_identity : false admin_group_object_ids : [] } } From cee4476febe8dc180a6dd6cd669e3f7923bd554b Mon Sep 17 00:00:00 2001 From: viniciusdc Date: Tue, 17 Dec 2024 14:54:15 -0300 Subject: [PATCH 4/4] disallow nullable variable values for azure_rbac --- .../template/azure/modules/kubernetes/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf index 74be51c062..7837f5b171 100644 --- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf +++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf @@ -89,4 +89,5 @@ variable "azure_rbac" { managed_identity : false admin_group_object_ids : [] } + nullable = false }