From 64b96bc64ff90c6cf310ae3d3f4646a614712617 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari <48932219+adityachoudhari26@users.noreply.github.com> Date: Wed, 26 Jun 2024 16:30:13 -0700 Subject: [PATCH 01/27] fix: Change ingress timeout to string (#80) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index fd2b0af..409963e 100644 --- a/main.tf +++ b/main.tf @@ -287,7 +287,7 @@ module "wandb" { "kubernetes.io/ingress.class" = "azure/application-gateway" "cert-manager.io/cluster-issuer" = "cert-issuer" "cert-manager.io/acme-challenge-type" = "http01" - "appgw.ingress.kubernetes.io/request-timeout" = 300 + "appgw.ingress.kubernetes.io/request-timeout" = "300" } tls = [ From b616fbdfad5fcdc8619a5e4788005f772e4fe55d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 26 Jun 2024 23:30:42 +0000 Subject: [PATCH 02/27] chore(release): version 2.11.2 [skip ci] ### [2.11.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.1...v2.11.2) (2024-06-26) ### Bug Fixes * Change ingress timeout to string ([#80](https://github.com/wandb/terraform-azurerm-wandb/issues/80)) ([64b96bc](https://github.com/wandb/terraform-azurerm-wandb/commit/64b96bc64ff90c6cf310ae3d3f4646a614712617)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 209713b..d98205c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [2.11.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.1...v2.11.2) (2024-06-26) + + +### Bug Fixes + +* Change ingress timeout to string ([#80](https://github.com/wandb/terraform-azurerm-wandb/issues/80)) ([64b96bc](https://github.com/wandb/terraform-azurerm-wandb/commit/64b96bc64ff90c6cf310ae3d3f4646a614712617)) + ### [2.11.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.0...v2.11.1) (2024-06-25) From 06066020ac57d2d93e406e4b103feffb260426e0 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Thu, 11 Jul 2024 07:29:40 -0700 Subject: [PATCH 03/27] fix: Pass cloudprovider value to the helm charts (#83) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 409963e..8c2bccb 100644 --- a/main.tf +++ b/main.tf @@ -237,7 +237,7 @@ module "wandb" { global = { host = local.url license = var.license - + cloudProvider = "azure" bucket = var.external_bucket == null ? { provider = "az" name = local.storage_account From 97643c95c9212ac9060459b446eedabe6d085f08 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 11 Jul 2024 14:30:08 +0000 Subject: [PATCH 04/27] chore(release): version 2.11.3 [skip ci] ### [2.11.3](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.2...v2.11.3) (2024-07-11) ### Bug Fixes * Pass cloudprovider value to the helm charts ([#83](https://github.com/wandb/terraform-azurerm-wandb/issues/83)) ([0606602](https://github.com/wandb/terraform-azurerm-wandb/commit/06066020ac57d2d93e406e4b103feffb260426e0)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d98205c..779f3f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [2.11.3](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.2...v2.11.3) (2024-07-11) + + +### Bug Fixes + +* Pass cloudprovider value to the helm charts ([#83](https://github.com/wandb/terraform-azurerm-wandb/issues/83)) ([0606602](https://github.com/wandb/terraform-azurerm-wandb/commit/06066020ac57d2d93e406e4b103feffb260426e0)) + ### [2.11.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.1...v2.11.2) (2024-06-26) From d582e7ccfb8bb12354f0bff1001bf4ed59e1d9d5 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Wed, 31 Jul 2024 15:49:01 -0700 Subject: [PATCH 05/27] feat: Bump operator image and chart versions (#85) --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 8c2bccb..75f7dba 100644 --- a/main.tf +++ b/main.tf @@ -229,8 +229,8 @@ module "wandb" { module.database, module.storage, ] - operator_chart_version = "1.1.2" - controller_image_tag = "1.10.1" + controller_image_tag = "1.12.0" + operator_chart_version = "1.2.0" spec = { values = { From a171dd3e858b3db0fe71e0ea1db6a3f5777654d6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 31 Jul 2024 22:49:25 +0000 Subject: [PATCH 06/27] chore(release): version 2.12.0 [skip ci] ## [2.12.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.3...v2.12.0) (2024-07-31) ### Features * Bump operator image and chart versions ([#85](https://github.com/wandb/terraform-azurerm-wandb/issues/85)) ([d582e7c](https://github.com/wandb/terraform-azurerm-wandb/commit/d582e7ccfb8bb12354f0bff1001bf4ed59e1d9d5)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 779f3f0..0cc6c38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [2.12.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.3...v2.12.0) (2024-07-31) + + +### Features + +* Bump operator image and chart versions ([#85](https://github.com/wandb/terraform-azurerm-wandb/issues/85)) ([d582e7c](https://github.com/wandb/terraform-azurerm-wandb/commit/d582e7ccfb8bb12354f0bff1001bf4ed59e1d9d5)) + ### [2.11.3](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.2...v2.11.3) (2024-07-11) From d6a38f22ab11cc131f7d7200f77ea39e6e53c8e3 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Wed, 31 Jul 2024 17:15:44 -0700 Subject: [PATCH 07/27] fix: Bump operator chart versions (#86) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 75f7dba..7feaa20 100644 --- a/main.tf +++ b/main.tf @@ -230,7 +230,7 @@ module "wandb" { module.storage, ] controller_image_tag = "1.12.0" - operator_chart_version = "1.2.0" + operator_chart_version = "1.2.1" spec = { values = { From abd33c8695fd89c2c409c53eef62238c5fd0b694 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 1 Aug 2024 00:16:14 +0000 Subject: [PATCH 08/27] chore(release): version 2.12.1 [skip ci] ### [2.12.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.0...v2.12.1) (2024-08-01) ### Bug Fixes * Bump operator chart versions ([#86](https://github.com/wandb/terraform-azurerm-wandb/issues/86)) ([d6a38f2](https://github.com/wandb/terraform-azurerm-wandb/commit/d6a38f22ab11cc131f7d7200f77ea39e6e53c8e3)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cc6c38..91bce58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [2.12.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.0...v2.12.1) (2024-08-01) + + +### Bug Fixes + +* Bump operator chart versions ([#86](https://github.com/wandb/terraform-azurerm-wandb/issues/86)) ([d6a38f2](https://github.com/wandb/terraform-azurerm-wandb/commit/d6a38f22ab11cc131f7d7200f77ea39e6e53c8e3)) + ## [2.12.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.3...v2.12.0) (2024-07-31) From 51e873629db3263a27beda2bcf3f40190cc7e0ae Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Thu, 1 Aug 2024 09:52:49 -0700 Subject: [PATCH 09/27] fix: Bump operator chart versions (#87) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 7feaa20..eadd9ec 100644 --- a/main.tf +++ b/main.tf @@ -230,7 +230,7 @@ module "wandb" { module.storage, ] controller_image_tag = "1.12.0" - operator_chart_version = "1.2.1" + operator_chart_version = "1.2.4" spec = { values = { From f6ad2d776bc1743628c230aa6fb07a11a32f255e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 1 Aug 2024 16:53:16 +0000 Subject: [PATCH 10/27] chore(release): version 2.12.2 [skip ci] ### [2.12.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.1...v2.12.2) (2024-08-01) ### Bug Fixes * Bump operator chart versions ([#87](https://github.com/wandb/terraform-azurerm-wandb/issues/87)) ([51e8736](https://github.com/wandb/terraform-azurerm-wandb/commit/51e873629db3263a27beda2bcf3f40190cc7e0ae)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91bce58..2ede379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [2.12.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.1...v2.12.2) (2024-08-01) + + +### Bug Fixes + +* Bump operator chart versions ([#87](https://github.com/wandb/terraform-azurerm-wandb/issues/87)) ([51e8736](https://github.com/wandb/terraform-azurerm-wandb/commit/51e873629db3263a27beda2bcf3f40190cc7e0ae)) + ### [2.12.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.0...v2.12.1) (2024-08-01) From 519c340fbf855743fe77b3ae075e6bfdb84740c2 Mon Sep 17 00:00:00 2001 From: Aastha Gupta <71313011+velotioaastha@users.noreply.github.com> Date: Thu, 1 Aug 2024 22:55:23 +0530 Subject: [PATCH 11/27] feat: Added for encrypting the database and blob storage with WB-managed key (#49) Co-authored-by: github-actions[bot] Co-authored-by: Aastha Gupta Co-authored-by: Aastha Gupta Co-authored-by: Zachary Blasczyk --- README.md | 9 +++++- examples/byob/README.md | 50 ++++++++++++++++++++++++++++++---- examples/byob/main.tf | 25 +++++++++++++---- examples/byob/terraform.tfvars | 19 +++++++++++-- examples/byob/variables.tf | 36 +++++++++++++++++++++++- main.tf | 50 +++++++++++++++++++++------------- modules/byob/main.tf | 43 +++++++++++++++++++++++++++-- modules/byob/outputs.tf | 10 ++++++- modules/byob/variables.tf | 37 +++++++++++++++++++++++-- modules/database/main.tf | 14 ++++++++++ modules/database/variables.tf | 14 ++++++++-- modules/identity/main.tf | 2 +- modules/identity/outputs.tf | 3 +- modules/identity/variables.tf | 4 +-- modules/storage/main.tf | 14 ++++++++++ modules/storage/variables.tf | 16 ++++++++++- modules/vault/main.tf | 36 ++++++++++++++++++------ modules/vault/outputs.tf | 14 +++++++++- modules/vault/variables.tf | 20 +++++++++++++- outputs.tf | 12 ++++++-- variables.tf | 49 +++++++++++++++++++++++++++++---- versions.tf | 3 +- 22 files changed, 412 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 7c0c086..0a65813 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ resources that lack official modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [allowed\_ip\_ranges](#input\_allowed\_ip\_ranges) | allowed public IP addresses or CIDR ranges. | `list(string)` | `[]` | no | +| [allowed\_ip\_ranges](#input\_allowed\_ip\_ranges) | Allowed public IP addresses or CIDR ranges. | `list(string)` | `[]` | no | | [allowed\_subscriptions](#input\_allowed\_subscriptions) | List of allowed customer subscriptions coma seperated values | `string` | `""` | no | | [app\_wandb\_env](#input\_app\_wandb\_env) | Extra environment variables for W&B | `map(string)` | `{}` | no | | [azuremonitor](#input\_azuremonitor) | # To support otel azure monitor sql and redis metrics need operator-wandb chart minimum version 0.14.0 | `bool` | `false` | no | @@ -78,11 +78,16 @@ resources that lack official modules. | [cluster\_sku\_tier](#input\_cluster\_sku\_tier) | The Azure AKS SKU Tier to use for this cluster (https://learn.microsoft.com/en-us/azure/aks/free-standard-pricing-tiers) | `string` | `"Free"` | no | | [create\_private\_link](#input\_create\_private\_link) | Use for the azure private link. | `bool` | `false` | no | | [create\_redis](#input\_create\_redis) | Boolean indicating whether to provision an redis instance (true) or not (false). | `bool` | `false` | no | +| [customer\_storage\_vault\_key\_id](#input\_customer\_storage\_vault\_key\_id) | The Azure Key Vault key ID for customer-provided storage encryption keys. Must match the pattern 'https://.vault.azure.net/keys//', or be null. | `string` | `null` | no | | [database\_availability\_mode](#input\_database\_availability\_mode) | n/a | `string` | `"SameZone"` | no | | [database\_sku\_name](#input\_database\_sku\_name) | Specifies the SKU Name for this MySQL Server | `string` | `"GP_Standard_D4ds_v4"` | no | | [database\_version](#input\_database\_version) | Version for MySQL | `string` | `"5.7"` | no | | [deletion\_protection](#input\_deletion\_protection) | If the instance should have deletion protection enabled. The database / Bucket can't be deleted when this value is set to `true`. | `bool` | `true` | no | +| [disable\_storage\_vault\_key\_id](#input\_disable\_storage\_vault\_key\_id) | Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation. | `bool` | `false` | no | | [domain\_name](#input\_domain\_name) | Domain for accessing the Weights & Biases UI. | `string` | `null` | no | +| [enable\_database\_vault\_key](#input\_enable\_database\_vault\_key) | Flag to enable managed key encryption for the database. Once enabled, cannot be disabled. | `bool` | `false` | no | +| [enable\_purge\_protection](#input\_enable\_purge\_protection) | Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled. | `bool` | `false` | no | +| [enable\_storage\_vault\_key](#input\_enable\_storage\_vault\_key) | Flag to enable managed key encryption for the storage account. | `bool` | `false` | no | | [external\_bucket](#input\_external\_bucket) | config an external bucket | `any` | `null` | no | | [kubernetes\_instance\_type](#input\_kubernetes\_instance\_type) | Use for the Kubernetes cluster. | `string` | `"Standard_D4a_v4"` | no | | [kubernetes\_node\_count](#input\_kubernetes\_node\_count) | n/a | `number` | `2` | no | @@ -116,6 +121,7 @@ resources that lack official modules. | [address](#output\_address) | n/a | | [aks\_node\_count](#output\_aks\_node\_count) | n/a | | [aks\_node\_instance\_type](#output\_aks\_node\_instance\_type) | n/a | +| [client\_id](#output\_client\_id) | n/a | | [cluster\_ca\_certificate](#output\_cluster\_ca\_certificate) | n/a | | [cluster\_client\_certificate](#output\_cluster\_client\_certificate) | n/a | | [cluster\_client\_key](#output\_cluster\_client\_key) | n/a | @@ -128,5 +134,6 @@ resources that lack official modules. | [standardized\_size](#output\_standardized\_size) | n/a | | [storage\_account](#output\_storage\_account) | n/a | | [storage\_container](#output\_storage\_container) | n/a | +| [tenant\_id](#output\_tenant\_id) | n/a | | [url](#output\_url) | The URL to the W&B application | diff --git a/examples/byob/README.md b/examples/byob/README.md index 2cb5de2..b4fcc0e 100644 --- a/examples/byob/README.md +++ b/examples/byob/README.md @@ -23,13 +23,53 @@ storage_key = To retrieve the storage key, you can use the Azure CLI installed previously like the example below. -```basb -az storage account keys list --account-name rgnamestorage --query '[].{key: value}' --output tsv +```bash +az storage account keys list --account-name --resource-group --query '[0].value' -o tsv 1111111111111122222222222333333333334444444555555555 -5555555554444444333333333332222222222211111111111111 ``` -You only need to provide one key. +This command will return the storage key, which you can then use for your deployment needs. Ensure you handle the storage key securely as it contains sensitive information. + +# Customer Managed Key Encryption + +The following section provides details on enabling Customer Managed Key (CMK) encryption for the Azure Blob Storage container which is disabled by default. + +To configure Customer Managed Key encryption, ensure you are using the latest version of out terraform which has the following added to the `variables.tf` file: + +- create_cmk +- disable_storage_vault_key_id +- enable_purge_protection +- tenant_id +- client_id + +You need to obtain the `tenant_id` and `client_id` from a Solutions Architect at W&B for an already instantiated instance of a Weights & Biases managed deployment. + +Set the follwoing new variabels to enable the CMK: + +```ini terraform.tfvars +create_cmk = true + +enable_purge_protection = true +disable_storage_vault_key_id = false + +tenant_id = "" +client_id = "" +``` + +After updating your `terraform.tfvars` configuration, run the Terraform commands to apply the changes: + +```bash +terraform init -upgrade +terraform apply -var-file=terraform.tfvars +``` + +Upon successful execution, you will receive the following output which needs to the shared with the SA at Weights & Biases. -* Note that all information about Storage Account and keys are mere examples, they are not valid. +```bash +blob_container = "/wandb" +command_to_get_storage_key = "az storage account keys list --account-name --resource-group --query '[0].value' -o tsv" +storage_key = +storage_vault_key_id = "https://.vault.azure.net/keys//" +``` +Retrieve the storage key as shown above and also share that with the SA at Weights & Biases. diff --git a/examples/byob/main.tf b/examples/byob/main.tf index 66fab24..f62f9ce 100644 --- a/examples/byob/main.tf +++ b/examples/byob/main.tf @@ -3,14 +3,19 @@ provider "azurerm" { } module "byob" { - source = "../../modules/byob" - resource_group_name = var.resource_group_name - location = var.location - prefix = var.prefix - deletion_protection = var.deletion_protection + source = "../../modules/byob" + resource_group_name = var.resource_group_name + location = var.location + prefix = var.prefix + deletion_protection = var.deletion_protection + create_cmk = var.create_cmk + enable_purge_protection = var.enable_purge_protection + client_id = var.client_id + tenant_id = var.tenant_id + tags = var.tags + disable_storage_vault_key_id = var.disable_storage_vault_key_id } - output "blob_container" { value = module.byob.blob_container } @@ -19,3 +24,11 @@ output "storage_key" { value = module.byob.azure_storage_key sensitive = true } + +output "storage_vault_key_id" { + value = module.byob.vault_key_id +} + +output "command_to_get_storage_key" { + value = "az storage account keys list --account-name ${module.byob.storage_account_name} --resource-group ${module.byob.resource_group_name.name} --query '[0].value' -o tsv" +} diff --git a/examples/byob/terraform.tfvars b/examples/byob/terraform.tfvars index ed42c15..62e6b1d 100644 --- a/examples/byob/terraform.tfvars +++ b/examples/byob/terraform.tfvars @@ -1,3 +1,16 @@ -resource_group_name = "rg-name" -location = "westeurope" -prefix = "byob-wandb" +resource_group_name = "" + +location = "" +prefix = "" +tags = { + "name" = "wandb" +} + +#To enable Azure Key Vault encryption uncomment the below lines +# create_cmk = true + +# enable_purge_protection = true +# disable_storage_vault_key_id = false + +# tenant_id = "" +# client_id = "" \ No newline at end of file diff --git a/examples/byob/variables.tf b/examples/byob/variables.tf index fafa958..7b265e5 100644 --- a/examples/byob/variables.tf +++ b/examples/byob/variables.tf @@ -1,6 +1,6 @@ variable "resource_group_name" { - description = "Resource Group name" type = string + description = "Resource Group" } variable "location" { @@ -18,3 +18,37 @@ variable "deletion_protection" { type = bool default = false } + +variable "create_cmk" { + type = bool + default = false + description = "Flag to create a Customer Managed Key for the Key Vault to encrypt the storage account." +} + +variable "disable_storage_vault_key_id" { + type = bool + default = false + description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." +} + +variable "enable_purge_protection" { + type = bool + default = false + description = "Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled." +} + +variable "tenant_id" { + type = string + description = "The tenant ID for the Key Vault Access Policy. Get from W&B SA" +} + +variable "client_id" { + type = string + description = "The client ID (object id) for the Key Vault Access Policy. Get from W&B SA" +} + +variable "tags" { + type = map(string) + description = "Map of tags for resource" + default = {} +} diff --git a/main.tf b/main.tf index eadd9ec..53c2349 100644 --- a/main.tf +++ b/main.tf @@ -2,6 +2,8 @@ locals { fqdn = var.subdomain == null ? var.domain_name : "${var.subdomain}.${var.domain_name}" url_prefix = var.ssl ? "https" : "http" url = "${local.url_prefix}://${local.fqdn}" + + enable_internal_storage = var.blob_container == "" && var.external_bucket == null } resource "azurerm_resource_group" "default" { @@ -27,12 +29,12 @@ module "networking" { private_link = var.create_private_link allowed_ip_ranges = var.allowed_ip_ranges tags = var.tags - } module "database" { - source = "./modules/database" - namespace = var.namespace + source = "./modules/database" + namespace = var.namespace + resource_group_name = azurerm_resource_group.default.name location = azurerm_resource_group.default.location @@ -40,10 +42,11 @@ module "database" { database_version = var.database_version database_private_dns_zone_id = module.networking.database_private_dns_zone.id database_subnet_id = module.networking.database_subnet.id + sku_name = try(local.deployment_size[var.size].db, var.database_sku_name) + deletion_protection = var.deletion_protection - sku_name = try(local.deployment_size[var.size].db, var.database_sku_name) - deletion_protection = var.deletion_protection - + database_key_id = try(module.vault.vault_internal_keys[module.vault.vault_key_map.database].id, null) + identity_ids = module.identity.identity.id tags = { "customer-ns" = var.namespace, "env" = "managed-install" @@ -64,23 +67,33 @@ module "redis" { module "vault" { source = "./modules/vault" - identity_object_id = module.identity.identity.principal_id - location = azurerm_resource_group.default.location - namespace = var.namespace - resource_group = azurerm_resource_group.default + identity_object_id = module.identity.identity.principal_id + location = azurerm_resource_group.default.location + namespace = var.namespace + resource_group = azurerm_resource_group.default + enable_purge_protection = var.enable_purge_protection + + enable_database_vault_key = var.enable_database_vault_key + enable_storage_vault_key = var.enable_storage_vault_key && local.enable_internal_storage tags = var.tags } module "storage" { - count = (var.blob_container == "" && var.external_bucket == null) ? 1 : 0 - source = "./modules/storage" + source = "./modules/storage" + + count = local.enable_internal_storage ? 1 : 0 namespace = var.namespace resource_group_name = azurerm_resource_group.default.name location = azurerm_resource_group.default.location create_queue = !var.use_internal_queue + identity_ids = module.identity.identity.id deletion_protection = var.deletion_protection - tags = var.tags + + storage_key_id = var.customer_storage_vault_key_id == null ? try(module.vault.vault_internal_keys[module.vault.vault_key_map.storage].id, null) : var.customer_storage_vault_key_id + disable_storage_vault_key_id = var.disable_storage_vault_key_id + + tags = var.tags } module "app_lb" { @@ -176,8 +189,9 @@ resource "azurerm_role_assignment" "gateway_role" { } module "cron_job" { - count = length(var.allowed_subscriptions) > 0 ? 1 : 0 # private endpoint approval application deployed as a cronjob in default namespace - source = "./modules/cron_job" + count = length(var.allowed_subscriptions) > 0 ? 1 : 0 # private endpoint approval application deployed as a cronjob in default namespace + source = "./modules/cron_job" + namespace = "default" client_id = module.pod_identity[0].identity.client_id serviceaccountName = local.private_endpoint_approval_sa @@ -186,7 +200,6 @@ module "cron_job" { applicationGatewayName = module.app_lb.gateway.name allowedSubscriptions = var.allowed_subscriptions depends_on = [module.app_lb, module.pod_identity] - } resource "azurerm_role_assignment" "otel_role" { @@ -194,7 +207,6 @@ resource "azurerm_role_assignment" "otel_role" { scope = azurerm_resource_group.default.id role_definition_name = "Contributor" principal_id = module.identity.otel_identity.principal_id - } resource "azurerm_federated_identity_credential" "otel_app" { @@ -235,8 +247,8 @@ module "wandb" { spec = { values = { global = { - host = local.url - license = var.license + host = local.url + license = var.license cloudProvider = "azure" bucket = var.external_bucket == null ? { provider = "az" diff --git a/modules/byob/main.tf b/modules/byob/main.tf index 72ba9b4..f16edb1 100644 --- a/modules/byob/main.tf +++ b/modules/byob/main.tf @@ -1,10 +1,47 @@ -module "storage" { - source = "../storage" +module "identity" { + count = var.create_cmk ? 1 : 0 + source = "../identity" + + namespace = var.prefix + resource_group = { name = "${var.resource_group_name}", id = "byob" } + location = var.location +} + +module "vault" { + count = var.create_cmk ? 1 : 0 + source = "../vault" + namespace = var.prefix + resource_group = { name = "${var.resource_group_name}", id = "byob" } + location = var.location + + identity_object_id = module.identity[0].identity.principal_id + depends_on = [module.identity] + tags = var.tags + enable_purge_protection = var.enable_purge_protection + enable_storage_vault_key = var.create_cmk +} + +resource "azurerm_key_vault_access_policy" "wandb" { + count = var.create_cmk ? 1 : 0 + key_vault_id = module.vault[0].vault.id + tenant_id = var.tenant_id + object_id = var.client_id + key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Recover", "Backup", "Restore"] + certificate_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Recover", "Backup", "Restore"] + secret_permissions = ["Get", "List", "Set", "Delete", "Recover", "Backup", "Restore"] +} + +module "storage" { + source = "../storage" create_queue = false namespace = var.prefix resource_group_name = var.resource_group_name location = var.location - deletion_protection = var.deletion_protection + storage_key_id = try(module.vault[0].vault_internal_keys[module.vault[0].vault_key_map.storage].id, null) + identity_ids = try(module.identity[0].identity.id, null) + + disable_storage_vault_key_id = var.disable_storage_vault_key_id } + diff --git a/modules/byob/outputs.tf b/modules/byob/outputs.tf index 3c3e8ac..4177455 100644 --- a/modules/byob/outputs.tf +++ b/modules/byob/outputs.tf @@ -2,6 +2,14 @@ output "azure_storage_key" { value = module.storage.account.primary_access_key } +output "storage_key_id" { + value = try(module.vault[0].vault_internal_keys[module.vault[0].vault_key_map.storage].id, null) +} + output "blob_container" { value = "${module.storage.account.name}/${module.storage.container.name}" -} \ No newline at end of file +} + +output "storage_account_name" { + value = module.storage.account.name +} diff --git a/modules/byob/variables.tf b/modules/byob/variables.tf index 57c9157..75e0888 100644 --- a/modules/byob/variables.tf +++ b/modules/byob/variables.tf @@ -1,6 +1,5 @@ variable "resource_group_name" { - description = "Resource Group name" - type = string + type = string } variable "location" { @@ -18,3 +17,37 @@ variable "deletion_protection" { type = bool default = true } + +variable "create_cmk" { + type = bool + default = false + description = "Flag to create a Customer Managed Key for the Key Vault to encrypt the storage account." +} + +variable "tags" { + type = map(string) + description = "Map of tags for resource" + default = {} +} + +variable "enable_purge_protection" { + type = bool + default = false + description = "Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled." +} + +variable "disable_storage_vault_key_id" { + type = bool + default = false + description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." +} + +variable "tenant_id" { + type = string + description = "The tenant ID for the Key Vault Access Policy. Get from W&B SA" +} + +variable "client_id" { + type = string + description = "The client ID (object id) for the Key Vault Access Policy. Get from W&B SA" +} diff --git a/modules/database/main.tf b/modules/database/main.tf index d5f5d12..4c4b9ae 100644 --- a/modules/database/main.tf +++ b/modules/database/main.tf @@ -52,6 +52,20 @@ resource "azurerm_mysql_flexible_server" "default" { ] } + dynamic "identity" { + for_each = var.database_key_id != null ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.identity_ids] + } + } + dynamic "customer_managed_key" { + for_each = var.database_key_id != null ? [1] : [] + content { + primary_user_assigned_identity_id = var.identity_ids + key_vault_key_id = var.database_key_id + } + } tags = var.tags } diff --git a/modules/database/variables.tf b/modules/database/variables.tf index a75ab3e..d01456f 100644 --- a/modules/database/variables.tf +++ b/modules/database/variables.tf @@ -53,6 +53,16 @@ variable "tags" { } variable "deletion_protection" { - description = "If the instance should have deletion protection enabled. The database can't be deleted when this value is set to `true`." type = bool -} \ No newline at end of file + description = "If the instance should have deletion protection enabled. The database can't be deleted when this value is set to `true`." +} + +variable "database_key_id" { + type = string + description = "The Azure Key Vault key identifier for the database encryption key." +} + +variable "identity_ids" { + type = string + description = "The identity ids to assign to the database when database_key_id is passed." +} diff --git a/modules/identity/main.tf b/modules/identity/main.tf index 6880b99..6a4b825 100644 --- a/modules/identity/main.tf +++ b/modules/identity/main.tf @@ -9,4 +9,4 @@ resource "azurerm_user_assigned_identity" "otel" { name = "${var.namespace}-identity-otel" location = var.location resource_group_name = var.resource_group.name -} \ No newline at end of file +} diff --git a/modules/identity/outputs.tf b/modules/identity/outputs.tf index 453d5a4..fa7c529 100644 --- a/modules/identity/outputs.tf +++ b/modules/identity/outputs.tf @@ -1,6 +1,7 @@ output "identity" { value = azurerm_user_assigned_identity.default } + output "otel_identity" { value = var.otel_identity ? azurerm_user_assigned_identity.otel[0] : null -} \ No newline at end of file +} diff --git a/modules/identity/variables.tf b/modules/identity/variables.tf index 54cb0e4..1522c50 100644 --- a/modules/identity/variables.tf +++ b/modules/identity/variables.tf @@ -14,6 +14,6 @@ variable "location" { } variable "otel_identity" { - type = bool + type = bool default = false -} \ No newline at end of file +} diff --git a/modules/storage/main.tf b/modules/storage/main.tf index b077e4d..0c55022 100644 --- a/modules/storage/main.tf +++ b/modules/storage/main.tf @@ -26,6 +26,20 @@ resource "azurerm_storage_account" "default" { } } + dynamic "identity" { + for_each = var.storage_key_id != null ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.identity_ids] + } + } + dynamic "customer_managed_key" { + for_each = var.storage_key_id != null && var.disable_storage_vault_key_id == false ? [1] : [] + content { + user_assigned_identity_id = var.identity_ids + key_vault_key_id = var.storage_key_id + } + } tags = var.tags } diff --git a/modules/storage/variables.tf b/modules/storage/variables.tf index 774306a..dab57e6 100644 --- a/modules/storage/variables.tf +++ b/modules/storage/variables.tf @@ -29,8 +29,22 @@ variable "deletion_protection" { type = bool } +variable "storage_key_id" { + type = string +} + +variable "disable_storage_vault_key_id" { + type = bool + default = false + description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." +} + +variable "identity_ids" { + type = string +} + variable "blob_container_name" { description = "Name of azure storage account container for storing blobs" type = string default = "wandb" -} \ No newline at end of file +} diff --git a/modules/vault/main.tf b/modules/vault/main.tf index e43290c..477c779 100644 --- a/modules/vault/main.tf +++ b/modules/vault/main.tf @@ -4,14 +4,21 @@ data "azurerm_client_config" "current" {} locals { vault_name = "${var.namespace}-vault" vault_truncated_name = substr(local.vault_name, 0, min(length(local.vault_name), 24)) + max_key_length = 127 + vault_key_map = { + database = var.enable_database_vault_key ? substr("database-key-${var.namespace}", 0, local.max_key_length) : null + storage = var.enable_storage_vault_key ? substr("storage-key-${var.namespace}", 0, local.max_key_length) : null + } } resource "azurerm_key_vault" "default" { - name = trim(local.vault_truncated_name, "-") - location = var.location - resource_group_name = var.resource_group.name - tenant_id = data.azurerm_client_config.current.tenant_id - + name = trim(local.vault_truncated_name, "-") + location = var.location + resource_group_name = var.resource_group.name + tenant_id = data.azurerm_client_config.current.tenant_id + purge_protection_enabled = var.enable_purge_protection + # https://learn.microsoft.com/en-us/azure/mysql/flexible-server/concepts-customer-managed-key#requirements-for-configuring-data-encryption-for-azure-database-for-mysql-flexible-server + soft_delete_retention_days = 90 # This must be 90 for azure msyql flex server encryption. enabled_for_disk_encryption = true sku_name = "standard" @@ -41,21 +48,32 @@ resource "azurerm_key_vault_access_policy" "identity" { tenant_id = data.azurerm_client_config.current.tenant_id object_id = var.identity_object_id - key_permissions = ["Create", "Decrypt", "Encrypt", "Get", "List"] + key_permissions = ["Create", "Decrypt", "Encrypt", "Get", "List", "UnwrapKey", "WrapKey"] secret_permissions = ["Delete", "Get", "List", "Purge", "Recover", "Restore", "Set"] storage_permissions = ["Get", "List"] - depends_on = [azurerm_key_vault.default] } resource "azurerm_key_vault_key" "etcd" { - depends_on = [azurerm_key_vault_access_policy.parent, azurerm_key_vault_access_policy.identity] - name = "generated-etcd-key" key_vault_id = azurerm_key_vault.default.id key_type = "RSA" key_size = 2048 key_opts = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey", ] + + depends_on = [azurerm_key_vault_access_policy.parent, azurerm_key_vault_access_policy.identity] +} + +resource "azurerm_key_vault_key" "intenral_encryption_keys" { + for_each = { for v in local.vault_key_map : v => v if v != null } + name = each.value + key_vault_id = azurerm_key_vault.default.id + key_type = "RSA" + key_size = 2048 + + key_opts = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey"] + + depends_on = [azurerm_key_vault_access_policy.parent, azurerm_key_vault_access_policy.identity] } diff --git a/modules/vault/outputs.tf b/modules/vault/outputs.tf index 9ec675d..0402867 100644 --- a/modules/vault/outputs.tf +++ b/modules/vault/outputs.tf @@ -4,4 +4,16 @@ output "etcd_key_id" { output "vault" { value = azurerm_key_vault.default -} \ No newline at end of file +} + +output "vault_id" { + value = azurerm_key_vault.default.id +} + +output "vault_internal_keys" { + value = azurerm_key_vault_key.intenral_encryption_keys +} + +output "vault_key_map" { + value = local.vault_key_map +} diff --git a/modules/vault/variables.tf b/modules/vault/variables.tf index f061a9a..693b9ca 100644 --- a/modules/vault/variables.tf +++ b/modules/vault/variables.tf @@ -20,4 +20,22 @@ variable "resource_group" { variable "tags" { type = map(string) description = "Map of tags for resource" -} \ No newline at end of file +} + +variable "enable_storage_vault_key" { + type = bool + default = false + description = "Flag to enable managed key encryption for the storage account." +} + +variable "enable_database_vault_key" { + type = bool + default = false + description = "Flag to enable managed key encryption for the database. Once enabled, cannot be disabled or you will loose access to the database." +} + +variable "enable_purge_protection" { + type = bool + default = false + description = "Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled." +} diff --git a/outputs.tf b/outputs.tf index 4a43945..40870ec 100644 --- a/outputs.tf +++ b/outputs.tf @@ -48,7 +48,7 @@ output "private_link_resource_id" { output "private_link_sub_resource_name" { value = module.app_lb.frontend_ip_configuration_names } - + output "standardized_size" { value = var.size } @@ -63,4 +63,12 @@ output "aks_node_instance_type" { output "database_instance_type" { value = try(local.deployment_size[var.size].db, var.database_sku_name) -} \ No newline at end of file +} + +output "client_id" { + value = module.identity.identity.client_id +} + +output "tenant_id" { + value = module.identity.identity.tenant_id +} diff --git a/variables.tf b/variables.tf index cdddcc2..4cfa373 100644 --- a/variables.tf +++ b/variables.tf @@ -145,9 +145,9 @@ variable "create_redis" { } variable "redis_capacity" { - type = number + type = number description = "Number indicating size of an redis instance" - default = 2 + default = 2 } ########################################## @@ -210,7 +210,7 @@ variable "node_pool_zones" { variable "node_max_pods" { type = number description = "Maximum number of pods per node" - default = 30 + default = 30 } ########################################### @@ -225,14 +225,14 @@ variable "create_private_link" { variable "allowed_subscriptions" { type = string description = "List of allowed customer subscriptions coma seperated values" - default = "" + default = "" } ########################################## # Network # ########################################## variable "allowed_ip_ranges" { - description = "allowed public IP addresses or CIDR ranges." + description = "Allowed public IP addresses or CIDR ranges." type = list(string) default = [] } @@ -255,6 +255,45 @@ variable "parquet_wandb_env" { default = {} } +########################################## +# vault key # +########################################## + +variable "enable_storage_vault_key" { + type = bool + default = false + description = "Flag to enable managed key encryption for the storage account." +} + +variable "disable_storage_vault_key_id" { + type = bool + default = false + description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." +} + +variable "customer_storage_vault_key_id" { + type = string + default = null + description = "The Azure Key Vault key ID for customer-provided storage encryption keys. Must match the pattern 'https://.vault.azure.net/keys//', or be null." + + validation { + condition = var.customer_storage_vault_key_id == null || can(regex("^https://[a-zA-Z0-9-]+.vault.azure.net/keys/[a-zA-Z0-9-]+/[a-f0-9]+$", var.customer_storage_vault_key_id)) + error_message = "The customer_storage_vault_key_id must be null or a valid Azure Key Vault key ID in the format 'https://.vault.azure.net/keys//'." + } +} + +variable "enable_database_vault_key" { + type = bool + default = false + description = "Flag to enable managed key encryption for the database. Once enabled, cannot be disabled." +} + +variable "enable_purge_protection" { + type = bool + default = false + description = "Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled." +} + ## To support otel azure monitor sql and redis metrics need operator-wandb chart minimum version 0.14.0 variable "azuremonitor" { type = bool diff --git a/versions.tf b/versions.tf index d740277..79b7958 100644 --- a/versions.tf +++ b/versions.tf @@ -14,5 +14,4 @@ terraform { version = "~> 2.6" } } -} - +} \ No newline at end of file From dd76d6936f62e753636e5595c93c0bf8de2a1541 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 1 Aug 2024 17:25:49 +0000 Subject: [PATCH 12/27] chore(release): version 2.13.0 [skip ci] ## [2.13.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.2...v2.13.0) (2024-08-01) ### Features * Added for encrypting the database and blob storage with WB-managed key ([#49](https://github.com/wandb/terraform-azurerm-wandb/issues/49)) ([519c340](https://github.com/wandb/terraform-azurerm-wandb/commit/519c340fbf855743fe77b3ae075e6bfdb84740c2)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ede379..6e42b2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [2.13.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.2...v2.13.0) (2024-08-01) + + +### Features + +* Added for encrypting the database and blob storage with WB-managed key ([#49](https://github.com/wandb/terraform-azurerm-wandb/issues/49)) ([519c340](https://github.com/wandb/terraform-azurerm-wandb/commit/519c340fbf855743fe77b3ae075e6bfdb84740c2)) + ### [2.12.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.1...v2.12.2) (2024-08-01) From 4832d247cdf8e75fe1ae75e7f4da8b528cde93e4 Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk <77289967+zacharyblasczyk@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:06:50 -0700 Subject: [PATCH 13/27] fix: Azure storage and Vault simplified (#89) Co-authored-by: github-actions[bot] --- README.md | 4 --- examples/byob/README.md | 8 ++--- examples/byob/main.tf | 1 - examples/byob/variables.tf | 10 ++---- main.tf | 67 +++++++++++++++++++++----------------- modules/byob/main.tf | 1 - modules/byob/variables.tf | 10 ++---- modules/vault/main.tf | 2 +- modules/vault/variables.tf | 6 ---- outputs.tf | 8 ----- variables.tf | 17 ---------- 11 files changed, 45 insertions(+), 89 deletions(-) diff --git a/README.md b/README.md index 0a65813..474b6d6 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,6 @@ resources that lack official modules. | [cluster\_sku\_tier](#input\_cluster\_sku\_tier) | The Azure AKS SKU Tier to use for this cluster (https://learn.microsoft.com/en-us/azure/aks/free-standard-pricing-tiers) | `string` | `"Free"` | no | | [create\_private\_link](#input\_create\_private\_link) | Use for the azure private link. | `bool` | `false` | no | | [create\_redis](#input\_create\_redis) | Boolean indicating whether to provision an redis instance (true) or not (false). | `bool` | `false` | no | -| [customer\_storage\_vault\_key\_id](#input\_customer\_storage\_vault\_key\_id) | The Azure Key Vault key ID for customer-provided storage encryption keys. Must match the pattern 'https://.vault.azure.net/keys//', or be null. | `string` | `null` | no | | [database\_availability\_mode](#input\_database\_availability\_mode) | n/a | `string` | `"SameZone"` | no | | [database\_sku\_name](#input\_database\_sku\_name) | Specifies the SKU Name for this MySQL Server | `string` | `"GP_Standard_D4ds_v4"` | no | | [database\_version](#input\_database\_version) | Version for MySQL | `string` | `"5.7"` | no | @@ -86,7 +85,6 @@ resources that lack official modules. | [disable\_storage\_vault\_key\_id](#input\_disable\_storage\_vault\_key\_id) | Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation. | `bool` | `false` | no | | [domain\_name](#input\_domain\_name) | Domain for accessing the Weights & Biases UI. | `string` | `null` | no | | [enable\_database\_vault\_key](#input\_enable\_database\_vault\_key) | Flag to enable managed key encryption for the database. Once enabled, cannot be disabled. | `bool` | `false` | no | -| [enable\_purge\_protection](#input\_enable\_purge\_protection) | Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled. | `bool` | `false` | no | | [enable\_storage\_vault\_key](#input\_enable\_storage\_vault\_key) | Flag to enable managed key encryption for the storage account. | `bool` | `false` | no | | [external\_bucket](#input\_external\_bucket) | config an external bucket | `any` | `null` | no | | [kubernetes\_instance\_type](#input\_kubernetes\_instance\_type) | Use for the Kubernetes cluster. | `string` | `"Standard_D4a_v4"` | no | @@ -132,8 +130,6 @@ resources that lack official modules. | [private\_link\_resource\_id](#output\_private\_link\_resource\_id) | n/a | | [private\_link\_sub\_resource\_name](#output\_private\_link\_sub\_resource\_name) | n/a | | [standardized\_size](#output\_standardized\_size) | n/a | -| [storage\_account](#output\_storage\_account) | n/a | -| [storage\_container](#output\_storage\_container) | n/a | | [tenant\_id](#output\_tenant\_id) | n/a | | [url](#output\_url) | The URL to the W&B application | diff --git a/examples/byob/README.md b/examples/byob/README.md index b4fcc0e..1c55d4d 100644 --- a/examples/byob/README.md +++ b/examples/byob/README.md @@ -38,18 +38,16 @@ To configure Customer Managed Key encryption, ensure you are using the latest ve - create_cmk - disable_storage_vault_key_id -- enable_purge_protection - tenant_id - client_id -You need to obtain the `tenant_id` and `client_id` from a Solutions Architect at W&B for an already instantiated instance of a Weights & Biases managed deployment. +You need to obtain the `tenant_id` and `client_id` from the `https://${WANDB_BASE_URL}/console/settings/advanced/spec/active` at W&B for an already instantiated instance of a Weights & Biases managed deployment. Set the follwoing new variabels to enable the CMK: ```ini terraform.tfvars create_cmk = true -enable_purge_protection = true disable_storage_vault_key_id = false tenant_id = "" @@ -63,7 +61,7 @@ terraform init -upgrade terraform apply -var-file=terraform.tfvars ``` -Upon successful execution, you will receive the following output which needs to the shared with the SA at Weights & Biases. +Upon successful execution, you will receive the following output which needs to be set in the system connection settings `https://${WANDB_BASE_URL}/console/settings/system` ```bash blob_container = "/wandb" @@ -72,4 +70,4 @@ storage_key = storage_vault_key_id = "https://.vault.azure.net/keys//" ``` -Retrieve the storage key as shown above and also share that with the SA at Weights & Biases. +Retrieve the storage key as shown above. diff --git a/examples/byob/main.tf b/examples/byob/main.tf index f62f9ce..414b02e 100644 --- a/examples/byob/main.tf +++ b/examples/byob/main.tf @@ -9,7 +9,6 @@ module "byob" { prefix = var.prefix deletion_protection = var.deletion_protection create_cmk = var.create_cmk - enable_purge_protection = var.enable_purge_protection client_id = var.client_id tenant_id = var.tenant_id tags = var.tags diff --git a/examples/byob/variables.tf b/examples/byob/variables.tf index 7b265e5..2c0555d 100644 --- a/examples/byob/variables.tf +++ b/examples/byob/variables.tf @@ -31,20 +31,14 @@ variable "disable_storage_vault_key_id" { description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." } -variable "enable_purge_protection" { - type = bool - default = false - description = "Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled." -} - variable "tenant_id" { type = string - description = "The tenant ID for the Key Vault Access Policy. Get from W&B SA" + description = "The tenant ID for the Key Vault Access Policy. Get from `https:///console/settings/advanced/spec/active`" } variable "client_id" { type = string - description = "The client ID (object id) for the Key Vault Access Policy. Get from W&B SA" + description = "The client ID (object id) for the Key Vault Access Policy. Get from `https:///console/settings/advanced/spec/active`" } variable "tags" { diff --git a/main.tf b/main.tf index 53c2349..c3005a0 100644 --- a/main.tf +++ b/main.tf @@ -2,8 +2,6 @@ locals { fqdn = var.subdomain == null ? var.domain_name : "${var.subdomain}.${var.domain_name}" url_prefix = var.ssl ? "https" : "http" url = "${local.url_prefix}://${local.fqdn}" - - enable_internal_storage = var.blob_container == "" && var.external_bucket == null } resource "azurerm_resource_group" "default" { @@ -67,14 +65,13 @@ module "redis" { module "vault" { source = "./modules/vault" - identity_object_id = module.identity.identity.principal_id - location = azurerm_resource_group.default.location - namespace = var.namespace - resource_group = azurerm_resource_group.default - enable_purge_protection = var.enable_purge_protection + identity_object_id = module.identity.identity.principal_id + location = azurerm_resource_group.default.location + namespace = var.namespace + resource_group = azurerm_resource_group.default enable_database_vault_key = var.enable_database_vault_key - enable_storage_vault_key = var.enable_storage_vault_key && local.enable_internal_storage + enable_storage_vault_key = var.enable_storage_vault_key tags = var.tags } @@ -82,7 +79,7 @@ module "vault" { module "storage" { source = "./modules/storage" - count = local.enable_internal_storage ? 1 : 0 + count = 1 namespace = var.namespace resource_group_name = azurerm_resource_group.default.name location = azurerm_resource_group.default.location @@ -90,7 +87,7 @@ module "storage" { identity_ids = module.identity.identity.id deletion_protection = var.deletion_protection - storage_key_id = var.customer_storage_vault_key_id == null ? try(module.vault.vault_internal_keys[module.vault.vault_key_map.storage].id, null) : var.customer_storage_vault_key_id + storage_key_id = try(module.vault.vault_internal_keys[module.vault.vault_key_map.storage].id, null) disable_storage_vault_key_id = var.disable_storage_vault_key_id tags = var.tags @@ -129,21 +126,6 @@ module "app_aks" { max_pods = var.node_max_pods tags = var.tags } - -locals { - container_name = try(module.storage[0].container.name, "") - account_name = try(module.storage[0].account.name, "") - access_key = try(module.storage[0].account.primary_access_key, "") - queue_name = try(module.storage[0].queue.name, "") - blob_container = var.external_bucket == null ? coalesce(var.blob_container, local.container_name) : "" - storage_account = var.external_bucket == null ? coalesce(var.storage_account, local.account_name) : "" - storage_key = var.external_bucket == null ? coalesce(var.storage_key, local.access_key) : "" - bucket = "az://${local.storage_account}/${local.blob_container}" - queue = (var.use_internal_queue || var.blob_container == "" || var.external_bucket == null) ? "internal://" : "az://${local.account_name}/${local.queue_name}" - - redis_connection_string = "redis://:${module.redis.instance.primary_access_key}@${module.redis.instance.hostname}:${module.redis.instance.port}" -} - locals { service_account_name = "wandb-app" private_endpoint_approval_sa = "private-endpoint-sa" @@ -231,6 +213,21 @@ module "cert_manager" { depends_on = [module.app_aks] } +locals { + use_customer_bucket = ( + var.storage_account != "" && + var.blob_container != "" && + var.storage_key != "" + ) + default_bucket_config = { + provider = "az" + name = var.storage_account + path = var.blob_container + accessKey = var.storage_key + } + bucket_config = var.external_bucket != null ? var.external_bucket : (local.use_customer_bucket ? local.default_bucket_config : null) +} + module "wandb" { source = "wandb/wandb/helm" version = "1.2.0" @@ -250,12 +247,22 @@ module "wandb" { host = local.url license = var.license cloudProvider = "azure" - bucket = var.external_bucket == null ? { + bucket = local.bucket_config == null ? { provider = "az" - name = local.storage_account - path = local.blob_container - accessKey = local.storage_key - } : var.external_bucket + name = module.storage[0].account.name + path = module.storage[0].container.name + accessKey = module.storage[0].account.primary_access_key + } : local.bucket_config + defaultBucket = { + provider = "az" + name = module.storage[0].account.name + path = module.storage[0].container.name + accessKey = module.storage[0].account.primary_access_key + } + azureIdentityForBucket = { + clientID = module.identity.identity.client_id + tenantID = module.identity.identity.tenant_id + } mysql = { host = module.database.address diff --git a/modules/byob/main.tf b/modules/byob/main.tf index f16edb1..4b0b81f 100644 --- a/modules/byob/main.tf +++ b/modules/byob/main.tf @@ -17,7 +17,6 @@ module "vault" { identity_object_id = module.identity[0].identity.principal_id depends_on = [module.identity] tags = var.tags - enable_purge_protection = var.enable_purge_protection enable_storage_vault_key = var.create_cmk } diff --git a/modules/byob/variables.tf b/modules/byob/variables.tf index 75e0888..c65700c 100644 --- a/modules/byob/variables.tf +++ b/modules/byob/variables.tf @@ -30,12 +30,6 @@ variable "tags" { default = {} } -variable "enable_purge_protection" { - type = bool - default = false - description = "Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled." -} - variable "disable_storage_vault_key_id" { type = bool default = false @@ -44,10 +38,10 @@ variable "disable_storage_vault_key_id" { variable "tenant_id" { type = string - description = "The tenant ID for the Key Vault Access Policy. Get from W&B SA" + description = "The tenant ID for the Key Vault Access Policy. Get from `https:///console/settings/advanced/spec/active`" } variable "client_id" { type = string - description = "The client ID (object id) for the Key Vault Access Policy. Get from W&B SA" + description = "The client ID (object id) for the Key Vault Access Policy. Get from `https:///console/settings/advanced/spec/active`" } diff --git a/modules/vault/main.tf b/modules/vault/main.tf index 477c779..7929c80 100644 --- a/modules/vault/main.tf +++ b/modules/vault/main.tf @@ -16,7 +16,7 @@ resource "azurerm_key_vault" "default" { location = var.location resource_group_name = var.resource_group.name tenant_id = data.azurerm_client_config.current.tenant_id - purge_protection_enabled = var.enable_purge_protection + purge_protection_enabled = true # https://learn.microsoft.com/en-us/azure/mysql/flexible-server/concepts-customer-managed-key#requirements-for-configuring-data-encryption-for-azure-database-for-mysql-flexible-server soft_delete_retention_days = 90 # This must be 90 for azure msyql flex server encryption. enabled_for_disk_encryption = true diff --git a/modules/vault/variables.tf b/modules/vault/variables.tf index 693b9ca..d3d1ced 100644 --- a/modules/vault/variables.tf +++ b/modules/vault/variables.tf @@ -33,9 +33,3 @@ variable "enable_database_vault_key" { default = false description = "Flag to enable managed key encryption for the database. Once enabled, cannot be disabled or you will loose access to the database." } - -variable "enable_purge_protection" { - type = bool - default = false - description = "Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled." -} diff --git a/outputs.tf b/outputs.tf index 40870ec..3176fb0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -33,14 +33,6 @@ output "oidc_issuer_url" { value = module.app_aks.oidc_issuer_url } -output "storage_account" { - value = var.external_bucket != null ? "" : coalesce(var.storage_account, local.account_name, "") -} - -output "storage_container" { - value = var.external_bucket != null ? "" : coalesce(var.blob_container, local.container_name) -} - output "private_link_resource_id" { value = module.app_lb.gateway_id } diff --git a/variables.tf b/variables.tf index 4cfa373..a8771c0 100644 --- a/variables.tf +++ b/variables.tf @@ -271,29 +271,12 @@ variable "disable_storage_vault_key_id" { description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." } -variable "customer_storage_vault_key_id" { - type = string - default = null - description = "The Azure Key Vault key ID for customer-provided storage encryption keys. Must match the pattern 'https://.vault.azure.net/keys//', or be null." - - validation { - condition = var.customer_storage_vault_key_id == null || can(regex("^https://[a-zA-Z0-9-]+.vault.azure.net/keys/[a-zA-Z0-9-]+/[a-f0-9]+$", var.customer_storage_vault_key_id)) - error_message = "The customer_storage_vault_key_id must be null or a valid Azure Key Vault key ID in the format 'https://.vault.azure.net/keys//'." - } -} - variable "enable_database_vault_key" { type = bool default = false description = "Flag to enable managed key encryption for the database. Once enabled, cannot be disabled." } -variable "enable_purge_protection" { - type = bool - default = false - description = "Flag to enable purge protection for the Azure Key Vault. Once enabled, cannot be disabled." -} - ## To support otel azure monitor sql and redis metrics need operator-wandb chart minimum version 0.14.0 variable "azuremonitor" { type = bool From 513ce08016fffbbe81e6b05672e564ee9562e98f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 5 Aug 2024 19:07:25 +0000 Subject: [PATCH 14/27] chore(release): version 2.13.1 [skip ci] ### [2.13.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.0...v2.13.1) (2024-08-05) ### Bug Fixes * Azure storage and Vault simplified ([#89](https://github.com/wandb/terraform-azurerm-wandb/issues/89)) ([4832d24](https://github.com/wandb/terraform-azurerm-wandb/commit/4832d247cdf8e75fe1ae75e7f4da8b528cde93e4)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e42b2f..54b0357 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [2.13.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.0...v2.13.1) (2024-08-05) + + +### Bug Fixes + +* Azure storage and Vault simplified ([#89](https://github.com/wandb/terraform-azurerm-wandb/issues/89)) ([4832d24](https://github.com/wandb/terraform-azurerm-wandb/commit/4832d247cdf8e75fe1ae75e7f4da8b528cde93e4)) + ## [2.13.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.2...v2.13.0) (2024-08-01) From 38d012f27a16c9a77d52e90e9bad99ae432bec83 Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk <77289967+zacharyblasczyk@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:47:11 -0700 Subject: [PATCH 15/27] fix: Max Length of Storage Account name (#90) --- modules/storage/main.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/storage/main.tf b/modules/storage/main.tf index 0c55022..c798a7d 100644 --- a/modules/storage/main.tf +++ b/modules/storage/main.tf @@ -1,5 +1,9 @@ +locals { + postfix = "storage" + truncated_namespace = substr(replace(var.namespace, "-", ""), 0, 24 - length(local.postfix)) +} resource "azurerm_storage_account" "default" { - name = replace("${var.namespace}-storage", "-", "") + name = "${local.truncated_namespace}${local.postfix}" resource_group_name = var.resource_group_name location = var.location account_tier = "Standard" From 2f166a3edb983dd32c38ba75831e6539e2934b81 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 5 Aug 2024 19:47:43 +0000 Subject: [PATCH 16/27] chore(release): version 2.13.2 [skip ci] ### [2.13.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.1...v2.13.2) (2024-08-05) ### Bug Fixes * Max Length of Storage Account name ([#90](https://github.com/wandb/terraform-azurerm-wandb/issues/90)) ([38d012f](https://github.com/wandb/terraform-azurerm-wandb/commit/38d012f27a16c9a77d52e90e9bad99ae432bec83)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54b0357..9aa131a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [2.13.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.1...v2.13.2) (2024-08-05) + + +### Bug Fixes + +* Max Length of Storage Account name ([#90](https://github.com/wandb/terraform-azurerm-wandb/issues/90)) ([38d012f](https://github.com/wandb/terraform-azurerm-wandb/commit/38d012f27a16c9a77d52e90e9bad99ae432bec83)) + ### [2.13.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.0...v2.13.1) (2024-08-05) From 2f430f25b98cac894c794edce12215d1847df475 Mon Sep 17 00:00:00 2001 From: levinandrew Date: Mon, 26 Aug 2024 08:05:33 -0700 Subject: [PATCH 17/27] feat: Add optional path var for instance level bucket path (#84) --- README.md | 1 + examples/public-dns/main.tf | 2 ++ examples/public-dns/variables.tf | 6 ++++++ main.tf | 6 +++--- variables.tf | 11 +++++++++++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 474b6d6..a48ea0e 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ resources that lack official modules. | [app\_wandb\_env](#input\_app\_wandb\_env) | Extra environment variables for W&B | `map(string)` | `{}` | no | | [azuremonitor](#input\_azuremonitor) | # To support otel azure monitor sql and redis metrics need operator-wandb chart minimum version 0.14.0 | `bool` | `false` | no | | [blob\_container](#input\_blob\_container) | Use an existing bucket. | `string` | `""` | no | +| [bucket\_path](#input\_bucket\_path) | path of where to store data for the instance-level bucket | `string` | `""` | no | | [cluster\_sku\_tier](#input\_cluster\_sku\_tier) | The Azure AKS SKU Tier to use for this cluster (https://learn.microsoft.com/en-us/azure/aks/free-standard-pricing-tiers) | `string` | `"Free"` | no | | [create\_private\_link](#input\_create\_private\_link) | Use for the azure private link. | `bool` | `false` | no | | [create\_redis](#input\_create\_redis) | Boolean indicating whether to provision an redis instance (true) or not (false). | `bool` | `false` | no | diff --git a/examples/public-dns/main.tf b/examples/public-dns/main.tf index 93215af..1ff8157 100644 --- a/examples/public-dns/main.tf +++ b/examples/public-dns/main.tf @@ -41,6 +41,8 @@ module "wandb" { deletion_protection = false + bucket_path = var.bucket_path + tags = { "Example" : "PublicDns" } diff --git a/examples/public-dns/variables.tf b/examples/public-dns/variables.tf index ebdb00d..2cc981b 100644 --- a/examples/public-dns/variables.tf +++ b/examples/public-dns/variables.tf @@ -52,3 +52,9 @@ variable "database_sku_name" { default = "GP_Standard_D4ds_v4" description = "Specifies the SKU Name for this MySQL Server" } + +variable "bucket_path" { + description = "path of where to store data for the instance-level bucket" + type = string + default = "" +} \ No newline at end of file diff --git a/main.tf b/main.tf index c3005a0..94e5e1b 100644 --- a/main.tf +++ b/main.tf @@ -222,7 +222,7 @@ locals { default_bucket_config = { provider = "az" name = var.storage_account - path = var.blob_container + path = "${var.blob_container}/${var.bucket_path}" accessKey = var.storage_key } bucket_config = var.external_bucket != null ? var.external_bucket : (local.use_customer_bucket ? local.default_bucket_config : null) @@ -250,13 +250,13 @@ module "wandb" { bucket = local.bucket_config == null ? { provider = "az" name = module.storage[0].account.name - path = module.storage[0].container.name + path = "${module.storage[0].container.name}/${var.bucket_path}" accessKey = module.storage[0].account.primary_access_key } : local.bucket_config defaultBucket = { provider = "az" name = module.storage[0].account.name - path = module.storage[0].container.name + path = "${module.storage[0].container.name}/${var.bucket_path}" accessKey = module.storage[0].account.primary_access_key } azureIdentityForBucket = { diff --git a/variables.tf b/variables.tf index a8771c0..7c0e2c4 100644 --- a/variables.tf +++ b/variables.tf @@ -180,6 +180,17 @@ variable "external_bucket" { default = null } +########################################## +# Bucket path # +########################################## +# This setting is meant for users who want to store all of their instance-level +# bucket's data at a specific path within their bucket. It can be set both for +# external buckets or the bucket created by this module. +variable "bucket_path" { + description = "path of where to store data for the instance-level bucket" + type = string + default = "" +} ########################################## # K8s # From a8fb8a2f3b8a176d4e78b2d7e846966c28c48600 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 26 Aug 2024 15:06:04 +0000 Subject: [PATCH 18/27] chore(release): version 2.14.0 [skip ci] ## [2.14.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.2...v2.14.0) (2024-08-26) ### Features * Add optional path var for instance level bucket path ([#84](https://github.com/wandb/terraform-azurerm-wandb/issues/84)) ([2f430f2](https://github.com/wandb/terraform-azurerm-wandb/commit/2f430f25b98cac894c794edce12215d1847df475)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa131a..9070131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [2.14.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.2...v2.14.0) (2024-08-26) + + +### Features + +* Add optional path var for instance level bucket path ([#84](https://github.com/wandb/terraform-azurerm-wandb/issues/84)) ([2f430f2](https://github.com/wandb/terraform-azurerm-wandb/commit/2f430f25b98cac894c794edce12215d1847df475)) + ### [2.13.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.1...v2.13.2) (2024-08-05) From c9b4d664dfc85c5f603e6a14b694923af8d1259d Mon Sep 17 00:00:00 2001 From: Jeff Raubitschek Date: Mon, 26 Aug 2024 08:51:08 -0700 Subject: [PATCH 19/27] feat: Add support for Private Link to ClickHouse (#93) --- README.md | 3 +++ main.tf | 13 ++++++++++ modules/clickhouse/main.tf | 43 ++++++++++++++++++++++++++++++ modules/clickhouse/variables.tf | 46 +++++++++++++++++++++++++++++++++ variables.tf | 15 +++++++++++ 5 files changed, 120 insertions(+) create mode 100644 modules/clickhouse/main.tf create mode 100644 modules/clickhouse/variables.tf diff --git a/README.md b/README.md index a48ea0e..56dafe2 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ resources that lack official modules. | [app\_aks](#module\_app\_aks) | ./modules/app_aks | n/a | | [app\_lb](#module\_app\_lb) | ./modules/app_lb | n/a | | [cert\_manager](#module\_cert\_manager) | ./modules/cert_manager | n/a | +| [clickhouse](#module\_clickhouse) | ./modules/clickhouse | n/a | | [cron\_job](#module\_cron\_job) | ./modules/cron_job | n/a | | [database](#module\_database) | ./modules/database | n/a | | [identity](#module\_identity) | ./modules/identity | n/a | @@ -76,6 +77,8 @@ resources that lack official modules. | [azuremonitor](#input\_azuremonitor) | # To support otel azure monitor sql and redis metrics need operator-wandb chart minimum version 0.14.0 | `bool` | `false` | no | | [blob\_container](#input\_blob\_container) | Use an existing bucket. | `string` | `""` | no | | [bucket\_path](#input\_bucket\_path) | path of where to store data for the instance-level bucket | `string` | `""` | no | +| [clickhouse\_private\_endpoint\_service\_name](#input\_clickhouse\_private\_endpoint\_service\_name) | ClickHouse private endpoint 'Service name' (ends in .azure.privatelinkservice). | `string` | `""` | no | +| [clickhouse\_region](#input\_clickhouse\_region) | ClickHouse region (eastus2, westus3, etc). | `string` | `""` | no | | [cluster\_sku\_tier](#input\_cluster\_sku\_tier) | The Azure AKS SKU Tier to use for this cluster (https://learn.microsoft.com/en-us/azure/aks/free-standard-pricing-tiers) | `string` | `"Free"` | no | | [create\_private\_link](#input\_create\_private\_link) | Use for the azure private link. | `bool` | `false` | no | | [create\_redis](#input\_create\_redis) | Boolean indicating whether to provision an redis instance (true) or not (false). | `bool` | `false` | no | diff --git a/main.tf b/main.tf index 94e5e1b..7f0f8ff 100644 --- a/main.tf +++ b/main.tf @@ -213,6 +213,19 @@ module "cert_manager" { depends_on = [module.app_aks] } +module "clickhouse" { + count = var.clickhouse_private_endpoint_service_name != "" ? 1 : 0 + source = "./modules/clickhouse" + namespace = var.namespace + resource_group_name = azurerm_resource_group.default.name + location = azurerm_resource_group.default.location + network_id = module.networking.network.id + private_subnet_id = module.networking.private_subnet.id + + clickhouse_private_endpoint_service_name = var.clickhouse_private_endpoint_service_name + clickhouse_region = var.clickhouse_region +} + locals { use_customer_bucket = ( var.storage_account != "" && diff --git a/modules/clickhouse/main.tf b/modules/clickhouse/main.tf new file mode 100644 index 0000000..01e5c9f --- /dev/null +++ b/modules/clickhouse/main.tf @@ -0,0 +1,43 @@ +locals { + dns_name_suffix = "privatelink.azure.clickhouse.cloud" +} + +resource "azurerm_private_endpoint" "clickhouse" { + name = "${var.namespace}-clickhouse-pe" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.private_subnet_id + custom_network_interface_name = "${var.namespace}-clickhouse-nic" + + private_service_connection { + name = "${var.namespace}-clickhouse-pl" + private_connection_resource_alias = var.clickhouse_private_endpoint_service_name + is_manual_connection = true + request_message = "ClickHouse Private Link" + } +} + +resource "azurerm_private_dns_zone" "clickhouse_cloud_private_link_zone" { + name = "${var.clickhouse_region}.${local.dns_name_suffix}" + resource_group_name = var.resource_group_name +} + +data "azurerm_network_interface" "clickhouse_nic" { + resource_group_name = var.resource_group_name + name = azurerm_private_endpoint.clickhouse.network_interface[0].name +} + +resource "azurerm_private_dns_a_record" "clickhouse_wildcard" { + name = "*" + zone_name = azurerm_private_dns_zone.clickhouse_cloud_private_link_zone.name + resource_group_name = var.resource_group_name + ttl = 300 + records = [data.azurerm_network_interface.clickhouse_nic.private_ip_address] +} + +resource "azurerm_private_dns_zone_virtual_network_link" "clickhouse_network" { + name = "network-link" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.clickhouse_cloud_private_link_zone.name + virtual_network_id = var.network_id +} diff --git a/modules/clickhouse/variables.tf b/modules/clickhouse/variables.tf new file mode 100644 index 0000000..4819ed0 --- /dev/null +++ b/modules/clickhouse/variables.tf @@ -0,0 +1,46 @@ +variable "namespace" { + type = string + description = "Friendly name prefix used for tagging and naming Azure resources." +} + +variable "resource_group_name" { + type = string + description = "The name of the resource group in which to create the network." +} + +variable "location" { + type = string + description = "Specifies the supported Azure location where the resource exists." +} + +variable "network_id" { + type = string + description = "The virtual network id used for all resources" +} + +variable "private_subnet_id" { + type = string + description = "Specifies the supported Azure subnet id where the resource exists." +} + +variable "clickhouse_private_endpoint_service_name" { + type = string + description = "ClickHouse private endpoint 'Service name' (ends in .azure.privatelinkservice)." + default = "" + + validation { + condition = can(regex("\\.azure\\.privatelinkservice$", var.clickhouse_private_endpoint_service_name)) + error_message = "ClickHouse Service name must end in '.azure.privatelinkservice'." + } +} + +variable "clickhouse_region" { + type = string + description = "ClickHouse region (eastus2, westus3, etc)." + default = "" + + validation { + condition = length(var.clickhouse_region) > 0 + error_message = "Clickhouse Region should always be set if the private endpoint service name is specified." + } +} diff --git a/variables.tf b/variables.tf index 7c0e2c4..e5e5060 100644 --- a/variables.tf +++ b/variables.tf @@ -293,3 +293,18 @@ variable "azuremonitor" { type = bool default = false } + +########################################### +# ClickHouse endpoint # +########################################### +variable "clickhouse_private_endpoint_service_name" { + type = string + description = "ClickHouse private endpoint 'Service name' (ends in .azure.privatelinkservice)." + default = "" +} + +variable "clickhouse_region" { + type = string + description = "ClickHouse region (eastus2, westus3, etc)." + default = "" +} From 615043c984471a4032c66f6550d1dd674bd40529 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 26 Aug 2024 15:51:36 +0000 Subject: [PATCH 20/27] chore(release): version 2.15.0 [skip ci] ## [2.15.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.14.0...v2.15.0) (2024-08-26) ### Features * Add support for Private Link to ClickHouse ([#93](https://github.com/wandb/terraform-azurerm-wandb/issues/93)) ([c9b4d66](https://github.com/wandb/terraform-azurerm-wandb/commit/c9b4d664dfc85c5f603e6a14b694923af8d1259d)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9070131..6e57437 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [2.15.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.14.0...v2.15.0) (2024-08-26) + + +### Features + +* Add support for Private Link to ClickHouse ([#93](https://github.com/wandb/terraform-azurerm-wandb/issues/93)) ([c9b4d66](https://github.com/wandb/terraform-azurerm-wandb/commit/c9b4d664dfc85c5f603e6a14b694923af8d1259d)) + ## [2.14.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.2...v2.14.0) (2024-08-26) From 76e25119c12157c8ee19c013c6544305ddd62b7c Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk <77289967+zacharyblasczyk@users.noreply.github.com> Date: Wed, 11 Sep 2024 21:17:14 -0500 Subject: [PATCH 21/27] fix: Bump operator chart and controller image (#99) --- main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 7f0f8ff..81fa724 100644 --- a/main.tf +++ b/main.tf @@ -251,8 +251,8 @@ module "wandb" { module.database, module.storage, ] - controller_image_tag = "1.12.0" - operator_chart_version = "1.2.4" + controller_image_tag = "1.13.0" + operator_chart_version = "1.3.1" spec = { values = { @@ -260,7 +260,7 @@ module "wandb" { host = local.url license = var.license cloudProvider = "azure" - bucket = local.bucket_config == null ? { + bucket = local.bucket_config == null ? { provider = "az" name = module.storage[0].account.name path = "${module.storage[0].container.name}/${var.bucket_path}" From 1020ee154b04b19dd99b3789254685d93e896cef Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 12 Sep 2024 02:17:41 +0000 Subject: [PATCH 22/27] chore(release): version 2.15.1 [skip ci] ### [2.15.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.15.0...v2.15.1) (2024-09-12) ### Bug Fixes * Bump operator chart and controller image ([#99](https://github.com/wandb/terraform-azurerm-wandb/issues/99)) ([76e2511](https://github.com/wandb/terraform-azurerm-wandb/commit/76e25119c12157c8ee19c013c6544305ddd62b7c)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e57437..f665774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +### [2.15.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.15.0...v2.15.1) (2024-09-12) + + +### Bug Fixes + +* Bump operator chart and controller image ([#99](https://github.com/wandb/terraform-azurerm-wandb/issues/99)) ([76e2511](https://github.com/wandb/terraform-azurerm-wandb/commit/76e25119c12157c8ee19c013c6544305ddd62b7c)) + ## [2.15.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.14.0...v2.15.0) (2024-08-26) From 8a17accd070035004e7665cb46a30c0fe41283fb Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Fri, 27 Sep 2024 11:48:21 -0700 Subject: [PATCH 23/27] feat!: Automatically select availability zones based on node type when not specified (#102) Co-authored-by: github-actions[bot] --- README.md | 6 +++++- examples/public-dns/main.tf | 27 +++++++++++++++++++++++++++ main.tf | 28 ++++++++++++++++++++++++++-- variables.tf | 8 +++++++- versions.tf | 4 ++++ 5 files changed, 69 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 56dafe2..96dd306 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ resources that lack official modules. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | ~> 1.0 | +| [azapi](#requirement\_azapi) | ~> 1.0 | | [azurerm](#requirement\_azurerm) | ~> 3.17 | | [helm](#requirement\_helm) | ~> 2.6 | | [kubernetes](#requirement\_kubernetes) | ~> 2.23 | @@ -41,6 +42,7 @@ resources that lack official modules. | Name | Version | |------|---------| +| [azapi](#provider\_azapi) | ~> 1.0 | | [azurerm](#provider\_azurerm) | ~> 3.17 | ## Modules @@ -65,6 +67,7 @@ resources that lack official modules. | Name | Type | |------|------| +| [azapi_resource_list.az_zones](https://registry.terraform.io/providers/azure/azapi/latest/docs/data-sources/resource_list) | data source | | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | ## Inputs @@ -97,7 +100,8 @@ resources that lack official modules. | [location](#input\_location) | n/a | `string` | n/a | yes | | [namespace](#input\_namespace) | String used for prefix resources. | `string` | n/a | yes | | [node\_max\_pods](#input\_node\_max\_pods) | Maximum number of pods per node | `number` | `30` | no | -| [node\_pool\_zones](#input\_node\_pool\_zones) | Availability zones for the node pool | `list(string)` | 
[
  "1",
  "2"
]
| no | +| [node\_pool\_num\_zones](#input\_node\_pool\_num\_zones) | Number of availability zones to use for the node pool when node\_pool\_zones is not set. | `number` | `2` | no | +| [node\_pool\_zones](#input\_node\_pool\_zones) | Availability zones for the node pool | `list(string)` | `null` | no | | [oidc\_auth\_method](#input\_oidc\_auth\_method) | OIDC auth method | `string` | `"implicit"` | no | | [oidc\_client\_id](#input\_oidc\_client\_id) | The Client ID of application in your identity provider | `string` | `""` | no | | [oidc\_issuer](#input\_oidc\_issuer) | A url to your Open ID Connect identity provider, i.e. https://cognito-idp.us-east-1.amazonaws.com/us-east-1_uiIFNdacd | `string` | `""` | no | diff --git a/examples/public-dns/main.tf b/examples/public-dns/main.tf index 1ff8157..0bb359d 100644 --- a/examples/public-dns/main.tf +++ b/examples/public-dns/main.tf @@ -1,8 +1,34 @@ +terraform { + required_version = "~> 1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.17" + } + azapi = { + source = "azure/azapi" + version = "~> 1.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.6" + } + } +} + provider "azurerm" { subscription_id = var.subscription_id features {} } +provider "azapi" { + subscription_id = var.subscription_id +} + data "azurerm_subscription" "current" {} provider "kubernetes" { @@ -46,6 +72,7 @@ module "wandb" { tags = { "Example" : "PublicDns" } + node_pool_num_zones = 2 } # # You'll want to update your DNS with the provisioned IP address diff --git a/main.tf b/main.tf index 81fa724..81a949a 100644 --- a/main.tf +++ b/main.tf @@ -107,6 +107,30 @@ module "app_lb" { tags = var.tags } +locals { + kubernetes_instance_type = try(local.deployment_size[var.size].node_type, var.kubernetes_instance_type) +} + +data "azapi_resource_list" "az_zones" { + parent_id = "/subscriptions/${data.azurerm_subscription.current.subscription_id}" + type = "Microsoft.Compute/skus@2021-07-01" + + response_export_values = ["value"] +} + +locals { + vm_skus = [ + for sku in jsondecode(data.azapi_resource_list.az_zones.output).value : + sku if( + sku.resourceType == "virtualMachines" && + lower(sku.locations[0]) == lower(azurerm_resource_group.default.location) && + sku.name == local.kubernetes_instance_type + ) + ] + num_zones = var.node_pool_zones != null ? length(var.node_pool_zones) : var.node_pool_num_zones + node_pool_zones = var.node_pool_zones != null ? var.node_pool_zones : slice(sort(local.vm_skus[0].locationInfo[0].zones), 0, local.num_zones) +} + module "app_aks" { source = "./modules/app_aks" depends_on = [module.app_lb] @@ -118,8 +142,8 @@ module "app_aks" { location = azurerm_resource_group.default.location namespace = var.namespace node_pool_vm_count = try(local.deployment_size[var.size].node_count, var.kubernetes_node_count) - node_pool_vm_size = try(local.deployment_size[var.size].node_instance, var.kubernetes_instance_type) - node_pool_zones = var.node_pool_zones + node_pool_vm_size = local.kubernetes_instance_type + node_pool_zones = local.node_pool_zones public_subnet = module.networking.public_subnet resource_group = azurerm_resource_group.default sku_tier = var.cluster_sku_tier diff --git a/variables.tf b/variables.tf index e5e5060..bc3efa3 100644 --- a/variables.tf +++ b/variables.tf @@ -215,7 +215,13 @@ variable "cluster_sku_tier" { variable "node_pool_zones" { type = list(string) description = "Availability zones for the node pool" - default = ["1", "2"] + default = null +} + +variable "node_pool_num_zones" { + type = number + description = "Number of availability zones to use for the node pool when node_pool_zones is not set." + default = 2 } variable "node_max_pods" { diff --git a/versions.tf b/versions.tf index 79b7958..59c2d3e 100644 --- a/versions.tf +++ b/versions.tf @@ -5,6 +5,10 @@ terraform { source = "hashicorp/azurerm" version = "~> 3.17" } + azapi = { + source = "azure/azapi" + version = "~> 1.0" + } kubernetes = { source = "hashicorp/kubernetes" version = "~> 2.23" From cbb7c0cefe7d36358f1518bc79491edf8e5556e9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 27 Sep 2024 18:48:48 +0000 Subject: [PATCH 24/27] chore(release): version 3.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [3.0.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.15.1...v3.0.0) (2024-09-27) ### ⚠ BREAKING CHANGES * Automatically select availability zones based on node type when not specified (#102) ### Features * Automatically select availability zones based on node type when not specified ([#102](https://github.com/wandb/terraform-azurerm-wandb/issues/102)) ([8a17acc](https://github.com/wandb/terraform-azurerm-wandb/commit/8a17accd070035004e7665cb46a30c0fe41283fb)) --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f665774..dd24607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. +## [3.0.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.15.1...v3.0.0) (2024-09-27) + + +### ⚠ BREAKING CHANGES + +* Automatically select availability zones based on node type when not specified (#102) + +### Features + +* Automatically select availability zones based on node type when not specified ([#102](https://github.com/wandb/terraform-azurerm-wandb/issues/102)) ([8a17acc](https://github.com/wandb/terraform-azurerm-wandb/commit/8a17accd070035004e7665cb46a30c0fe41283fb)) + ### [2.15.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.15.0...v2.15.1) (2024-09-12) From df95c36a30f024ed1a4f650fc665dc87619577c2 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Fri, 27 Sep 2024 13:30:23 -0700 Subject: [PATCH 25/27] fix: Add info for users moving from 2.x to 3.x (#103) --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 96dd306..5235a5d 100644 --- a/README.md +++ b/README.md @@ -141,3 +141,31 @@ resources that lack official modules. | [tenant\_id](#output\_tenant\_id) | n/a | | [url](#output\_url) | The URL to the W&B application | + +## Migrations + +### Upgrading from 2.x to 3.x + +When upgrading from 2.x to 3.x, the following changes are required: + +1. Add the `azapi` provider to the `required_providers` block: + +```hcl +terraform { + required_providers { + azapi = { + source = "azure/azapi" + version = "~> 1.0" + } + } +} +``` + +2. Add the `azapi` provider to the `provider` block: + +```hcl +provider "azapi" { + # azapi provider configuration should be the same as azurerm provider configuration +} +``` + From d910a411637352673c1df11a7f0ce8e461eb24db Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Mon, 30 Sep 2024 16:34:16 -0700 Subject: [PATCH 26/27] fix: Reference the correct value from deployment sizes (#104) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 81a949a..1c386c7 100644 --- a/main.tf +++ b/main.tf @@ -108,7 +108,7 @@ module "app_lb" { } locals { - kubernetes_instance_type = try(local.deployment_size[var.size].node_type, var.kubernetes_instance_type) + kubernetes_instance_type = try(local.deployment_size[var.size].node_instance, var.kubernetes_instance_type) } data "azapi_resource_list" "az_zones" { From a228e17bb124d0746e6ded36dfeaff76d419b436 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 30 Sep 2024 23:34:41 +0000 Subject: [PATCH 27/27] chore(release): version 3.0.1 [skip ci] ### [3.0.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v3.0.0...v3.0.1) (2024-09-30) ### Bug Fixes * Add info for users moving from 2.x to 3.x ([#103](https://github.com/wandb/terraform-azurerm-wandb/issues/103)) ([df95c36](https://github.com/wandb/terraform-azurerm-wandb/commit/df95c36a30f024ed1a4f650fc665dc87619577c2)) * Reference the correct value from deployment sizes ([#104](https://github.com/wandb/terraform-azurerm-wandb/issues/104)) ([d910a41](https://github.com/wandb/terraform-azurerm-wandb/commit/d910a411637352673c1df11a7f0ce8e461eb24db)) --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd24607..53ddd0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. +### [3.0.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v3.0.0...v3.0.1) (2024-09-30) + + +### Bug Fixes + +* Add info for users moving from 2.x to 3.x ([#103](https://github.com/wandb/terraform-azurerm-wandb/issues/103)) ([df95c36](https://github.com/wandb/terraform-azurerm-wandb/commit/df95c36a30f024ed1a4f650fc665dc87619577c2)) +* Reference the correct value from deployment sizes ([#104](https://github.com/wandb/terraform-azurerm-wandb/issues/104)) ([d910a41](https://github.com/wandb/terraform-azurerm-wandb/commit/d910a411637352673c1df11a7f0ce8e461eb24db)) + ## [3.0.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.15.1...v3.0.0) (2024-09-27)