diff --git a/CHANGELOG.md b/CHANGELOG.md index 209713b..53ddd0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,102 @@ All notable changes to this project will be documented in this file. +### [3.0.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v3.0.0...v3.0.1) (2024-09-30) + + +### Bug Fixes + +* Add info for users moving from 2.x to 3.x ([#103](https://github.com/wandb/terraform-azurerm-wandb/issues/103)) ([df95c36](https://github.com/wandb/terraform-azurerm-wandb/commit/df95c36a30f024ed1a4f650fc665dc87619577c2)) +* Reference the correct value from deployment sizes ([#104](https://github.com/wandb/terraform-azurerm-wandb/issues/104)) ([d910a41](https://github.com/wandb/terraform-azurerm-wandb/commit/d910a411637352673c1df11a7f0ce8e461eb24db)) + +## [3.0.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.15.1...v3.0.0) (2024-09-27) + + +### ⚠ BREAKING CHANGES + +* Automatically select availability zones based on node type when not specified (#102) + +### Features + +* Automatically select availability zones based on node type when not specified ([#102](https://github.com/wandb/terraform-azurerm-wandb/issues/102)) ([8a17acc](https://github.com/wandb/terraform-azurerm-wandb/commit/8a17accd070035004e7665cb46a30c0fe41283fb)) + +### [2.15.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.15.0...v2.15.1) (2024-09-12) + + +### Bug Fixes + +* Bump operator chart and controller image ([#99](https://github.com/wandb/terraform-azurerm-wandb/issues/99)) ([76e2511](https://github.com/wandb/terraform-azurerm-wandb/commit/76e25119c12157c8ee19c013c6544305ddd62b7c)) + +## [2.15.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.14.0...v2.15.0) (2024-08-26) + + +### Features + +* Add support for Private Link to ClickHouse ([#93](https://github.com/wandb/terraform-azurerm-wandb/issues/93)) ([c9b4d66](https://github.com/wandb/terraform-azurerm-wandb/commit/c9b4d664dfc85c5f603e6a14b694923af8d1259d)) + +## [2.14.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.2...v2.14.0) (2024-08-26) + + +### Features + +* Add optional path var for instance level bucket path ([#84](https://github.com/wandb/terraform-azurerm-wandb/issues/84)) ([2f430f2](https://github.com/wandb/terraform-azurerm-wandb/commit/2f430f25b98cac894c794edce12215d1847df475)) + +### [2.13.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.1...v2.13.2) (2024-08-05) + + +### Bug Fixes + +* Max Length of Storage Account name ([#90](https://github.com/wandb/terraform-azurerm-wandb/issues/90)) ([38d012f](https://github.com/wandb/terraform-azurerm-wandb/commit/38d012f27a16c9a77d52e90e9bad99ae432bec83)) + +### [2.13.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.13.0...v2.13.1) (2024-08-05) + + +### Bug Fixes + +* Azure storage and Vault simplified ([#89](https://github.com/wandb/terraform-azurerm-wandb/issues/89)) ([4832d24](https://github.com/wandb/terraform-azurerm-wandb/commit/4832d247cdf8e75fe1ae75e7f4da8b528cde93e4)) + +## [2.13.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.2...v2.13.0) (2024-08-01) + + +### Features + +* Added for encrypting the database and blob storage with WB-managed key ([#49](https://github.com/wandb/terraform-azurerm-wandb/issues/49)) ([519c340](https://github.com/wandb/terraform-azurerm-wandb/commit/519c340fbf855743fe77b3ae075e6bfdb84740c2)) + +### [2.12.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.1...v2.12.2) (2024-08-01) + + +### Bug Fixes + +* Bump operator chart versions ([#87](https://github.com/wandb/terraform-azurerm-wandb/issues/87)) ([51e8736](https://github.com/wandb/terraform-azurerm-wandb/commit/51e873629db3263a27beda2bcf3f40190cc7e0ae)) + +### [2.12.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.12.0...v2.12.1) (2024-08-01) + + +### Bug Fixes + +* Bump operator chart versions ([#86](https://github.com/wandb/terraform-azurerm-wandb/issues/86)) ([d6a38f2](https://github.com/wandb/terraform-azurerm-wandb/commit/d6a38f22ab11cc131f7d7200f77ea39e6e53c8e3)) + +## [2.12.0](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.3...v2.12.0) (2024-07-31) + + +### Features + +* Bump operator image and chart versions ([#85](https://github.com/wandb/terraform-azurerm-wandb/issues/85)) ([d582e7c](https://github.com/wandb/terraform-azurerm-wandb/commit/d582e7ccfb8bb12354f0bff1001bf4ed59e1d9d5)) + +### [2.11.3](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.2...v2.11.3) (2024-07-11) + + +### Bug Fixes + +* Pass cloudprovider value to the helm charts ([#83](https://github.com/wandb/terraform-azurerm-wandb/issues/83)) ([0606602](https://github.com/wandb/terraform-azurerm-wandb/commit/06066020ac57d2d93e406e4b103feffb260426e0)) + +### [2.11.2](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.1...v2.11.2) (2024-06-26) + + +### Bug Fixes + +* Change ingress timeout to string ([#80](https://github.com/wandb/terraform-azurerm-wandb/issues/80)) ([64b96bc](https://github.com/wandb/terraform-azurerm-wandb/commit/64b96bc64ff90c6cf310ae3d3f4646a614712617)) + ### [2.11.1](https://github.com/wandb/terraform-azurerm-wandb/compare/v2.11.0...v2.11.1) (2024-06-25) diff --git a/README.md b/README.md index 7c0c086..5235a5d 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ resources that lack official modules. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | ~> 1.0 | +| [azapi](#requirement\_azapi) | ~> 1.0 | | [azurerm](#requirement\_azurerm) | ~> 3.17 | | [helm](#requirement\_helm) | ~> 2.6 | | [kubernetes](#requirement\_kubernetes) | ~> 2.23 | @@ -41,6 +42,7 @@ resources that lack official modules. | Name | Version | |------|---------| +| [azapi](#provider\_azapi) | ~> 1.0 | | [azurerm](#provider\_azurerm) | ~> 3.17 | ## Modules @@ -50,6 +52,7 @@ resources that lack official modules. | [app\_aks](#module\_app\_aks) | ./modules/app_aks | n/a | | [app\_lb](#module\_app\_lb) | ./modules/app_lb | n/a | | [cert\_manager](#module\_cert\_manager) | ./modules/cert_manager | n/a | +| [clickhouse](#module\_clickhouse) | ./modules/clickhouse | n/a | | [cron\_job](#module\_cron\_job) | ./modules/cron_job | n/a | | [database](#module\_database) | ./modules/database | n/a | | [identity](#module\_identity) | ./modules/identity | n/a | @@ -64,17 +67,21 @@ resources that lack official modules. | Name | Type | |------|------| +| [azapi_resource_list.az_zones](https://registry.terraform.io/providers/azure/azapi/latest/docs/data-sources/resource_list) | data source | | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [allowed\_ip\_ranges](#input\_allowed\_ip\_ranges) | allowed public IP addresses or CIDR ranges. | `list(string)` | `[]` | no | +| [allowed\_ip\_ranges](#input\_allowed\_ip\_ranges) | Allowed public IP addresses or CIDR ranges. | `list(string)` | `[]` | no | | [allowed\_subscriptions](#input\_allowed\_subscriptions) | List of allowed customer subscriptions coma seperated values | `string` | `""` | no | | [app\_wandb\_env](#input\_app\_wandb\_env) | Extra environment variables for W&B | `map(string)` | `{}` | no | | [azuremonitor](#input\_azuremonitor) | # To support otel azure monitor sql and redis metrics need operator-wandb chart minimum version 0.14.0 | `bool` | `false` | no | | [blob\_container](#input\_blob\_container) | Use an existing bucket. | `string` | `""` | no | +| [bucket\_path](#input\_bucket\_path) | path of where to store data for the instance-level bucket | `string` | `""` | no | +| [clickhouse\_private\_endpoint\_service\_name](#input\_clickhouse\_private\_endpoint\_service\_name) | ClickHouse private endpoint 'Service name' (ends in .azure.privatelinkservice). | `string` | `""` | no | +| [clickhouse\_region](#input\_clickhouse\_region) | ClickHouse region (eastus2, westus3, etc). | `string` | `""` | no | | [cluster\_sku\_tier](#input\_cluster\_sku\_tier) | The Azure AKS SKU Tier to use for this cluster (https://learn.microsoft.com/en-us/azure/aks/free-standard-pricing-tiers) | `string` | `"Free"` | no | | [create\_private\_link](#input\_create\_private\_link) | Use for the azure private link. | `bool` | `false` | no | | [create\_redis](#input\_create\_redis) | Boolean indicating whether to provision an redis instance (true) or not (false). | `bool` | `false` | no | @@ -82,7 +89,10 @@ resources that lack official modules. | [database\_sku\_name](#input\_database\_sku\_name) | Specifies the SKU Name for this MySQL Server | `string` | `"GP_Standard_D4ds_v4"` | no | | [database\_version](#input\_database\_version) | Version for MySQL | `string` | `"5.7"` | no | | [deletion\_protection](#input\_deletion\_protection) | If the instance should have deletion protection enabled. The database / Bucket can't be deleted when this value is set to `true`. | `bool` | `true` | no | +| [disable\_storage\_vault\_key\_id](#input\_disable\_storage\_vault\_key\_id) | Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation. | `bool` | `false` | no | | [domain\_name](#input\_domain\_name) | Domain for accessing the Weights & Biases UI. | `string` | `null` | no | +| [enable\_database\_vault\_key](#input\_enable\_database\_vault\_key) | Flag to enable managed key encryption for the database. Once enabled, cannot be disabled. | `bool` | `false` | no | +| [enable\_storage\_vault\_key](#input\_enable\_storage\_vault\_key) | Flag to enable managed key encryption for the storage account. | `bool` | `false` | no | | [external\_bucket](#input\_external\_bucket) | config an external bucket | `any` | `null` | no | | [kubernetes\_instance\_type](#input\_kubernetes\_instance\_type) | Use for the Kubernetes cluster. | `string` | `"Standard_D4a_v4"` | no | | [kubernetes\_node\_count](#input\_kubernetes\_node\_count) | n/a | `number` | `2` | no | @@ -90,7 +100,8 @@ resources that lack official modules. | [location](#input\_location) | n/a | `string` | n/a | yes | | [namespace](#input\_namespace) | String used for prefix resources. | `string` | n/a | yes | | [node\_max\_pods](#input\_node\_max\_pods) | Maximum number of pods per node | `number` | `30` | no | -| [node\_pool\_zones](#input\_node\_pool\_zones) | Availability zones for the node pool | `list(string)` | 
[
  "1",
  "2"
]
| no | +| [node\_pool\_num\_zones](#input\_node\_pool\_num\_zones) | Number of availability zones to use for the node pool when node\_pool\_zones is not set. | `number` | `2` | no | +| [node\_pool\_zones](#input\_node\_pool\_zones) | Availability zones for the node pool | `list(string)` | `null` | no | | [oidc\_auth\_method](#input\_oidc\_auth\_method) | OIDC auth method | `string` | `"implicit"` | no | | [oidc\_client\_id](#input\_oidc\_client\_id) | The Client ID of application in your identity provider | `string` | `""` | no | | [oidc\_issuer](#input\_oidc\_issuer) | A url to your Open ID Connect identity provider, i.e. https://cognito-idp.us-east-1.amazonaws.com/us-east-1_uiIFNdacd | `string` | `""` | no | @@ -116,6 +127,7 @@ resources that lack official modules. | [address](#output\_address) | n/a | | [aks\_node\_count](#output\_aks\_node\_count) | n/a | | [aks\_node\_instance\_type](#output\_aks\_node\_instance\_type) | n/a | +| [client\_id](#output\_client\_id) | n/a | | [cluster\_ca\_certificate](#output\_cluster\_ca\_certificate) | n/a | | [cluster\_client\_certificate](#output\_cluster\_client\_certificate) | n/a | | [cluster\_client\_key](#output\_cluster\_client\_key) | n/a | @@ -126,7 +138,34 @@ resources that lack official modules. | [private\_link\_resource\_id](#output\_private\_link\_resource\_id) | n/a | | [private\_link\_sub\_resource\_name](#output\_private\_link\_sub\_resource\_name) | n/a | | [standardized\_size](#output\_standardized\_size) | n/a | -| [storage\_account](#output\_storage\_account) | n/a | -| [storage\_container](#output\_storage\_container) | n/a | +| [tenant\_id](#output\_tenant\_id) | n/a | | [url](#output\_url) | The URL to the W&B application | + +## Migrations + +### Upgrading from 2.x to 3.x + +When upgrading from 2.x to 3.x, the following changes are required: + +1. Add the `azapi` provider to the `required_providers` block: + +```hcl +terraform { + required_providers { + azapi = { + source = "azure/azapi" + version = "~> 1.0" + } + } +} +``` + +2. Add the `azapi` provider to the `provider` block: + +```hcl +provider "azapi" { + # azapi provider configuration should be the same as azurerm provider configuration +} +``` + diff --git a/examples/byob/README.md b/examples/byob/README.md index 2cb5de2..1c55d4d 100644 --- a/examples/byob/README.md +++ b/examples/byob/README.md @@ -23,13 +23,51 @@ storage_key = To retrieve the storage key, you can use the Azure CLI installed previously like the example below. -```basb -az storage account keys list --account-name rgnamestorage --query '[].{key: value}' --output tsv +```bash +az storage account keys list --account-name --resource-group --query '[0].value' -o tsv 1111111111111122222222222333333333334444444555555555 -5555555554444444333333333332222222222211111111111111 ``` -You only need to provide one key. +This command will return the storage key, which you can then use for your deployment needs. Ensure you handle the storage key securely as it contains sensitive information. + +# Customer Managed Key Encryption + +The following section provides details on enabling Customer Managed Key (CMK) encryption for the Azure Blob Storage container which is disabled by default. + +To configure Customer Managed Key encryption, ensure you are using the latest version of out terraform which has the following added to the `variables.tf` file: + +- create_cmk +- disable_storage_vault_key_id +- tenant_id +- client_id + +You need to obtain the `tenant_id` and `client_id` from the `https://${WANDB_BASE_URL}/console/settings/advanced/spec/active` at W&B for an already instantiated instance of a Weights & Biases managed deployment. + +Set the follwoing new variabels to enable the CMK: + +```ini terraform.tfvars +create_cmk = true + +disable_storage_vault_key_id = false + +tenant_id = "" +client_id = "" +``` + +After updating your `terraform.tfvars` configuration, run the Terraform commands to apply the changes: + +```bash +terraform init -upgrade +terraform apply -var-file=terraform.tfvars +``` + +Upon successful execution, you will receive the following output which needs to be set in the system connection settings `https://${WANDB_BASE_URL}/console/settings/system` -* Note that all information about Storage Account and keys are mere examples, they are not valid. +```bash +blob_container = "/wandb" +command_to_get_storage_key = "az storage account keys list --account-name --resource-group --query '[0].value' -o tsv" +storage_key = +storage_vault_key_id = "https://.vault.azure.net/keys//" +``` +Retrieve the storage key as shown above. diff --git a/examples/byob/main.tf b/examples/byob/main.tf index 66fab24..414b02e 100644 --- a/examples/byob/main.tf +++ b/examples/byob/main.tf @@ -3,14 +3,18 @@ provider "azurerm" { } module "byob" { - source = "../../modules/byob" - resource_group_name = var.resource_group_name - location = var.location - prefix = var.prefix - deletion_protection = var.deletion_protection + source = "../../modules/byob" + resource_group_name = var.resource_group_name + location = var.location + prefix = var.prefix + deletion_protection = var.deletion_protection + create_cmk = var.create_cmk + client_id = var.client_id + tenant_id = var.tenant_id + tags = var.tags + disable_storage_vault_key_id = var.disable_storage_vault_key_id } - output "blob_container" { value = module.byob.blob_container } @@ -19,3 +23,11 @@ output "storage_key" { value = module.byob.azure_storage_key sensitive = true } + +output "storage_vault_key_id" { + value = module.byob.vault_key_id +} + +output "command_to_get_storage_key" { + value = "az storage account keys list --account-name ${module.byob.storage_account_name} --resource-group ${module.byob.resource_group_name.name} --query '[0].value' -o tsv" +} diff --git a/examples/byob/terraform.tfvars b/examples/byob/terraform.tfvars index ed42c15..62e6b1d 100644 --- a/examples/byob/terraform.tfvars +++ b/examples/byob/terraform.tfvars @@ -1,3 +1,16 @@ -resource_group_name = "rg-name" -location = "westeurope" -prefix = "byob-wandb" +resource_group_name = "" + +location = "" +prefix = "" +tags = { + "name" = "wandb" +} + +#To enable Azure Key Vault encryption uncomment the below lines +# create_cmk = true + +# enable_purge_protection = true +# disable_storage_vault_key_id = false + +# tenant_id = "" +# client_id = "" \ No newline at end of file diff --git a/examples/byob/variables.tf b/examples/byob/variables.tf index fafa958..2c0555d 100644 --- a/examples/byob/variables.tf +++ b/examples/byob/variables.tf @@ -1,6 +1,6 @@ variable "resource_group_name" { - description = "Resource Group name" type = string + description = "Resource Group" } variable "location" { @@ -18,3 +18,31 @@ variable "deletion_protection" { type = bool default = false } + +variable "create_cmk" { + type = bool + default = false + description = "Flag to create a Customer Managed Key for the Key Vault to encrypt the storage account." +} + +variable "disable_storage_vault_key_id" { + type = bool + default = false + description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." +} + +variable "tenant_id" { + type = string + description = "The tenant ID for the Key Vault Access Policy. Get from `https:///console/settings/advanced/spec/active`" +} + +variable "client_id" { + type = string + description = "The client ID (object id) for the Key Vault Access Policy. Get from `https:///console/settings/advanced/spec/active`" +} + +variable "tags" { + type = map(string) + description = "Map of tags for resource" + default = {} +} diff --git a/examples/public-dns/main.tf b/examples/public-dns/main.tf index 93215af..0bb359d 100644 --- a/examples/public-dns/main.tf +++ b/examples/public-dns/main.tf @@ -1,8 +1,34 @@ +terraform { + required_version = "~> 1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.17" + } + azapi = { + source = "azure/azapi" + version = "~> 1.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.6" + } + } +} + provider "azurerm" { subscription_id = var.subscription_id features {} } +provider "azapi" { + subscription_id = var.subscription_id +} + data "azurerm_subscription" "current" {} provider "kubernetes" { @@ -41,9 +67,12 @@ module "wandb" { deletion_protection = false + bucket_path = var.bucket_path + tags = { "Example" : "PublicDns" } + node_pool_num_zones = 2 } # # You'll want to update your DNS with the provisioned IP address diff --git a/examples/public-dns/variables.tf b/examples/public-dns/variables.tf index ebdb00d..2cc981b 100644 --- a/examples/public-dns/variables.tf +++ b/examples/public-dns/variables.tf @@ -52,3 +52,9 @@ variable "database_sku_name" { default = "GP_Standard_D4ds_v4" description = "Specifies the SKU Name for this MySQL Server" } + +variable "bucket_path" { + description = "path of where to store data for the instance-level bucket" + type = string + default = "" +} \ No newline at end of file diff --git a/main.tf b/main.tf index 76e1641..b8269c7 100644 --- a/main.tf +++ b/main.tf @@ -27,12 +27,12 @@ module "networking" { private_link = var.create_private_link allowed_ip_ranges = var.allowed_ip_ranges tags = var.tags - } module "database" { - source = "./modules/database" - namespace = var.namespace + source = "./modules/database" + namespace = var.namespace + resource_group_name = azurerm_resource_group.default.name location = azurerm_resource_group.default.location @@ -40,10 +40,11 @@ module "database" { database_version = var.database_version database_private_dns_zone_id = module.networking.database_private_dns_zone.id database_subnet_id = module.networking.database_subnet.id + sku_name = try(local.deployment_size[var.size].db, var.database_sku_name) + deletion_protection = var.deletion_protection - sku_name = try(local.deployment_size[var.size].db, var.database_sku_name) - deletion_protection = var.deletion_protection - + database_key_id = try(module.vault.vault_internal_keys[module.vault.vault_key_map.database].id, null) + identity_ids = module.identity.identity.id tags = { "customer-ns" = var.namespace, "env" = "managed-install" @@ -69,18 +70,27 @@ module "vault" { namespace = var.namespace resource_group = azurerm_resource_group.default + enable_database_vault_key = var.enable_database_vault_key + enable_storage_vault_key = var.enable_storage_vault_key + tags = var.tags } module "storage" { - count = (var.blob_container == "" && var.external_bucket == null) ? 1 : 0 - source = "./modules/storage" + source = "./modules/storage" + + count = 1 namespace = var.namespace resource_group_name = azurerm_resource_group.default.name location = azurerm_resource_group.default.location create_queue = !var.use_internal_queue + identity_ids = module.identity.identity.id deletion_protection = var.deletion_protection - tags = var.tags + + storage_key_id = try(module.vault.vault_internal_keys[module.vault.vault_key_map.storage].id, null) + disable_storage_vault_key_id = var.disable_storage_vault_key_id + + tags = var.tags } module "app_lb" { @@ -97,6 +107,30 @@ module "app_lb" { tags = var.tags } +locals { + kubernetes_instance_type = try(local.deployment_size[var.size].node_instance, var.kubernetes_instance_type) +} + +data "azapi_resource_list" "az_zones" { + parent_id = "/subscriptions/${data.azurerm_subscription.current.subscription_id}" + type = "Microsoft.Compute/skus@2021-07-01" + + response_export_values = ["value"] +} + +locals { + vm_skus = [ + for sku in jsondecode(data.azapi_resource_list.az_zones.output).value : + sku if( + sku.resourceType == "virtualMachines" && + lower(sku.locations[0]) == lower(azurerm_resource_group.default.location) && + sku.name == local.kubernetes_instance_type + ) + ] + num_zones = var.node_pool_zones != null ? length(var.node_pool_zones) : var.node_pool_num_zones + node_pool_zones = var.node_pool_zones != null ? var.node_pool_zones : slice(sort(local.vm_skus[0].locationInfo[0].zones), 0, local.num_zones) +} + module "app_aks" { source = "./modules/app_aks" depends_on = [module.app_lb] @@ -108,29 +142,14 @@ module "app_aks" { location = azurerm_resource_group.default.location namespace = var.namespace node_pool_vm_count = try(local.deployment_size[var.size].node_count, var.kubernetes_node_count) - node_pool_vm_size = try(local.deployment_size[var.size].node_instance, var.kubernetes_instance_type) - node_pool_zones = var.node_pool_zones + node_pool_vm_size = local.kubernetes_instance_type + node_pool_zones = local.node_pool_zones public_subnet = module.networking.public_subnet resource_group = azurerm_resource_group.default sku_tier = var.cluster_sku_tier max_pods = var.node_max_pods tags = var.tags } - -locals { - container_name = try(module.storage[0].container.name, "") - account_name = try(module.storage[0].account.name, "") - access_key = try(module.storage[0].account.primary_access_key, "") - queue_name = try(module.storage[0].queue.name, "") - blob_container = var.external_bucket == null ? coalesce(var.blob_container, local.container_name) : "" - storage_account = var.external_bucket == null ? coalesce(var.storage_account, local.account_name) : "" - storage_key = var.external_bucket == null ? coalesce(var.storage_key, local.access_key) : "" - bucket = "az://${local.storage_account}/${local.blob_container}" - queue = (var.use_internal_queue || var.blob_container == "" || var.external_bucket == null) ? "internal://" : "az://${local.account_name}/${local.queue_name}" - - redis_connection_string = "redis://:${module.redis.instance.primary_access_key}@${module.redis.instance.hostname}:${module.redis.instance.port}" -} - locals { service_account_name = "wandb-app" private_endpoint_approval_sa = "private-endpoint-sa" @@ -176,8 +195,9 @@ resource "azurerm_role_assignment" "gateway_role" { } module "cron_job" { - count = length(var.allowed_subscriptions) > 0 ? 1 : 0 # private endpoint approval application deployed as a cronjob in default namespace - source = "./modules/cron_job" + count = length(var.allowed_subscriptions) > 0 ? 1 : 0 # private endpoint approval application deployed as a cronjob in default namespace + source = "./modules/cron_job" + namespace = "default" client_id = module.pod_identity[0].identity.client_id serviceaccountName = local.private_endpoint_approval_sa @@ -186,7 +206,6 @@ module "cron_job" { applicationGatewayName = module.app_lb.gateway.name allowedSubscriptions = var.allowed_subscriptions depends_on = [module.app_lb, module.pod_identity] - } resource "azurerm_role_assignment" "otel_role" { @@ -194,7 +213,6 @@ resource "azurerm_role_assignment" "otel_role" { scope = azurerm_resource_group.default.id role_definition_name = "Contributor" principal_id = module.identity.otel_identity.principal_id - } resource "azurerm_federated_identity_credential" "otel_app" { @@ -219,6 +237,34 @@ module "cert_manager" { depends_on = [module.app_aks] } +module "clickhouse" { + count = var.clickhouse_private_endpoint_service_name != "" ? 1 : 0 + source = "./modules/clickhouse" + namespace = var.namespace + resource_group_name = azurerm_resource_group.default.name + location = azurerm_resource_group.default.location + network_id = module.networking.network.id + private_subnet_id = module.networking.private_subnet.id + + clickhouse_private_endpoint_service_name = var.clickhouse_private_endpoint_service_name + clickhouse_region = var.clickhouse_region +} + +locals { + use_customer_bucket = ( + var.storage_account != "" && + var.blob_container != "" && + var.storage_key != "" + ) + default_bucket_config = { + provider = "az" + name = var.storage_account + path = "${var.blob_container}/${var.bucket_path}" + accessKey = var.storage_key + } + bucket_config = var.external_bucket != null ? var.external_bucket : (local.use_customer_bucket ? local.default_bucket_config : null) +} + module "wandb" { source = "wandb/wandb/helm" version = "1.2.0" @@ -232,18 +278,29 @@ module "wandb" { operator_chart_version = var.operator_chart_version controller_image_tag = var.controller_image_tag + spec = { values = { global = { - host = local.url - license = var.license - - bucket = var.external_bucket == null ? { + host = local.url + license = var.license + cloudProvider = "azure" + bucket = local.bucket_config == null ? { + provider = "az" + name = module.storage[0].account.name + path = "${module.storage[0].container.name}/${var.bucket_path}" + accessKey = module.storage[0].account.primary_access_key + } : local.bucket_config + defaultBucket = { provider = "az" - name = local.storage_account - path = local.blob_container - accessKey = local.storage_key - } : var.external_bucket + name = module.storage[0].account.name + path = "${module.storage[0].container.name}/${var.bucket_path}" + accessKey = module.storage[0].account.primary_access_key + } + azureIdentityForBucket = { + clientID = module.identity.identity.client_id + tenantID = module.identity.identity.tenant_id + } mysql = { host = module.database.address diff --git a/modules/byob/main.tf b/modules/byob/main.tf index 72ba9b4..4b0b81f 100644 --- a/modules/byob/main.tf +++ b/modules/byob/main.tf @@ -1,10 +1,46 @@ -module "storage" { - source = "../storage" +module "identity" { + count = var.create_cmk ? 1 : 0 + source = "../identity" + + namespace = var.prefix + resource_group = { name = "${var.resource_group_name}", id = "byob" } + location = var.location +} + +module "vault" { + count = var.create_cmk ? 1 : 0 + source = "../vault" + namespace = var.prefix + resource_group = { name = "${var.resource_group_name}", id = "byob" } + location = var.location + + identity_object_id = module.identity[0].identity.principal_id + depends_on = [module.identity] + tags = var.tags + enable_storage_vault_key = var.create_cmk +} + +resource "azurerm_key_vault_access_policy" "wandb" { + count = var.create_cmk ? 1 : 0 + key_vault_id = module.vault[0].vault.id + tenant_id = var.tenant_id + object_id = var.client_id + key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Recover", "Backup", "Restore"] + certificate_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Recover", "Backup", "Restore"] + secret_permissions = ["Get", "List", "Set", "Delete", "Recover", "Backup", "Restore"] +} + +module "storage" { + source = "../storage" create_queue = false namespace = var.prefix resource_group_name = var.resource_group_name location = var.location - deletion_protection = var.deletion_protection + storage_key_id = try(module.vault[0].vault_internal_keys[module.vault[0].vault_key_map.storage].id, null) + identity_ids = try(module.identity[0].identity.id, null) + + disable_storage_vault_key_id = var.disable_storage_vault_key_id } + diff --git a/modules/byob/outputs.tf b/modules/byob/outputs.tf index 3c3e8ac..4177455 100644 --- a/modules/byob/outputs.tf +++ b/modules/byob/outputs.tf @@ -2,6 +2,14 @@ output "azure_storage_key" { value = module.storage.account.primary_access_key } +output "storage_key_id" { + value = try(module.vault[0].vault_internal_keys[module.vault[0].vault_key_map.storage].id, null) +} + output "blob_container" { value = "${module.storage.account.name}/${module.storage.container.name}" -} \ No newline at end of file +} + +output "storage_account_name" { + value = module.storage.account.name +} diff --git a/modules/byob/variables.tf b/modules/byob/variables.tf index 57c9157..c65700c 100644 --- a/modules/byob/variables.tf +++ b/modules/byob/variables.tf @@ -1,6 +1,5 @@ variable "resource_group_name" { - description = "Resource Group name" - type = string + type = string } variable "location" { @@ -18,3 +17,31 @@ variable "deletion_protection" { type = bool default = true } + +variable "create_cmk" { + type = bool + default = false + description = "Flag to create a Customer Managed Key for the Key Vault to encrypt the storage account." +} + +variable "tags" { + type = map(string) + description = "Map of tags for resource" + default = {} +} + +variable "disable_storage_vault_key_id" { + type = bool + default = false + description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." +} + +variable "tenant_id" { + type = string + description = "The tenant ID for the Key Vault Access Policy. Get from `https:///console/settings/advanced/spec/active`" +} + +variable "client_id" { + type = string + description = "The client ID (object id) for the Key Vault Access Policy. Get from `https:///console/settings/advanced/spec/active`" +} diff --git a/modules/clickhouse/main.tf b/modules/clickhouse/main.tf new file mode 100644 index 0000000..01e5c9f --- /dev/null +++ b/modules/clickhouse/main.tf @@ -0,0 +1,43 @@ +locals { + dns_name_suffix = "privatelink.azure.clickhouse.cloud" +} + +resource "azurerm_private_endpoint" "clickhouse" { + name = "${var.namespace}-clickhouse-pe" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.private_subnet_id + custom_network_interface_name = "${var.namespace}-clickhouse-nic" + + private_service_connection { + name = "${var.namespace}-clickhouse-pl" + private_connection_resource_alias = var.clickhouse_private_endpoint_service_name + is_manual_connection = true + request_message = "ClickHouse Private Link" + } +} + +resource "azurerm_private_dns_zone" "clickhouse_cloud_private_link_zone" { + name = "${var.clickhouse_region}.${local.dns_name_suffix}" + resource_group_name = var.resource_group_name +} + +data "azurerm_network_interface" "clickhouse_nic" { + resource_group_name = var.resource_group_name + name = azurerm_private_endpoint.clickhouse.network_interface[0].name +} + +resource "azurerm_private_dns_a_record" "clickhouse_wildcard" { + name = "*" + zone_name = azurerm_private_dns_zone.clickhouse_cloud_private_link_zone.name + resource_group_name = var.resource_group_name + ttl = 300 + records = [data.azurerm_network_interface.clickhouse_nic.private_ip_address] +} + +resource "azurerm_private_dns_zone_virtual_network_link" "clickhouse_network" { + name = "network-link" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.clickhouse_cloud_private_link_zone.name + virtual_network_id = var.network_id +} diff --git a/modules/clickhouse/variables.tf b/modules/clickhouse/variables.tf new file mode 100644 index 0000000..4819ed0 --- /dev/null +++ b/modules/clickhouse/variables.tf @@ -0,0 +1,46 @@ +variable "namespace" { + type = string + description = "Friendly name prefix used for tagging and naming Azure resources." +} + +variable "resource_group_name" { + type = string + description = "The name of the resource group in which to create the network." +} + +variable "location" { + type = string + description = "Specifies the supported Azure location where the resource exists." +} + +variable "network_id" { + type = string + description = "The virtual network id used for all resources" +} + +variable "private_subnet_id" { + type = string + description = "Specifies the supported Azure subnet id where the resource exists." +} + +variable "clickhouse_private_endpoint_service_name" { + type = string + description = "ClickHouse private endpoint 'Service name' (ends in .azure.privatelinkservice)." + default = "" + + validation { + condition = can(regex("\\.azure\\.privatelinkservice$", var.clickhouse_private_endpoint_service_name)) + error_message = "ClickHouse Service name must end in '.azure.privatelinkservice'." + } +} + +variable "clickhouse_region" { + type = string + description = "ClickHouse region (eastus2, westus3, etc)." + default = "" + + validation { + condition = length(var.clickhouse_region) > 0 + error_message = "Clickhouse Region should always be set if the private endpoint service name is specified." + } +} diff --git a/modules/database/main.tf b/modules/database/main.tf index d5f5d12..4c4b9ae 100644 --- a/modules/database/main.tf +++ b/modules/database/main.tf @@ -52,6 +52,20 @@ resource "azurerm_mysql_flexible_server" "default" { ] } + dynamic "identity" { + for_each = var.database_key_id != null ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.identity_ids] + } + } + dynamic "customer_managed_key" { + for_each = var.database_key_id != null ? [1] : [] + content { + primary_user_assigned_identity_id = var.identity_ids + key_vault_key_id = var.database_key_id + } + } tags = var.tags } diff --git a/modules/database/variables.tf b/modules/database/variables.tf index a75ab3e..d01456f 100644 --- a/modules/database/variables.tf +++ b/modules/database/variables.tf @@ -53,6 +53,16 @@ variable "tags" { } variable "deletion_protection" { - description = "If the instance should have deletion protection enabled. The database can't be deleted when this value is set to `true`." type = bool -} \ No newline at end of file + description = "If the instance should have deletion protection enabled. The database can't be deleted when this value is set to `true`." +} + +variable "database_key_id" { + type = string + description = "The Azure Key Vault key identifier for the database encryption key." +} + +variable "identity_ids" { + type = string + description = "The identity ids to assign to the database when database_key_id is passed." +} diff --git a/modules/identity/main.tf b/modules/identity/main.tf index 6880b99..6a4b825 100644 --- a/modules/identity/main.tf +++ b/modules/identity/main.tf @@ -9,4 +9,4 @@ resource "azurerm_user_assigned_identity" "otel" { name = "${var.namespace}-identity-otel" location = var.location resource_group_name = var.resource_group.name -} \ No newline at end of file +} diff --git a/modules/identity/outputs.tf b/modules/identity/outputs.tf index 453d5a4..fa7c529 100644 --- a/modules/identity/outputs.tf +++ b/modules/identity/outputs.tf @@ -1,6 +1,7 @@ output "identity" { value = azurerm_user_assigned_identity.default } + output "otel_identity" { value = var.otel_identity ? azurerm_user_assigned_identity.otel[0] : null -} \ No newline at end of file +} diff --git a/modules/identity/variables.tf b/modules/identity/variables.tf index 54cb0e4..1522c50 100644 --- a/modules/identity/variables.tf +++ b/modules/identity/variables.tf @@ -14,6 +14,6 @@ variable "location" { } variable "otel_identity" { - type = bool + type = bool default = false -} \ No newline at end of file +} diff --git a/modules/storage/main.tf b/modules/storage/main.tf index b077e4d..c798a7d 100644 --- a/modules/storage/main.tf +++ b/modules/storage/main.tf @@ -1,5 +1,9 @@ +locals { + postfix = "storage" + truncated_namespace = substr(replace(var.namespace, "-", ""), 0, 24 - length(local.postfix)) +} resource "azurerm_storage_account" "default" { - name = replace("${var.namespace}-storage", "-", "") + name = "${local.truncated_namespace}${local.postfix}" resource_group_name = var.resource_group_name location = var.location account_tier = "Standard" @@ -26,6 +30,20 @@ resource "azurerm_storage_account" "default" { } } + dynamic "identity" { + for_each = var.storage_key_id != null ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.identity_ids] + } + } + dynamic "customer_managed_key" { + for_each = var.storage_key_id != null && var.disable_storage_vault_key_id == false ? [1] : [] + content { + user_assigned_identity_id = var.identity_ids + key_vault_key_id = var.storage_key_id + } + } tags = var.tags } diff --git a/modules/storage/variables.tf b/modules/storage/variables.tf index 774306a..dab57e6 100644 --- a/modules/storage/variables.tf +++ b/modules/storage/variables.tf @@ -29,8 +29,22 @@ variable "deletion_protection" { type = bool } +variable "storage_key_id" { + type = string +} + +variable "disable_storage_vault_key_id" { + type = bool + default = false + description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." +} + +variable "identity_ids" { + type = string +} + variable "blob_container_name" { description = "Name of azure storage account container for storing blobs" type = string default = "wandb" -} \ No newline at end of file +} diff --git a/modules/vault/main.tf b/modules/vault/main.tf index e43290c..7929c80 100644 --- a/modules/vault/main.tf +++ b/modules/vault/main.tf @@ -4,14 +4,21 @@ data "azurerm_client_config" "current" {} locals { vault_name = "${var.namespace}-vault" vault_truncated_name = substr(local.vault_name, 0, min(length(local.vault_name), 24)) + max_key_length = 127 + vault_key_map = { + database = var.enable_database_vault_key ? substr("database-key-${var.namespace}", 0, local.max_key_length) : null + storage = var.enable_storage_vault_key ? substr("storage-key-${var.namespace}", 0, local.max_key_length) : null + } } resource "azurerm_key_vault" "default" { - name = trim(local.vault_truncated_name, "-") - location = var.location - resource_group_name = var.resource_group.name - tenant_id = data.azurerm_client_config.current.tenant_id - + name = trim(local.vault_truncated_name, "-") + location = var.location + resource_group_name = var.resource_group.name + tenant_id = data.azurerm_client_config.current.tenant_id + purge_protection_enabled = true + # https://learn.microsoft.com/en-us/azure/mysql/flexible-server/concepts-customer-managed-key#requirements-for-configuring-data-encryption-for-azure-database-for-mysql-flexible-server + soft_delete_retention_days = 90 # This must be 90 for azure msyql flex server encryption. enabled_for_disk_encryption = true sku_name = "standard" @@ -41,21 +48,32 @@ resource "azurerm_key_vault_access_policy" "identity" { tenant_id = data.azurerm_client_config.current.tenant_id object_id = var.identity_object_id - key_permissions = ["Create", "Decrypt", "Encrypt", "Get", "List"] + key_permissions = ["Create", "Decrypt", "Encrypt", "Get", "List", "UnwrapKey", "WrapKey"] secret_permissions = ["Delete", "Get", "List", "Purge", "Recover", "Restore", "Set"] storage_permissions = ["Get", "List"] - depends_on = [azurerm_key_vault.default] } resource "azurerm_key_vault_key" "etcd" { - depends_on = [azurerm_key_vault_access_policy.parent, azurerm_key_vault_access_policy.identity] - name = "generated-etcd-key" key_vault_id = azurerm_key_vault.default.id key_type = "RSA" key_size = 2048 key_opts = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey", ] + + depends_on = [azurerm_key_vault_access_policy.parent, azurerm_key_vault_access_policy.identity] +} + +resource "azurerm_key_vault_key" "intenral_encryption_keys" { + for_each = { for v in local.vault_key_map : v => v if v != null } + name = each.value + key_vault_id = azurerm_key_vault.default.id + key_type = "RSA" + key_size = 2048 + + key_opts = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey"] + + depends_on = [azurerm_key_vault_access_policy.parent, azurerm_key_vault_access_policy.identity] } diff --git a/modules/vault/outputs.tf b/modules/vault/outputs.tf index 9ec675d..0402867 100644 --- a/modules/vault/outputs.tf +++ b/modules/vault/outputs.tf @@ -4,4 +4,16 @@ output "etcd_key_id" { output "vault" { value = azurerm_key_vault.default -} \ No newline at end of file +} + +output "vault_id" { + value = azurerm_key_vault.default.id +} + +output "vault_internal_keys" { + value = azurerm_key_vault_key.intenral_encryption_keys +} + +output "vault_key_map" { + value = local.vault_key_map +} diff --git a/modules/vault/variables.tf b/modules/vault/variables.tf index f061a9a..d3d1ced 100644 --- a/modules/vault/variables.tf +++ b/modules/vault/variables.tf @@ -20,4 +20,16 @@ variable "resource_group" { variable "tags" { type = map(string) description = "Map of tags for resource" -} \ No newline at end of file +} + +variable "enable_storage_vault_key" { + type = bool + default = false + description = "Flag to enable managed key encryption for the storage account." +} + +variable "enable_database_vault_key" { + type = bool + default = false + description = "Flag to enable managed key encryption for the database. Once enabled, cannot be disabled or you will loose access to the database." +} diff --git a/outputs.tf b/outputs.tf index 4a43945..3176fb0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -33,14 +33,6 @@ output "oidc_issuer_url" { value = module.app_aks.oidc_issuer_url } -output "storage_account" { - value = var.external_bucket != null ? "" : coalesce(var.storage_account, local.account_name, "") -} - -output "storage_container" { - value = var.external_bucket != null ? "" : coalesce(var.blob_container, local.container_name) -} - output "private_link_resource_id" { value = module.app_lb.gateway_id } @@ -48,7 +40,7 @@ output "private_link_resource_id" { output "private_link_sub_resource_name" { value = module.app_lb.frontend_ip_configuration_names } - + output "standardized_size" { value = var.size } @@ -63,4 +55,12 @@ output "aks_node_instance_type" { output "database_instance_type" { value = try(local.deployment_size[var.size].db, var.database_sku_name) -} \ No newline at end of file +} + +output "client_id" { + value = module.identity.identity.client_id +} + +output "tenant_id" { + value = module.identity.identity.tenant_id +} diff --git a/variables.tf b/variables.tf index 94765eb..aed9b8d 100644 --- a/variables.tf +++ b/variables.tf @@ -194,6 +194,17 @@ variable "external_bucket" { default = null } +########################################## +# Bucket path # +########################################## +# This setting is meant for users who want to store all of their instance-level +# bucket's data at a specific path within their bucket. It can be set both for +# external buckets or the bucket created by this module. +variable "bucket_path" { + description = "path of where to store data for the instance-level bucket" + type = string + default = "" +} ########################################## # K8s # @@ -218,7 +229,13 @@ variable "cluster_sku_tier" { variable "node_pool_zones" { type = list(string) description = "Availability zones for the node pool" - default = ["1", "2"] + default = null +} + +variable "node_pool_num_zones" { + type = number + description = "Number of availability zones to use for the node pool when node_pool_zones is not set." + default = 2 } variable "node_max_pods" { @@ -246,7 +263,7 @@ variable "allowed_subscriptions" { ########################################## variable "allowed_ip_ranges" { - description = "allowed public IP addresses or CIDR ranges." + description = "Allowed public IP addresses or CIDR ranges." type = list(string) default = [] } @@ -269,8 +286,45 @@ variable "parquet_wandb_env" { default = {} } +########################################## +# vault key # +########################################## + +variable "enable_storage_vault_key" { + type = bool + default = false + description = "Flag to enable managed key encryption for the storage account." +} + +variable "disable_storage_vault_key_id" { + type = bool + default = false + description = "Flag to disable the `customer_managed_key` block, the properties 'encryption.identity, encryption.keyvaultproperties' cannot be updated in a single operation." +} + +variable "enable_database_vault_key" { + type = bool + default = false + description = "Flag to enable managed key encryption for the database. Once enabled, cannot be disabled." +} + ## To support otel azure monitor sql and redis metrics need operator-wandb chart minimum version 0.14.0 variable "azuremonitor" { type = bool default = false } + +########################################### +# ClickHouse endpoint # +########################################### +variable "clickhouse_private_endpoint_service_name" { + type = string + description = "ClickHouse private endpoint 'Service name' (ends in .azure.privatelinkservice)." + default = "" +} + +variable "clickhouse_region" { + type = string + description = "ClickHouse region (eastus2, westus3, etc)." + default = "" +} diff --git a/versions.tf b/versions.tf index d740277..59c2d3e 100644 --- a/versions.tf +++ b/versions.tf @@ -5,6 +5,10 @@ terraform { source = "hashicorp/azurerm" version = "~> 3.17" } + azapi = { + source = "azure/azapi" + version = "~> 1.0" + } kubernetes = { source = "hashicorp/kubernetes" version = "~> 2.23" @@ -14,5 +18,4 @@ terraform { version = "~> 2.6" } } -} - +} \ No newline at end of file