From 9d89fce6c182df62d85a80cf95eb86ae646b21e4 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Tue, 3 Sep 2024 15:28:15 -0700 Subject: [PATCH 01/13] feat: set default availability zones to null and find azs that support the instance type --- main.tf | 8 ++++++++ variables.tf | 2 +- vmtype_to_az.sh | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100755 vmtype_to_az.sh diff --git a/main.tf b/main.tf index 81fa724..1f6c3f5 100644 --- a/main.tf +++ b/main.tf @@ -107,6 +107,14 @@ module "app_lb" { tags = var.tags } +data "external" "az_zones" { + program = ["bash", "${path.module}/vmtype_to_az.sh", local.kubernetes_instance_type, azurerm_resource_group.default.location] +} + +locals { + node_pool_zones = (var.node_pool_zones == null) ? jsondecode(data.external.az_zones.result.zones) : var.node_pool_zones +} + module "app_aks" { source = "./modules/app_aks" depends_on = [module.app_lb] diff --git a/variables.tf b/variables.tf index e5e5060..8c9c7c9 100644 --- a/variables.tf +++ b/variables.tf @@ -215,7 +215,7 @@ variable "cluster_sku_tier" { variable "node_pool_zones" { type = list(string) description = "Availability zones for the node pool" - default = ["1", "2"] + default = null } variable "node_max_pods" { diff --git a/vmtype_to_az.sh b/vmtype_to_az.sh new file mode 100755 index 0000000..b69a196 --- /dev/null +++ b/vmtype_to_az.sh @@ -0,0 +1,32 @@ +#! /usr/bin/env bash + +# Given a Azure VM instance type and a region return the availability zones that support the instance type + +# Example: +# ./vmtype_to_az.sh Standard_D2_v3 westeurope +# +# Output: +# ["1", "2", "3"] + +# Copy script arguments to named environment variables +VM_TYPE="$1" +REGION="$2" + +# Check if both arguments are provided +if [ -z "$VM_TYPE" ] || [ -z "$REGION" ]; then + echo "Error: Both VM type and region must be provided." >&2 + echo "Usage: $0 " >&2 + exit 1 +fi + +# Query Azure CLI for availability zones in the region for the specified VM type +ZONES=$(az vm list-skus --location "$REGION" --size "$VM_TYPE" --query "[0].locationInfo[0].zones" -o json | jq -r -c 'sort | .[0:3]') + +# Check if the query returned any results +if [ -z "$ZONES" ] || [ "$ZONES" == "null" ]; then + echo "Error: No availability zones found for VM type $VM_TYPE in region $REGION." >&2 + exit 1 +fi + +# Output the result +jq -n --arg zones "$ZONES" '{"zones":$zones}' From 6d9c0e68939719a846485f8e98fb94ad284cc08d Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Mon, 16 Sep 2024 15:01:57 -0700 Subject: [PATCH 02/13] fix: allow specifying number of AZ to use when autoselecting zones --- main.tf | 2 +- variables.tf | 6 ++++++ vmtype_to_az.sh | 10 ++++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 1f6c3f5..febd734 100644 --- a/main.tf +++ b/main.tf @@ -108,7 +108,7 @@ module "app_lb" { } data "external" "az_zones" { - program = ["bash", "${path.module}/vmtype_to_az.sh", local.kubernetes_instance_type, azurerm_resource_group.default.location] + program = ["bash", "${path.module}/vmtype_to_az.sh", local.kubernetes_instance_type, azurerm_resource_group.default.location, var.node_pool_num_zones] } locals { diff --git a/variables.tf b/variables.tf index 8c9c7c9..c543998 100644 --- a/variables.tf +++ b/variables.tf @@ -218,6 +218,12 @@ variable "node_pool_zones" { default = null } +variable "node_pool_num_zones" { + type = number + description = "Number of availability zones to use for the node pool when node_pool_zones is not set. If neither are set, 3 zones will be used" + default = null +} + variable "node_max_pods" { type = number description = "Maximum number of pods per node" diff --git a/vmtype_to_az.sh b/vmtype_to_az.sh index b69a196..ae74523 100755 --- a/vmtype_to_az.sh +++ b/vmtype_to_az.sh @@ -11,16 +11,22 @@ # Copy script arguments to named environment variables VM_TYPE="$1" REGION="$2" +NUM_ZONES="$3" # Check if both arguments are provided if [ -z "$VM_TYPE" ] || [ -z "$REGION" ]; then echo "Error: Both VM type and region must be provided." >&2 - echo "Usage: $0 " >&2 + echo "Usage: $0 " >&2 exit 1 fi +# Default to 3 zones if not specified +if [ -z "$NUM_ZONES" ]; then + NUM_ZONES=3 +fi + # Query Azure CLI for availability zones in the region for the specified VM type -ZONES=$(az vm list-skus --location "$REGION" --size "$VM_TYPE" --query "[0].locationInfo[0].zones" -o json | jq -r -c 'sort | .[0:3]') +ZONES=$(az vm list-skus --location "$REGION" --size "$VM_TYPE" --query "[0].locationInfo[0].zones" -o json | jq -r -c "sort | .[0:$NUM_ZONES]") # Check if the query returned any results if [ -z "$ZONES" ] || [ "$ZONES" == "null" ]; then From 0dd52bdaa3922cca88a47baa66e1026a320542c0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Sep 2024 17:09:55 +0000 Subject: [PATCH 03/13] terraform-docs: automated action --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 56dafe2..5c9084c 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ resources that lack official modules. | Name | Version | |------|---------| | [azurerm](#provider\_azurerm) | ~> 3.17 | +| [external](#provider\_external) | n/a | ## Modules @@ -66,6 +67,7 @@ resources that lack official modules. | Name | Type | |------|------| | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | +| [external_external.az_zones](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source | ## Inputs @@ -97,7 +99,8 @@ resources that lack official modules. | [location](#input\_location) | n/a | `string` | n/a | yes | | [namespace](#input\_namespace) | String used for prefix resources. | `string` | n/a | yes | | [node\_max\_pods](#input\_node\_max\_pods) | Maximum number of pods per node | `number` | `30` | no | -| [node\_pool\_zones](#input\_node\_pool\_zones) | Availability zones for the node pool | `list(string)` |
[
"1",
"2"
]
| no | +| [node\_pool\_num\_zones](#input\_node\_pool\_num\_zones) | Number of availability zones to use for the node pool when node\_pool\_zones is not set. If neither are set, 3 zones will be used | `number` | `null` | no | +| [node\_pool\_zones](#input\_node\_pool\_zones) | Availability zones for the node pool | `list(string)` | `null` | no | | [oidc\_auth\_method](#input\_oidc\_auth\_method) | OIDC auth method | `string` | `"implicit"` | no | | [oidc\_client\_id](#input\_oidc\_client\_id) | The Client ID of application in your identity provider | `string` | `""` | no | | [oidc\_issuer](#input\_oidc\_issuer) | A url to your Open ID Connect identity provider, i.e. https://cognito-idp.us-east-1.amazonaws.com/us-east-1_uiIFNdacd | `string` | `""` | no | From ceb45552364f8afd94e523fa2983a750a71f85e7 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Thu, 26 Sep 2024 10:12:40 -0700 Subject: [PATCH 04/13] fix: change default for node_pool_num_zones to keep the same number of zones by default --- variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/variables.tf b/variables.tf index c543998..bc3efa3 100644 --- a/variables.tf +++ b/variables.tf @@ -220,8 +220,8 @@ variable "node_pool_zones" { variable "node_pool_num_zones" { type = number - description = "Number of availability zones to use for the node pool when node_pool_zones is not set. If neither are set, 3 zones will be used" - default = null + description = "Number of availability zones to use for the node pool when node_pool_zones is not set." + default = 2 } variable "node_max_pods" { From e2417ee5c683a1af6911b38d4202fec78bdcb021 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Sep 2024 17:13:30 +0000 Subject: [PATCH 05/13] terraform-docs: automated action --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c9084c..24b4006 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ resources that lack official modules. | [location](#input\_location) | n/a | `string` | n/a | yes | | [namespace](#input\_namespace) | String used for prefix resources. | `string` | n/a | yes | | [node\_max\_pods](#input\_node\_max\_pods) | Maximum number of pods per node | `number` | `30` | no | -| [node\_pool\_num\_zones](#input\_node\_pool\_num\_zones) | Number of availability zones to use for the node pool when node\_pool\_zones is not set. If neither are set, 3 zones will be used | `number` | `null` | no | +| [node\_pool\_num\_zones](#input\_node\_pool\_num\_zones) | Number of availability zones to use for the node pool when node\_pool\_zones is not set. | `number` | `2` | no | | [node\_pool\_zones](#input\_node\_pool\_zones) | Availability zones for the node pool | `list(string)` | `null` | no | | [oidc\_auth\_method](#input\_oidc\_auth\_method) | OIDC auth method | `string` | `"implicit"` | no | | [oidc\_client\_id](#input\_oidc\_client\_id) | The Client ID of application in your identity provider | `string` | `""` | no | From 8e0bf3a1ae5d568b3a57aad41a72b51d2168a77c Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Thu, 26 Sep 2024 10:28:17 -0700 Subject: [PATCH 06/13] fix: move instance type into local --- main.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index febd734..cb364d0 100644 --- a/main.tf +++ b/main.tf @@ -107,6 +107,10 @@ module "app_lb" { tags = var.tags } +locals { + kubernetes_instance_type = try(local.deployment_size[var.size].node_type, var.kubernetes_instance_type) +} + data "external" "az_zones" { program = ["bash", "${path.module}/vmtype_to_az.sh", local.kubernetes_instance_type, azurerm_resource_group.default.location, var.node_pool_num_zones] } @@ -126,7 +130,7 @@ module "app_aks" { location = azurerm_resource_group.default.location namespace = var.namespace node_pool_vm_count = try(local.deployment_size[var.size].node_count, var.kubernetes_node_count) - node_pool_vm_size = try(local.deployment_size[var.size].node_instance, var.kubernetes_instance_type) + node_pool_vm_size = local.kubernetes_instance_type node_pool_zones = var.node_pool_zones public_subnet = module.networking.public_subnet resource_group = azurerm_resource_group.default From a6b18c1e89da85717e5711ef0dd35e12579f7dc2 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Thu, 26 Sep 2024 10:51:18 -0700 Subject: [PATCH 07/13] fix: install az client --- main.tf | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/main.tf b/main.tf index cb364d0..81d19a8 100644 --- a/main.tf +++ b/main.tf @@ -111,6 +111,19 @@ locals { kubernetes_instance_type = try(local.deployment_size[var.size].node_type, var.kubernetes_instance_type) } +resource "null_resource" "install_az_cli" { + provisioner "local-exec" { + command = < Date: Thu, 26 Sep 2024 17:51:40 +0000 Subject: [PATCH 08/13] terraform-docs: automated action --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 24b4006..65bf703 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ resources that lack official modules. |------|---------| | [azurerm](#provider\_azurerm) | ~> 3.17 | | [external](#provider\_external) | n/a | +| [null](#provider\_null) | n/a | ## Modules From c081926222192c0b8e58962efffd7b8ef6c20c0e Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Fri, 27 Sep 2024 10:41:52 -0700 Subject: [PATCH 09/13] fix: Query zone info via azapi instead of using external data provider --- examples/public-dns/main.tf | 27 +++++++++++++++++++++++++++ main.tf | 31 +++++++++++++++---------------- versions.tf | 4 ++++ 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/examples/public-dns/main.tf b/examples/public-dns/main.tf index 1ff8157..b23c588 100644 --- a/examples/public-dns/main.tf +++ b/examples/public-dns/main.tf @@ -1,8 +1,34 @@ +terraform { + required_version = "~> 1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.17" + } + azapi = { + source = "azure/azapi" + version = "~> 1.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.6" + } + } +} + provider "azurerm" { subscription_id = var.subscription_id features {} } +provider "azapi" { + subscription_id = var.subscription_id +} + data "azurerm_subscription" "current" {} provider "kubernetes" { @@ -46,6 +72,7 @@ module "wandb" { tags = { "Example" : "PublicDns" } + node_pool_num_zones = 2 } # # You'll want to update your DNS with the provisioned IP address diff --git a/main.tf b/main.tf index 81d19a8..1adf086 100644 --- a/main.tf +++ b/main.tf @@ -111,25 +111,24 @@ locals { kubernetes_instance_type = try(local.deployment_size[var.size].node_type, var.kubernetes_instance_type) } -resource "null_resource" "install_az_cli" { - provisioner "local-exec" { - command = < Date: Fri, 27 Sep 2024 17:43:09 +0000 Subject: [PATCH 10/13] terraform-docs: automated action --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 65bf703..96dd306 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ resources that lack official modules. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | ~> 1.0 | +| [azapi](#requirement\_azapi) | ~> 1.0 | | [azurerm](#requirement\_azurerm) | ~> 3.17 | | [helm](#requirement\_helm) | ~> 2.6 | | [kubernetes](#requirement\_kubernetes) | ~> 2.23 | @@ -41,9 +42,8 @@ resources that lack official modules. | Name | Version | |------|---------| +| [azapi](#provider\_azapi) | ~> 1.0 | | [azurerm](#provider\_azurerm) | ~> 3.17 | -| [external](#provider\_external) | n/a | -| [null](#provider\_null) | n/a | ## Modules @@ -67,8 +67,8 @@ resources that lack official modules. | Name | Type | |------|------| +| [azapi_resource_list.az_zones](https://registry.terraform.io/providers/azure/azapi/latest/docs/data-sources/resource_list) | data source | | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | -| [external_external.az_zones](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source | ## Inputs From 5f1edfd86ea2d0e4d4211bb3bf8440ccd4f6ff87 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Fri, 27 Sep 2024 11:04:20 -0700 Subject: [PATCH 11/13] chore: delete unused file --- vmtype_to_az.sh | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100755 vmtype_to_az.sh diff --git a/vmtype_to_az.sh b/vmtype_to_az.sh deleted file mode 100755 index ae74523..0000000 --- a/vmtype_to_az.sh +++ /dev/null @@ -1,38 +0,0 @@ -#! /usr/bin/env bash - -# Given a Azure VM instance type and a region return the availability zones that support the instance type - -# Example: -# ./vmtype_to_az.sh Standard_D2_v3 westeurope -# -# Output: -# ["1", "2", "3"] - -# Copy script arguments to named environment variables -VM_TYPE="$1" -REGION="$2" -NUM_ZONES="$3" - -# Check if both arguments are provided -if [ -z "$VM_TYPE" ] || [ -z "$REGION" ]; then - echo "Error: Both VM type and region must be provided." >&2 - echo "Usage: $0 " >&2 - exit 1 -fi - -# Default to 3 zones if not specified -if [ -z "$NUM_ZONES" ]; then - NUM_ZONES=3 -fi - -# Query Azure CLI for availability zones in the region for the specified VM type -ZONES=$(az vm list-skus --location "$REGION" --size "$VM_TYPE" --query "[0].locationInfo[0].zones" -o json | jq -r -c "sort | .[0:$NUM_ZONES]") - -# Check if the query returned any results -if [ -z "$ZONES" ] || [ "$ZONES" == "null" ]; then - echo "Error: No availability zones found for VM type $VM_TYPE in region $REGION." >&2 - exit 1 -fi - -# Output the result -jq -n --arg zones "$ZONES" '{"zones":$zones}' From 45c9aa5a0e20b4aa6a9b6de8cc9edb2b5ebddcee Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Fri, 27 Sep 2024 11:04:53 -0700 Subject: [PATCH 12/13] chore: terraform fmt --- examples/public-dns/main.tf | 2 +- main.tf | 18 +++++++++--------- versions.tf | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/public-dns/main.tf b/examples/public-dns/main.tf index b23c588..0bb359d 100644 --- a/examples/public-dns/main.tf +++ b/examples/public-dns/main.tf @@ -6,7 +6,7 @@ terraform { version = "~> 3.17" } azapi = { - source = "azure/azapi" + source = "azure/azapi" version = "~> 1.0" } kubernetes = { diff --git a/main.tf b/main.tf index 1adf086..d51ec42 100644 --- a/main.tf +++ b/main.tf @@ -108,7 +108,7 @@ module "app_lb" { } locals { - kubernetes_instance_type = try(local.deployment_size[var.size].node_type, var.kubernetes_instance_type) + kubernetes_instance_type = try(local.deployment_size[var.size].node_type, var.kubernetes_instance_type) } data "azapi_resource_list" "az_zones" { @@ -119,15 +119,15 @@ data "azapi_resource_list" "az_zones" { } locals { - vm_skus = [ - for sku in jsondecode(data.azapi_resource_list.az_zones.output).value : - sku if ( - sku.resourceType == "virtualMachines" && - contains(sku.locations, azurerm_resource_group.default.location) && - sku.name == local.kubernetes_instance_type - ) + vm_skus = [ + for sku in jsondecode(data.azapi_resource_list.az_zones.output).value : + sku if( + sku.resourceType == "virtualMachines" && + contains(sku.locations, azurerm_resource_group.default.location) && + sku.name == local.kubernetes_instance_type + ) ] - num_zones = var.node_pool_zones != null ? length(var.node_pool_zones) : var.node_pool_num_zones + num_zones = var.node_pool_zones != null ? length(var.node_pool_zones) : var.node_pool_num_zones node_pool_zones = var.node_pool_zones != null ? var.node_pool_zones : slice(sort(local.vm_skus[0].locationInfo[0].zones), 0, local.num_zones) } diff --git a/versions.tf b/versions.tf index ddccf7c..59c2d3e 100644 --- a/versions.tf +++ b/versions.tf @@ -6,7 +6,7 @@ terraform { version = "~> 3.17" } azapi = { - source = "azure/azapi" + source = "azure/azapi" version = "~> 1.0" } kubernetes = { From 1ba1aac1573eaf691eb2cb4bd7d73e120304a45c Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Fri, 27 Sep 2024 11:26:59 -0700 Subject: [PATCH 13/13] fix: need to case normalize the zone name --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index d51ec42..81a949a 100644 --- a/main.tf +++ b/main.tf @@ -123,7 +123,7 @@ locals { for sku in jsondecode(data.azapi_resource_list.az_zones.output).value : sku if( sku.resourceType == "virtualMachines" && - contains(sku.locations, azurerm_resource_group.default.location) && + lower(sku.locations[0]) == lower(azurerm_resource_group.default.location) && sku.name == local.kubernetes_instance_type ) ]