From c7d443d4e102a4afff3b3e1bbb05a676b449de4a Mon Sep 17 00:00:00 2001 From: Daniel Butler Date: Mon, 11 Mar 2024 17:11:10 +0000 Subject: [PATCH] feat: added cluster addon support to the OCP DA and module. It can be configured using the `cluster_addons` and `manage_all_cluster_addons` variables (#733) --- README.md | 4 ++- cluster.tf | 60 +++++++++++++++++++++++++++++++ patterns/roks/main.tf | 2 ++ patterns/roks/module/config.tf | 2 ++ patterns/roks/module/variables.tf | 21 +++++++++++ patterns/roks/variables.tf | 21 +++++++++++ variables.tf | 10 ++++++ 7 files changed, 119 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1883d55fd..9a25f871d 100644 --- a/README.md +++ b/README.md @@ -866,6 +866,7 @@ module "cluster_pattern" { | [ibm_appid_redirect_urls.urls](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/appid_redirect_urls) | resource | | [ibm_atracker_route.atracker_route](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/atracker_route) | resource | | [ibm_atracker_target.atracker_target](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/atracker_target) | resource | +| [ibm_container_addons.addons](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/container_addons) | resource | | [ibm_container_vpc_cluster.cluster](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource | | [ibm_container_vpc_worker_pool.pool](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/container_vpc_worker_pool) | resource | | [ibm_cos_bucket.buckets](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/cos_bucket) | resource | @@ -890,6 +891,7 @@ module "cluster_pattern" { | [random_string.random_cos_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [time_sleep.wait_30_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [time_sleep.wait_for_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [ibm_container_addons.existing_addons](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/container_addons) | data source | | [ibm_container_cluster_versions.cluster_versions](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/container_cluster_versions) | data source | | [ibm_iam_account_settings.iam_account_settings](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/iam_account_settings) | data source | | [ibm_is_image.image](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/is_image) | data source | @@ -903,7 +905,7 @@ module "cluster_pattern" { |------|-------------|------|---------|:--------:| | [appid](#input\_appid) | The App ID instance to be used for the teleport vsi deployments | 
object({
    name           = optional(string)
    resource_group = optional(string)
    use_data       = optional(bool)
    keys           = optional(list(string))
    use_appid      = bool
  })
| 
{
  "use_appid": false
}
| no | | [atracker](#input\_atracker) | atracker variables | 
object({
    resource_group        = string
    receive_global_events = bool
    collector_bucket_name = string
    add_route             = bool
  })
| n/a | yes | -| [clusters](#input\_clusters) | A list describing clusters workloads to create | 
list(
    object({
      name                    = string           # Name of Cluster
      vpc_name                = string           # Name of VPC
      subnet_names            = list(string)     # List of vpc subnets for cluster
      workers_per_subnet      = number           # Worker nodes per subnet.
      machine_type            = string           # Worker node flavor
      kube_type               = string           # iks or openshift
      kube_version            = optional(string) # Can be a version from `ibmcloud ks versions` or `default`
      entitlement             = optional(string) # entitlement option for openshift
      secondary_storage       = optional(string) # Secondary storage type
      pod_subnet              = optional(string) # Portable subnet for pods
      service_subnet          = optional(string) # Portable subnet for services
      resource_group          = string           # Resource Group used for cluster
      cos_name                = optional(string) # Name of COS instance Required only for OpenShift clusters
      access_tags             = optional(list(string), [])
      boot_volume_crk_name    = optional(string)     # Boot volume encryption key name
      disable_public_endpoint = optional(bool, true) # disable cluster public, leaving only private endpoint
      kms_config = optional(
        object({
          crk_name         = string         # Name of key
          private_endpoint = optional(bool) # Private endpoint
        })
      )
      worker_pools = optional(
        list(
          object({
            name                 = string           # Worker pool name
            vpc_name             = string           # VPC name
            workers_per_subnet   = number           # Worker nodes per subnet
            flavor               = string           # Worker node flavor
            subnet_names         = list(string)     # List of vpc subnets for worker pool
            entitlement          = optional(string) # entitlement option for openshift
            secondary_storage    = optional(string) # Secondary storage type
            boot_volume_crk_name = optional(string) # Boot volume encryption key name
          })
        )
      )
    })
  )
| n/a | yes | +| [clusters](#input\_clusters) | A list describing clusters workloads to create | 
list(
    object({
      name                    = string           # Name of Cluster
      vpc_name                = string           # Name of VPC
      subnet_names            = list(string)     # List of vpc subnets for cluster
      workers_per_subnet      = number           # Worker nodes per subnet.
      machine_type            = string           # Worker node flavor
      kube_type               = string           # iks or openshift
      kube_version            = optional(string) # Can be a version from `ibmcloud ks versions` or `default`
      entitlement             = optional(string) # entitlement option for openshift
      secondary_storage       = optional(string) # Secondary storage type
      pod_subnet              = optional(string) # Portable subnet for pods
      service_subnet          = optional(string) # Portable subnet for services
      resource_group          = string           # Resource Group used for cluster
      cos_name                = optional(string) # Name of COS instance Required only for OpenShift clusters
      access_tags             = optional(list(string), [])
      boot_volume_crk_name    = optional(string)     # Boot volume encryption key name
      disable_public_endpoint = optional(bool, true) # disable cluster public, leaving only private endpoint
      addons = optional(object({                     # Map of OCP cluster add-on versions to install
        debug-tool                = optional(string)
        image-key-synchronizer    = optional(string)
        openshift-data-foundation = optional(string)
        vpc-file-csi-driver       = optional(string)
        static-route              = optional(string)
        cluster-autoscaler        = optional(string)
        vpc-block-csi-driver      = optional(string)
      }), {})
      manage_all_addons = optional(bool, false) # Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources.
      kms_config = optional(
        object({
          crk_name         = string         # Name of key
          private_endpoint = optional(bool) # Private endpoint
        })
      )
      worker_pools = optional(
        list(
          object({
            name                 = string           # Worker pool name
            vpc_name             = string           # VPC name
            workers_per_subnet   = number           # Worker nodes per subnet
            flavor               = string           # Worker node flavor
            subnet_names         = list(string)     # List of vpc subnets for worker pool
            entitlement          = optional(string) # entitlement option for openshift
            secondary_storage    = optional(string) # Secondary storage type
            boot_volume_crk_name = optional(string) # Boot volume encryption key name
          })
        )
      )
    })
  )
| n/a | yes | | [cos](#input\_cos) | Object describing the cloud object storage instance, buckets, and keys. Set `use_data` to false to create instance | 
list(
    object({
      name           = string
      use_data       = optional(bool)
      resource_group = string
      plan           = optional(string)
      random_suffix  = optional(bool) # Use a random suffix for COS instance
      access_tags    = optional(list(string), [])
      buckets = list(object({
        name                  = string
        storage_class         = string
        endpoint_type         = string
        force_delete          = bool
        single_site_location  = optional(string)
        region_location       = optional(string)
        cross_region_location = optional(string)
        kms_key               = optional(string)
        access_tags           = optional(list(string), [])
        allowed_ip            = optional(list(string))
        hard_quota            = optional(number)
        archive_rule = optional(object({
          days    = number
          enable  = bool
          rule_id = optional(string)
          type    = string
        }))
        expire_rule = optional(object({
          days                         = optional(number)
          date                         = optional(string)
          enable                       = bool
          expired_object_delete_marker = optional(string)
          prefix                       = optional(string)
          rule_id                      = optional(string)
        }))
        activity_tracking = optional(object({
          activity_tracker_crn = string
          read_data_events     = bool
          write_data_events    = bool
        }))
        metrics_monitoring = optional(object({
          metrics_monitoring_crn  = string
          request_metrics_enabled = optional(bool)
          usage_metrics_enabled   = optional(bool)
        }))
      }))
      keys = optional(
        list(object({
          name        = string
          role        = string
          enable_HMAC = bool
        }))
      )

    })
  )
| n/a | yes | | [enable\_transit\_gateway](#input\_enable\_transit\_gateway) | Create transit gateway | `bool` | `true` | no | | [f5\_template\_data](#input\_f5\_template\_data) | Data for all f5 templates | 
object({
    tmos_admin_password     = optional(string)
    license_type            = optional(string)
    byol_license_basekey    = optional(string)
    license_host            = optional(string)
    license_username        = optional(string)
    license_password        = optional(string)
    license_pool            = optional(string)
    license_sku_keyword_1   = optional(string)
    license_sku_keyword_2   = optional(string)
    license_unit_of_measure = optional(string)
    do_declaration_url      = optional(string)
    as3_declaration_url     = optional(string)
    ts_declaration_url      = optional(string)
    phone_home_url          = optional(string)
    template_source         = optional(string)
    template_version        = optional(string)
    app_id                  = optional(string)
    tgactive_url            = optional(string)
    tgstandby_url           = optional(string)
    tgrefresh_url           = optional(string)
  })
| 
{
  "license_type": "none"
}
| no | diff --git a/cluster.tf b/cluster.tf index ec0891ec9..6a08138c2 100644 --- a/cluster.tf +++ b/cluster.tf @@ -127,3 +127,63 @@ resource "ibm_container_vpc_worker_pool" "pool" { } ############################################################################## + +############################################################################## +# Addons +############################################################################## + +# Lookup the current default csi-driver version +data "ibm_container_addons" "existing_addons" { + for_each = ibm_container_vpc_cluster.cluster + cluster = each.value.id +} + +locals { + csi_driver_version = { + for cluster in ibm_container_vpc_cluster.cluster : cluster.name => ( + length(data.ibm_container_addons.existing_addons[cluster.name].addons) > 0 && + data.ibm_container_addons.existing_addons[cluster.name].addons[0].name == "vpc-block-csi-driver" ? + data.ibm_container_addons.existing_addons[cluster.name].addons[0].version : "" + ) + } + + + # addons_list = var.addons != null ? { for k, v in var.addons : k => v if v != null } : {} + # addons = lookup(local.addons_list, "vpc-block-csi-driver", null) == null ? merge(local.addons_list, { vpc-block-csi-driver = local.csi_driver_version[0] }) : local.addons_list + # for each cluster in the clusters_map, get the addons and their versions and create an addons map including the corosponding csi_driver_version + cluster_addons = { + for cluster in var.clusters : "${var.prefix}-${cluster.name}" => { + id = ibm_container_vpc_cluster.cluster["${var.prefix}-${cluster.name}"].id + resource_group_id = ibm_container_vpc_cluster.cluster["${var.prefix}-${cluster.name}"].resource_group_id + addons = merge( + { for addon_name, addon_version in(cluster.addons != null ? cluster.addons : {}) : addon_name => addon_version if addon_version != null }, + local.csi_driver_version["${var.prefix}-${cluster.name}"] != null ? { vpc-block-csi-driver = local.csi_driver_version["${var.prefix}-${cluster.name}"] } : {} + ) + } + } +} + +resource "ibm_container_addons" "addons" { + # Worker pool creation can start before the 'ibm_container_vpc_cluster' completes since there is no explicit + # depends_on in 'ibm_container_vpc_worker_pool', just an implicit depends_on on the cluster ID. Cluster ID can exist before + # 'ibm_container_vpc_cluster' completes, so hence need to add explicit depends on against 'ibm_container_vpc_cluster' here. + depends_on = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_worker_pool.pool] + for_each = local.cluster_addons + cluster = each.value.id + resource_group_id = each.value.resource_group_id + + # setting to false means we do not want Terraform to manage addons that are managed elsewhere + manage_all_addons = local.clusters_map[each.key].manage_all_addons + + dynamic "addons" { + for_each = local.cluster_addons[each.key].addons + content { + name = addons.key + version = addons.value + } + } + + timeouts { + create = "1h" + } +} diff --git a/patterns/roks/main.tf b/patterns/roks/main.tf index 4512022a9..d48a88952 100644 --- a/patterns/roks/main.tf +++ b/patterns/roks/main.tf @@ -35,6 +35,8 @@ module "roks_landing_zone" { workers_per_zone = var.workers_per_zone flavor = var.flavor kube_version = var.kube_version + cluster_addons = var.cluster_addons + manage_all_cluster_addons = var.manage_all_cluster_addons add_atracker_route = var.add_atracker_route hs_crypto_instance_name = var.hs_crypto_instance_name hs_crypto_resource_group = var.hs_crypto_resource_group diff --git a/patterns/roks/module/config.tf b/patterns/roks/module/config.tf index 5c6748ce9..dc3f754e5 100644 --- a/patterns/roks/module/config.tf +++ b/patterns/roks/module/config.tf @@ -87,6 +87,8 @@ locals { cos_name = "cos" entitlement = var.entitlement secondary_storage = var.secondary_storage + addons = var.cluster_addons + manage_all_addons = var.manage_all_cluster_addons boot_volume_crk_name = "${var.prefix}-roks-key" # By default, create dedicated pool for logging worker_pools = [ diff --git a/patterns/roks/module/variables.tf b/patterns/roks/module/variables.tf index 61a13add6..542cd4c9b 100644 --- a/patterns/roks/module/variables.tf +++ b/patterns/roks/module/variables.tf @@ -163,6 +163,27 @@ variable "secondary_storage" { default = null } +variable "cluster_addons" { + type = object({ + debug-tool = optional(string) + image-key-synchronizer = optional(string) + openshift-data-foundation = optional(string) + vpc-file-csi-driver = optional(string) + static-route = optional(string) + cluster-autoscaler = optional(string) + vpc-block-csi-driver = optional(string) + }) + description = "Map of OCP cluster add-on versions to install (NOTE: The 'vpc-block-csi-driver' add-on is installed by default for VPC clusters, however you can explicitly specify it here if you wish to choose a later version than the default one). For full list of all supported add-ons and versions, see https://cloud.ibm.com/docs/containers?topic=containers-supported-cluster-addon-versions" + default = null +} + +variable "manage_all_cluster_addons" { + type = bool + default = false + nullable = false # null values are set to default value + description = "Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources." +} + ############################################################################## diff --git a/patterns/roks/variables.tf b/patterns/roks/variables.tf index 34eab9a2d..db8aa3154 100644 --- a/patterns/roks/variables.tf +++ b/patterns/roks/variables.tf @@ -171,6 +171,27 @@ variable "entitlement" { default = null } +variable "cluster_addons" { + type = object({ + debug-tool = optional(string) + image-key-synchronizer = optional(string) + openshift-data-foundation = optional(string) + vpc-file-csi-driver = optional(string) + static-route = optional(string) + cluster-autoscaler = optional(string) + vpc-block-csi-driver = optional(string) + }) + description = "Map of OCP cluster add-on versions to install (NOTE: The 'vpc-block-csi-driver' add-on is installed by default for VPC clusters, however you can explicitly specify it here if you wish to choose a later version than the default one). For full list of all supported add-ons and versions, see https://cloud.ibm.com/docs/containers?topic=containers-supported-cluster-addon-versions" + default = null +} + +variable "manage_all_cluster_addons" { + type = bool + default = false + nullable = false # null values are set to default value + description = "Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources." +} + ############################################################################## diff --git a/variables.tf b/variables.tf index 672e4d0dd..7c04ac89b 100644 --- a/variables.tf +++ b/variables.tf @@ -827,6 +827,16 @@ variable "clusters" { access_tags = optional(list(string), []) boot_volume_crk_name = optional(string) # Boot volume encryption key name disable_public_endpoint = optional(bool, true) # disable cluster public, leaving only private endpoint + addons = optional(object({ # Map of OCP cluster add-on versions to install + debug-tool = optional(string) + image-key-synchronizer = optional(string) + openshift-data-foundation = optional(string) + vpc-file-csi-driver = optional(string) + static-route = optional(string) + cluster-autoscaler = optional(string) + vpc-block-csi-driver = optional(string) + }), {}) + manage_all_addons = optional(bool, false) # Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources. kms_config = optional( object({ crk_name = string # Name of key