feat: added cluster addon support to the OCP DA and module. It can be…

… configured using the `cluster_addons` and `manage_all_cluster_addons` variables (#733)

README.md

@@ -866,6 +866,7 @@ module "cluster_pattern" {
 | [ibm_appid_redirect_urls.urls](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/appid_redirect_urls) | resource |
 | [ibm_atracker_route.atracker_route](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/atracker_route) | resource |
 | [ibm_atracker_target.atracker_target](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/atracker_target) | resource |
+| [ibm_container_addons.addons](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/container_addons) | resource |
 | [ibm_container_vpc_cluster.cluster](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/container_vpc_cluster) | resource |
 | [ibm_container_vpc_worker_pool.pool](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/container_vpc_worker_pool) | resource |
 | [ibm_cos_bucket.buckets](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/cos_bucket) | resource |
@@ -890,6 +891,7 @@ module "cluster_pattern" {
 | [random_string.random_cos_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
 | [time_sleep.wait_30_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
 | [time_sleep.wait_for_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
+| [ibm_container_addons.existing_addons](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/container_addons) | data source |
 | [ibm_container_cluster_versions.cluster_versions](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/container_cluster_versions) | data source |
 | [ibm_iam_account_settings.iam_account_settings](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/iam_account_settings) | data source |
 | [ibm_is_image.image](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/is_image) | data source |
@@ -903,7 +905,7 @@ module "cluster_pattern" {
 |------|-------------|------|---------|:--------:|
 | <a name="input_appid"></a> [appid](#input\_appid) | The App ID instance to be used for the teleport vsi deployments | <pre>object({<br>    name           = optional(string)<br>    resource_group = optional(string)<br>    use_data       = optional(bool)<br>    keys           = optional(list(string))<br>    use_appid      = bool<br>  })</pre> | <pre>{<br>  "use_appid": false<br>}</pre> | no |
 | <a name="input_atracker"></a> [atracker](#input\_atracker) | atracker variables | <pre>object({<br>    resource_group        = string<br>    receive_global_events = bool<br>    collector_bucket_name = string<br>    add_route             = bool<br>  })</pre> | n/a | yes |
-| <a name="input_clusters"></a> [clusters](#input\_clusters) | A list describing clusters workloads to create | <pre>list(<br>    object({<br>      name                    = string           # Name of Cluster<br>      vpc_name                = string           # Name of VPC<br>      subnet_names            = list(string)     # List of vpc subnets for cluster<br>      workers_per_subnet      = number           # Worker nodes per subnet.<br>      machine_type            = string           # Worker node flavor<br>      kube_type               = string           # iks or openshift<br>      kube_version            = optional(string) # Can be a version from `ibmcloud ks versions` or `default`<br>      entitlement             = optional(string) # entitlement option for openshift<br>      secondary_storage       = optional(string) # Secondary storage type<br>      pod_subnet              = optional(string) # Portable subnet for pods<br>      service_subnet          = optional(string) # Portable subnet for services<br>      resource_group          = string           # Resource Group used for cluster<br>      cos_name                = optional(string) # Name of COS instance Required only for OpenShift clusters<br>      access_tags             = optional(list(string), [])<br>      boot_volume_crk_name    = optional(string)     # Boot volume encryption key name<br>      disable_public_endpoint = optional(bool, true) # disable cluster public, leaving only private endpoint<br>      kms_config = optional(<br>        object({<br>          crk_name         = string         # Name of key<br>          private_endpoint = optional(bool) # Private endpoint<br>        })<br>      )<br>      worker_pools = optional(<br>        list(<br>          object({<br>            name                 = string           # Worker pool name<br>            vpc_name             = string           # VPC name<br>            workers_per_subnet   = number           # Worker nodes per subnet<br>            flavor               = string           # Worker node flavor<br>            subnet_names         = list(string)     # List of vpc subnets for worker pool<br>            entitlement          = optional(string) # entitlement option for openshift<br>            secondary_storage    = optional(string) # Secondary storage type<br>            boot_volume_crk_name = optional(string) # Boot volume encryption key name<br>          })<br>        )<br>      )<br>    })<br>  )</pre> | n/a | yes |
+| <a name="input_clusters"></a> [clusters](#input\_clusters) | A list describing clusters workloads to create | <pre>list(<br>    object({<br>      name                    = string           # Name of Cluster<br>      vpc_name                = string           # Name of VPC<br>      subnet_names            = list(string)     # List of vpc subnets for cluster<br>      workers_per_subnet      = number           # Worker nodes per subnet.<br>      machine_type            = string           # Worker node flavor<br>      kube_type               = string           # iks or openshift<br>      kube_version            = optional(string) # Can be a version from `ibmcloud ks versions` or `default`<br>      entitlement             = optional(string) # entitlement option for openshift<br>      secondary_storage       = optional(string) # Secondary storage type<br>      pod_subnet              = optional(string) # Portable subnet for pods<br>      service_subnet          = optional(string) # Portable subnet for services<br>      resource_group          = string           # Resource Group used for cluster<br>      cos_name                = optional(string) # Name of COS instance Required only for OpenShift clusters<br>      access_tags             = optional(list(string), [])<br>      boot_volume_crk_name    = optional(string)     # Boot volume encryption key name<br>      disable_public_endpoint = optional(bool, true) # disable cluster public, leaving only private endpoint<br>      addons = optional(object({                     # Map of OCP cluster add-on versions to install<br>        debug-tool                = optional(string)<br>        image-key-synchronizer    = optional(string)<br>        openshift-data-foundation = optional(string)<br>        vpc-file-csi-driver       = optional(string)<br>        static-route              = optional(string)<br>        cluster-autoscaler        = optional(string)<br>        vpc-block-csi-driver      = optional(string)<br>      }), {})<br>      manage_all_addons = optional(bool, false) # Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources.<br>      kms_config = optional(<br>        object({<br>          crk_name         = string         # Name of key<br>          private_endpoint = optional(bool) # Private endpoint<br>        })<br>      )<br>      worker_pools = optional(<br>        list(<br>          object({<br>            name                 = string           # Worker pool name<br>            vpc_name             = string           # VPC name<br>            workers_per_subnet   = number           # Worker nodes per subnet<br>            flavor               = string           # Worker node flavor<br>            subnet_names         = list(string)     # List of vpc subnets for worker pool<br>            entitlement          = optional(string) # entitlement option for openshift<br>            secondary_storage    = optional(string) # Secondary storage type<br>            boot_volume_crk_name = optional(string) # Boot volume encryption key name<br>          })<br>        )<br>      )<br>    })<br>  )</pre> | n/a | yes |
 | <a name="input_cos"></a> [cos](#input\_cos) | Object describing the cloud object storage instance, buckets, and keys. Set `use_data` to false to create instance | <pre>list(<br>    object({<br>      name           = string<br>      use_data       = optional(bool)<br>      resource_group = string<br>      plan           = optional(string)<br>      random_suffix  = optional(bool) # Use a random suffix for COS instance<br>      access_tags    = optional(list(string), [])<br>      buckets = list(object({<br>        name                  = string<br>        storage_class         = string<br>        endpoint_type         = string<br>        force_delete          = bool<br>        single_site_location  = optional(string)<br>        region_location       = optional(string)<br>        cross_region_location = optional(string)<br>        kms_key               = optional(string)<br>        access_tags           = optional(list(string), [])<br>        allowed_ip            = optional(list(string))<br>        hard_quota            = optional(number)<br>        archive_rule = optional(object({<br>          days    = number<br>          enable  = bool<br>          rule_id = optional(string)<br>          type    = string<br>        }))<br>        expire_rule = optional(object({<br>          days                         = optional(number)<br>          date                         = optional(string)<br>          enable                       = bool<br>          expired_object_delete_marker = optional(string)<br>          prefix                       = optional(string)<br>          rule_id                      = optional(string)<br>        }))<br>        activity_tracking = optional(object({<br>          activity_tracker_crn = string<br>          read_data_events     = bool<br>          write_data_events    = bool<br>        }))<br>        metrics_monitoring = optional(object({<br>          metrics_monitoring_crn  = string<br>          request_metrics_enabled = optional(bool)<br>          usage_metrics_enabled   = optional(bool)<br>        }))<br>      }))<br>      keys = optional(<br>        list(object({<br>          name        = string<br>          role        = string<br>          enable_HMAC = bool<br>        }))<br>      )<br><br>    })<br>  )</pre> | n/a | yes |
 | <a name="input_enable_transit_gateway"></a> [enable\_transit\_gateway](#input\_enable\_transit\_gateway) | Create transit gateway | `bool` | `true` | no |
 | <a name="input_f5_template_data"></a> [f5\_template\_data](#input\_f5\_template\_data) | Data for all f5 templates | <pre>object({<br>    tmos_admin_password     = optional(string)<br>    license_type            = optional(string)<br>    byol_license_basekey    = optional(string)<br>    license_host            = optional(string)<br>    license_username        = optional(string)<br>    license_password        = optional(string)<br>    license_pool            = optional(string)<br>    license_sku_keyword_1   = optional(string)<br>    license_sku_keyword_2   = optional(string)<br>    license_unit_of_measure = optional(string)<br>    do_declaration_url      = optional(string)<br>    as3_declaration_url     = optional(string)<br>    ts_declaration_url      = optional(string)<br>    phone_home_url          = optional(string)<br>    template_source         = optional(string)<br>    template_version        = optional(string)<br>    app_id                  = optional(string)<br>    tgactive_url            = optional(string)<br>    tgstandby_url           = optional(string)<br>    tgrefresh_url           = optional(string)<br>  })</pre> | <pre>{<br>  "license_type": "none"<br>}</pre> | no |

cluster.tf

@@ -127,3 +127,63 @@ resource "ibm_container_vpc_worker_pool" "pool" {
 }
 
 ##############################################################################
+
+##############################################################################
+# Addons
+##############################################################################
+
+# Lookup the current default csi-driver version
+data "ibm_container_addons" "existing_addons" {
+  for_each = ibm_container_vpc_cluster.cluster
+  cluster  = each.value.id
+}
+
+locals {
+  csi_driver_version = {
+    for cluster in ibm_container_vpc_cluster.cluster : cluster.name => (
+      length(data.ibm_container_addons.existing_addons[cluster.name].addons) > 0 &&
+      data.ibm_container_addons.existing_addons[cluster.name].addons[0].name == "vpc-block-csi-driver" ?
+      data.ibm_container_addons.existing_addons[cluster.name].addons[0].version : ""
+    )
+  }
+
+
+  #  addons_list = var.addons != null ? { for k, v in var.addons : k => v if v != null } : {}
+  #  addons      = lookup(local.addons_list, "vpc-block-csi-driver", null) == null ? merge(local.addons_list, { vpc-block-csi-driver = local.csi_driver_version[0] }) : local.addons_list
+  # for each cluster in the clusters_map, get the addons and their versions and create an addons map including the corosponding csi_driver_version
+  cluster_addons = {
+    for cluster in var.clusters : "${var.prefix}-${cluster.name}" => {
+      id                = ibm_container_vpc_cluster.cluster["${var.prefix}-${cluster.name}"].id
+      resource_group_id = ibm_container_vpc_cluster.cluster["${var.prefix}-${cluster.name}"].resource_group_id
+      addons = merge(
+        { for addon_name, addon_version in(cluster.addons != null ? cluster.addons : {}) : addon_name => addon_version if addon_version != null },
+        local.csi_driver_version["${var.prefix}-${cluster.name}"] != null ? { vpc-block-csi-driver = local.csi_driver_version["${var.prefix}-${cluster.name}"] } : {}
+      )
+    }
+  }
+}
+
+resource "ibm_container_addons" "addons" {
+  # Worker pool creation can start before the 'ibm_container_vpc_cluster' completes since there is no explicit
+  # depends_on in 'ibm_container_vpc_worker_pool', just an implicit depends_on on the cluster ID. Cluster ID can exist before
+  # 'ibm_container_vpc_cluster' completes, so hence need to add explicit depends on against 'ibm_container_vpc_cluster' here.
+  depends_on        = [ibm_container_vpc_cluster.cluster, ibm_container_vpc_worker_pool.pool]
+  for_each          = local.cluster_addons
+  cluster           = each.value.id
+  resource_group_id = each.value.resource_group_id
+
+  # setting to false means we do not want Terraform to manage addons that are managed elsewhere
+  manage_all_addons = local.clusters_map[each.key].manage_all_addons
+
+  dynamic "addons" {
+    for_each = local.cluster_addons[each.key].addons
+    content {
+      name    = addons.key
+      version = addons.value
+    }
+  }
+
+  timeouts {
+    create = "1h"
+  }
+}

patterns/roks/main.tf

@@ -35,6 +35,8 @@ module "roks_landing_zone" {
   workers_per_zone                       = var.workers_per_zone
   flavor                                 = var.flavor
   kube_version                           = var.kube_version
+  cluster_addons                         = var.cluster_addons
+  manage_all_cluster_addons              = var.manage_all_cluster_addons
   add_atracker_route                     = var.add_atracker_route
   hs_crypto_instance_name                = var.hs_crypto_instance_name
   hs_crypto_resource_group               = var.hs_crypto_resource_group

patterns/roks/module/config.tf

@@ -87,6 +87,8 @@ locals {
         cos_name             = "cos"
         entitlement          = var.entitlement
         secondary_storage    = var.secondary_storage
+        addons               = var.cluster_addons
+        manage_all_addons    = var.manage_all_cluster_addons
         boot_volume_crk_name = "${var.prefix}-roks-key"
         # By default, create dedicated pool for logging
         worker_pools = [

patterns/roks/module/variables.tf

@@ -163,6 +163,27 @@ variable "secondary_storage" {
   default     = null
 }
 
+variable "cluster_addons" {
+  type = object({
+    debug-tool                = optional(string)
+    image-key-synchronizer    = optional(string)
+    openshift-data-foundation = optional(string)
+    vpc-file-csi-driver       = optional(string)
+    static-route              = optional(string)
+    cluster-autoscaler        = optional(string)
+    vpc-block-csi-driver      = optional(string)
+  })
+  description = "Map of OCP cluster add-on versions to install (NOTE: The 'vpc-block-csi-driver' add-on is installed by default for VPC clusters, however you can explicitly specify it here if you wish to choose a later version than the default one). For full list of all supported add-ons and versions, see https://cloud.ibm.com/docs/containers?topic=containers-supported-cluster-addon-versions"
+  default     = null
+}
+
+variable "manage_all_cluster_addons" {
+  type        = bool
+  default     = false
+  nullable    = false # null values are set to default value
+  description = "Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources."
+}
+
 ##############################################################################
 
 

patterns/roks/variables.tf

@@ -171,6 +171,27 @@ variable "entitlement" {
   default     = null
 }
 
+variable "cluster_addons" {
+  type = object({
+    debug-tool                = optional(string)
+    image-key-synchronizer    = optional(string)
+    openshift-data-foundation = optional(string)
+    vpc-file-csi-driver       = optional(string)
+    static-route              = optional(string)
+    cluster-autoscaler        = optional(string)
+    vpc-block-csi-driver      = optional(string)
+  })
+  description = "Map of OCP cluster add-on versions to install (NOTE: The 'vpc-block-csi-driver' add-on is installed by default for VPC clusters, however you can explicitly specify it here if you wish to choose a later version than the default one). For full list of all supported add-ons and versions, see https://cloud.ibm.com/docs/containers?topic=containers-supported-cluster-addon-versions"
+  default     = null
+}
+
+variable "manage_all_cluster_addons" {
+  type        = bool
+  default     = false
+  nullable    = false # null values are set to default value
+  description = "Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources."
+}
+
 ##############################################################################
 
 

variables.tf

@@ -827,6 +827,16 @@ variable "clusters" {
       access_tags             = optional(list(string), [])
       boot_volume_crk_name    = optional(string)     # Boot volume encryption key name
       disable_public_endpoint = optional(bool, true) # disable cluster public, leaving only private endpoint
+      addons = optional(object({                     # Map of OCP cluster add-on versions to install
+        debug-tool                = optional(string)
+        image-key-synchronizer    = optional(string)
+        openshift-data-foundation = optional(string)
+        vpc-file-csi-driver       = optional(string)
+        static-route              = optional(string)
+        cluster-autoscaler        = optional(string)
+        vpc-block-csi-driver      = optional(string)
+      }), {})
+      manage_all_addons = optional(bool, false) # Instructs Terraform to manage all cluster addons, even if addons were installed outside of the module. If set to 'true' this module will destroy any addons that were installed by other sources.
       kms_config = optional(
         object({
           crk_name         = string         # Name of key