From c8abd68950cd24d375749fca5da9488c61b41610 Mon Sep 17 00:00:00 2001 From: Brendan Luchen Date: Sat, 19 Nov 2022 12:47:54 -0500 Subject: [PATCH] feat: Add `data_tiering_enabled` (#175) * Plumb through data_tiering_enabled (#2) ## what * Introduce var.data_tiering_enabled and plumb it through to the aws provider ## why * We need to set `data_tiering_enabled` true to provision our desired node type [![DEVOPS-2155](https://img.shields.io/badge/DEVOPS-2155-blue.svg)](https://teikametrics.atlassian.net/browse/DEVOPS-2155) * This flag `data_tiering_enabled` doesn't currently exist in the `terraform-aws-elasticache-redis` module * After verifying this change, we plan to send it back upstream in a PR to https://github.com/cloudposse/terraform-aws-elasticache-redis ## testing Against @teikametrics/bid-orchestrator#202, made locally the following changes: ```diff diff --git a/terraform-redis-bidder-common/redis.tf b/terraform-redis-bidder-common/redis.tf index fc582ae..1bdb9df 100644 --- a/terraform-redis-bidder-common/redis.tf +++ b/terraform-redis-bidder-common/redis.tf @@ -48,7 +48,7 @@ resource "random_password" "password" { module "redis" { # https://registry.terraform.io/modules/cloudposse/elasticache-redis/aws/latest # source = "cloudposse/elasticache-redis/aws" ## FIXME: DEVOPS-2155 - source = "git@github.com:teikametrics/terraform-aws-elasticache-redis.git" + source = "git@github.com:teikametrics/terraform-aws-elasticache-redis.git?ref=bml.DEVOPS-2155.cloudposse-fork-data-tiering" # Auth token for password protecting redis, `transit_encryption_enabled` must be set to `true`. auth_token = random_password.password.result name = var.redis_name @@ -63,7 +63,7 @@ module "redis" { automatic_failover_enabled = var.redis_automatic_failover cluster_size = var.redis_cluster_size # Must be set to true to enable data tiering - # data_tiering_enabled = var.redis_data_tiering ## FIXME: DEVOPS-2155 + data_tiering_enabled = var.redis_data_tiering # The instance class used --> https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/CacheNodes.SupportedTypes.html instance_type = var.redis_instance_type # Apply changes immediately ``` Staging plan reports no changes (because `redis_data_tiering` is false in staging); production plan reports the same currently-unapplied changes as before, plus including `data_tiering_enabled`. Previously the plan errored out as this module did not support the flag. ## terraform plans ### Staging `tm-terraform staging staging plan` ``` No changes. Infrastructure is up-to-date. ``` ### Production `tm-terraform production production plan` ``` Terraform will perform the following actions: # module.redis.aws_elasticache_parameter_group.default[0] will be updated in-place ~ resource "aws_elasticache_parameter_group" "default" { id = "bidder-redis-cluster-production" name = "bidder-redis-cluster-production" ~ tags = { + "Name" = "bidder-redis-cluster-production" + "teikametrics.com/app" = "bid-orchestrator" + "teikametrics.com/deploy/stage" = "production" + "teikametrics.com/name" = "bidder-redis-cluster-production" + "teikametrics.com/team" = "artificial-intelligence" + "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } ~ tags_all = { + "Name" = "bidder-redis-cluster-production" + "teikametrics.com/app" = "bid-orchestrator" + "teikametrics.com/deploy/stage" = "production" + "teikametrics.com/name" = "bidder-redis-cluster-production" + "teikametrics.com/team" = "artificial-intelligence" + "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } # (3 unchanged attributes hidden) # (1 unchanged block hidden) } # module.redis.aws_elasticache_replication_group.default[0] will be created + resource "aws_elasticache_replication_group" "default" { + apply_immediately = true + arn = (known after apply) + at_rest_encryption_enabled = false + auth_token = (sensitive value) + auto_minor_version_upgrade = (known after apply) + automatic_failover_enabled = true + cluster_enabled = (known after apply) + configuration_endpoint_address = (known after apply) + data_tiering_enabled = true + description = "bidder-redis-cluster-production" + engine = "redis" + engine_version = "6.x" + engine_version_actual = (known after apply) + global_replication_group_id = (known after apply) + id = (known after apply) + maintenance_window = "sun:03:00-sun:04:00" + member_clusters = (known after apply) + multi_az_enabled = false + node_type = "cache.r6gd.xlarge" + num_cache_clusters = (known after apply) + num_node_groups = 2 + number_cache_clusters = (known after apply) + parameter_group_name = "bidder-redis-cluster-production" + port = 6379 + primary_endpoint_address = (known after apply) + reader_endpoint_address = (known after apply) + replicas_per_node_group = 1 + replication_group_description = (known after apply) + replication_group_id = "bidder-redis-cluster-production" + security_group_ids = (known after apply) + security_group_names = (known after apply) + snapshot_retention_limit = 7 + snapshot_window = "04:00-05:00" + subnet_group_name = "bidder-redis-cluster-production" + tags = { + "Name" = "bidder-redis-cluster-production" + "teikametrics.com/app" = "bid-orchestrator" + "teikametrics.com/deploy/stage" = "production" + "teikametrics.com/name" = "bidder-redis-cluster-production" + "teikametrics.com/team" = "artificial-intelligence" + "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } + tags_all = { + "Name" = "bidder-redis-cluster-production" + "teikametrics.com/app" = "bid-orchestrator" + "teikametrics.com/deploy/stage" = "production" + "teikametrics.com/name" = "bidder-redis-cluster-production" + "teikametrics.com/team" = "artificial-intelligence" + "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } + transit_encryption_enabled = true + cluster_mode { + num_node_groups = (known after apply) + replicas_per_node_group = (known after apply) } } # module.redis.aws_elasticache_subnet_group.default[0] will be updated in-place ~ resource "aws_elasticache_subnet_group" "default" { ~ description = "Managed by Terraform" -> "Elasticache subnet group for bidder-redis-cluster-production" id = "bidder-redis-cluster-production" name = "bidder-redis-cluster-production" ~ tags = { + "Name" = "bidder-redis-cluster-production" + "teikametrics.com/app" = "bid-orchestrator" + "teikametrics.com/deploy/stage" = "production" + "teikametrics.com/name" = "bidder-redis-cluster-production" + "teikametrics.com/team" = "artificial-intelligence" + "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } ~ tags_all = { + "Name" = "bidder-redis-cluster-production" + "teikametrics.com/app" = "bid-orchestrator" + "teikametrics.com/deploy/stage" = "production" + "teikametrics.com/name" = "bidder-redis-cluster-production" + "teikametrics.com/team" = "artificial-intelligence" + "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } # (2 unchanged attributes hidden) } # module.redis.module.aws_security_group.aws_security_group.cbd[0] will be created + resource "aws_security_group" "cbd" { + arn = (known after apply) + description = "Security group for Elasticache Redis" + egress = (known after apply) + id = (known after apply) + ingress = (known after apply) + name = (known after apply) + name_prefix = "bidder-redis-cluster-production-" + owner_id = (known after apply) + revoke_rules_on_delete = false + tags = { + "Name" = "bidder-redis-cluster-production" + "teikametrics.com/app" = "bid-orchestrator" + "teikametrics.com/deploy/stage" = "production" + "teikametrics.com/name" = "bidder-redis-cluster-production" + "teikametrics.com/team" = "artificial-intelligence" + "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } + tags_all = { + "Name" = "bidder-redis-cluster-production" + "teikametrics.com/app" = "bid-orchestrator" + "teikametrics.com/deploy/stage" = "production" + "teikametrics.com/name" = "bidder-redis-cluster-production" + "teikametrics.com/team" = "artificial-intelligence" + "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } + vpc_id = "vpc-0134131a45fbf70e6" + timeouts { + create = "10m" + delete = "15m" } } # module.redis.module.aws_security_group.aws_security_group_rule.keyed["_allow_all_egress_"] will be created + resource "aws_security_group_rule" "keyed" { + cidr_blocks = [ + "0.0.0.0/0", ] + description = "Allow all egress" + from_port = 0 + id = (known after apply) + ipv6_cidr_blocks = [ + "::/0", ] + prefix_list_ids = [] + protocol = "-1" + security_group_id = (known after apply) + self = false + source_security_group_id = (known after apply) + to_port = 0 + type = "egress" } # module.redis.module.aws_security_group.aws_security_group_rule.keyed["extra[0]"] will be created + resource "aws_security_group_rule" "keyed" { + cidr_blocks = [ + "10.76.0.0/16", + "10.60.0.0/16", + "10.78.0.0/16", + "10.30.0.0/16", ] + description = "Allow TCP inbound traffic on port 6379 from Convox and the vpc which is hosting this redis." + from_port = 6379 + id = (known after apply) + prefix_list_ids = [] + protocol = "tcp" + security_group_id = (known after apply) + self = false + source_security_group_id = (known after apply) + to_port = 6379 + type = "ingress" } # module.redis.module.security_group.aws_security_group.default[0] will be destroyed - resource "aws_security_group" "default" { - arn = "arn:aws:ec2:us-east-1:659641375152:security-group/sg-02edcb2360f6895b0" -> null - description = "ElastiCache Security Group" -> null - egress = [ - { - cidr_blocks = [ - "0.0.0.0/0", ] - description = "Allow all outbound traffic" - from_port = 0 - ipv6_cidr_blocks = [] - prefix_list_ids = [] - protocol = "-1" - security_groups = [] - self = false - to_port = 0 }, ] -> null - id = "sg-02edcb2360f6895b0" -> null - ingress = [ - { - cidr_blocks = [ - "10.76.0.0/16", - "10.60.0.0/16", - "10.78.0.0/16", - "10.30.0.0/16", ] - description = "Allow TCP inbound traffic on port 6379 from Convox and the vpc which is hosting this redis." - from_port = 6379 - ipv6_cidr_blocks = [] - prefix_list_ids = [] - protocol = "tcp" - security_groups = [] - self = false - to_port = 6379 }, ] -> null - name = "bidder-redis-cluster-production" -> null - owner_id = "659641375152" -> null - revoke_rules_on_delete = false -> null - tags = { - "Name" = "bidder-redis-cluster-production" - "teikametrics.com/app" = "bid-orchestrator" - "teikametrics.com/deploy/stage" = "production" - "teikametrics.com/name" = "bidder-redis-cluster-production" - "teikametrics.com/team" = "artificial-intelligence" - "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } -> null - tags_all = { - "Name" = "bidder-redis-cluster-production" - "teikametrics.com/app" = "bid-orchestrator" - "teikametrics.com/deploy/stage" = "production" - "teikametrics.com/name" = "bidder-redis-cluster-production" - "teikametrics.com/team" = "artificial-intelligence" - "teikametrics.com/terraform/gitpath" = "teikametrics/bid-orchestrator/terraform-redis" } -> null - vpc_id = "vpc-0134131a45fbf70e6" -> null } # module.redis.module.security_group.aws_security_group_rule.default["egress--1-0-0-9c87e5e1ed040a443ce1ac8e6d6cf159"] will be destroyed - resource "aws_security_group_rule" "default" { - cidr_blocks = [ - "0.0.0.0/0", ] -> null - description = "Allow all outbound traffic" -> null - from_port = 0 -> null - id = "sgrule-3785251229" -> null - protocol = "-1" -> null - security_group_id = "sg-02edcb2360f6895b0" -> null - self = false -> null - to_port = 0 -> null - type = "egress" -> null } # module.redis.module.security_group.aws_security_group_rule.default["ingress-tcp-6379-6379-c992040fb21cf75967d80aa440691f00"] will be destroyed - resource "aws_security_group_rule" "default" { - cidr_blocks = [ - "10.76.0.0/16", - "10.60.0.0/16", - "10.78.0.0/16", - "10.30.0.0/16", ] -> null - description = "Allow TCP inbound traffic on port 6379 from Convox and the vpc which is hosting this redis." -> null - from_port = 6379 -> null - id = "sgrule-236626820" -> null - protocol = "tcp" -> null - security_group_id = "sg-02edcb2360f6895b0" -> null - self = false -> null - to_port = 6379 -> null - type = "ingress" -> null } Plan: 4 to add, 2 to change, 3 to destroy. Changes to Outputs: + redis_master_endpoint = (known after apply) ``` ## references * https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/elasticache_replication_group#data_tiering_enabled * make pr/auto-format * Update variables.tf per @nitrocode Co-authored-by: nitrocode <7775707+nitrocode@users.noreply.github.com> * make pr/auto-format Co-authored-by: Brendan Luchen Co-authored-by: nitrocode <7775707+nitrocode@users.noreply.github.com> --- README.md | 1 + docs/terraform.md | 1 + main.tf | 1 + variables.tf | 6 ++++++ 4 files changed, 9 insertions(+) diff --git a/README.md b/README.md index fe91f13..79c5ed9 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,7 @@ Available targets: | [cluster\_size](#input\_cluster\_size) | Number of nodes in cluster. *Ignored when `cluster_mode_enabled` == `true`* | `number` | `1` | no | | [context](#input\_context) | Single object for setting entire context at once.
See description of individual variables for details.
Leave string and numeric variables as `null` to use default value.
Individual variable settings (non-null) override settings in context object,
except for attributes, tags, and additional\_tag\_map, which are merged. | `any` | 
{
  "additional_tag_map": {},
  "attributes": [],
  "delimiter": null,
  "descriptor_formats": {},
  "enabled": true,
  "environment": null,
  "id_length_limit": null,
  "label_key_case": null,
  "label_order": [],
  "label_value_case": null,
  "labels_as_tags": [
    "unset"
  ],
  "name": null,
  "namespace": null,
  "regex_replace_chars": null,
  "stage": null,
  "tags": {},
  "tenant": null
}
| no | | [create\_security\_group](#input\_create\_security\_group) | Set `true` to create and configure a new security group. If false, `associated_security_group_ids` must be provided. | `bool` | `true` | no | +| [data\_tiering\_enabled](#input\_data\_tiering\_enabled) | Enables data tiering. Data tiering is only supported for replication groups using the r6gd node type. | `bool` | `false` | no | | [delimiter](#input\_delimiter) | Delimiter to be used between ID elements.
Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no | | [description](#input\_description) | Description of elasticache replication group | `string` | `null` | no | | [descriptor\_formats](#input\_descriptor\_formats) | Describe additional descriptors to be output in the `descriptors` output map.
Map of maps. Keys are names of descriptors. Values are maps of the form
`{
format = string
labels = list(string)
}`
(Type is `any` so the map values can later be enhanced to provide additional options.)
`format` is a Terraform format string to be passed to the `format()` function.
`labels` is a list of labels, in order, to pass to `format()` function.
Label values will be normalized before being passed to `format()` so they will be
identical to how they appear in `id`.
Default is `{}` (`descriptors` output will be empty). | `any` | `{}` | no | diff --git a/docs/terraform.md b/docs/terraform.md index 8f489ca..b34f8dd 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -58,6 +58,7 @@ | [cluster\_size](#input\_cluster\_size) | Number of nodes in cluster. *Ignored when `cluster_mode_enabled` == `true`* | `number` | `1` | no | | [context](#input\_context) | Single object for setting entire context at once.
See description of individual variables for details.
Leave string and numeric variables as `null` to use default value.
Individual variable settings (non-null) override settings in context object,
except for attributes, tags, and additional\_tag\_map, which are merged. | `any` | 
{
  "additional_tag_map": {},
  "attributes": [],
  "delimiter": null,
  "descriptor_formats": {},
  "enabled": true,
  "environment": null,
  "id_length_limit": null,
  "label_key_case": null,
  "label_order": [],
  "label_value_case": null,
  "labels_as_tags": [
    "unset"
  ],
  "name": null,
  "namespace": null,
  "regex_replace_chars": null,
  "stage": null,
  "tags": {},
  "tenant": null
}
| no | | [create\_security\_group](#input\_create\_security\_group) | Set `true` to create and configure a new security group. If false, `associated_security_group_ids` must be provided. | `bool` | `true` | no | +| [data\_tiering\_enabled](#input\_data\_tiering\_enabled) | Enables data tiering. Data tiering is only supported for replication groups using the r6gd node type. | `bool` | `false` | no | | [delimiter](#input\_delimiter) | Delimiter to be used between ID elements.
Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no | | [description](#input\_description) | Description of elasticache replication group | `string` | `null` | no | | [descriptor\_formats](#input\_descriptor\_formats) | Describe additional descriptors to be output in the `descriptors` output map.
Map of maps. Keys are names of descriptors. Values are maps of the form
`{
format = string
labels = list(string)
}`
(Type is `any` so the map values can later be enhanced to provide additional options.)
`format` is a Terraform format string to be passed to the `format()` function.
`labels` is a list of labels, in order, to pass to `format()` function.
Label values will be normalized before being passed to `format()` so they will be
identical to how they appear in `id`.
Default is `{}` (`descriptors` output will be empty). | `any` | `{}` | no | diff --git a/main.tf b/main.tf index fe8dd88..eddfdcc 100644 --- a/main.tf +++ b/main.tf @@ -142,6 +142,7 @@ resource "aws_elasticache_replication_group" "default" { snapshot_retention_limit = var.snapshot_retention_limit final_snapshot_identifier = var.final_snapshot_identifier apply_immediately = var.apply_immediately + data_tiering_enabled = var.data_tiering_enabled auto_minor_version_upgrade = var.auto_minor_version_upgrade dynamic "log_delivery_configuration" { diff --git a/variables.tf b/variables.tf index 4d7480c..40b0c9c 100644 --- a/variables.tf +++ b/variables.tf @@ -112,6 +112,12 @@ variable "apply_immediately" { description = "Apply changes immediately" } +variable "data_tiering_enabled" { + type = bool + default = false + description = "Enables data tiering. Data tiering is only supported for replication groups using the r6gd node type." +} + variable "automatic_failover_enabled" { type = bool default = false