feat(kafka): Add amazon-s3-sing-connector-url (#98)

* feat(kafka): Add amazon-s3-sing-connector-url * feat(kafka): Add amazon-s3-sing-connector-url * feat(kafka): Change security grops * feat(kafka): Add server properties * feat(kafka): Add server properties * feat(kafka): Fix var reference Signed-off-by: [email protected] <[email protected]> * feat(kafka): Fix typo * feat(kafka): Add documentation * feat(kafka): Add documentation * feat(kafka): Add documentation * feat(kafka): Add variable --------- Signed-off-by: [email protected] <[email protected]>
TradrAPI · Dec 9, 2024 · 0a85ca8 · 0a85ca8
1 parent 45b71a3
commit 0a85ca8
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 13 deletions.
diff --git a/aws/kafka/README.md b/aws/kafka/README.md
@@ -52,6 +52,7 @@ No requirements.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_allowed_cidr_blocks"></a> [allowed\_cidr\_blocks](#input\_allowed\_cidr\_blocks) | Allowed CIDR blocks. | `list(string)` | `[]` | no |
+| <a name="input_amazon-s3-sink-connector-url"></a> [amazon-s3-sink-connector-url](#input\_amazon-s3-sink-connector-url) | Amazon S3 Sink Connector URL | `string` | `"https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/kafka-connect-s3/versions/10.5.2/confluentinc-kafka-connect-s3-10.5.2.zip"` | no |
 | <a name="input_client_subnets"></a> [client\_subnets](#input\_client\_subnets) | Client subnets. | `list(string)` | `[]` | no |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | msk cluster name. | `string` | `null` | no |
 | <a name="input_ebs_volume_size"></a> [ebs\_volume\_size](#input\_ebs\_volume\_size) | EBS volume size. | `number` | `10` | no |
@@ -62,7 +63,7 @@ No requirements.
 | <a name="input_platform"></a> [platform](#input\_platform) | Platform name. | `string` | `null` | no |
 | <a name="input_private_subnets"></a> [private\_subnets](#input\_private\_subnets) | Private subnets. | `list(string)` | `[]` | no |
 | <a name="input_region"></a> [region](#input\_region) | AWS region. | `string` | `null` | no |
-| <a name="input_security_groups"></a> [security\_groups](#input\_security\_groups) | Security groups. | `list(string)` | `[]` | no |
+| <a name="input_server_properties"></a> [server\_properties](#input\_server\_properties) | Server properties. | `string` | `null` | no |
 | <a name="input_users"></a> [users](#input\_users) | MSK users. | `list(string)` | `[]` | no |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | VPC id. where the MSK cluster will be created. | `string` | `null` | no |
 
@@ -72,4 +73,4 @@ No requirements.
 |------|-------------|
 | <a name="output_msk_brokers_by_auth_method"></a> [msk\_brokers\_by\_auth\_method](#output\_msk\_brokers\_by\_auth\_method) | n/a |
 | <a name="output_msk_sasl_scram_users"></a> [msk\_sasl\_scram\_users](#output\_msk\_sasl\_scram\_users) | n/a |
-<!-- END_TF_DOCS -->
+<!-- END_TF_DOCS -->
diff --git a/aws/kafka/main.tf b/aws/kafka/main.tf
@@ -5,12 +5,12 @@ locals {
   ))
 
   _server_properties_good_defaults = [
-    "num.partitions=${3 * local.var.msk.number_of_broker_nodes}",
+    "num.partitions=${3 * var.number_of_broker_nodes}",
     "log.retention.hours=168",
   ]
 
   _server_properties_without_locally_provided = [
-    for property in split("\n", local.var.msk.server_properties) :
+    for property in split("\n", var.server_properties) :
     property
     if(
       property != ""
@@ -24,7 +24,7 @@ locals {
 
 
 resource "aws_msk_configuration" "this" {
-  kafka_versions = [var.msk.kafka_version]
+  kafka_versions = [var.kafka_version]
 
   name = "${var.platform}-${var.environment}-${replace(var.kafka_version, ".", "")}"
 
@@ -66,7 +66,7 @@ resource "aws_msk_cluster" "this" {
       }
     }
 
-    security_groups = var.security_groups
+    security_groups = [aws_security_group.msk.id]
   }
 
   client_authentication {
@@ -105,7 +105,7 @@ resource "aws_msk_cluster" "this" {
 }
 
 resource "aws_security_group" "msk" {
-  name        = "${var.platform}-${var.cluster_name}-${var.environment}-msk"
+  name        = var.cluster_name
   description = "MSK security group"
   vpc_id      = var.vpc_id
 

diff --git a/aws/kafka/plugins.tf b/aws/kafka/plugins.tf
@@ -2,7 +2,7 @@ locals {
   plugins = {
     # Write data from kafka to s3 backup
     "amazon-s3-sink-connector" = {
-      url = "https://d1i4a15mxbxib1.cloudfront.net/api/plugins/confluentinc/kafka-connect-s3/versions/10.5.5/confluentinc-kafka-connect-s3-10.5.5.zip"
+      url = "${var.amazon-s3-sink-connector-url}"
     }
     # Write data from s3 to kafka restore
     "amazon-s3-source-connector" = {

diff --git a/aws/kafka/variable.tf b/aws/kafka/variable.tf
@@ -1,4 +1,9 @@
 
+variable "amazon-s3-sink-connector-url" {
+  description = "Amazon S3 Sink Connector URL"
+  type        = string
+  default     = "https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/kafka-connect-s3/versions/10.5.2/confluentinc-kafka-connect-s3-10.5.2.zip"
+}
 variable "platform" {
   description = "Platform name."
   type        = string
@@ -59,11 +64,7 @@ variable "vpc_id" {
   default     = null
 }
 
-variable "security_groups" {
-  description = "Security groups."
-  type        = list(string)
-  default     = []
-}
+
 
 
 variable "allowed_cidr_blocks" {
@@ -89,6 +90,15 @@ variable "region" {
   default     = null
 }
 
+
+variable "server_properties" {
+  description = "Server properties."
+  type        = string
+  default     = null
+
+}
+
+
 variable "private_subnets" {
   description = "Private subnets."
   type        = list(string)