Move NLB to Switch (#627)

somleng · Jun 16, 2024 · 2aa7d2c · 2aa7d2c
1 parent 77a0434
commit 2aa7d2c
Show file tree

Hide file tree

Showing 6 changed files with 131 additions and 17 deletions.
diff --git a/components/gateway/public_gateway/docker-entrypoint.sh b/components/gateway/public_gateway/docker-entrypoint.sh
@@ -11,7 +11,7 @@ if [ "$1" = 'opensips' ]; then
   DATABASE_URL="${DATABASE_URL:="postgres://postgres:@localhost:5432/opensips"}"
   SIP_ADVERTISED_IP="${SIP_ADVERTISED_IP:="$(hostname -i)"}"
   LOCAL_IP="$(hostname -i)"
-  INTERFACE_NAME="$(ip route list | grep default | grep -E  'dev (\w+)' -o | awk '{print $2}')"
+  INTERFACE_NAME="$(ip route list | grep default | grep -E 'dev (\w+)' -o | awk '{print $2}')"
 
   if [ -n "$DATABASE_HOST" ]; then
     DATABASE_URL="postgres://$DATABASE_USERNAME:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -5,7 +5,6 @@
 # Postgres is not included in this docker compose configuration
 # because of the complexity of adding code to wait for postgres to start.
 
-version: "3.9"
 volumes:
   workspace:
   opensips_fifo:
@@ -145,7 +144,7 @@ services:
       test:
         [
           "CMD-SHELL",
-          "wget --server-response --spider --quiet http://localhost:8080/health_checks 2>&1 | grep '200 OK' > /dev/null",
+          "wget --server-response --spider --quiet http://0.0.0.0:8080/health_checks 2>&1 | grep '200 OK' > /dev/null",
         ]
       interval: 10s
       timeout: 5s

diff --git a/infrastructure/modules/somleng_switch/public_gateway.tf b/infrastructure/modules/somleng_switch/public_gateway.tf
@@ -69,6 +69,33 @@ resource "aws_security_group_rule" "public_gateway_egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
+# Global Accelerator
+
+resource "aws_globalaccelerator_listener" "public_gateway" {
+  accelerator_arn = var.global_accelerator.id
+  protocol        = "UDP"
+
+  port_range {
+    from_port = var.sip_port
+    to_port   = var.sip_port
+  }
+
+  port_range {
+    from_port = var.sip_alternative_port
+    to_port   = var.sip_alternative_port
+  }
+}
+
+resource "aws_globalaccelerator_endpoint_group" "public_gateway" {
+  count        = var.public_gateway_min_tasks > 0 ? 1 : 0
+  listener_arn = aws_globalaccelerator_listener.public_gateway.id
+
+  endpoint_configuration {
+    endpoint_id                    = aws_lb.public_gateway_nlb[count.index].arn
+    client_ip_preservation_enabled = true
+  }
+}
+
 # IAM
 resource "aws_iam_role" "public_gateway_task_role" {
   name = "${var.public_gateway_identifier}-ecsTaskRole"
@@ -225,7 +252,7 @@ resource "aws_ecs_task_definition" "public_gateway" {
         },
         {
           name  = "SIP_ADVERTISED_IP",
-          value = tostring(var.external_sip_ip)
+          value = tostring(var.global_accelerator.ip_sets[0].ip_addresses[0])
         }
       ]
     },
@@ -252,14 +279,15 @@ resource "aws_ecs_task_definition" "public_gateway" {
     }
   ])
 
-  memory = module.public_gateway_container_instances.ec2_instance_type.memory_size - 512
+  memory = max((module.public_gateway_container_instances.ec2_instance_type.memory_size - 512), 128)
 
   volume {
     name = "opensips"
   }
 }
 
 resource "aws_ecs_service" "public_gateway" {
+  count           = var.public_gateway_min_tasks > 0 ? 1 : 0
   name            = aws_ecs_task_definition.public_gateway.family
   cluster         = aws_ecs_cluster.cluster.id
   task_definition = aws_ecs_task_definition.public_gateway.arn
@@ -300,6 +328,86 @@ resource "aws_ecs_service" "public_gateway" {
 }
 
 # Load Balancer
+
+resource "aws_security_group" "public_gateway_nlb" {
+  name   = "${var.public_gateway_identifier}-nlb"
+  vpc_id = var.vpc.vpc_id
+}
+
+resource "aws_security_group_rule" "public_gateway_nlb_sip_ingress" {
+  type        = "ingress"
+  from_port   = var.sip_port
+  to_port     = var.sip_port
+  protocol    = "udp"
+  cidr_blocks = ["0.0.0.0/0"]
+
+  security_group_id = aws_security_group.public_gateway_nlb.id
+}
+
+resource "aws_security_group_rule" "public_gateway_nlb_sip_alternative_ingress" {
+  type        = "ingress"
+  from_port   = var.sip_alternative_port
+  to_port     = var.sip_alternative_port
+  protocol    = "udp"
+  cidr_blocks = ["0.0.0.0/0"]
+
+  security_group_id = aws_security_group.public_gateway_nlb.id
+}
+
+resource "aws_security_group_rule" "public_gateway_nlb_udp_egress" {
+  type        = "egress"
+  from_port   = 0
+  to_port     = 65535
+  protocol    = "udp"
+  cidr_blocks = ["0.0.0.0/0"]
+
+  security_group_id = aws_security_group.public_gateway_nlb.id
+}
+
+resource "aws_security_group_rule" "public_gateway_nlb_tcp_egress" {
+  type        = "egress"
+  from_port   = 0
+  to_port     = 65535
+  protocol    = "tcp"
+  cidr_blocks = ["0.0.0.0/0"]
+
+  security_group_id = aws_security_group.public_gateway_nlb.id
+}
+
+resource "aws_eip" "public_gateway_nlb" {
+  count  = var.public_gateway_min_tasks > 0 ? length(var.vpc.public_subnets) : 0
+  domain = "vpc"
+
+  tags = {
+    Name = "Public Gateway NLB IP"
+  }
+}
+
+resource "aws_lb" "public_gateway_nlb" {
+  count                            = var.public_gateway_min_tasks > 0 ? 1 : 0
+  name                             = var.public_gateway_identifier
+  load_balancer_type               = "network"
+  enable_cross_zone_load_balancing = true
+
+  security_groups = [aws_security_group.public_gateway_nlb.id]
+
+  access_logs {
+    bucket  = var.logs_bucket.id
+    prefix  = var.public_gateway_identifier
+    enabled = true
+  }
+
+  dynamic "subnet_mapping" {
+    for_each = var.vpc.public_subnets
+    content {
+      subnet_id     = subnet_mapping.value
+      allocation_id = aws_eip.public_gateway_nlb.*.id[subnet_mapping.key]
+    }
+  }
+}
+
+# Target Groups
+
 resource "aws_lb_target_group" "sip" {
   name        = "${var.public_gateway_identifier}-sip"
   port        = var.sip_port
@@ -318,7 +426,8 @@ resource "aws_lb_target_group" "sip" {
 }
 
 resource "aws_lb_listener" "sip" {
-  load_balancer_arn = var.network_load_balancer.arn
+  count             = var.public_gateway_min_tasks > 0 ? 1 : 0
+  load_balancer_arn = aws_lb.public_gateway_nlb[count.index].arn
   port              = var.sip_port
   protocol          = "UDP"
 
@@ -346,7 +455,8 @@ resource "aws_lb_target_group" "sip_alternative" {
 }
 
 resource "aws_lb_listener" "sip_alternative" {
-  load_balancer_arn = var.network_load_balancer.arn
+  count             = var.public_gateway_min_tasks > 0 ? 1 : 0
+  load_balancer_arn = aws_lb.public_gateway_nlb[count.index].arn
   port              = var.sip_alternative_port
   protocol          = "UDP"
 
@@ -358,10 +468,11 @@ resource "aws_lb_listener" "sip_alternative" {
 
 # Autoscaling
 resource "aws_appautoscaling_policy" "public_gateway_policy" {
+  count              = var.public_gateway_min_tasks > 0 ? 1 : 0
   name               = var.public_gateway_identifier
-  service_namespace  = aws_appautoscaling_target.public_gateway_scale_target.service_namespace
-  resource_id        = aws_appautoscaling_target.public_gateway_scale_target.resource_id
-  scalable_dimension = aws_appautoscaling_target.public_gateway_scale_target.scalable_dimension
+  service_namespace  = aws_appautoscaling_target.public_gateway_scale_target[count.index].service_namespace
+  resource_id        = aws_appautoscaling_target.public_gateway_scale_target[count.index].resource_id
+  scalable_dimension = aws_appautoscaling_target.public_gateway_scale_target[count.index].scalable_dimension
   policy_type        = "TargetTrackingScaling"
 
   target_tracking_scaling_policy_configuration {
@@ -376,8 +487,9 @@ resource "aws_appautoscaling_policy" "public_gateway_policy" {
 }
 
 resource "aws_appautoscaling_target" "public_gateway_scale_target" {
+  count              = var.public_gateway_min_tasks > 0 ? 1 : 0
   service_namespace  = "ecs"
-  resource_id        = "service/${aws_ecs_cluster.cluster.name}/${aws_ecs_service.public_gateway.name}"
+  resource_id        = "service/${aws_ecs_cluster.cluster.name}/${aws_ecs_service.public_gateway[count.index].name}"
   scalable_dimension = "ecs:service:DesiredCount"
   min_capacity       = var.public_gateway_min_tasks
   max_capacity       = var.public_gateway_max_tasks

diff --git a/infrastructure/modules/somleng_switch/variables.tf b/infrastructure/modules/somleng_switch/variables.tf
@@ -20,14 +20,16 @@ variable "freeswitch_event_logger_image" {}
 variable "s3_mpeg_ecr_repository_url" {}
 variable "services_ecr_repository_url" {}
 variable "internal_load_balancer" {}
-variable "network_load_balancer" {}
 variable "internal_listener" {}
 variable "switch_subdomain" {}
 variable "client_gateway_subdomain" {}
 variable "route53_zone" {}
 variable "internal_route53_zone" {}
 variable "recordings_bucket_name" {}
+variable "logs_bucket" {}
 variable "efs_cache_name" {}
+variable "global_accelerator" {}
+
 variable "container_insights_enabled" {
   default = false
 }
@@ -135,7 +137,6 @@ variable "db_username" {}
 variable "db_password_parameter_arn" {}
 
 variable "json_cdr_password_parameter_arn" {}
-variable "external_sip_ip" {}
 variable "external_rtp_ip" {}
 variable "alternative_sip_outbound_ip" {}
 variable "alternative_rtp_ip" {}

diff --git a/infrastructure/production/main.tf b/infrastructure/production/main.tf
@@ -31,7 +31,6 @@ module "somleng_switch" {
 
   json_cdr_password_parameter_arn = data.aws_ssm_parameter.somleng_services_password.arn
   json_cdr_url                    = "https://api.internal.somleng.org/services/call_data_records"
-  external_sip_ip                 = data.terraform_remote_state.core_infrastructure.outputs.nlb_eips[0].public_ip
   external_rtp_ip                 = data.terraform_remote_state.core_infrastructure.outputs.vpc.nat_public_ips[0]
 
   alternative_sip_outbound_ip = data.terraform_remote_state.core_infrastructure.outputs.nat_instance_ip
@@ -47,8 +46,10 @@ module "somleng_switch" {
   db_security_group         = data.terraform_remote_state.core_infrastructure.outputs.db_security_group.id
 
   internal_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.internal_application_load_balancer
-  network_load_balancer  = data.terraform_remote_state.core_infrastructure.outputs.network_load_balancer
   internal_listener      = data.terraform_remote_state.core_infrastructure.outputs.internal_https_listener
+  global_accelerator     = data.terraform_remote_state.core_infrastructure.outputs.global_accelerator
+
+  logs_bucket = data.terraform_remote_state.core_infrastructure.outputs.logs_bucket
 
   route53_zone          = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_somleng_org
   internal_route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_internal_somleng_org

diff --git a/infrastructure/staging/main.tf b/infrastructure/staging/main.tf
@@ -32,7 +32,6 @@ module "somleng_switch_staging" {
 
   json_cdr_password_parameter_arn = data.aws_ssm_parameter.somleng_services_password.arn
   json_cdr_url                    = "https://api-staging.internal.somleng.org/services/call_data_records"
-  external_sip_ip                 = data.terraform_remote_state.core_infrastructure.outputs.nlb_eips[0].public_ip
   external_rtp_ip                 = data.terraform_remote_state.core_infrastructure.outputs.vpc.nat_public_ips[0]
 
   alternative_sip_outbound_ip = data.terraform_remote_state.core_infrastructure.outputs.nat_instance_ip
@@ -48,11 +47,13 @@ module "somleng_switch_staging" {
   db_security_group         = data.terraform_remote_state.core_infrastructure.outputs.db_security_group.id
 
   internal_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.internal_application_load_balancer
-  network_load_balancer  = data.terraform_remote_state.core_infrastructure.outputs.network_load_balancer
   internal_listener      = data.terraform_remote_state.core_infrastructure.outputs.internal_https_listener
 
   route53_zone          = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_somleng_org
   internal_route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_internal_somleng_org
+  global_accelerator    = data.terraform_remote_state.core_infrastructure.outputs.global_accelerator
+
+  logs_bucket = data.terraform_remote_state.core_infrastructure.outputs.logs_bucket
 
   switch_subdomain         = "switch-staging"
   client_gateway_subdomain = "sip-staging"