From 2aa7d2c376b98d67b81a6033e024baef54bed107 Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sun, 16 Jun 2024 08:57:11 +0700 Subject: [PATCH] Move NLB to Switch (#627) --- .../public_gateway/docker-entrypoint.sh | 2 +- docker-compose.yml | 3 +- .../modules/somleng_switch/public_gateway.tf | 128 ++++++++++++++++-- .../modules/somleng_switch/variables.tf | 5 +- infrastructure/production/main.tf | 5 +- infrastructure/staging/main.tf | 5 +- 6 files changed, 131 insertions(+), 17 deletions(-) diff --git a/components/gateway/public_gateway/docker-entrypoint.sh b/components/gateway/public_gateway/docker-entrypoint.sh index 55c2757b4..dbb15f4da 100755 --- a/components/gateway/public_gateway/docker-entrypoint.sh +++ b/components/gateway/public_gateway/docker-entrypoint.sh @@ -11,7 +11,7 @@ if [ "$1" = 'opensips' ]; then DATABASE_URL="${DATABASE_URL:="postgres://postgres:@localhost:5432/opensips"}" SIP_ADVERTISED_IP="${SIP_ADVERTISED_IP:="$(hostname -i)"}" LOCAL_IP="$(hostname -i)" - INTERFACE_NAME="$(ip route list | grep default | grep -E 'dev (\w+)' -o | awk '{print $2}')" + INTERFACE_NAME="$(ip route list | grep default | grep -E 'dev (\w+)' -o | awk '{print $2}')" if [ -n "$DATABASE_HOST" ]; then DATABASE_URL="postgres://$DATABASE_USERNAME:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME" diff --git a/docker-compose.yml b/docker-compose.yml index ce6f83bfb..faa65ff45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,6 @@ # Postgres is not included in this docker compose configuration # because of the complexity of adding code to wait for postgres to start. -version: "3.9" volumes: workspace: opensips_fifo: @@ -145,7 +144,7 @@ services: test: [ "CMD-SHELL", - "wget --server-response --spider --quiet http://localhost:8080/health_checks 2>&1 | grep '200 OK' > /dev/null", + "wget --server-response --spider --quiet http://0.0.0.0:8080/health_checks 2>&1 | grep '200 OK' > /dev/null", ] interval: 10s timeout: 5s diff --git a/infrastructure/modules/somleng_switch/public_gateway.tf b/infrastructure/modules/somleng_switch/public_gateway.tf index e95121114..b890810b6 100644 --- a/infrastructure/modules/somleng_switch/public_gateway.tf +++ b/infrastructure/modules/somleng_switch/public_gateway.tf @@ -69,6 +69,33 @@ resource "aws_security_group_rule" "public_gateway_egress" { cidr_blocks = ["0.0.0.0/0"] } +# Global Accelerator + +resource "aws_globalaccelerator_listener" "public_gateway" { + accelerator_arn = var.global_accelerator.id + protocol = "UDP" + + port_range { + from_port = var.sip_port + to_port = var.sip_port + } + + port_range { + from_port = var.sip_alternative_port + to_port = var.sip_alternative_port + } +} + +resource "aws_globalaccelerator_endpoint_group" "public_gateway" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 + listener_arn = aws_globalaccelerator_listener.public_gateway.id + + endpoint_configuration { + endpoint_id = aws_lb.public_gateway_nlb[count.index].arn + client_ip_preservation_enabled = true + } +} + # IAM resource "aws_iam_role" "public_gateway_task_role" { name = "${var.public_gateway_identifier}-ecsTaskRole" @@ -225,7 +252,7 @@ resource "aws_ecs_task_definition" "public_gateway" { }, { name = "SIP_ADVERTISED_IP", - value = tostring(var.external_sip_ip) + value = tostring(var.global_accelerator.ip_sets[0].ip_addresses[0]) } ] }, @@ -252,7 +279,7 @@ resource "aws_ecs_task_definition" "public_gateway" { } ]) - memory = module.public_gateway_container_instances.ec2_instance_type.memory_size - 512 + memory = max((module.public_gateway_container_instances.ec2_instance_type.memory_size - 512), 128) volume { name = "opensips" @@ -260,6 +287,7 @@ resource "aws_ecs_task_definition" "public_gateway" { } resource "aws_ecs_service" "public_gateway" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 name = aws_ecs_task_definition.public_gateway.family cluster = aws_ecs_cluster.cluster.id task_definition = aws_ecs_task_definition.public_gateway.arn @@ -300,6 +328,86 @@ resource "aws_ecs_service" "public_gateway" { } # Load Balancer + +resource "aws_security_group" "public_gateway_nlb" { + name = "${var.public_gateway_identifier}-nlb" + vpc_id = var.vpc.vpc_id +} + +resource "aws_security_group_rule" "public_gateway_nlb_sip_ingress" { + type = "ingress" + from_port = var.sip_port + to_port = var.sip_port + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.public_gateway_nlb.id +} + +resource "aws_security_group_rule" "public_gateway_nlb_sip_alternative_ingress" { + type = "ingress" + from_port = var.sip_alternative_port + to_port = var.sip_alternative_port + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.public_gateway_nlb.id +} + +resource "aws_security_group_rule" "public_gateway_nlb_udp_egress" { + type = "egress" + from_port = 0 + to_port = 65535 + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.public_gateway_nlb.id +} + +resource "aws_security_group_rule" "public_gateway_nlb_tcp_egress" { + type = "egress" + from_port = 0 + to_port = 65535 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.public_gateway_nlb.id +} + +resource "aws_eip" "public_gateway_nlb" { + count = var.public_gateway_min_tasks > 0 ? length(var.vpc.public_subnets) : 0 + domain = "vpc" + + tags = { + Name = "Public Gateway NLB IP" + } +} + +resource "aws_lb" "public_gateway_nlb" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 + name = var.public_gateway_identifier + load_balancer_type = "network" + enable_cross_zone_load_balancing = true + + security_groups = [aws_security_group.public_gateway_nlb.id] + + access_logs { + bucket = var.logs_bucket.id + prefix = var.public_gateway_identifier + enabled = true + } + + dynamic "subnet_mapping" { + for_each = var.vpc.public_subnets + content { + subnet_id = subnet_mapping.value + allocation_id = aws_eip.public_gateway_nlb.*.id[subnet_mapping.key] + } + } +} + +# Target Groups + resource "aws_lb_target_group" "sip" { name = "${var.public_gateway_identifier}-sip" port = var.sip_port @@ -318,7 +426,8 @@ resource "aws_lb_target_group" "sip" { } resource "aws_lb_listener" "sip" { - load_balancer_arn = var.network_load_balancer.arn + count = var.public_gateway_min_tasks > 0 ? 1 : 0 + load_balancer_arn = aws_lb.public_gateway_nlb[count.index].arn port = var.sip_port protocol = "UDP" @@ -346,7 +455,8 @@ resource "aws_lb_target_group" "sip_alternative" { } resource "aws_lb_listener" "sip_alternative" { - load_balancer_arn = var.network_load_balancer.arn + count = var.public_gateway_min_tasks > 0 ? 1 : 0 + load_balancer_arn = aws_lb.public_gateway_nlb[count.index].arn port = var.sip_alternative_port protocol = "UDP" @@ -358,10 +468,11 @@ resource "aws_lb_listener" "sip_alternative" { # Autoscaling resource "aws_appautoscaling_policy" "public_gateway_policy" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 name = var.public_gateway_identifier - service_namespace = aws_appautoscaling_target.public_gateway_scale_target.service_namespace - resource_id = aws_appautoscaling_target.public_gateway_scale_target.resource_id - scalable_dimension = aws_appautoscaling_target.public_gateway_scale_target.scalable_dimension + service_namespace = aws_appautoscaling_target.public_gateway_scale_target[count.index].service_namespace + resource_id = aws_appautoscaling_target.public_gateway_scale_target[count.index].resource_id + scalable_dimension = aws_appautoscaling_target.public_gateway_scale_target[count.index].scalable_dimension policy_type = "TargetTrackingScaling" target_tracking_scaling_policy_configuration { @@ -376,8 +487,9 @@ resource "aws_appautoscaling_policy" "public_gateway_policy" { } resource "aws_appautoscaling_target" "public_gateway_scale_target" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 service_namespace = "ecs" - resource_id = "service/${aws_ecs_cluster.cluster.name}/${aws_ecs_service.public_gateway.name}" + resource_id = "service/${aws_ecs_cluster.cluster.name}/${aws_ecs_service.public_gateway[count.index].name}" scalable_dimension = "ecs:service:DesiredCount" min_capacity = var.public_gateway_min_tasks max_capacity = var.public_gateway_max_tasks diff --git a/infrastructure/modules/somleng_switch/variables.tf b/infrastructure/modules/somleng_switch/variables.tf index 2afcdffa7..1d11601e9 100644 --- a/infrastructure/modules/somleng_switch/variables.tf +++ b/infrastructure/modules/somleng_switch/variables.tf @@ -20,14 +20,16 @@ variable "freeswitch_event_logger_image" {} variable "s3_mpeg_ecr_repository_url" {} variable "services_ecr_repository_url" {} variable "internal_load_balancer" {} -variable "network_load_balancer" {} variable "internal_listener" {} variable "switch_subdomain" {} variable "client_gateway_subdomain" {} variable "route53_zone" {} variable "internal_route53_zone" {} variable "recordings_bucket_name" {} +variable "logs_bucket" {} variable "efs_cache_name" {} +variable "global_accelerator" {} + variable "container_insights_enabled" { default = false } @@ -135,7 +137,6 @@ variable "db_username" {} variable "db_password_parameter_arn" {} variable "json_cdr_password_parameter_arn" {} -variable "external_sip_ip" {} variable "external_rtp_ip" {} variable "alternative_sip_outbound_ip" {} variable "alternative_rtp_ip" {} diff --git a/infrastructure/production/main.tf b/infrastructure/production/main.tf index e549d6aae..261c95017 100644 --- a/infrastructure/production/main.tf +++ b/infrastructure/production/main.tf @@ -31,7 +31,6 @@ module "somleng_switch" { json_cdr_password_parameter_arn = data.aws_ssm_parameter.somleng_services_password.arn json_cdr_url = "https://api.internal.somleng.org/services/call_data_records" - external_sip_ip = data.terraform_remote_state.core_infrastructure.outputs.nlb_eips[0].public_ip external_rtp_ip = data.terraform_remote_state.core_infrastructure.outputs.vpc.nat_public_ips[0] alternative_sip_outbound_ip = data.terraform_remote_state.core_infrastructure.outputs.nat_instance_ip @@ -47,8 +46,10 @@ module "somleng_switch" { db_security_group = data.terraform_remote_state.core_infrastructure.outputs.db_security_group.id internal_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.internal_application_load_balancer - network_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.network_load_balancer internal_listener = data.terraform_remote_state.core_infrastructure.outputs.internal_https_listener + global_accelerator = data.terraform_remote_state.core_infrastructure.outputs.global_accelerator + + logs_bucket = data.terraform_remote_state.core_infrastructure.outputs.logs_bucket route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_somleng_org internal_route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_internal_somleng_org diff --git a/infrastructure/staging/main.tf b/infrastructure/staging/main.tf index ddc36e848..34075e347 100644 --- a/infrastructure/staging/main.tf +++ b/infrastructure/staging/main.tf @@ -32,7 +32,6 @@ module "somleng_switch_staging" { json_cdr_password_parameter_arn = data.aws_ssm_parameter.somleng_services_password.arn json_cdr_url = "https://api-staging.internal.somleng.org/services/call_data_records" - external_sip_ip = data.terraform_remote_state.core_infrastructure.outputs.nlb_eips[0].public_ip external_rtp_ip = data.terraform_remote_state.core_infrastructure.outputs.vpc.nat_public_ips[0] alternative_sip_outbound_ip = data.terraform_remote_state.core_infrastructure.outputs.nat_instance_ip @@ -48,11 +47,13 @@ module "somleng_switch_staging" { db_security_group = data.terraform_remote_state.core_infrastructure.outputs.db_security_group.id internal_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.internal_application_load_balancer - network_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.network_load_balancer internal_listener = data.terraform_remote_state.core_infrastructure.outputs.internal_https_listener route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_somleng_org internal_route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_internal_somleng_org + global_accelerator = data.terraform_remote_state.core_infrastructure.outputs.global_accelerator + + logs_bucket = data.terraform_remote_state.core_infrastructure.outputs.logs_bucket switch_subdomain = "switch-staging" client_gateway_subdomain = "sip-staging"