From f48b2438d248a6f12d6db4870b5d754842b3e24a Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sat, 15 Jun 2024 11:59:23 +0700 Subject: [PATCH 01/11] Fix integration tests --- .github/workflows/integration_tests.yml | 2 +- components/gateway/public_gateway/docker-entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d035ba57b..58f4659c0 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -145,7 +145,7 @@ jobs: - name: Run docker compose run: | docker compose run -e DATABASE_NAME=opensips_${{ inputs.context }}_test gateway-bootstrap create_db ${{ inputs.context }} - docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing + docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing || docker logs switch-app - name: Run ${{ inputs.context }} Tests run: | diff --git a/components/gateway/public_gateway/docker-entrypoint.sh b/components/gateway/public_gateway/docker-entrypoint.sh index 55c2757b4..dbb15f4da 100755 --- a/components/gateway/public_gateway/docker-entrypoint.sh +++ b/components/gateway/public_gateway/docker-entrypoint.sh @@ -11,7 +11,7 @@ if [ "$1" = 'opensips' ]; then DATABASE_URL="${DATABASE_URL:="postgres://postgres:@localhost:5432/opensips"}" SIP_ADVERTISED_IP="${SIP_ADVERTISED_IP:="$(hostname -i)"}" LOCAL_IP="$(hostname -i)" - INTERFACE_NAME="$(ip route list | grep default | grep -E 'dev (\w+)' -o | awk '{print $2}')" + INTERFACE_NAME="$(ip route list | grep default | grep -E 'dev (\w+)' -o | awk '{print $2}')" if [ -n "$DATABASE_HOST" ]; then DATABASE_URL="postgres://$DATABASE_USERNAME:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME" From 4e46d49468baa469b06725950bde2373674a1386 Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sat, 15 Jun 2024 12:10:59 +0700 Subject: [PATCH 02/11] WIP --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 58f4659c0..bf3faae14 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -145,7 +145,7 @@ jobs: - name: Run docker compose run: | docker compose run -e DATABASE_NAME=opensips_${{ inputs.context }}_test gateway-bootstrap create_db ${{ inputs.context }} - docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing || docker logs switch-app + docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing || docker logs - name: Run ${{ inputs.context }} Tests run: | From 9d446463326ff62350ec50b706f2bbbc2e707cc9 Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sat, 15 Jun 2024 12:17:39 +0700 Subject: [PATCH 03/11] WIP --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index bf3faae14..6ea649150 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -145,7 +145,7 @@ jobs: - name: Run docker compose run: | docker compose run -e DATABASE_NAME=opensips_${{ inputs.context }}_test gateway-bootstrap create_db ${{ inputs.context }} - docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing || docker logs + docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing || docker logs somleng-switch-switch-app-1 - name: Run ${{ inputs.context }} Tests run: | From 25325a330f017b3251e0a6cce229b5b342a831b2 Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sat, 15 Jun 2024 12:27:05 +0700 Subject: [PATCH 04/11] WIP --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6ea649150..1c04c45b5 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -145,7 +145,7 @@ jobs: - name: Run docker compose run: | docker compose run -e DATABASE_NAME=opensips_${{ inputs.context }}_test gateway-bootstrap create_db ${{ inputs.context }} - docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing || docker logs somleng-switch-switch-app-1 + docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing || docker inspect somleng-switch-switch-app-1 - name: Run ${{ inputs.context }} Tests run: | From b5a0afe7bdd2b20d185a7d080e0727a5bee3ea19 Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sat, 15 Jun 2024 12:42:43 +0700 Subject: [PATCH 05/11] WIP --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index ce6f83bfb..2e35e6fb4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -150,6 +150,7 @@ services: interval: 10s timeout: 5s retries: 10 + start_period: 60s expose: - "8080/tcp" From 77dc360ba28fb85cf1dd58f3584b76d0b8a19537 Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sat, 15 Jun 2024 12:49:05 +0700 Subject: [PATCH 06/11] WIP --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 1c04c45b5..d035ba57b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -145,7 +145,7 @@ jobs: - name: Run docker compose run: | docker compose run -e DATABASE_NAME=opensips_${{ inputs.context }}_test gateway-bootstrap create_db ${{ inputs.context }} - docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing || docker inspect somleng-switch-switch-app-1 + docker compose up --wait -d ${{ inputs.context }} media_proxy freeswitch freeswitch_event_logger switch-app redis testing - name: Run ${{ inputs.context }} Tests run: | From acd6b08d7d89708d1161436b9060dc1b6b47106b Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sat, 15 Jun 2024 12:57:41 +0700 Subject: [PATCH 07/11] WIP --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2e35e6fb4..732768594 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -147,8 +147,8 @@ services: "CMD-SHELL", "wget --server-response --spider --quiet http://localhost:8080/health_checks 2>&1 | grep '200 OK' > /dev/null", ] - interval: 10s - timeout: 5s + interval: 30s + timeout: 10s retries: 10 start_period: 60s expose: From 96f0b0b5bbd8bd24b5601483877eacc2d67d31bc Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sat, 15 Jun 2024 21:45:10 +0700 Subject: [PATCH 08/11] Move NLB infra --- .../modules/somleng_switch/public_gateway.tf | 128 ++++++++++++++++-- .../modules/somleng_switch/variables.tf | 5 +- infrastructure/production/main.tf | 5 +- infrastructure/staging/main.tf | 5 +- 4 files changed, 129 insertions(+), 14 deletions(-) diff --git a/infrastructure/modules/somleng_switch/public_gateway.tf b/infrastructure/modules/somleng_switch/public_gateway.tf index e95121114..b890810b6 100644 --- a/infrastructure/modules/somleng_switch/public_gateway.tf +++ b/infrastructure/modules/somleng_switch/public_gateway.tf @@ -69,6 +69,33 @@ resource "aws_security_group_rule" "public_gateway_egress" { cidr_blocks = ["0.0.0.0/0"] } +# Global Accelerator + +resource "aws_globalaccelerator_listener" "public_gateway" { + accelerator_arn = var.global_accelerator.id + protocol = "UDP" + + port_range { + from_port = var.sip_port + to_port = var.sip_port + } + + port_range { + from_port = var.sip_alternative_port + to_port = var.sip_alternative_port + } +} + +resource "aws_globalaccelerator_endpoint_group" "public_gateway" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 + listener_arn = aws_globalaccelerator_listener.public_gateway.id + + endpoint_configuration { + endpoint_id = aws_lb.public_gateway_nlb[count.index].arn + client_ip_preservation_enabled = true + } +} + # IAM resource "aws_iam_role" "public_gateway_task_role" { name = "${var.public_gateway_identifier}-ecsTaskRole" @@ -225,7 +252,7 @@ resource "aws_ecs_task_definition" "public_gateway" { }, { name = "SIP_ADVERTISED_IP", - value = tostring(var.external_sip_ip) + value = tostring(var.global_accelerator.ip_sets[0].ip_addresses[0]) } ] }, @@ -252,7 +279,7 @@ resource "aws_ecs_task_definition" "public_gateway" { } ]) - memory = module.public_gateway_container_instances.ec2_instance_type.memory_size - 512 + memory = max((module.public_gateway_container_instances.ec2_instance_type.memory_size - 512), 128) volume { name = "opensips" @@ -260,6 +287,7 @@ resource "aws_ecs_task_definition" "public_gateway" { } resource "aws_ecs_service" "public_gateway" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 name = aws_ecs_task_definition.public_gateway.family cluster = aws_ecs_cluster.cluster.id task_definition = aws_ecs_task_definition.public_gateway.arn @@ -300,6 +328,86 @@ resource "aws_ecs_service" "public_gateway" { } # Load Balancer + +resource "aws_security_group" "public_gateway_nlb" { + name = "${var.public_gateway_identifier}-nlb" + vpc_id = var.vpc.vpc_id +} + +resource "aws_security_group_rule" "public_gateway_nlb_sip_ingress" { + type = "ingress" + from_port = var.sip_port + to_port = var.sip_port + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.public_gateway_nlb.id +} + +resource "aws_security_group_rule" "public_gateway_nlb_sip_alternative_ingress" { + type = "ingress" + from_port = var.sip_alternative_port + to_port = var.sip_alternative_port + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.public_gateway_nlb.id +} + +resource "aws_security_group_rule" "public_gateway_nlb_udp_egress" { + type = "egress" + from_port = 0 + to_port = 65535 + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.public_gateway_nlb.id +} + +resource "aws_security_group_rule" "public_gateway_nlb_tcp_egress" { + type = "egress" + from_port = 0 + to_port = 65535 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.public_gateway_nlb.id +} + +resource "aws_eip" "public_gateway_nlb" { + count = var.public_gateway_min_tasks > 0 ? length(var.vpc.public_subnets) : 0 + domain = "vpc" + + tags = { + Name = "Public Gateway NLB IP" + } +} + +resource "aws_lb" "public_gateway_nlb" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 + name = var.public_gateway_identifier + load_balancer_type = "network" + enable_cross_zone_load_balancing = true + + security_groups = [aws_security_group.public_gateway_nlb.id] + + access_logs { + bucket = var.logs_bucket.id + prefix = var.public_gateway_identifier + enabled = true + } + + dynamic "subnet_mapping" { + for_each = var.vpc.public_subnets + content { + subnet_id = subnet_mapping.value + allocation_id = aws_eip.public_gateway_nlb.*.id[subnet_mapping.key] + } + } +} + +# Target Groups + resource "aws_lb_target_group" "sip" { name = "${var.public_gateway_identifier}-sip" port = var.sip_port @@ -318,7 +426,8 @@ resource "aws_lb_target_group" "sip" { } resource "aws_lb_listener" "sip" { - load_balancer_arn = var.network_load_balancer.arn + count = var.public_gateway_min_tasks > 0 ? 1 : 0 + load_balancer_arn = aws_lb.public_gateway_nlb[count.index].arn port = var.sip_port protocol = "UDP" @@ -346,7 +455,8 @@ resource "aws_lb_target_group" "sip_alternative" { } resource "aws_lb_listener" "sip_alternative" { - load_balancer_arn = var.network_load_balancer.arn + count = var.public_gateway_min_tasks > 0 ? 1 : 0 + load_balancer_arn = aws_lb.public_gateway_nlb[count.index].arn port = var.sip_alternative_port protocol = "UDP" @@ -358,10 +468,11 @@ resource "aws_lb_listener" "sip_alternative" { # Autoscaling resource "aws_appautoscaling_policy" "public_gateway_policy" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 name = var.public_gateway_identifier - service_namespace = aws_appautoscaling_target.public_gateway_scale_target.service_namespace - resource_id = aws_appautoscaling_target.public_gateway_scale_target.resource_id - scalable_dimension = aws_appautoscaling_target.public_gateway_scale_target.scalable_dimension + service_namespace = aws_appautoscaling_target.public_gateway_scale_target[count.index].service_namespace + resource_id = aws_appautoscaling_target.public_gateway_scale_target[count.index].resource_id + scalable_dimension = aws_appautoscaling_target.public_gateway_scale_target[count.index].scalable_dimension policy_type = "TargetTrackingScaling" target_tracking_scaling_policy_configuration { @@ -376,8 +487,9 @@ resource "aws_appautoscaling_policy" "public_gateway_policy" { } resource "aws_appautoscaling_target" "public_gateway_scale_target" { + count = var.public_gateway_min_tasks > 0 ? 1 : 0 service_namespace = "ecs" - resource_id = "service/${aws_ecs_cluster.cluster.name}/${aws_ecs_service.public_gateway.name}" + resource_id = "service/${aws_ecs_cluster.cluster.name}/${aws_ecs_service.public_gateway[count.index].name}" scalable_dimension = "ecs:service:DesiredCount" min_capacity = var.public_gateway_min_tasks max_capacity = var.public_gateway_max_tasks diff --git a/infrastructure/modules/somleng_switch/variables.tf b/infrastructure/modules/somleng_switch/variables.tf index 2afcdffa7..1d11601e9 100644 --- a/infrastructure/modules/somleng_switch/variables.tf +++ b/infrastructure/modules/somleng_switch/variables.tf @@ -20,14 +20,16 @@ variable "freeswitch_event_logger_image" {} variable "s3_mpeg_ecr_repository_url" {} variable "services_ecr_repository_url" {} variable "internal_load_balancer" {} -variable "network_load_balancer" {} variable "internal_listener" {} variable "switch_subdomain" {} variable "client_gateway_subdomain" {} variable "route53_zone" {} variable "internal_route53_zone" {} variable "recordings_bucket_name" {} +variable "logs_bucket" {} variable "efs_cache_name" {} +variable "global_accelerator" {} + variable "container_insights_enabled" { default = false } @@ -135,7 +137,6 @@ variable "db_username" {} variable "db_password_parameter_arn" {} variable "json_cdr_password_parameter_arn" {} -variable "external_sip_ip" {} variable "external_rtp_ip" {} variable "alternative_sip_outbound_ip" {} variable "alternative_rtp_ip" {} diff --git a/infrastructure/production/main.tf b/infrastructure/production/main.tf index e549d6aae..261c95017 100644 --- a/infrastructure/production/main.tf +++ b/infrastructure/production/main.tf @@ -31,7 +31,6 @@ module "somleng_switch" { json_cdr_password_parameter_arn = data.aws_ssm_parameter.somleng_services_password.arn json_cdr_url = "https://api.internal.somleng.org/services/call_data_records" - external_sip_ip = data.terraform_remote_state.core_infrastructure.outputs.nlb_eips[0].public_ip external_rtp_ip = data.terraform_remote_state.core_infrastructure.outputs.vpc.nat_public_ips[0] alternative_sip_outbound_ip = data.terraform_remote_state.core_infrastructure.outputs.nat_instance_ip @@ -47,8 +46,10 @@ module "somleng_switch" { db_security_group = data.terraform_remote_state.core_infrastructure.outputs.db_security_group.id internal_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.internal_application_load_balancer - network_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.network_load_balancer internal_listener = data.terraform_remote_state.core_infrastructure.outputs.internal_https_listener + global_accelerator = data.terraform_remote_state.core_infrastructure.outputs.global_accelerator + + logs_bucket = data.terraform_remote_state.core_infrastructure.outputs.logs_bucket route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_somleng_org internal_route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_internal_somleng_org diff --git a/infrastructure/staging/main.tf b/infrastructure/staging/main.tf index ddc36e848..34075e347 100644 --- a/infrastructure/staging/main.tf +++ b/infrastructure/staging/main.tf @@ -32,7 +32,6 @@ module "somleng_switch_staging" { json_cdr_password_parameter_arn = data.aws_ssm_parameter.somleng_services_password.arn json_cdr_url = "https://api-staging.internal.somleng.org/services/call_data_records" - external_sip_ip = data.terraform_remote_state.core_infrastructure.outputs.nlb_eips[0].public_ip external_rtp_ip = data.terraform_remote_state.core_infrastructure.outputs.vpc.nat_public_ips[0] alternative_sip_outbound_ip = data.terraform_remote_state.core_infrastructure.outputs.nat_instance_ip @@ -48,11 +47,13 @@ module "somleng_switch_staging" { db_security_group = data.terraform_remote_state.core_infrastructure.outputs.db_security_group.id internal_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.internal_application_load_balancer - network_load_balancer = data.terraform_remote_state.core_infrastructure.outputs.network_load_balancer internal_listener = data.terraform_remote_state.core_infrastructure.outputs.internal_https_listener route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_somleng_org internal_route53_zone = data.terraform_remote_state.core_infrastructure.outputs.route53_zone_internal_somleng_org + global_accelerator = data.terraform_remote_state.core_infrastructure.outputs.global_accelerator + + logs_bucket = data.terraform_remote_state.core_infrastructure.outputs.logs_bucket switch_subdomain = "switch-staging" client_gateway_subdomain = "sip-staging" From 0b0f61a7106172cc81f04a9bfc64d6cc4f4c223e Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sun, 16 Jun 2024 08:23:15 +0700 Subject: [PATCH 09/11] WIP --- .github/workflows/integration_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d035ba57b..14151a878 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -142,6 +142,9 @@ jobs: cache-from: type=gha,scope=integration-tests-testing cache-to: type=gha,mode=max,scope=integration-tests-testing + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Run docker compose run: | docker compose run -e DATABASE_NAME=opensips_${{ inputs.context }}_test gateway-bootstrap create_db ${{ inputs.context }} From 88fb96a849a2f7c3e2e08bc400d02e789eda00b1 Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sun, 16 Jun 2024 08:38:11 +0700 Subject: [PATCH 10/11] WIP --- .github/workflows/integration_tests.yml | 3 --- docker-compose.yml | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 14151a878..d035ba57b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -142,9 +142,6 @@ jobs: cache-from: type=gha,scope=integration-tests-testing cache-to: type=gha,mode=max,scope=integration-tests-testing - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Run docker compose run: | docker compose run -e DATABASE_NAME=opensips_${{ inputs.context }}_test gateway-bootstrap create_db ${{ inputs.context }} diff --git a/docker-compose.yml b/docker-compose.yml index 732768594..b2c4b2f21 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,6 @@ # Postgres is not included in this docker compose configuration # because of the complexity of adding code to wait for postgres to start. -version: "3.9" volumes: workspace: opensips_fifo: @@ -145,12 +144,11 @@ services: test: [ "CMD-SHELL", - "wget --server-response --spider --quiet http://localhost:8080/health_checks 2>&1 | grep '200 OK' > /dev/null", + "wget --server-response --spider --quiet http://0.0.0.0:8080/health_checks 2>&1 | grep '200 OK' > /dev/null", ] interval: 30s timeout: 10s retries: 10 - start_period: 60s expose: - "8080/tcp" From b540d8d2aa6defaff76f111e27a0681faf423878 Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Sun, 16 Jun 2024 08:51:25 +0700 Subject: [PATCH 11/11] WIP --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b2c4b2f21..faa65ff45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -146,8 +146,8 @@ services: "CMD-SHELL", "wget --server-response --spider --quiet http://0.0.0.0:8080/health_checks 2>&1 | grep '200 OK' > /dev/null", ] - interval: 30s - timeout: 10s + interval: 10s + timeout: 5s retries: 10 expose: - "8080/tcp"