From b8e7ce2ec61298ceb992ad53973a3e0695fc773e Mon Sep 17 00:00:00 2001 From: David Wilkie Date: Thu, 29 Aug 2024 23:57:41 +0700 Subject: [PATCH] Handle multi-region tasks in services --- .github/workflows/services.yml | 24 +++++++------- components/services/.rubocop.yml | 1 + .../services/app/parsers/ecs_event_parser.rb | 32 +++++++++++++------ .../spec/parsers/ecs_event_parser_spec.rb | 13 ++++++++ .../services/spec/support/event_helpers.rb | 2 ++ infrastructure/modules/switch/ecs.tf | 5 +++ infrastructure/staging/switch.tf | 2 +- 7 files changed, 57 insertions(+), 22 deletions(-) create mode 100644 components/services/.rubocop.yml diff --git a/.github/workflows/services.yml b/.github/workflows/services.yml index 149c5da44..9951c1d31 100644 --- a/.github/workflows/services.yml +++ b/.github/workflows/services.yml @@ -61,7 +61,7 @@ jobs: { "identifier": "switch-services-staging", "environment": "staging", - "branch": "develop", + "branch": "modularize_infrastructure", "image_tag": "staging" }, { @@ -74,8 +74,8 @@ jobs: EOF ) matrix=$(echo $matrixSource | jq --arg branchName "$branchName" 'map(. | select((.branch==$branchName)) )') - echo ::set-output name=matrix::{\"include\":$(echo $matrix)}\" - echo ::set-output name=matrixLength::$(echo $matrix | jq length) + echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT + echo "matrixLength=$(echo $matrix | jq length)" >> $GITHUB_OUTPUT deploy: name: Deploy @@ -102,15 +102,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Create Sentry release - uses: getsentry/action-release@v1 - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_ORG: somleng - SENTRY_PROJECT: somleng-switch-services - with: - environment: ${{ matrix.environment }} - - name: Configure AWS credentials id: aws-login uses: aws-actions/configure-aws-credentials@v4 @@ -160,3 +151,12 @@ jobs: --image-uri ${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} \ --architectures "arm64" \ --publish + + - name: Create Sentry release + uses: getsentry/action-release@v1 + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: somleng + SENTRY_PROJECT: somleng-switch-services + with: + environment: ${{ matrix.environment }} diff --git a/components/services/.rubocop.yml b/components/services/.rubocop.yml new file mode 100644 index 000000000..1248a2f82 --- /dev/null +++ b/components/services/.rubocop.yml @@ -0,0 +1 @@ +inherit_from: ../../.rubocop.yml diff --git a/components/services/app/parsers/ecs_event_parser.rb b/components/services/app/parsers/ecs_event_parser.rb index 53135da32..3d96299cc 100644 --- a/components/services/app/parsers/ecs_event_parser.rb +++ b/components/services/app/parsers/ecs_event_parser.rb @@ -73,6 +73,7 @@ def eni_private_ip_details def private_ip return eni_private_ip unless eni_private_ip.nil? + ec2_instance_private_ip unless container_instance_arn.nil? end @@ -104,13 +105,19 @@ def cluster_arn detail.fetch("clusterArn") end + def region + event.fetch("region") + end + def container_instance_details - @container_instance_details ||= ecs_client.describe_container_instances( - cluster: cluster_arn, - container_instances: [ - container_instance_arn - ] - ).to_h + @container_instance_details ||= with_aws_client(ecs_client, region:) do |client| + client.describe_container_instances( + cluster: cluster_arn, + container_instances: [ + container_instance_arn + ] + ).to_h + end end def ec2_instance_id @@ -118,9 +125,11 @@ def ec2_instance_id end def ec2_instance_details - @ec2_instance_details ||= ec2_client.describe_instances( - instance_ids: [ ec2_instance_id ] - ).to_h + @ec2_instance_details ||= with_aws_client(ec2_client, region:) do |client| + client.describe_instances( + instance_ids: [ ec2_instance_id ] + ).to_h + end end def ec2_instance_private_ip @@ -130,4 +139,9 @@ def ec2_instance_private_ip def ec2_instance_public_ip ec2_instance_details.dig(:reservations, 0, :instances, 0, :public_ip_address) end + + def with_aws_client(client, region:) + client.config.region = region + yield(client) + end end diff --git a/components/services/spec/parsers/ecs_event_parser_spec.rb b/components/services/spec/parsers/ecs_event_parser_spec.rb index d19dbde20..6854fd4dc 100644 --- a/components/services/spec/parsers/ecs_event_parser_spec.rb +++ b/components/services/spec/parsers/ecs_event_parser_spec.rb @@ -111,6 +111,17 @@ ) end + it "requests to the correct regional endpoint" do + ecs_client, ec2_client = stub_aws_clients(region: "ap-southeast-1") + event = build_ecs_event_payload(region: "us-east-1") + parser = ECSEventParser.new(event, ecs_client:, ec2_client:) + + parser.parse_event + + expect(ecs_client.api_requests.first.fetch(:context).client.config.region).to eq("us-east-1") + expect(ec2_client.api_requests.first.fetch(:context).client.config.region).to eq("us-east-1") + end + it "handles public instances" do ecs_client, ec2_client = stub_aws_clients(private_ip_address: "10.0.0.1", public_ip_address: "54.251.92.249") event = build_ecs_event_payload @@ -126,6 +137,7 @@ def stub_aws_clients(options = {}) ecs_client = Aws::ECS::Client.new( + region: options.fetch(:region, "ap-southeast-1"), stub_responses: { describe_container_instances: { container_instances: [ @@ -138,6 +150,7 @@ def stub_aws_clients(options = {}) ) ec2_client = Aws::EC2::Client.new( + region: options.fetch(:region, "ap-southeast-1"), stub_responses: { describe_instances: { reservations: [ diff --git a/components/services/spec/support/event_helpers.rb b/components/services/spec/support/event_helpers.rb index 85ad70231..07ffb1088 100644 --- a/components/services/spec/support/event_helpers.rb +++ b/components/services/spec/support/event_helpers.rb @@ -25,6 +25,7 @@ def build_sqs_message_event_payload(data = {}) def build_ecs_event_payload(data = {}) data = { + region: "us-west-2", eni_private_ip: "10.0.0.1", eni_status: "ATTACHED", last_status: "RUNNING", @@ -51,6 +52,7 @@ def build_ecs_event_payload(data = {}) payload = JSON.parse(file_fixture("task_state_change_event.json").read) overrides = { + "region" => data.fetch(:region), "detail" => { "attachments" => data.fetch(:attachments), "lastStatus" => data.fetch(:last_status), diff --git a/infrastructure/modules/switch/ecs.tf b/infrastructure/modules/switch/ecs.tf index 9c65487f5..bc41b2ce4 100644 --- a/infrastructure/modules/switch/ecs.tf +++ b/infrastructure/modules/switch/ecs.tf @@ -364,6 +364,11 @@ resource "aws_ecs_service" "this" { container_port = var.webserver_port } + deployment_circuit_breaker { + enable = true + rollback = true + } + lifecycle { ignore_changes = [task_definition, desired_count] } diff --git a/infrastructure/staging/switch.tf b/infrastructure/staging/switch.tf index bf751207c..4a3543422 100644 --- a/infrastructure/staging/switch.tf +++ b/infrastructure/staging/switch.tf @@ -11,7 +11,7 @@ module "switch" { freeswitch_event_socket_password_parameter_name = "somleng-switch.${var.app_environment}.freeswitch_event_socket_password" recordings_bucket_access_key_id_parameter_name = "somleng-switch.${var.app_environment}.recordings_bucket_access_key_id" recordings_bucket_secret_access_key_parameter_name = "somleng-switch.${var.app_environment}.recordings_bucket_secret_access_key" - min_tasks = 0 + min_tasks = 1 max_tasks = 2 lb_rule_index = 120 identifier = var.switch_identifier