Handle multi-region tasks in services

somleng · Aug 29, 2024 · b8e7ce2 · b8e7ce2
1 parent dbd593a
commit b8e7ce2
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 22 deletions.
diff --git a/.github/workflows/services.yml b/.github/workflows/services.yml
@@ -61,7 +61,7 @@ jobs:
             {
               "identifier": "switch-services-staging",
               "environment": "staging",
-              "branch": "develop",
+              "branch": "modularize_infrastructure",
               "image_tag": "staging"
             },
             {
@@ -74,8 +74,8 @@ jobs:
           EOF
           )
           matrix=$(echo $matrixSource | jq --arg branchName "$branchName" 'map(. | select((.branch==$branchName)) )')
-          echo ::set-output name=matrix::{\"include\":$(echo $matrix)}\"
-          echo ::set-output name=matrixLength::$(echo $matrix | jq length)
+          echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
+          echo "matrixLength=$(echo $matrix | jq length)" >> $GITHUB_OUTPUT
 
   deploy:
     name: Deploy
@@ -102,15 +102,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Create Sentry release
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: somleng
-          SENTRY_PROJECT: somleng-switch-services
-        with:
-          environment: ${{ matrix.environment }}
-
       - name: Configure AWS credentials
         id: aws-login
         uses: aws-actions/configure-aws-credentials@v4
@@ -160,3 +151,12 @@ jobs:
                                           --image-uri ${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} \
                                           --architectures "arm64" \
                                           --publish
+
+      - name: Create Sentry release
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: somleng
+          SENTRY_PROJECT: somleng-switch-services
+        with:
+          environment: ${{ matrix.environment }}
diff --git a/components/services/.rubocop.yml b/components/services/.rubocop.yml
@@ -0,0 +1 @@
+inherit_from: ../../.rubocop.yml
diff --git a/components/services/app/parsers/ecs_event_parser.rb b/components/services/app/parsers/ecs_event_parser.rb
@@ -73,6 +73,7 @@ def eni_private_ip_details
 
   def private_ip
     return eni_private_ip unless eni_private_ip.nil?
+
     ec2_instance_private_ip unless container_instance_arn.nil?
   end
 
@@ -104,23 +105,31 @@ def cluster_arn
     detail.fetch("clusterArn")
   end
 
+  def region
+    event.fetch("region")
+  end
+
   def container_instance_details
-    @container_instance_details ||= ecs_client.describe_container_instances(
-      cluster: cluster_arn,
-      container_instances: [
-        container_instance_arn
-      ]
-    ).to_h
+    @container_instance_details ||= with_aws_client(ecs_client, region:) do |client|
+      client.describe_container_instances(
+        cluster: cluster_arn,
+        container_instances: [
+          container_instance_arn
+        ]
+      ).to_h
+    end
   end
 
   def ec2_instance_id
     container_instance_details.dig(:container_instances, 0, :ec2_instance_id)
   end
 
   def ec2_instance_details
-    @ec2_instance_details ||= ec2_client.describe_instances(
-      instance_ids: [ ec2_instance_id ]
-    ).to_h
+    @ec2_instance_details ||= with_aws_client(ec2_client, region:) do |client|
+      client.describe_instances(
+        instance_ids: [ ec2_instance_id ]
+      ).to_h
+    end
   end
 
   def ec2_instance_private_ip
@@ -130,4 +139,9 @@ def ec2_instance_private_ip
   def ec2_instance_public_ip
     ec2_instance_details.dig(:reservations, 0, :instances, 0, :public_ip_address)
   end
+
+  def with_aws_client(client, region:)
+    client.config.region = region
+    yield(client)
+  end
 end
diff --git a/components/services/spec/parsers/ecs_event_parser_spec.rb b/components/services/spec/parsers/ecs_event_parser_spec.rb
@@ -111,6 +111,17 @@
     )
   end
 
+  it "requests to the correct regional endpoint" do
+    ecs_client, ec2_client = stub_aws_clients(region: "ap-southeast-1")
+    event = build_ecs_event_payload(region: "us-east-1")
+    parser = ECSEventParser.new(event, ecs_client:, ec2_client:)
+
+    parser.parse_event
+
+    expect(ecs_client.api_requests.first.fetch(:context).client.config.region).to eq("us-east-1")
+    expect(ec2_client.api_requests.first.fetch(:context).client.config.region).to eq("us-east-1")
+  end
+
   it "handles public instances" do
     ecs_client, ec2_client = stub_aws_clients(private_ip_address: "10.0.0.1", public_ip_address: "54.251.92.249")
     event = build_ecs_event_payload
@@ -126,6 +137,7 @@
 
   def stub_aws_clients(options = {})
     ecs_client = Aws::ECS::Client.new(
+      region: options.fetch(:region, "ap-southeast-1"),
       stub_responses: {
         describe_container_instances: {
           container_instances: [
@@ -138,6 +150,7 @@ def stub_aws_clients(options = {})
     )
 
     ec2_client = Aws::EC2::Client.new(
+      region: options.fetch(:region, "ap-southeast-1"),
       stub_responses: {
         describe_instances: {
           reservations: [

diff --git a/components/services/spec/support/event_helpers.rb b/components/services/spec/support/event_helpers.rb
@@ -25,6 +25,7 @@ def build_sqs_message_event_payload(data = {})
 
   def build_ecs_event_payload(data = {})
     data = {
+      region: "us-west-2",
       eni_private_ip: "10.0.0.1",
       eni_status: "ATTACHED",
       last_status: "RUNNING",
@@ -51,6 +52,7 @@ def build_ecs_event_payload(data = {})
     payload = JSON.parse(file_fixture("task_state_change_event.json").read)
 
     overrides = {
+      "region" => data.fetch(:region),
       "detail" => {
         "attachments" => data.fetch(:attachments),
         "lastStatus" => data.fetch(:last_status),

diff --git a/infrastructure/modules/switch/ecs.tf b/infrastructure/modules/switch/ecs.tf
@@ -364,6 +364,11 @@ resource "aws_ecs_service" "this" {
     container_port   = var.webserver_port
   }
 
+  deployment_circuit_breaker {
+    enable   = true
+    rollback = true
+  }
+
   lifecycle {
     ignore_changes = [task_definition, desired_count]
   }

diff --git a/infrastructure/staging/switch.tf b/infrastructure/staging/switch.tf
@@ -11,7 +11,7 @@ module "switch" {
   freeswitch_event_socket_password_parameter_name    = "somleng-switch.${var.app_environment}.freeswitch_event_socket_password"
   recordings_bucket_access_key_id_parameter_name     = "somleng-switch.${var.app_environment}.recordings_bucket_access_key_id"
   recordings_bucket_secret_access_key_parameter_name = "somleng-switch.${var.app_environment}.recordings_bucket_secret_access_key"
-  min_tasks                                          = 0
+  min_tasks                                          = 1
   max_tasks                                          = 2
   lb_rule_index                                      = 120
   identifier                                         = var.switch_identifier