Add increased alerting for when acceptance tests fail

This posts to the #mission-control channel when the acceptance tests fail, these are infrequent so I don't expect things to spam up the channel.
thoughtbot · Apr 11, 2024 · 2f73cb5 · 2f73cb5
1 parent b85c710
commit 2f73cb5
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 2 deletions.
diff --git a/.github/workflows/acceptance.yaml b/.github/workflows/acceptance.yaml
@@ -18,6 +18,7 @@ jobs:
       module: network/sandbox
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   apply_cluster:
     name: Apply Cluster
@@ -29,6 +30,7 @@ jobs:
       module: cluster/sandbox-v1
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   apply_ingress:
     name: Apply Ingress
@@ -40,6 +42,7 @@ jobs:
       module: ingress/sandbox
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   apply_platform:
     name: Apply Platform
@@ -51,6 +54,7 @@ jobs:
       module: platform/sandbox-v1
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   tests:
     name: Run Tests
@@ -60,6 +64,7 @@ jobs:
     concurrency: ${{ github.ref_name }}-platform
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   cleanup_platform:
     name: Cleanup Platform
@@ -72,6 +77,7 @@ jobs:
       terraform_command: destroy
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   cleanup_ingress:
     name: Cleanup Ingress
@@ -84,6 +90,7 @@ jobs:
       terraform_command: destroy
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   cleanup_cluster:
     name: Cleanup Cluster
@@ -96,6 +103,7 @@ jobs:
       terraform_command: destroy
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   cleanup_network:
     name: Cleanup Network
@@ -108,3 +116,4 @@ jobs:
       terraform_command: destroy
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/.github/workflows/terraform-dispatch.yaml b/.github/workflows/terraform-dispatch.yaml
@@ -36,4 +36,4 @@ jobs:
       terraform_version: ${{ github.event.inputs.terraform_version }}
     secrets:
       AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
-
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml
@@ -15,6 +15,12 @@ on:
       AWS_ACCOUNT_ID:
         description: AWS account in which Terraform will run
         required: true
+      SLACK_BOT_TOKEN:
+        description: Token used to send messages to Slack
+        required: true
+
+env:
+  SLACK_CHANNEL: '#mission-control'
 
 jobs:
   plan:
@@ -26,7 +32,6 @@ jobs:
         working-directory: ${{ inputs.module }}
 
     steps:
-
     - name: Checkout
       uses: actions/checkout@v4
       with:
@@ -92,3 +97,11 @@ jobs:
       run: |
         terraform ${{ inputs.terraform_command }} -auto-approve
 
+    - name: Notify Slack
+      if: failure()
+      uses: slackapi/slack-github-action@007b2c3c751a190b6f0f040e47ed024deaa72844 # v1.23.0
+      with:
+        channel-id: ${{ env.SLACK_CHANNEL }}
+        slack-message: "Flightdeck acceptance terraform has failed to ${{ inputs.terraform_command }} on the ${{ inputs.module }} module."
+      env:
+        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -5,9 +5,13 @@ on:
       AWS_ACCOUNT_ID:
         description: AWS account in which tests will run
         required: true
+      SLACK_BOT_TOKEN:
+        description: Token used to send messages to Slack
+        required: true
 
 env:
   KUBECONFIG: /tmp/kubeconfig
+  SLACK_CHANNEL: '#mission-control'
 
 jobs:
   plan:
@@ -68,3 +72,12 @@ jobs:
         make tests \
           ADDRESS=https://${{ github.ref_name }}.flightdeck-test.thoughtbot.com \
           CLUSTER="$CLUSTER"
+
+    - name: Notify Slack
+      if: failure()
+      uses: slackapi/slack-github-action@007b2c3c751a190b6f0f040e47ed024deaa72844 # v1.23.0
+      with:
+        channel-id: ${{ env.SLACK_CHANNEL }}
+        slack-message: "Flightdeck acceptance tests have failed on the testing cluster."
+      env:
+        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}