-
Notifications
You must be signed in to change notification settings - Fork 63
174 lines (156 loc) · 8.94 KB
/
deploy_delay_notifications.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
name: Deploy Delay Notifications
on:
schedule:
- cron: "*/10 * * * *" # Runs every 10 minutes
jobs:
check-deployment:
runs-on: ubuntu-latest
outputs:
dev_summary: ${{ steps.check-dev-status.outputs.dev_summary }}
staging_summary: ${{ steps.check-staging-status.outputs.staging_summary }}
steps:
- name: Get latest commit SHA and time from master branch
id: git-info
run: |
latest_commit_info=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
"https://api.github.com/repos/${{ github.repository }}/commits/master")
latest_sha=$(echo "${latest_commit_info}" | jq -r '.sha')
echo "latest_sha=${latest_sha}" >> $GITHUB_ENV
echo "latest_sha: ${latest_sha}"
commit_time=$(echo "${latest_commit_info}" | jq -r '.commit.committer.date')
echo "commit_time=${commit_time}" >> $GITHUB_ENV
echo "commit_time: ${commit_time}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Get deployed SHA for development
id: dev-deploy-sha
run: |
deployed_sha=$(curl -s https://dev-api.va.gov/v0/status | jq -r .git_revision)
echo "dev_deployed_sha=${deployed_sha}" >> $GITHUB_ENV
- name: Get deployed SHA for staging
id: staging-deploy-sha
run: |
deployed_sha=$(curl -s https://staging-api.va.gov/v0/status | jq -r .git_revision)
echo "staging_deployed_sha=${deployed_sha}" >> $GITHUB_ENV
- name: Check deployment status for development
if: ${{ env.latest_sha != '' && env.dev_deployed_sha != '' }}
id: check-dev-status
run: |
latest_sha=${{ env.latest_sha }}
commit_time=${{ env.commit_time }}
deployed_sha=${{ env.dev_deployed_sha }}
info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to development"
action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-dev|ArgoCD dev>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>"
if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then
echo "${info_message} has been deployed."
echo "dev_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '45 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 45 minutes. Skipping notification."
echo "dev_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '30 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 30 minutes."
echo "Current commit on development is ${deployed_sha:0:8}."
echo "dev_summary=${info_message} has been delayed for more than 30 minutes. Current commit on development is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT
exit 1
else
echo "Awaiting deployment of ${info_message}."
echo "dev_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT
fi
- name: Check deployment status for staging
if: ${{ always() && env.latest_sha != '' && env.staging_deployed_sha != '' }}
id: check-staging-status
run: |
latest_sha=${{ env.latest_sha }}
commit_time=${{ env.commit_time }}
deployed_sha=${{ env.staging_deployed_sha }}
info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to staging"
action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-staging|ArgoCD staging>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>"
if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then
echo "${info_message} has been deployed."
echo "staging_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '45 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 45 minutes. Skipping notification."
echo "staging_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT
elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '30 minutes ago' +%s)" ]; then
echo "${info_message} has been delayed for more than 30 minutes."
echo "Current commit on staging is ${deployed_sha:0:8}."
echo "staging_summary=${info_message} has been delayed for more than 30 minutes. Current commit on staging is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT
exit 1
else
echo "Awaiting deployment of ${info_message}."
echo "staging_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT
fi
notify-on-failure:
runs-on: ubuntu-latest
needs: [check-deployment]
env:
dev_summary: ${{ needs.check-deployment.outputs.dev_summary }}
staging_summary: ${{ needs.check-deployment.outputs.staging_summary }}
if: ${{ failure() }}
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: "us-gov-west-1"
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN
env_variable_name: VA_VSP_BOT_GITHUB_TOKEN
- name: Checkout VSP actions
uses: actions/checkout@v4
with:
repository: department-of-veterans-affairs/vsp-github-actions
ref: refs/heads/main
token: ${{ env.VA_VSP_BOT_GITHUB_TOKEN }}
persist-credentials: false
path: ./.github/actions/vsp-github-actions
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/github_actions_slack_socket_token
env_variable_name: SLACK_APP_TOKEN
- uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb
with:
ssm_parameter: /devops/github_actions_slack_bot_user_token
env_variable_name: SLACK_BOT_TOKEN
- name: Notify for deployment failure
if: ${{ env.dev_summary != '' || env.staging_summary != '' }}
uses: ./.github/actions/vsp-github-actions/slack-socket
with:
slack_app_token: ${{ env.SLACK_APP_TOKEN }}
slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
message: "Vets API Deployment Delay:"
blocks: |
[
{ "type": "divider" },
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>\n\n *Development Summary:*\n${{ env.dev_summary }}\n\n *Staging Summary:*\n${{ env.staging_summary }}"
}
},
{ "type": "divider" }
]
channel_id: "C039HRTHXDH"
- name: Notify for other failure
if: ${{ env.dev_summary == '' && env.staging_summary == '' }}
uses: ./.github/actions/vsp-github-actions/slack-socket
with:
slack_app_token: ${{ env.SLACK_APP_TOKEN }}
slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
message: "Vets API Deployment Delay:"
blocks: |
[
{ "type": "divider" },
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n\n Unknown error occured. See logs:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>"
}
},
{ "type": "divider" }
]
channel_id: "C039HRTHXDH"