Check Vets API Prod Deploy goes out #46
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check Vets API Prod Deploy goes out | |
| on: | |
| schedule: | |
| - cron: '0 17 * * *' # Run at 1:00 PM ET (17:00 UTC) every day | |
| jobs: | |
| check-api-status: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| status_summary: ${{ steps.check-api.outputs.status_summary }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: "us-gov-west-1" | |
| - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb | |
| with: | |
| ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN | |
| env_variable_name: VA_VSP_BOT_GITHUB_TOKEN | |
| - name: Checkout VSP infra ArgoCD repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: department-of-veterans-affairs/vsp-infra-argocd | |
| ref: refs/heads/main | |
| token: ${{ env.VA_VSP_BOT_GITHUB_TOKEN }} | |
| persist-credentials: false | |
| path: ./vsp-infra-argocd | |
| - name: Install yq | |
| run: | | |
| sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | |
| sudo chmod a+x /usr/local/bin/yq | |
| - name: Check if today is a valid day | |
| id: check-day | |
| run: | | |
| set -x # Enable debug mode | |
| today=$(date +'%Y-%m-%d') | |
| current_time=$(date +'%H:%M') | |
| day_of_week=$(date +'%u') | |
| # Parse the values.yaml file | |
| sync_windows=$(yq e '.projects[] | select(.name == "vets-api") | .sync_windows[]' ./vsp-infra-argocd/chart/values.yaml) | |
| # Debug: Print the extracted sync_windows | |
| echo "Extracted sync_windows:" | |
| echo "$sync_windows" | |
| # Check if sync_windows is empty | |
| if [ -z "$sync_windows" ]; then | |
| echo "Error: No sync windows found for vets-api project" | |
| echo "run_check=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| # Check for deny windows first | |
| deny_active=false | |
| while IFS= read -r window; do | |
| kind=$(echo "$window" | yq e '.kind' -) | |
| schedule=$(echo "$window" | yq e '.schedule' -) | |
| if [ "$kind" = "deny" ]; then | |
| month=$(echo "$schedule" | awk '{print $4}') | |
| day=$(echo "$schedule" | awk '{print $3}') | |
| if [[ "$(date +'%b' | tr '[:lower:]' '[:upper:]')" == "$month" && "$(date +'%d')" == "$day" ]]; then | |
| echo "Deny window active today" | |
| deny_active=true | |
| break | |
| fi | |
| fi | |
| done <<< "$sync_windows" | |
| # If no deny window is active, check for allow window | |
| if [ "$deny_active" = false ]; then | |
| if [[ $day_of_week -le 5 ]]; then | |
| while IFS= read -r window; do | |
| kind=$(echo "$window" | yq e '.kind' -) | |
| if [ "$kind" = "allow" ]; then | |
| schedule=$(echo "$window" | yq e '.schedule' -) | |
| duration=$(echo "$window" | yq e '.duration' -) | |
| allow_time=$(echo "$schedule" | awk '{print $2}') | |
| # Convert allow_time to minutes since midnight | |
| IFS=: read allow_hour allow_minute <<< "$allow_time" | |
| allow_minutes=$((10#$allow_hour * 60 + 10#$allow_minute)) | |
| # Convert current_time to minutes since midnight | |
| IFS=: read current_hour current_minute <<< "$current_time" | |
| current_minutes=$((10#$current_hour * 60 + 10#$current_minute)) | |
| # Convert duration to minutes | |
| duration_minutes=$(echo "$duration" | sed 's/m//') | |
| # Check if current time is within the allow window | |
| if ((current_minutes >= allow_minutes && current_minutes < allow_minutes + duration_minutes)); then | |
| echo "Weekday within allowed time window" | |
| echo "run_check=true" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| fi | |
| done <<< "$sync_windows" | |
| fi | |
| fi | |
| echo "Not within allowed window or deny window active" | |
| echo "run_check=false" >> $GITHUB_OUTPUT | |
| shell: /usr/bin/bash -e {0} | |
| - name: Check API status | |
| if: steps.check-day.outputs.run_check == 'true' | |
| id: check-api | |
| run: | | |
| initial_response=$(curl -s https://api.va.gov/v0/status) | |
| initial_revision=$(echo $initial_response | jq -r .git_revision) | |
| echo "Initial git_revision: $initial_revision" | |
| sleep 600 # 99% of deploys are done in 10 minutes | |
| final_response=$(curl -s https://api.va.gov/v0/status) | |
| final_revision=$(echo $final_response | jq -r .git_revision) | |
| echo "Final git_revision: $final_revision" | |
| if [ "$initial_revision" == "$final_revision" ]; then | |
| echo "status_summary=The git_revision at https://api.va.gov/v0/status did not change between 1:00 PM and 1:10 PM ET." >> $GITHUB_OUTPUT | |
| exit 1 # Fail the job if git_revision didn't change | |
| else | |
| echo "status_summary=The git_revision changed from $initial_revision to $final_revision." >> $GITHUB_OUTPUT | |
| fi | |
| notify-on-failure: | |
| runs-on: ubuntu-latest | |
| needs: [check-api-status] | |
| if: ${{ failure() }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: "us-gov-west-1" | |
| - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb | |
| with: | |
| ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN | |
| env_variable_name: VA_VSP_BOT_GITHUB_TOKEN | |
| - name: Checkout VSP actions | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: department-of-veterans-affairs/vsp-github-actions | |
| ref: refs/heads/main | |
| token: ${{ env.VA_VSP_BOT_GITHUB_TOKEN }} | |
| persist-credentials: false | |
| path: ./.github/actions/vsp-github-actions | |
| - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb | |
| with: | |
| ssm_parameter: /devops/github_actions_slack_socket_token | |
| env_variable_name: SLACK_APP_TOKEN | |
| - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb | |
| with: | |
| ssm_parameter: /devops/github_actions_slack_bot_user_token | |
| env_variable_name: SLACK_BOT_TOKEN | |
| - name: Slack notify | |
| uses: ./.github/actions/vsp-github-actions/slack-socket | |
| with: | |
| slack_app_token: ${{ env.SLACK_APP_TOKEN }} | |
| slack_bot_token: ${{ env.SLACK_BOT_TOKEN }} | |
| message: "Vets API Deployment Delay:" | |
| blocks: "[{\"type\": \"divider\"}, {\"type\": \"section\", \"text\": { \"type\": \"mrkdwn\", \"text\": \":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>\n\n*Status Summary:*\n${{ needs.check-api-status.outputs.status_summary }}\"}}, {\"type\": \"divider\"}]" | |
| channel_id: "C039HRTHXDH" |