Read-only #211
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Read-only | |
| # Prepare for Delius down-time by entering "read-only" mode. | |
| # Switches off message consumers, blocks any write APIs, and re-points everything else at the snapshot standby database. | |
| # Note: In the test environment, where there is no snapshot standby database, the services are stopped completely. | |
| on: | |
| workflow_call: | |
| inputs: | |
| environment: | |
| description: Environment | |
| required: true | |
| type: string | |
| action: | |
| description: Enable or disable read-only mode? | |
| required: true | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: Environment | |
| default: prod | |
| required: true | |
| type: choice | |
| options: | |
| - test | |
| - preprod | |
| - prod | |
| action: | |
| description: Enable or disable read-only mode? | |
| default: enable | |
| required: true | |
| type: choice | |
| options: | |
| - enable | |
| - disable | |
| jobs: | |
| unblock-pipeline: | |
| name: Unblock deployment pipeline | |
| runs-on: ubuntu-latest | |
| steps: | |
| - if: inputs.environment != 'prod' && inputs.action == 'disable' | |
| uses: actions/create-github-app-token@v1 | |
| id: app-token | |
| with: | |
| app-id: ${{ secrets.BOT_APP_ID }} | |
| private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }} | |
| - if: inputs.environment != 'prod' && inputs.action == 'disable' | |
| run: jq -n "$reviewers" | gh api -XPUT '/repos/ministryofjustice/hmpps-probation-integration-services/environments/${{ inputs.environment }}' --input - | |
| env: | |
| reviewers: '{"reviewers":[]}' | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| get-projects: | |
| name: Get projects | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| needs: unblock-pipeline | |
| outputs: | |
| projects: ${{ steps.kubectl.outputs.projects }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/cloud-platform-auth | |
| with: | |
| api: ${{ secrets.KUBE_ENV_API }} | |
| cert: ${{ secrets.KUBE_CERT }} | |
| cluster: ${{ secrets.KUBE_CLUSTER }} | |
| namespace: ${{ secrets.KUBE_NAMESPACE }} | |
| token: ${{ secrets.KUBE_TOKEN }} | |
| - id: kubectl | |
| run: | | |
| json=$( | |
| kubectl get deployments -o jsonpath='{.items[*].metadata.name}' | xargs -n1 \ | |
| | grep -v domain-events-and-delius \ | |
| | grep -v offender-events-and-delius \ | |
| | jq --raw-input . | jq --slurp --compact-output . | |
| ) | |
| echo "projects=$json" | tee -a "$GITHUB_OUTPUT" | |
| # Event publishers always require write access to the DB, so stop them while in read-only mode | |
| event-publishers: | |
| name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} event publishers | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| needs: get-projects | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/cloud-platform-auth | |
| with: | |
| api: ${{ secrets.KUBE_ENV_API }} | |
| cert: ${{ secrets.KUBE_CERT }} | |
| cluster: ${{ secrets.KUBE_CLUSTER }} | |
| namespace: ${{ secrets.KUBE_NAMESPACE }} | |
| token: ${{ secrets.KUBE_TOKEN }} | |
| - run: | | |
| kubectl scale deploy domain-events-and-delius --replicas "$replicas" | |
| kubectl scale deploy offender-events-and-delius --replicas "$replicas" | |
| env: | |
| replicas: ${{ inputs.action == 'enable' && '0' || '1' }} | |
| # There is no standby database in the test environment, so stop all deployments (except auth) | |
| stop-start: | |
| if: inputs.environment == 'test' | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| needs: get-projects | |
| strategy: | |
| matrix: | |
| project: ${{ fromJson(needs.get-projects.outputs.projects) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/cloud-platform-auth | |
| with: | |
| api: ${{ secrets.KUBE_ENV_API }} | |
| cert: ${{ secrets.KUBE_CERT }} | |
| cluster: ${{ secrets.KUBE_CLUSTER }} | |
| namespace: ${{ secrets.KUBE_NAMESPACE }} | |
| token: ${{ secrets.KUBE_TOKEN }} | |
| - name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} deployments | |
| run: kubectl scale deploy '${{ matrix.project }}' --replicas "${{ inputs.action == 'enable' && '0' || '2' }}" | |
| - name: ${{ inputs.action == 'enable' && 'Block' || 'Unblock' }} ingresses | |
| uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 | |
| with: | |
| max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step | |
| timeout_minutes: 15 | |
| command: | | |
| ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}') | |
| kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=${{ inputs.action == 'enable' && 'limit_except GET { deny all; }' || '' }}' --overwrite | |
| # Block updates at the ingress and switch to the standby database for read operations | |
| switch-database: | |
| if: inputs.environment != 'test' | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| needs: get-projects | |
| strategy: | |
| matrix: | |
| project: ${{ fromJson(needs.get-projects.outputs.projects) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/cloud-platform-auth | |
| with: | |
| api: ${{ secrets.KUBE_ENV_API }} | |
| cert: ${{ secrets.KUBE_CERT }} | |
| cluster: ${{ secrets.KUBE_CLUSTER }} | |
| namespace: ${{ secrets.KUBE_NAMESPACE }} | |
| token: ${{ secrets.KUBE_TOKEN }} | |
| - name: Block updates | |
| if: inputs.action == 'enable' && matrix.project != 'hmpps-auth-and-delius' && matrix.project != 'probation-search-and-delius' && matrix.project != 'feature-flags' # Note: hmpps-auth is excluded here to continue allowing 'POST /authenticate' requests, which are read-only | |
| uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 | |
| with: | |
| max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step | |
| timeout_minutes: 15 | |
| command: | | |
| ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}') | |
| kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=limit_except GET { deny all; }' --overwrite | |
| - name: Switch to ${{ inputs.action == 'enable' && 'standby' || 'primary' }} database | |
| env: | |
| MESSAGING_CONSUMER_ENABLED: ${{ inputs.action == 'enable' && 'false' || 'true' }} | |
| SPRING_DATASOURCE_URL: ${{ inputs.action == 'enable' && 'DB_STANDBY_URL' || 'DB_URL' }} | |
| run: | | |
| kubectl get deployment "${{ matrix.project }}" -o json \ | |
| | jq --arg name MESSAGING_CONSUMER_ENABLED --arg value "$MESSAGING_CONSUMER_ENABLED" \ | |
| '.spec.template.spec.containers[0].env |= if any(.[]; .name == $name) then map(if .name == $name then . + {"value":$value} else . end) else . + [{"name":$name,"value":$value}] end' \ | |
| | jq --arg name SPRING_DATASOURCE_URL --arg value "$SPRING_DATASOURCE_URL" \ | |
| '.spec.template.spec.containers[0].env |= map(if .name == $name then .valueFrom.secretKeyRef.key = $value else . end)' \ | |
| | kubectl apply -f - | |
| - name: Unblock updates | |
| if: inputs.action == 'disable' && matrix.project != 'hmpps-auth-and-delius' && matrix.project != 'probation-search-and-delius' && matrix.project != 'feature-flags' | |
| uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 | |
| with: | |
| max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step | |
| timeout_minutes: 15 | |
| command: | | |
| ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}') | |
| kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=' --overwrite | |
| block-pipeline: | |
| name: Block deployment pipeline | |
| needs: | |
| - event-publishers | |
| - stop-start | |
| - switch-database | |
| if: always() && !failure() && !cancelled() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - if: inputs.environment != 'prod' && inputs.action == 'enable' | |
| uses: actions/create-github-app-token@v1 | |
| id: app-token | |
| with: | |
| app-id: ${{ secrets.BOT_APP_ID }} | |
| private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }} | |
| - if: inputs.environment != 'prod' && inputs.action == 'enable' | |
| run: jq -n "$reviewers" | gh api -XPUT '/repos/ministryofjustice/hmpps-probation-integration-services/environments/${{ inputs.environment }}' --input - | |
| env: | |
| reviewers: '{"reviewers":[{"type":"Team","id":5521382}]}' | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| notify: | |
| if: always() && !cancelled() | |
| runs-on: ubuntu-latest | |
| needs: | |
| - event-publishers | |
| - stop-start | |
| - switch-database | |
| - block-pipeline | |
| steps: | |
| - name: Send message to Slack | |
| uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | |
| if: ${{ !contains(needs.*.result, 'failure') }} | |
| with: | |
| method: chat.postMessage | |
| token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| payload: | | |
| { | |
| "channel": "probation-integration-notifications", | |
| "text": "Read-only mode alert", | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "${{ inputs.action == 'enable' && (inputs.environment == 'test' && '🔴 Offline' || '🚫 Read-only') || '🟢 Online' }}" | |
| } | |
| }, | |
| { | |
| "type": "context", | |
| "elements": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "The *${{ inputs.environment }}* integration services ${{ inputs.action == 'enable' && (inputs.environment == 'test' && 'have been switched off for a Delius deployment' || 'are in read-only mode') || 'are back online' }}." | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "actions", | |
| "elements": [ | |
| { | |
| "type": "button", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "↩️ Switch back" | |
| }, | |
| "url": "${{ github.server_url }}/${{ github.repository }}/actions/workflows/readonly.yml" | |
| }, | |
| { | |
| "type": "button", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "📝 Logs" | |
| }, | |
| "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| - name: Send failure message to Slack | |
| uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | |
| if: ${{ contains(needs.*.result, 'failure') }} | |
| with: | |
| method: chat.postMessage | |
| token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| payload: | | |
| { | |
| "channel": "probation-integration-notifications", | |
| "text": "Read-only mode failure", | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "❌ Failed to ${{ inputs.action }} read-only mode" | |
| } | |
| }, | |
| { | |
| "type": "context", | |
| "elements": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "The *${{ inputs.environment }}* integration services may be in the wrong state. Please check the logs and re-run the workflow." | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "actions", | |
| "elements": [ | |
| { | |
| "type": "button", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "📝 Logs" | |
| }, | |
| "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |