diff --git a/.github/workflows/readonly.yml b/.github/workflows/readonly.yml index ab248d3b73..9866456905 100644 --- a/.github/workflows/readonly.yml +++ b/.github/workflows/readonly.yml @@ -1,6 +1,7 @@ name: Read-only # Prepare for Delius down-time by entering "read-only" mode. # Switches off message consumers, blocks any write APIs, and re-points everything else at the snapshot standby database. +# Note: In the test environment, where there is no snapshot standby database, the services are stopped completely. on: workflow_call: @@ -34,12 +35,14 @@ on: - disable jobs: - readonly: + get-projects: + name: Get projects runs-on: ubuntu-latest environment: ${{ inputs.environment }} + outputs: + projects: ${{ steps.kubectl.outputs.projects }} steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/cloud-platform-auth with: api: ${{ secrets.KUBE_ENV_API }} @@ -47,77 +50,132 @@ jobs: cluster: ${{ secrets.KUBE_CLUSTER }} namespace: ${{ secrets.KUBE_NAMESPACE }} token: ${{ secrets.KUBE_TOKEN }} + - id: kubectl + run: | + json=$( + kubectl get deployments -o jsonpath='{.items[*].metadata.name}' | xargs -n1 \ + | grep -v domain-events-and-delius \ + | grep -v offender-events-and-delius \ + | jq --raw-input . | jq --slurp --compact-output . + ) + echo "projects=$json" | tee -a "$GITHUB_OUTPUT" - - name: Patch deployments - switch back to primary database - if: inputs.environment != 'test' && inputs.action == 'disable' + # Event publishers always require write access to the DB, so stop them while in read-only mode + event-publishers: + name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} event publishers + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + needs: get-projects + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/cloud-platform-auth + with: + api: ${{ secrets.KUBE_ENV_API }} + cert: ${{ secrets.KUBE_CERT }} + cluster: ${{ secrets.KUBE_CLUSTER }} + namespace: ${{ secrets.KUBE_NAMESPACE }} + token: ${{ secrets.KUBE_TOKEN }} + - run: | + kubectl scale deploy domain-events-and-delius --replicas "$replicas" + kubectl scale deploy offender-events-and-delius --replicas "$replicas" env: - MESSAGING_CONSUMER_ENABLED: 'true' - SPRING_DATASOURCE_URL: 'DB_URL' - run: | - deployments=$(kubectl get deployments -o jsonpath='{.items[*].metadata.name}') - for deployment in $deployments; do - kubectl get deployment "$deployment" -o json \ - | jq --arg name MESSAGING_CONSUMER_ENABLED --arg value "$MESSAGING_CONSUMER_ENABLED" \ - '.spec.template.spec.containers[0].env |= if any(.[]; .name == $name) then map(if .name == $name then . + {"value":$value} else . end) else . + [{"name":$name,"value":$value}] end' \ - | jq --arg name SPRING_DATASOURCE_URL --arg value "$SPRING_DATASOURCE_URL" \ - '.spec.template.spec.containers[0].env |= map(if .name == $name then .valueFrom.secretKeyRef.key = $value else . end)' \ - | kubectl apply -f - - done + replicas: ${{ inputs.action == 'enable' && '0' || '1' }} - - name: Patch ingresses + # There is no standby database in the test environment, so stop all deployments (except auth) + stop-start: + if: inputs.environment == 'test' + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + needs: get-projects + strategy: + matrix: + project: ${{ fromJson(needs.get-projects.outputs.projects) }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/cloud-platform-auth + with: + api: ${{ secrets.KUBE_ENV_API }} + cert: ${{ secrets.KUBE_CERT }} + cluster: ${{ secrets.KUBE_CLUSTER }} + namespace: ${{ secrets.KUBE_NAMESPACE }} + token: ${{ secrets.KUBE_TOKEN }} + + - name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} deployments + if: matrix.project != 'hmpps-auth-and-delius' + run: kubectl scale deploy '${{ matrix.project }}' --replicas "${{ inputs.action == 'enable' && '0' || '2' }}" + + - name: ${{ inputs.action == 'enable' && 'Block' || 'Unblock' }} ingresses + if: matrix.project != 'hmpps-auth-and-delius' uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0 with: max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step timeout_minutes: 15 command: | - ingresses=$(kubectl get ingresses -o jsonpath='{.items[*].metadata.name}') - for ingress in $ingresses; do - if [[ "$ingress" != hmpps-auth-and-delius* ]]; then - kubectl annotate ingress "$ingress" "nginx.ingress.kubernetes.io/configuration-snippet=$configuration_snippet" --overwrite - fi - done - env: - configuration_snippet: ${{ inputs.action == 'enable' && 'limit_except GET { deny all; }' || '' }} + ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}') + kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=${{ inputs.action == 'enable' && 'limit_except GET { deny all; }' || '' }}' --overwrite - - name: Patch deployments - switch to standby database - if: inputs.environment != 'test' && inputs.action == 'enable' - env: - MESSAGING_CONSUMER_ENABLED: 'false' - SPRING_DATASOURCE_URL: 'DB_STANDBY_URL' - run: | - deployments=$(kubectl get deployments -o jsonpath='{.items[*].metadata.name}') - for deployment in $deployments; do - kubectl get deployment "$deployment" -o json \ - | jq --arg name MESSAGING_CONSUMER_ENABLED --arg value "$MESSAGING_CONSUMER_ENABLED" \ - '.spec.template.spec.containers[0].env |= if any(.[]; .name == $name) then map(if .name == $name then . + {"value":$value} else . end) else . + [{"name":$name,"value":$value}] end' \ - | jq --arg name SPRING_DATASOURCE_URL --arg value "$SPRING_DATASOURCE_URL" \ - '.spec.template.spec.containers[0].env |= map(if .name == $name then .valueFrom.secretKeyRef.key = $value else . end)' \ - | kubectl apply -f - - done + # Block updates at the ingress and switch to the standby database for read operations + switch-database: + if: inputs.environment != 'test' + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + needs: get-projects + strategy: + matrix: + project: ${{ fromJson(needs.get-projects.outputs.projects) }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/cloud-platform-auth + with: + api: ${{ secrets.KUBE_ENV_API }} + cert: ${{ secrets.KUBE_CERT }} + cluster: ${{ secrets.KUBE_CLUSTER }} + namespace: ${{ secrets.KUBE_NAMESPACE }} + token: ${{ secrets.KUBE_TOKEN }} - - name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} deployments - no standby database - if: inputs.environment == 'test' - env: - replicas: ${{ inputs.action == 'enable' && '0' || '2' }} - run: | - deployments=$(kubectl get deployments -o jsonpath='{.items[*].metadata.name}') - for deployment in $deployments; do - if [ "$deployment" != 'hmpps-auth-and-delius' ] && \ - [ "$deployment" != 'domain-events-and-delius' ] && \ - [ "$deployment" != 'offender-events-and-delius' ]; then - kubectl scale deploy "$deployment" --replicas "$replicas" - fi - done + - name: Block updates + if: inputs.action == 'enable' && matrix.project != 'hmpps-auth-and-delius' + uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0 + with: + max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step + timeout_minutes: 15 + command: | + ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}') + kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=limit_except GET { deny all; }' --overwrite - - name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} event publishers + - name: Switch to ${{ inputs.action == 'enable' && 'standby' || 'primary' }} database env: - replicas: ${{ inputs.action == 'enable' && '0' || '1' }} + MESSAGING_CONSUMER_ENABLED: ${{ inputs.action == 'enable' && 'false' || 'true' }} + SPRING_DATASOURCE_URL: ${{ inputs.action == 'enable' && 'DB_STANDBY_URL' || 'DB_URL' }} run: | - kubectl scale deploy domain-events-and-delius --replicas "$replicas" - kubectl scale deploy offender-events-and-delius --replicas "$replicas" + kubectl get deployment "${{ matrix.project }}" -o json \ + | jq --arg name MESSAGING_CONSUMER_ENABLED --arg value "$MESSAGING_CONSUMER_ENABLED" \ + '.spec.template.spec.containers[0].env |= if any(.[]; .name == $name) then map(if .name == $name then . + {"value":$value} else . end) else . + [{"name":$name,"value":$value}] end' \ + | jq --arg name SPRING_DATASOURCE_URL --arg value "$SPRING_DATASOURCE_URL" \ + '.spec.template.spec.containers[0].env |= map(if .name == $name then .valueFrom.secretKeyRef.key = $value else . end)' \ + | kubectl apply -f - + - name: Unblock updates + if: inputs.action == 'disable' && matrix.project != 'hmpps-auth-and-delius' + uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0 + with: + max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step + timeout_minutes: 15 + command: | + ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}') + kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=' --overwrite + + notify: + if: always() + runs-on: ubuntu-latest + needs: + - event-publishers + - stop-start + - switch-database + steps: - name: Send message to Slack uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + if: ${{ !contains(needs.*.result, 'failure') }} with: channel-id: probation-integration-notifications payload: | @@ -167,7 +225,7 @@ jobs: - name: Send failure message to Slack uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - if: failure() + if: ${{ contains(needs.*.result, 'failure') }} with: channel-id: probation-integration-notifications payload: |