PI-1790 Toggle read-only mode in parallel by service (#3150)

.github/workflows/readonly.yml

@@ -1,6 +1,7 @@
 name: Read-only
 # Prepare for Delius down-time by entering "read-only" mode.
 # Switches off message consumers, blocks any write APIs, and re-points everything else at the snapshot standby database.
+# Note: In the test environment, where there is no snapshot standby database, the services are stopped completely.
 
 on:
   workflow_call:
@@ -34,90 +35,147 @@ on:
           - disable
 
 jobs:
-  readonly:
+  get-projects:
+    name: Get projects
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
+    outputs:
+      projects: ${{ steps.kubectl.outputs.projects }}
     steps:
       - uses: actions/checkout@v4
-
       - uses: ./.github/actions/cloud-platform-auth
         with:
           api: ${{ secrets.KUBE_ENV_API }}
           cert: ${{ secrets.KUBE_CERT }}
           cluster: ${{ secrets.KUBE_CLUSTER }}
           namespace: ${{ secrets.KUBE_NAMESPACE }}
           token: ${{ secrets.KUBE_TOKEN }}
+      - id: kubectl
+        run: |
+          json=$(
+            kubectl get deployments -o jsonpath='{.items[*].metadata.name}' | xargs -n1 \
+            | grep -v domain-events-and-delius \
+            | grep -v offender-events-and-delius \
+            | jq --raw-input . | jq --slurp --compact-output .
+          )
+          echo "projects=$json" | tee -a "$GITHUB_OUTPUT"
 
-      - name: Patch deployments - switch back to primary database
-        if: inputs.environment != 'test' && inputs.action == 'disable'
+  # Event publishers always require write access to the DB, so stop them while in read-only mode
+  event-publishers:
+    name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} event publishers
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    needs: get-projects
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/cloud-platform-auth
+        with:
+          api: ${{ secrets.KUBE_ENV_API }}
+          cert: ${{ secrets.KUBE_CERT }}
+          cluster: ${{ secrets.KUBE_CLUSTER }}
+          namespace: ${{ secrets.KUBE_NAMESPACE }}
+          token: ${{ secrets.KUBE_TOKEN }}
+      - run: |
+          kubectl scale deploy domain-events-and-delius --replicas "$replicas"
+          kubectl scale deploy offender-events-and-delius --replicas "$replicas"
         env:
-          MESSAGING_CONSUMER_ENABLED: 'true'
-          SPRING_DATASOURCE_URL: 'DB_URL'
-        run: |
-          deployments=$(kubectl get deployments -o jsonpath='{.items[*].metadata.name}')
-          for deployment in $deployments; do
-            kubectl get deployment "$deployment" -o json \
-            | jq --arg name MESSAGING_CONSUMER_ENABLED --arg value "$MESSAGING_CONSUMER_ENABLED" \
-              '.spec.template.spec.containers[0].env |= if any(.[]; .name == $name) then map(if .name == $name then . + {"value":$value} else . end) else . + [{"name":$name,"value":$value}] end' \
-            | jq --arg name SPRING_DATASOURCE_URL --arg value "$SPRING_DATASOURCE_URL" \
-              '.spec.template.spec.containers[0].env |= map(if .name == $name then .valueFrom.secretKeyRef.key = $value else . end)' \
-            | kubectl apply -f -
-          done
+          replicas: ${{ inputs.action == 'enable' && '0' || '1' }}
 
-      - name: Patch ingresses
+  # There is no standby database in the test environment, so stop all deployments (except auth)
+  stop-start:
+    if: inputs.environment == 'test'
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    needs: get-projects
+    strategy:
+      matrix:
+        project: ${{ fromJson(needs.get-projects.outputs.projects) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/cloud-platform-auth
+        with:
+          api: ${{ secrets.KUBE_ENV_API }}
+          cert: ${{ secrets.KUBE_CERT }}
+          cluster: ${{ secrets.KUBE_CLUSTER }}
+          namespace: ${{ secrets.KUBE_NAMESPACE }}
+          token: ${{ secrets.KUBE_TOKEN }}
+
+      - name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} deployments
+        if: matrix.project != 'hmpps-auth-and-delius'
+        run: kubectl scale deploy '${{ matrix.project }}' --replicas "${{ inputs.action == 'enable' && '0' || '2' }}"
+
+      - name: ${{ inputs.action == 'enable' && 'Block' || 'Unblock' }} ingresses
+        if: matrix.project != 'hmpps-auth-and-delius'
         uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
         with:
           max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step
           timeout_minutes: 15
           command: |
-            ingresses=$(kubectl get ingresses -o jsonpath='{.items[*].metadata.name}')
-            for ingress in $ingresses; do
-              if [[ "$ingress" != hmpps-auth-and-delius* ]]; then
-                kubectl annotate ingress "$ingress" "nginx.ingress.kubernetes.io/configuration-snippet=$configuration_snippet" --overwrite
-              fi
-            done
-        env:
-          configuration_snippet: ${{ inputs.action == 'enable' && 'limit_except GET { deny all; }' || '' }}
+            ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}')
+            kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=${{ inputs.action == 'enable' && 'limit_except GET { deny all; }' || '' }}' --overwrite 
 
-      - name: Patch deployments - switch to standby database
-        if: inputs.environment != 'test' && inputs.action == 'enable'
-        env:
-          MESSAGING_CONSUMER_ENABLED: 'false'
-          SPRING_DATASOURCE_URL: 'DB_STANDBY_URL'
-        run: |
-          deployments=$(kubectl get deployments -o jsonpath='{.items[*].metadata.name}')
-          for deployment in $deployments; do
-            kubectl get deployment "$deployment" -o json \
-            | jq --arg name MESSAGING_CONSUMER_ENABLED --arg value "$MESSAGING_CONSUMER_ENABLED" \
-              '.spec.template.spec.containers[0].env |= if any(.[]; .name == $name) then map(if .name == $name then . + {"value":$value} else . end) else . + [{"name":$name,"value":$value}] end' \
-            | jq --arg name SPRING_DATASOURCE_URL --arg value "$SPRING_DATASOURCE_URL" \
-              '.spec.template.spec.containers[0].env |= map(if .name == $name then .valueFrom.secretKeyRef.key = $value else . end)' \
-            | kubectl apply -f -
-          done
+  # Block updates at the ingress and switch to the standby database for read operations
+  switch-database:
+    if: inputs.environment != 'test'
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    needs: get-projects
+    strategy:
+      matrix:
+        project: ${{ fromJson(needs.get-projects.outputs.projects) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/cloud-platform-auth
+        with:
+          api: ${{ secrets.KUBE_ENV_API }}
+          cert: ${{ secrets.KUBE_CERT }}
+          cluster: ${{ secrets.KUBE_CLUSTER }}
+          namespace: ${{ secrets.KUBE_NAMESPACE }}
+          token: ${{ secrets.KUBE_TOKEN }}
 
-      - name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} deployments - no standby database
-        if: inputs.environment == 'test'
-        env:
-          replicas: ${{ inputs.action == 'enable' && '0' || '2' }}
-        run: |
-          deployments=$(kubectl get deployments -o jsonpath='{.items[*].metadata.name}')
-          for deployment in $deployments; do
-            if [ "$deployment" != 'hmpps-auth-and-delius' ] && \
-               [ "$deployment" != 'domain-events-and-delius' ] && \
-               [ "$deployment" != 'offender-events-and-delius' ]; then
-              kubectl scale deploy "$deployment" --replicas "$replicas"
-            fi
-          done
+      - name: Block updates
+        if: inputs.action == 'enable' && matrix.project != 'hmpps-auth-and-delius'
+        uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
+        with:
+          max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step
+          timeout_minutes: 15
+          command: |
+            ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}')
+            kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=limit_except GET { deny all; }' --overwrite
 
-      - name: ${{ inputs.action == 'enable' && 'Stop' || 'Start' }} event publishers
+      - name: Switch to ${{ inputs.action == 'enable' && 'standby' || 'primary' }} database
         env:
-          replicas: ${{ inputs.action == 'enable' && '0' || '1' }}
+          MESSAGING_CONSUMER_ENABLED: ${{ inputs.action == 'enable' && 'false' || 'true' }}
+          SPRING_DATASOURCE_URL: ${{ inputs.action == 'enable' && 'DB_STANDBY_URL' || 'DB_URL' }}
         run: |
-          kubectl scale deploy domain-events-and-delius --replicas "$replicas"
-          kubectl scale deploy offender-events-and-delius --replicas "$replicas"
+          kubectl get deployment "${{ matrix.project }}" -o json \
+          | jq --arg name MESSAGING_CONSUMER_ENABLED --arg value "$MESSAGING_CONSUMER_ENABLED" \
+            '.spec.template.spec.containers[0].env |= if any(.[]; .name == $name) then map(if .name == $name then . + {"value":$value} else . end) else . + [{"name":$name,"value":$value}] end' \
+          | jq --arg name SPRING_DATASOURCE_URL --arg value "$SPRING_DATASOURCE_URL" \
+            '.spec.template.spec.containers[0].env |= map(if .name == $name then .valueFrom.secretKeyRef.key = $value else . end)' \
+          | kubectl apply -f -
 
+      - name: Unblock updates
+        if: inputs.action == 'disable' && matrix.project != 'hmpps-auth-and-delius'
+        uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
+        with:
+          max_attempts: 3 # Patching ingresses intermittently fails on MOJ Cloud Platform, so we retry this step
+          timeout_minutes: 15
+          command: |
+            ingress=$(kubectl get ingress -o jsonpath='{.items[*].metadata.name}' -l 'app=${{ matrix.project }}')
+            kubectl annotate ingress "$ingress" 'nginx.ingress.kubernetes.io/configuration-snippet=' --overwrite
+
+  notify:
+    if: always()
+    runs-on: ubuntu-latest
+    needs:
+      - event-publishers
+      - stop-start
+      - switch-database
+    steps:
       - name: Send message to Slack
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        if: ${{ !contains(needs.*.result, 'failure') }}
         with:
           channel-id: probation-integration-notifications
           payload: |
@@ -167,7 +225,7 @@ jobs:
 
       - name: Send failure message to Slack
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        if: failure()
+        if: ${{ contains(needs.*.result, 'failure') }}
         with:
           channel-id: probation-integration-notifications
           payload: |