[OSDEV-1177] Templates for generating OpenSearch index cleaning scrip…

…ts (#348) [[OSDEV-1177](https://opensupplyhub.atlassian.net/browse/OSDEV-1177)] Script templates for running on a bastion host so that changes can be made in one place and not in each pipeline separately. Stop/start Logstash and clearing OpenSearch indexes moved to separate jobs of pipelines. Stop/start Logstash and clearing OpenSearch indexes now runs on ubuntu-latest runner. [OSDEV-1177]: https://opensupplyhub.atlassian.net/browse/OSDEV-1177?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
opensupplyhub · Sep 16, 2024 · f8bd103 · f8bd103
1 parent b894180
commit f8bd103
Show file tree

Hide file tree

Showing 5 changed files with 223 additions and 143 deletions.
diff --git a/.github/workflows/db_apply_anonimized.yml b/.github/workflows/db_apply_anonimized.yml
@@ -13,10 +13,31 @@ on:
         default: Test
 
 jobs:
+  stop_logstash:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.deploy-env || 'Test' }}
+    steps:
+      - name: Get Environment Name for ${{ vars.ENV_NAME }}
+        id: get_env_name
+        uses: Entepotenz/change-string-case-action-min-dependencies@v1
+        with:
+          string: ${{ vars.ENV_NAME }}
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Stop Logstash for ${{ vars.ENV_NAME }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: "eu-west-1"
+        run: |
+          aws \
+              ecs update-service --desired-count 0 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \
+              --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash
+
   apply-anonymized-db:
     runs-on: self-hosted
     environment: ${{ inputs.deploy-env || 'Test' }}
-
+    needs: stop_logstash
     steps:
       - name: Get Environment Name for ${{ vars.ENV_NAME }}
         id: get_env_name
@@ -25,97 +46,101 @@ jobs:
           string: ${{ vars.ENV_NAME }}
       - name: Checkout repo
         uses: actions/checkout@v4
-      - name: Stop Logstash for ${{ vars.ENV_NAME }}
+      - name: Restore database for ${{ vars.ENV_NAME }}
         run: |
-          docker run --rm \
+          cd ./src/anon-tools
+          mkdir -p ./keys
+          echo "${{ secrets.KEY_FILE }}" > ./keys/key
+          docker build -t restore -f Dockerfile.restore .
+          docker run -v ./keys/key:/keys/key --shm-size=2gb --rm \
               -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
               -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
               -e AWS_DEFAULT_REGION=eu-west-1 \
-              amazon/aws-cli \
-              ecs update-service --desired-count 0 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \
-              --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash  --region=eu-west-1
+              -e ENVIRONMENT=${{ vars.ENV_NAME }} \
+              -e DATABASE_NAME=opensupplyhub \
+              -e DATABASE_USERNAME=opensupplyhub \
+              -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \
+              restore
+
+  post_deploy:
+    needs: apply-anonymized-db
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.deploy-env || 'Test' }}
+    steps:
+      - name: Get Environment Name for ${{ vars.ENV_NAME }}
+        id: get_env_name
+        uses: Entepotenz/change-string-case-action-min-dependencies@v1
+        with:
+          string: ${{ vars.ENV_NAME }}
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Run migrations for ${{ vars.ENV_NAME }}
+        run: |
+          ./deployment/run_cli_task ${{ vars.ENV_NAME }} "post_deployment"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: "eu-west-1"
+
+  clear_opensearch:
+    needs: post_deploy
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.deploy-env || 'Test' }}
+    steps:
+      - name: Get Environment Name for ${{ vars.ENV_NAME }}
+        id: get_env_name
+        uses: Entepotenz/change-string-case-action-min-dependencies@v1
+        with:
+          string: ${{ vars.ENV_NAME }}
+      - name: Checkout repo
+        uses: actions/checkout@v4
       - name: Get OpenSearch domain, filesystem and access point IDs for ${{ vars.ENV_NAME }}
         id: export_variables
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: "eu-west-1"
         run: |
           OS_DOMAIN_NAME=$(echo "${{ vars.ENV_NAME }}-os-domain" | tr '[:upper:]' '[:lower:]')
-          OPENSEARCH_DOMAIN=$(docker run --rm \
-                       -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-                       -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-                       -e AWS_DEFAULT_REGION=eu-west-1 \
-                       amazon/aws-cli \
+          OPENSEARCH_DOMAIN=$(aws \
                        es describe-elasticsearch-domains --domain-names $OS_DOMAIN_NAME \
                        --query "DomainStatusList[].Endpoints.vpc" --output text)
-          EFS_ID=$(docker run --rm \
-                       -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-                       -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-                       -e AWS_DEFAULT_REGION=eu-west-1 \
-                       amazon/aws-cli \
+          EFS_ID=$(aws \
                        efs describe-file-systems \
                        --query "FileSystems[?Tags[?Key=='Environment' && Value=='${{ vars.ENV_NAME }}']].FileSystemId" \
                        --output text)
-          EFS_AP_ID=$(docker run --rm \
-                          -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-                          -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-                          -e AWS_DEFAULT_REGION=eu-west-1 \
-                          amazon/aws-cli \
+          EFS_AP_ID=$(aws \
                           efs describe-access-points \
                           --query "AccessPoints[?FileSystemId=='$EFS_ID'].AccessPointId" \
                           --output text)
           echo "EFS_ID=$EFS_ID" >> $GITHUB_OUTPUT
           echo "EFS_AP_ID=$EFS_AP_ID" >> $GITHUB_OUTPUT
           echo "OPENSEARCH_DOMAIN=$OPENSEARCH_DOMAIN" >> $GITHUB_OUTPUT
       - name: Clear the OpenSearch indexes for ${{ vars.ENV_NAME }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          OPENSEARCH_DOMAIN: ${{ steps.export_variables.outputs.OPENSEARCH_DOMAIN }}
+          EFS_AP_ID: ${{ steps.export_variables.outputs.EFS_AP_ID }}
+          EFS_ID: ${{ steps.export_variables.outputs.EFS_ID }}
+          BASTION_IP: ${{ vars.BASTION_IP }}
         run: |
+          cd ./deployment/clear_opensearch
           mkdir -p script
           mkdir -p ssh
           echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ssh/config
           printf "%s\n" "${{ secrets.SSH_PRIVATE_KEY }}" > ssh/id_rsa
           echo "" >> ssh/id_rsa
           echo -n ${{ vars.BASTION_IP }} > script/.env
-          cat <<EOF > script/clear_opensearch.sh
-          #!/bin/bash
-          curl -X DELETE https://${{ steps.export_variables.outputs.OPENSEARCH_DOMAIN }}/production-locations --aws-sigv4 "aws:amz:eu-west-1:es" --user "${{ secrets.AWS_ACCESS_KEY_ID }}:${{ secrets.AWS_SECRET_ACCESS_KEY }}"
-          sudo mount -t efs -o tls,accesspoint=${{ steps.export_variables.outputs.EFS_AP_ID }} ${{ steps.export_variables.outputs.EFS_ID }}:/ /mnt
-          sudo rm /mnt/logstash_jdbc_last_run
-          sudo umount /mnt
-          EOF
-          cat <<EOF > script/run.sh
-          chmod 700 /root/.ssh
-          chmod 400 /root/.ssh/id_rsa
-          chmod +x /script/clear_opensearch.sh
-          scp /script/clear_opensearch.sh ec2-user@${{ vars.BASTION_IP }}:
-          ssh ec2-user@${{ vars.BASTION_IP }} ./clear_opensearch.sh
-          EOF
+          envsubst < clear_opensearch.sh.tpl > script/clear_opensearch.sh
+          envsubst < run.sh.tpl > script/run.sh
           docker run --rm  \
               -v ./script:/script \
               -v ./ssh:/root/.ssh \
               kroniak/ssh-client bash /script/run.sh
-      - name: Restore database for ${{ vars.ENV_NAME }}
-        run: |
-          cd ./src/anon-tools
-          mkdir -p ./keys
-          echo "${{ secrets.KEY_FILE }}" > ./keys/key
-          docker build -t restore -f Dockerfile.restore .
-          docker run -v ./keys/key:/keys/key --shm-size=2gb --rm \
-              -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-              -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-              -e AWS_DEFAULT_REGION=eu-west-1 \
-              -e ENVIRONMENT=${{ vars.ENV_NAME }} \
-              -e DATABASE_NAME=opensupplyhub \
-              -e DATABASE_USERNAME=opensupplyhub \
-              -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \
-              restore
-      - name: Start Logstash for ${{ vars.ENV_NAME }}
-        run: |
-          docker run --rm \
-              -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-              -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-              -e AWS_DEFAULT_REGION=eu-west-1 \
-              amazon/aws-cli \
-              ecs update-service --desired-count 1 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \
-              --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash  --region=eu-west-1
-  post_deploy:
-    needs: apply-anonymized-db
+
+  start_logstash:
+    needs: clear_opensearch
     runs-on: ubuntu-latest
     environment: ${{ inputs.deploy-env || 'Test' }}
     steps:
@@ -126,10 +151,12 @@ jobs:
           string: ${{ vars.ENV_NAME }}
       - name: Checkout repo
         uses: actions/checkout@v4
-      - name: Run migrations for ${{ vars.ENV_NAME }}
-        run: |
-          ./deployment/run_cli_task ${{ vars.ENV_NAME }} "post_deployment"
+      - name: Start Logstash for ${{ vars.ENV_NAME }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: "eu-west-1"
+        run: |
+          aws \
+              ecs update-service --desired-count 1 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \
+              --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash