diff --git a/.github/workflows/db_apply_anonimized.yml b/.github/workflows/db_apply_anonimized.yml index 7a2db620b..86fcaf19f 100644 --- a/.github/workflows/db_apply_anonimized.yml +++ b/.github/workflows/db_apply_anonimized.yml @@ -13,10 +13,31 @@ on: default: Test jobs: + stop_logstash: + runs-on: ubuntu-latest + environment: ${{ inputs.deploy-env || 'Test' }} + steps: + - name: Get Environment Name for ${{ vars.ENV_NAME }} + id: get_env_name + uses: Entepotenz/change-string-case-action-min-dependencies@v1 + with: + string: ${{ vars.ENV_NAME }} + - name: Checkout repo + uses: actions/checkout@v4 + - name: Stop Logstash for ${{ vars.ENV_NAME }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" + run: | + aws \ + ecs update-service --desired-count 0 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \ + --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash + apply-anonymized-db: runs-on: self-hosted environment: ${{ inputs.deploy-env || 'Test' }} - + needs: stop_logstash steps: - name: Get Environment Name for ${{ vars.ENV_NAME }} id: get_env_name @@ -25,39 +46,70 @@ jobs: string: ${{ vars.ENV_NAME }} - name: Checkout repo uses: actions/checkout@v4 - - name: Stop Logstash for ${{ vars.ENV_NAME }} + - name: Restore database for ${{ vars.ENV_NAME }} run: | - docker run --rm \ + cd ./src/anon-tools + mkdir -p ./keys + echo "${{ secrets.KEY_FILE }}" > ./keys/key + docker build -t restore -f Dockerfile.restore . + docker run -v ./keys/key:/keys/key --shm-size=2gb --rm \ -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ - ecs update-service --desired-count 0 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \ - --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash --region=eu-west-1 + -e ENVIRONMENT=${{ vars.ENV_NAME }} \ + -e DATABASE_NAME=opensupplyhub \ + -e DATABASE_USERNAME=opensupplyhub \ + -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \ + restore + + post_deploy: + needs: apply-anonymized-db + runs-on: ubuntu-latest + environment: ${{ inputs.deploy-env || 'Test' }} + steps: + - name: Get Environment Name for ${{ vars.ENV_NAME }} + id: get_env_name + uses: Entepotenz/change-string-case-action-min-dependencies@v1 + with: + string: ${{ vars.ENV_NAME }} + - name: Checkout repo + uses: actions/checkout@v4 + - name: Run migrations for ${{ vars.ENV_NAME }} + run: | + ./deployment/run_cli_task ${{ vars.ENV_NAME }} "post_deployment" + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" + + clear_opensearch: + needs: post_deploy + runs-on: ubuntu-latest + environment: ${{ inputs.deploy-env || 'Test' }} + steps: + - name: Get Environment Name for ${{ vars.ENV_NAME }} + id: get_env_name + uses: Entepotenz/change-string-case-action-min-dependencies@v1 + with: + string: ${{ vars.ENV_NAME }} + - name: Checkout repo + uses: actions/checkout@v4 - name: Get OpenSearch domain, filesystem and access point IDs for ${{ vars.ENV_NAME }} id: export_variables + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" run: | OS_DOMAIN_NAME=$(echo "${{ vars.ENV_NAME }}-os-domain" | tr '[:upper:]' '[:lower:]') - OPENSEARCH_DOMAIN=$(docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ + OPENSEARCH_DOMAIN=$(aws \ es describe-elasticsearch-domains --domain-names $OS_DOMAIN_NAME \ --query "DomainStatusList[].Endpoints.vpc" --output text) - EFS_ID=$(docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ + EFS_ID=$(aws \ efs describe-file-systems \ --query "FileSystems[?Tags[?Key=='Environment' && Value=='${{ vars.ENV_NAME }}']].FileSystemId" \ --output text) - EFS_AP_ID=$(docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ + EFS_AP_ID=$(aws \ efs describe-access-points \ --query "AccessPoints[?FileSystemId=='$EFS_ID'].AccessPointId" \ --output text) @@ -65,57 +117,30 @@ jobs: echo "EFS_AP_ID=$EFS_AP_ID" >> $GITHUB_OUTPUT echo "OPENSEARCH_DOMAIN=$OPENSEARCH_DOMAIN" >> $GITHUB_OUTPUT - name: Clear the OpenSearch indexes for ${{ vars.ENV_NAME }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + OPENSEARCH_DOMAIN: ${{ steps.export_variables.outputs.OPENSEARCH_DOMAIN }} + EFS_AP_ID: ${{ steps.export_variables.outputs.EFS_AP_ID }} + EFS_ID: ${{ steps.export_variables.outputs.EFS_ID }} + BASTION_IP: ${{ vars.BASTION_IP }} run: | + cd ./deployment/clear_opensearch mkdir -p script mkdir -p ssh echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ssh/config printf "%s\n" "${{ secrets.SSH_PRIVATE_KEY }}" > ssh/id_rsa echo "" >> ssh/id_rsa echo -n ${{ vars.BASTION_IP }} > script/.env - cat < script/clear_opensearch.sh - #!/bin/bash - curl -X DELETE https://${{ steps.export_variables.outputs.OPENSEARCH_DOMAIN }}/production-locations --aws-sigv4 "aws:amz:eu-west-1:es" --user "${{ secrets.AWS_ACCESS_KEY_ID }}:${{ secrets.AWS_SECRET_ACCESS_KEY }}" - sudo mount -t efs -o tls,accesspoint=${{ steps.export_variables.outputs.EFS_AP_ID }} ${{ steps.export_variables.outputs.EFS_ID }}:/ /mnt - sudo rm /mnt/logstash_jdbc_last_run - sudo umount /mnt - EOF - cat < script/run.sh - chmod 700 /root/.ssh - chmod 400 /root/.ssh/id_rsa - chmod +x /script/clear_opensearch.sh - scp /script/clear_opensearch.sh ec2-user@${{ vars.BASTION_IP }}: - ssh ec2-user@${{ vars.BASTION_IP }} ./clear_opensearch.sh - EOF + envsubst < clear_opensearch.sh.tpl > script/clear_opensearch.sh + envsubst < run.sh.tpl > script/run.sh docker run --rm \ -v ./script:/script \ -v ./ssh:/root/.ssh \ kroniak/ssh-client bash /script/run.sh - - name: Restore database for ${{ vars.ENV_NAME }} - run: | - cd ./src/anon-tools - mkdir -p ./keys - echo "${{ secrets.KEY_FILE }}" > ./keys/key - docker build -t restore -f Dockerfile.restore . - docker run -v ./keys/key:/keys/key --shm-size=2gb --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - -e ENVIRONMENT=${{ vars.ENV_NAME }} \ - -e DATABASE_NAME=opensupplyhub \ - -e DATABASE_USERNAME=opensupplyhub \ - -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \ - restore - - name: Start Logstash for ${{ vars.ENV_NAME }} - run: | - docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ - ecs update-service --desired-count 1 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \ - --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash --region=eu-west-1 - post_deploy: - needs: apply-anonymized-db + + start_logstash: + needs: clear_opensearch runs-on: ubuntu-latest environment: ${{ inputs.deploy-env || 'Test' }} steps: @@ -126,10 +151,12 @@ jobs: string: ${{ vars.ENV_NAME }} - name: Checkout repo uses: actions/checkout@v4 - - name: Run migrations for ${{ vars.ENV_NAME }} - run: | - ./deployment/run_cli_task ${{ vars.ENV_NAME }} "post_deployment" + - name: Start Logstash for ${{ vars.ENV_NAME }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: "eu-west-1" + run: | + aws \ + ecs update-service --desired-count 1 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \ + --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash diff --git a/.github/workflows/deploy_to_aws.yml b/.github/workflows/deploy_to_aws.yml index 0ca53b26c..44feb4f18 100644 --- a/.github/workflows/deploy_to_aws.yml +++ b/.github/workflows/deploy_to_aws.yml @@ -31,6 +31,11 @@ on: required: false type: boolean default: false + clear-opensearch: + description: 'Clear OpenSearch indexes' + required: false + type: boolean + default: false jobs: init-and-plan: @@ -248,11 +253,11 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: "eu-west-1" - restore_database: + stop_logstash: needs: create_kafka_topic - runs-on: self-hosted + runs-on: ubuntu-latest environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') || (startsWith(github.ref_name, 'production-') && 'Production') || (startsWith(github.ref_name, 'sandbox-') && 'Staging') || 'None' }} - if: ${{ inputs.deploy-plan-only == false }} + if: ${{ inputs.deploy-plan-only == false }} steps: - name: Get Environment Name for ${{ vars.ENV_NAME }} id: get_env_name @@ -262,73 +267,29 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - name: Stop Logstash for ${{ vars.ENV_NAME }} - if: ${{ (vars.ENV_NAME != 'Production' && vars.ENV_NAME != 'Staging' && vars.ENV_NAME) && inputs.restore-db == true}} + if: ${{ inputs.restore-db == true || inputs.clear-opensearch == true}} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" run: | - docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ + aws \ ecs update-service --desired-count 0 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \ - --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash --region=eu-west-1 - - name: Get OpenSearch domain, filesystem and access point IDs for ${{ vars.ENV_NAME }} - if: ${{ (vars.ENV_NAME != 'Production' && vars.ENV_NAME != 'Staging' && vars.ENV_NAME) && inputs.restore-db == true}} - id: export_variables - run: | - OS_DOMAIN_NAME=$(echo "${{ vars.ENV_NAME }}-os-domain" | tr '[:upper:]' '[:lower:]') - OPENSEARCH_DOMAIN=$(docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ - es describe-elasticsearch-domains --domain-names $OS_DOMAIN_NAME \ - --query "DomainStatusList[].Endpoints.vpc" --output text) - EFS_ID=$(docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ - efs describe-file-systems \ - --query "FileSystems[?Tags[?Key=='Environment' && Value=='${{ vars.ENV_NAME }}']].FileSystemId" \ - --output text) - EFS_AP_ID=$(docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ - efs describe-access-points \ - --query "AccessPoints[?FileSystemId=='$EFS_ID'].AccessPointId" \ - --output text) - echo "EFS_ID=$EFS_ID" >> $GITHUB_OUTPUT - echo "EFS_AP_ID=$EFS_AP_ID" >> $GITHUB_OUTPUT - echo "OPENSEARCH_DOMAIN=$OPENSEARCH_DOMAIN" >> $GITHUB_OUTPUT - - name: Clear the OpenSearch indexes for ${{ vars.ENV_NAME }} - if: ${{ (vars.ENV_NAME != 'Production' && vars.ENV_NAME != 'Staging' && vars.ENV_NAME) && inputs.restore-db == true}} - run: | - mkdir -p script - mkdir -p ssh - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ssh/config - printf "%s\n" "${{ secrets.SSH_PRIVATE_KEY }}" > ssh/id_rsa - echo "" >> ssh/id_rsa - echo -n ${{ vars.BASTION_IP }} > script/.env - cat < script/clear_opensearch.sh - #!/bin/bash - curl -X DELETE https://${{ steps.export_variables.outputs.OPENSEARCH_DOMAIN }}/production-locations --aws-sigv4 "aws:amz:eu-west-1:es" --user "${{ secrets.AWS_ACCESS_KEY_ID }}:${{ secrets.AWS_SECRET_ACCESS_KEY }}" - sudo mount -t efs -o tls,accesspoint=${{ steps.export_variables.outputs.EFS_AP_ID }} ${{ steps.export_variables.outputs.EFS_ID }}:/ /mnt - sudo rm /mnt/logstash_jdbc_last_run - sudo umount /mnt - EOF - cat < script/run.sh - chmod 700 /root/.ssh - chmod 400 /root/.ssh/id_rsa - chmod +x /script/clear_opensearch.sh - scp /script/clear_opensearch.sh ec2-user@${{ vars.BASTION_IP }}: - ssh ec2-user@${{ vars.BASTION_IP }} ./clear_opensearch.sh - EOF - docker run --rm \ - -v ./script:/script \ - -v ./ssh:/root/.ssh \ - kroniak/ssh-client bash /script/run.sh + --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash + + restore_database: + needs: stop_logstash + runs-on: self-hosted + environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') || (startsWith(github.ref_name, 'production-') && 'Production') || (startsWith(github.ref_name, 'sandbox-') && 'Staging') || 'None' }} + if: ${{ inputs.deploy-plan-only == false }} + steps: + - name: Get Environment Name for ${{ vars.ENV_NAME }} + id: get_env_name + uses: Entepotenz/change-string-case-action-min-dependencies@v1 + with: + string: ${{ vars.ENV_NAME }} + - name: Checkout repo + uses: actions/checkout@v4 - name: Restore database for ${{ vars.ENV_NAME }} if: ${{ (vars.ENV_NAME != 'Production' && vars.ENV_NAME != 'Staging' && vars.ENV_NAME != 'Development') && inputs.restore-db == true}} run: | @@ -354,16 +315,6 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: "eu-west-1" - - name: Start Logstash for ${{ vars.ENV_NAME }} - if: ${{ (vars.ENV_NAME != 'Production' && vars.ENV_NAME != 'Staging' && vars.ENV_NAME) && inputs.restore-db == true}} - run: | - docker run --rm \ - -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - amazon/aws-cli \ - ecs update-service --desired-count 1 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \ - --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash --region=eu-west-1 update_services: needs: restore_database @@ -412,3 +363,86 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: "eu-west-1" + clear_opensearch: + needs: post_deploy + runs-on: ubuntu-latest + environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') || (startsWith(github.ref_name, 'production-') && 'Production') || (startsWith(github.ref_name, 'sandbox-') && 'Staging') || 'None' }} + if: ${{ inputs.deploy-plan-only == false }} + steps: + - name: Get Environment Name for ${{ vars.ENV_NAME }} + id: get_env_name + uses: Entepotenz/change-string-case-action-min-dependencies@v1 + with: + string: ${{ vars.ENV_NAME }} + - name: Checkout repo + uses: actions/checkout@v4 + - name: Get OpenSearch domain, filesystem and access point IDs for ${{ vars.ENV_NAME }} + if: ${{ inputs.restore-db == true || inputs.clear-opensearch == true}} + id: export_variables + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" + run: | + OS_DOMAIN_NAME=$(echo "${{ vars.ENV_NAME }}-os-domain" | tr '[:upper:]' '[:lower:]') + OPENSEARCH_DOMAIN=$(aws \ + es describe-elasticsearch-domains --domain-names $OS_DOMAIN_NAME \ + --query "DomainStatusList[].Endpoints.vpc" --output text) + EFS_ID=$(aws \ + efs describe-file-systems \ + --query "FileSystems[?Tags[?Key=='Environment' && Value=='${{ vars.ENV_NAME }}']].FileSystemId" \ + --output text) + EFS_AP_ID=$(aws \ + efs describe-access-points \ + --query "AccessPoints[?FileSystemId=='$EFS_ID'].AccessPointId" \ + --output text) + echo "EFS_ID=$EFS_ID" >> $GITHUB_OUTPUT + echo "EFS_AP_ID=$EFS_AP_ID" >> $GITHUB_OUTPUT + echo "OPENSEARCH_DOMAIN=$OPENSEARCH_DOMAIN" >> $GITHUB_OUTPUT + - name: Clear the OpenSearch indexes for ${{ vars.ENV_NAME }} + if: ${{ inputs.restore-db == true || inputs.clear-opensearch == true}} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + OPENSEARCH_DOMAIN: ${{ steps.export_variables.outputs.OPENSEARCH_DOMAIN }} + EFS_AP_ID: ${{ steps.export_variables.outputs.EFS_AP_ID }} + EFS_ID: ${{ steps.export_variables.outputs.EFS_ID }} + BASTION_IP: ${{ vars.BASTION_IP }} + run: | + cd ./deployment/clear_opensearch + mkdir -p script + mkdir -p ssh + echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ssh/config + printf "%s\n" "${{ secrets.SSH_PRIVATE_KEY }}" > ssh/id_rsa + echo "" >> ssh/id_rsa + echo -n ${{ vars.BASTION_IP }} > script/.env + envsubst < clear_opensearch.sh.tpl > script/clear_opensearch.sh + envsubst < run.sh.tpl > script/run.sh + docker run --rm \ + -v ./script:/script \ + -v ./ssh:/root/.ssh \ + kroniak/ssh-client bash /script/run.sh + + start_logstash: + needs: clear_opensearch + runs-on: ubuntu-latest + environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') || (startsWith(github.ref_name, 'production-') && 'Production') || (startsWith(github.ref_name, 'sandbox-') && 'Staging') || 'None' }} + if: ${{ inputs.deploy-plan-only == false }} + steps: + - name: Get Environment Name for ${{ vars.ENV_NAME }} + id: get_env_name + uses: Entepotenz/change-string-case-action-min-dependencies@v1 + with: + string: ${{ vars.ENV_NAME }} + - name: Checkout repo + uses: actions/checkout@v4 + - name: Start Logstash for ${{ vars.ENV_NAME }} + if: ${{ inputs.restore-db == true || inputs.clear-opensearch == true}} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" + run: | + aws \ + ecs update-service --desired-count 1 --cluster=ecsOpenSupplyHub${{vars.ENV_NAME}}Cluster \ + --service=OpenSupplyHub${{vars.ENV_NAME}}AppLogstash diff --git a/deployment/clear_opensearch/clear_opensearch.sh.tpl b/deployment/clear_opensearch/clear_opensearch.sh.tpl new file mode 100644 index 000000000..82f0556f3 --- /dev/null +++ b/deployment/clear_opensearch/clear_opensearch.sh.tpl @@ -0,0 +1,7 @@ +#!/bin/bash +echo -e "\nDelete OpenSearch indexes\n" +curl -X DELETE https://$OPENSEARCH_DOMAIN/production-locations --aws-sigv4 "aws:amz:eu-west-1:es" --user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" +echo -e "\nRemove lock file\n" +sudo mount -t efs -o tls,accesspoint=$EFS_AP_ID $EFS_ID:/ /mnt +sudo rm /mnt/logstash_jdbc_last_run +sudo umount /mnt diff --git a/deployment/clear_opensearch/run.sh.tpl b/deployment/clear_opensearch/run.sh.tpl new file mode 100644 index 000000000..e83c42c44 --- /dev/null +++ b/deployment/clear_opensearch/run.sh.tpl @@ -0,0 +1,9 @@ +chmod 700 /root/.ssh +chmod 600 /root/.ssh/id_rsa +chmod 600 /root/.ssh/config +chown -R root:root /root/.ssh +chmod +x /script/clear_opensearch.sh +echo -e "\nCopy script to Bastion host\n" +scp /script/clear_opensearch.sh ec2-user@$BASTION_IP: +echo -e "\nRun cleaning OpenSearch indexes\n" +ssh ec2-user@$BASTION_IP ./clear_opensearch.sh diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index d06869f1f..ca3517c5c 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -22,7 +22,10 @@ To make this possible, the `sync_production_locations.sql` script, which generat Additionally, a `historical_os_id` filter was added to the `sync_production_locations.conf`, ensuring that the `historical_os_id` is included in the index document only when the `historical_os_id_value` is not empty. ### Architecture/Environment changes -* *Describe architecture/environment changes here.* +* [OSDEV-1177](https://opensupplyhub.atlassian.net/browse/OSDEV-1177) + - Improved OpenSearch indexes cleanup step in the _Deploy to AWS_ and _DB - Apply Anonymized DB_ pipelines to use script templates so that changes can be made in one place rather than in each pipeline separately + - Stop/start Logstash and clearing OpenSearch indexes moved to separate jobs of _Deploy to AWS_ and _DB - Apply Anonymized DB_ pipelines. + - Stop/start Logstash and clearing OpenSearch indexes now runs on ubuntu-latest runner. ### Bugfix * [OSDEV-1177](https://opensupplyhub.atlassian.net/browse/OSDEV-1177) - The following changes have been made: