From f8ee523677ad60d54b457cd1a95071fa72ac1332 Mon Sep 17 00:00:00 2001 From: vladsha-dev Date: Fri, 6 Dec 2024 13:45:17 +0100 Subject: [PATCH] Fix OpenSearch index mapping updates (#445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was found that clearing OpenSearch indexes didn’t work properly because the templates weren’t cleared. After updating the index mappings within the index template files, the index template remained unchanged because only the indexes were deleted during deployment, not both the indexes and their templates. This caused conflicts and prevented developers' updates from being applied to the OpenSearch indexes. This issue has been fixed by adding additional requests to delete the appropriate index templates to the `clear_opensearch.sh.tpl` script, which is triggered when clearing OpenSearch during deployment to any environment. --- .github/workflows/db_apply_anonimized.yml | 2 +- .github/workflows/deploy_to_aws.yml | 4 ++-- .github/workflows/release-deploy.yml | 2 +- .../clear_opensearch/clear_opensearch.sh.tpl | 20 +++++++++++++++++-- deployment/clear_opensearch/run.sh.tpl | 2 ++ doc/release/RELEASE-NOTES.md | 3 +++ doc/release/RELEASE-PROTOCOL.md | 8 ++++---- 7 files changed, 31 insertions(+), 10 deletions(-) diff --git a/.github/workflows/db_apply_anonimized.yml b/.github/workflows/db_apply_anonimized.yml index 86fcaf19f..47efc9bfe 100644 --- a/.github/workflows/db_apply_anonimized.yml +++ b/.github/workflows/db_apply_anonimized.yml @@ -116,7 +116,7 @@ jobs: echo "EFS_ID=$EFS_ID" >> $GITHUB_OUTPUT echo "EFS_AP_ID=$EFS_AP_ID" >> $GITHUB_OUTPUT echo "OPENSEARCH_DOMAIN=$OPENSEARCH_DOMAIN" >> $GITHUB_OUTPUT - - name: Clear the OpenSearch indexes for ${{ vars.ENV_NAME }} + - name: Clear the custom OpenSearch indexes and templates for ${{ vars.ENV_NAME }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/deploy_to_aws.yml b/.github/workflows/deploy_to_aws.yml index 7111d6efa..2def34ede 100644 --- a/.github/workflows/deploy_to_aws.yml +++ b/.github/workflows/deploy_to_aws.yml @@ -30,7 +30,7 @@ on: type: boolean default: false clear-opensearch: - description: 'Clear OpenSearch indexes' + description: 'Clear the custom OpenSearch indexes and templates' required: false type: boolean default: false @@ -413,7 +413,7 @@ jobs: echo "EFS_ID=$EFS_ID" >> $GITHUB_OUTPUT echo "EFS_AP_ID=$EFS_AP_ID" >> $GITHUB_OUTPUT echo "OPENSEARCH_DOMAIN=$OPENSEARCH_DOMAIN" >> $GITHUB_OUTPUT - - name: Clear the OpenSearch indexes for ${{ vars.ENV_NAME }} + - name: Clear the custom OpenSearch indexes and templates for ${{ vars.ENV_NAME }} if: ${{ inputs.restore-db == true || inputs.clear-opensearch == true}} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/release-deploy.yml b/.github/workflows/release-deploy.yml index 5b25faaf5..130ae5dad 100644 --- a/.github/workflows/release-deploy.yml +++ b/.github/workflows/release-deploy.yml @@ -16,7 +16,7 @@ on: - production default: sandbox clear-opensearch: - description: 'Clear OpenSearch indexes' + description: 'Clear the custom OpenSearch indexes and templates' required: false type: boolean default: false diff --git a/deployment/clear_opensearch/clear_opensearch.sh.tpl b/deployment/clear_opensearch/clear_opensearch.sh.tpl index ae005fa5f..817f141d1 100644 --- a/deployment/clear_opensearch/clear_opensearch.sh.tpl +++ b/deployment/clear_opensearch/clear_opensearch.sh.tpl @@ -1,9 +1,25 @@ #!/bin/bash -echo -e "\nDelete OpenSearch indexes\n" + +# This script is necessary to delete the custom OpenSearch indexes, +# templates, and Logstash pipeline lock files in order to apply +# new index mappings or to refresh the OpenSearch cluster after +# restarting Logstash, with the lock files deleted from EFS +# storage for each pipeline. +# +# The script can be modified to delete only specific templates, +# indexes, and Logstash pipeline lock files, allowing for a more +# targeted refresh without affecting the entire OpenSearch cluster. +# This can help speed up the deployment process of new changes. + +echo -e "\nDelete the custom OpenSearch indexes\n" curl -X DELETE https://$OPENSEARCH_DOMAIN/production-locations --aws-sigv4 "aws:amz:eu-west-1:es" --user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" curl -X DELETE https://$OPENSEARCH_DOMAIN/moderation-events --aws-sigv4 "aws:amz:eu-west-1:es" --user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" -echo -e "\nRemove the JDBC input lock files from the EFS connected to Logstash\n" +echo -e "\nDelete the custom OpenSearch templates\n" +curl -X DELETE https://$OPENSEARCH_DOMAIN/_index_template/production_locations_template --aws-sigv4 "aws:amz:eu-west-1:es" --user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" +curl -X DELETE https://$OPENSEARCH_DOMAIN/_index_template/moderation_events_template --aws-sigv4 "aws:amz:eu-west-1:es" --user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" + +echo -e "\nRemove the JDBC input lock files from the EFS storage connected to Logstash\n" sudo mount -t efs -o tls,accesspoint=$EFS_AP_ID $EFS_ID:/ /mnt sudo rm /mnt/production_locations_jdbc_last_run sudo rm /mnt/moderation_events_jdbc_last_run diff --git a/deployment/clear_opensearch/run.sh.tpl b/deployment/clear_opensearch/run.sh.tpl index e83c42c44..0fa0a0769 100644 --- a/deployment/clear_opensearch/run.sh.tpl +++ b/deployment/clear_opensearch/run.sh.tpl @@ -1,3 +1,5 @@ +#!/bin/bash + chmod 700 /root/.ssh chmod 600 /root/.ssh/id_rsa chmod 600 /root/.ssh/config diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index 2f5ebdec9..40128950a 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -50,6 +50,8 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ### Bugfix * [OSDEV-1388](https://opensupplyhub.atlassian.net/browse/OSDEV-1388) - The waiter from boto3 cannot wait more than half an hour so we replaced it with our own. +* It was found that clearing OpenSearch indexes didn’t work properly because the templates weren’t cleared. After updating the index mappings within the index template files, the index template remained unchanged because only the indexes were deleted during deployment, not both the indexes and their templates. This caused conflicts and prevented developers' updates from being applied to the OpenSearch indexes. +This issue has been fixed by adding additional requests to delete the appropriate index templates to the `clear_opensearch.sh.tpl` script, which is triggered when clearing OpenSearch during deployment to any environment. ### What's new * [OSDEV-1175](https://opensupplyhub.atlassian.net/browse/OSDEV-1175) - New Moderation Queue Page was integrated with `GET api/v1/moderation-events/` endpoint that include pagination, sorting and filtering. @@ -58,6 +60,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * Ensure that the following commands are included in the `post_deployment` command: * `migrate` * `reindex_database` +* Run `[Release] Deploy` pipeline for the target environment with the flag `Clear custom OpenSearch indexes and templates` set to true - to refresh the index mappings for the `production-locations` and `moderation-events` indexes after fixing the process of clearing the custom OpenSearch indexes. ## Release 1.25.0 diff --git a/doc/release/RELEASE-PROTOCOL.md b/doc/release/RELEASE-PROTOCOL.md index 6e4793c0b..e144f3c01 100644 --- a/doc/release/RELEASE-PROTOCOL.md +++ b/doc/release/RELEASE-PROTOCOL.md @@ -112,12 +112,12 @@ Each new feature should reside in its own branch, which can be pushed to the cen Once `main` has acquired enough features for a release (or a predetermined release date is approaching), you run the `Release [Init]` GitHub workflow that creates a new release branch with a version number for the release. The release version number for release branches includes only the major and minor versions. When the release branch is ready for release, the `Release [Deploy]` workflow should be run for each environment, such as sandbox and production. This workflow will create two Git tags, each with a version number. -This workflow will also initiate the Deploy to AWS workflow and pass it the necessary parameters. If you need to clear OpenSearch indexes during deployment, you need to select the Clear OpenSearch indexes checkbox. +This workflow will also initiate the Deploy to AWS workflow and pass it the necessary parameters. If you need to clear the custom OpenSearch indexes and tenplates during deployment, you need to select the `Clear the custom OpenSearch indexes and templates` checkbox. #### Hotfix branches Hotfix branches are utilized to quickly patch production and sandbox releases. They resemble release branches and feature branches, except they are based on a release branch instead of `main`. This is the branch that should fork directly from a release branch. As soon as the fix is complete, it should be merged into the release branch and, if the fix isn't dirty, into `main` as well. After manually running the `Release [Deploy]` workflow, two new tags with increased patch versions will be created, and the new version will be shipped to sandbox and production environments. -This workflow will also initiate the Deploy to AWS workflow and pass it the necessary parameters. If you need to clear OpenSearch indexes during deployment, you need to select the Clear OpenSearch indexes checkbox. +This workflow will also initiate the Deploy to AWS workflow and pass it the necessary parameters. If you need to clear the custom OpenSearch indexes and templates during deployment, you need to select the `Clear the custom OpenSearch indexes and templates` checkbox. #### Quick-fix branches @@ -159,7 +159,7 @@ Make sure that: 1. To enhance communication within the team, the responsible person for the release must notify all stakeholders about the release two working days before its scheduled date and in 1-2 hours to prevent any actions on the environment on which the deployment is carried out. 2. The responsible person have to take db snapshot manually via Amazon RDS in the `Snapshots` tab with name `env-db-date` (examples: `stg-db-05-18-2024` and `prd-db-05-18-2024`). 3. On the designated time and day, before triggering workflow on Production environment the responsible person have manually make active the `disable_list_uploading` switch, as mentioned in [Block loading of new production locations](#block-loading-production-locations). -4. Then the responsible person runs the `Release [Deploy]` workflow for the sandbox and production environments from the release branch. They need to fill in the full release tag version (`X.Y.Z`) and choose the environment. If the responsible person need to clear OpenSearch indexes during deployment, they must select the Clear OpenSearch indexes checkbox. +4. Then the responsible person runs the `Release [Deploy]` workflow for the sandbox and production environments from the release branch. They need to fill in the full release tag version (`X.Y.Z`) and choose the environment. If the responsible person need to clear the custom OpenSearch indexes and templates during deployment, they must select the `Clear the custom OpenSearch indexes and templates` checkbox. ℹ️ Note, that `Deploy to AWS` workflow will be triggered automatically for the sandbox and production environments respectively. 5. After completing the triggered workflows, the responsible person must open the AWS console and verify that all tasks of the `OpenSupplyHubStagingAppDD`, `OpenSupplyHubStagingApp`, `OpenSupplyHubStagingAppLogstash`, `OpenSupplyHubProductionAppDD`, `OpenSupplyHubProductionApp` and `OpenSupplyHubProductionAppLogstash` services in the `ecsOpenSupplyHubStagingCluster` and `ecsOpenSupplyHubProductionCluster` Amazon ECS clusters, respectively, have been restarted. 6. Additionally, it is necessary to check the OpenSearch domains and their statuses, such as Domain Processing Status, Configuration Change Status, and Cluster Health, to ensure they are green (which indicates that everything is fine). Use the Amazon OpenSearch Service console to check this. @@ -175,7 +175,7 @@ In case there is a need to run additional command in the terminal of the Django ### Hotfixes - To deploy a hotfix to pre-prod, you should fork from the latest release branch, and after preparing the fix, merge it back. Merging will trigger the `Deploy to AWS` workflow that will deploy the hotfix to the **running** pre-prod environment. -- To release a hotfix to production and staging, you should fork from the latest release branch, and after preparing the fix, merge it back. For production you have to make active the `disable_list_uploading` switch. The last step is to execute the `Release [Deploy]` workflow for each environment separately, which will deploy the fix to these two environments. If you need to clear OpenSearch indexes during deployment, you must select the Clear OpenSearch indexes checkbox. At the end for production make inactive the `disable_list_uploading` switch. +- To release a hotfix to production and staging, you should fork from the latest release branch, and after preparing the fix, merge it back. For production you have to make active the `disable_list_uploading` switch. The last step is to execute the `Release [Deploy]` workflow for each environment separately, which will deploy the fix to these two environments. If you need to clear the custom OpenSearch indexes and templates during deployment, you must select the `Clear the custom OpenSearch indexes and templates` checkbox. At the end for production make inactive the `disable_list_uploading` switch. ### Quick-fixes