diff --git a/.github/workflows/deploy_to_aws.yml b/.github/workflows/deploy_to_aws.yml index 2def34ede..a4c791814 100644 --- a/.github/workflows/deploy_to_aws.yml +++ b/.github/workflows/deploy_to_aws.yml @@ -224,8 +224,8 @@ jobs: uses: docker/build-push-action@v2 if: ${{ steps.get_env_name.outputs.lowercase == 'production' }} with: - context: deployment/terraform/database_anonymizer_sheduled_task/docker - file: deployment/terraform/database_anonymizer_sheduled_task/docker/Dockerfile + context: deployment/terraform/database_anonymizer_scheduled_task/docker + file: deployment/terraform/database_anonymizer_scheduled_task/docker/Dockerfile push: true tags: ${{ vars.ECR_REGISTRY }}/${{ vars.IMAGE_NAME }}-database-anonymizer-${{ steps.get_env_name.outputs.lowercase }}:${{ env.GIT_COMMIT }} diff --git a/deployment/environments/terraform-development.tfvars b/deployment/environments/terraform-development.tfvars index 8a593143b..d25e2c638 100644 --- a/deployment/environments/terraform-development.tfvars +++ b/deployment/environments/terraform-development.tfvars @@ -14,15 +14,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "128" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.t3.micro" rds_database_identifier = "opensupplyhub-enc-stg" rds_database_name = "opensupplyhub" rds_multi_az = false rds_storage_encrypted = true -snapshot_identifier = "" -rds_deletion_protection = true +rds_allow_major_version_upgrade = true +rds_apply_immediately = true app_ecs_desired_count = "1" app_ecs_deployment_min_percent = "100" diff --git a/deployment/environments/terraform-preprod.tfvars b/deployment/environments/terraform-preprod.tfvars index 380714645..6b65c6bab 100644 --- a/deployment/environments/terraform-preprod.tfvars +++ b/deployment/environments/terraform-preprod.tfvars @@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.m6in.4xlarge" rds_database_identifier = "opensupplyhub-enc-pp" rds_database_name = "opensupplyhub" diff --git a/deployment/environments/terraform-production.tfvars b/deployment/environments/terraform-production.tfvars index 933cc1ebb..c77987158 100644 --- a/deployment/environments/terraform-production.tfvars +++ b/deployment/environments/terraform-production.tfvars @@ -13,13 +13,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.m6in.4xlarge" rds_database_identifier = "opensupplyhub-enc-prd" rds_database_name = "opensupplyhub" rds_multi_az = false rds_storage_encrypted = true +rds_allow_major_version_upgrade = true +rds_apply_immediately = true app_ecs_desired_count = "10" app_ecs_deployment_min_percent = "100" diff --git a/deployment/environments/terraform-staging.tfvars b/deployment/environments/terraform-staging.tfvars index fd586e28c..616369e98 100644 --- a/deployment/environments/terraform-staging.tfvars +++ b/deployment/environments/terraform-staging.tfvars @@ -12,13 +12,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "128" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.t3.large" rds_database_identifier = "opensupplyhub-enc-stg" rds_database_name = "opensupplyhub" rds_multi_az = false rds_storage_encrypted = true +rds_allow_major_version_upgrade = true +rds_apply_immediately = true app_ecs_desired_count = "4" app_ecs_deployment_min_percent = "100" diff --git a/deployment/environments/terraform-test.tfvars b/deployment/environments/terraform-test.tfvars index b2a7c4b25..adccf0196 100644 --- a/deployment/environments/terraform-test.tfvars +++ b/deployment/environments/terraform-test.tfvars @@ -14,13 +14,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.t3.xlarge" rds_database_identifier = "opensupplyhub-enc-tst" rds_database_name = "opensupplyhub" rds_multi_az = false rds_storage_encrypted = true +rds_allow_major_version_upgrade = true +rds_apply_immediately = true anonymized_database_instance_type = "db.t3.2xlarge" anonymized_database_identifier = "database-anonymizer" diff --git a/deployment/terraform/anonymize_db_job.tf b/deployment/terraform/anonymize_db_job.tf index eb859113e..7bd8d7fe6 100644 --- a/deployment/terraform/anonymize_db_job.tf +++ b/deployment/terraform/anonymize_db_job.tf @@ -1,7 +1,7 @@ module "database_anonymizer" { count = var.database_anonymizer_enabled == true ? 1 : 0 - source = "./database_anonymizer_sheduled_task" + source = "./database_anonymizer_scheduled_task" rds_database_identifier = var.rds_database_identifier rds_database_name = var.rds_database_name diff --git a/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile b/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile index 6e0b7e9b9..fc2f8e174 100644 --- a/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile +++ b/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM postgis/postgis:12-3.4-alpine +FROM postgis/postgis:13-3.4-alpine WORKDIR /opt/ diff --git a/deployment/terraform/database.tf b/deployment/terraform/database.tf index 4e703b814..540d6af62 100644 --- a/deployment/terraform/database.tf +++ b/deployment/terraform/database.tf @@ -78,28 +78,30 @@ resource "aws_db_parameter_group" "default" { module "database_enc" { source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.0.3" - vpc_id = module.vpc.id - allocated_storage = var.rds_allocated_storage - engine_version = var.rds_engine_version - instance_type = var.rds_instance_type - storage_type = var.rds_storage_type - database_identifier = var.rds_database_identifier - database_name = var.rds_database_name - database_username = var.rds_database_username - database_password = var.rds_database_password - backup_retention_period = var.rds_backup_retention_period - backup_window = var.rds_backup_window - maintenance_window = var.rds_maintenance_window - auto_minor_version_upgrade = var.rds_auto_minor_version_upgrade - final_snapshot_identifier = join("-", [var.rds_final_snapshot_identifier, formatdate("YYYYMMDDhhmmss", timestamp())]) - skip_final_snapshot = var.rds_skip_final_snapshot - copy_tags_to_snapshot = var.rds_copy_tags_to_snapshot - multi_availability_zone = var.rds_multi_az - storage_encrypted = var.rds_storage_encrypted - subnet_group = aws_db_subnet_group.default.name - parameter_group = aws_db_parameter_group.default.name - deletion_protection = var.rds_deletion_protection - snapshot_identifier = var.snapshot_identifier + vpc_id = module.vpc.id + allocated_storage = var.rds_allocated_storage + engine_version = var.rds_engine_version + instance_type = var.rds_instance_type + storage_type = var.rds_storage_type + database_identifier = var.rds_database_identifier + database_name = var.rds_database_name + database_username = var.rds_database_username + database_password = var.rds_database_password + backup_retention_period = var.rds_backup_retention_period + backup_window = var.rds_backup_window + maintenance_window = var.rds_maintenance_window + auto_minor_version_upgrade = var.rds_auto_minor_version_upgrade + allow_major_version_upgrade = var.rds_allow_major_version_upgrade + apply_immediately = var.rds_apply_immediately + final_snapshot_identifier = join("-", [var.rds_final_snapshot_identifier, formatdate("YYYYMMDDhhmmss", timestamp())]) + skip_final_snapshot = var.rds_skip_final_snapshot + copy_tags_to_snapshot = var.rds_copy_tags_to_snapshot + multi_availability_zone = var.rds_multi_az + storage_encrypted = var.rds_storage_encrypted + subnet_group = aws_db_subnet_group.default.name + parameter_group = aws_db_parameter_group.default.name + deletion_protection = var.rds_deletion_protection + snapshot_identifier = var.snapshot_identifier alarm_cpu_threshold = var.rds_cpu_threshold_percent alarm_disk_queue_threshold = var.rds_disk_queue_threshold diff --git a/deployment/terraform/database_anonymizer_sheduled_task/.gitignore b/deployment/terraform/database_anonymizer_scheduled_task/.gitignore similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/.gitignore rename to deployment/terraform/database_anonymizer_scheduled_task/.gitignore diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/.dockerignore b/deployment/terraform/database_anonymizer_scheduled_task/docker/.dockerignore similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/.dockerignore rename to deployment/terraform/database_anonymizer_scheduled_task/docker/.dockerignore diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/Dockerfile b/deployment/terraform/database_anonymizer_scheduled_task/docker/Dockerfile similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/Dockerfile rename to deployment/terraform/database_anonymizer_scheduled_task/docker/Dockerfile diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/anonymize_script.sql b/deployment/terraform/database_anonymizer_scheduled_task/docker/anonymize_script.sql similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/anonymize_script.sql rename to deployment/terraform/database_anonymizer_scheduled_task/docker/anonymize_script.sql diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/database_anonymizer.py b/deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py similarity index 98% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/database_anonymizer.py rename to deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py index 6a7e9a99a..754c9ad7e 100755 --- a/deployment/terraform/database_anonymizer_sheduled_task/docker/database_anonymizer.py +++ b/deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py @@ -77,9 +77,6 @@ ) db = pg8000.native.Connection(**connection_information) -# cur = db.cursor() -# cur.execute(open("anonymize_script.sql", "r").read()) -# cur.commit() db.run(open("anonymize_script.sql", "r").read()) print('Database anonymized successfully!') diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/requirements.txt b/deployment/terraform/database_anonymizer_scheduled_task/docker/requirements.txt similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/requirements.txt rename to deployment/terraform/database_anonymizer_scheduled_task/docker/requirements.txt diff --git a/deployment/terraform/database_anonymizer_sheduled_task/kms.tf b/deployment/terraform/database_anonymizer_scheduled_task/kms.tf similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/kms.tf rename to deployment/terraform/database_anonymizer_scheduled_task/kms.tf diff --git a/deployment/terraform/database_anonymizer_sheduled_task/locals.tf b/deployment/terraform/database_anonymizer_scheduled_task/locals.tf similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/locals.tf rename to deployment/terraform/database_anonymizer_scheduled_task/locals.tf diff --git a/deployment/terraform/database_anonymizer_sheduled_task/main.tf b/deployment/terraform/database_anonymizer_scheduled_task/main.tf similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/main.tf rename to deployment/terraform/database_anonymizer_scheduled_task/main.tf diff --git a/deployment/terraform/database_anonymizer_sheduled_task/variables.tf b/deployment/terraform/database_anonymizer_scheduled_task/variables.tf similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/variables.tf rename to deployment/terraform/database_anonymizer_scheduled_task/variables.tf diff --git a/deployment/terraform/variables.tf b/deployment/terraform/variables.tf index e275140bd..a6eae78a3 100644 --- a/deployment/terraform/variables.tf +++ b/deployment/terraform/variables.tf @@ -70,11 +70,11 @@ variable "rds_allocated_storage" { } variable "rds_engine_version" { - default = "12.4" + default = "13" } variable "rds_parameter_group_family" { - default = "postgres12" + default = "postgres13" } variable "rds_instance_type" { @@ -114,6 +114,18 @@ variable "rds_auto_minor_version_upgrade" { default = true } +variable "rds_allow_major_version_upgrade" { + default = false + type = bool + description = "Indicates that major PostgreSQL engine version upgrades are allowed." +} + +variable "rds_apply_immediately" { + default = false + type = bool + description = "Specifies whether any database modifications are applied immediately, or during the next maintenance window." +} + variable "rds_final_snapshot_identifier" { default = "osh-rds-snapshot" } diff --git a/src/anon-tools/Dockerfile.dump b/src/anon-tools/Dockerfile.dump index 52478c73e..8a0d946d4 100644 --- a/src/anon-tools/Dockerfile.dump +++ b/src/anon-tools/Dockerfile.dump @@ -1,4 +1,4 @@ -FROM postgis/postgis:12-3.4-alpine +FROM postgis/postgis:13-3.4-alpine WORKDIR /opt/ @@ -31,9 +31,8 @@ RUN chmod 644 ~/.ssh/known_hosts COPY ./do_dump.sh ./do_dump.sh -COPY ./initdb.sql /docker-entrypoint-initdb.d VOLUME /keys -# ENTRYPOINT [ "docker-entrypoint.sh" ] + CMD ["sh", "do_dump.sh"] diff --git a/src/anon-tools/Dockerfile.restore b/src/anon-tools/Dockerfile.restore index a8fb26843..7236a5f5c 100644 --- a/src/anon-tools/Dockerfile.restore +++ b/src/anon-tools/Dockerfile.restore @@ -1,4 +1,4 @@ -FROM postgis/postgis:12-3.4-alpine +FROM postgis/postgis:13-3.4-alpine WORKDIR /opt/ diff --git a/src/anon-tools/anon.sql b/src/anon-tools/anon.sql deleted file mode 100644 index d71a213d3..000000000 --- a/src/anon-tools/anon.sql +++ /dev/null @@ -1,35 +0,0 @@ --- DECLARE --- current_table text; --- column_exists_email boolean; --- column_exists_username boolean; --- column_exists_password boolean; --- column_exists_phone_number boolean; --- BEGIN --- FOR current_table IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') --- LOOP --- EXECUTE 'SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = ''' || current_table || ''' AND column_name = ''email'')' INTO column_exists_email; --- EXECUTE 'SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = ''' || current_table || ''' AND column_name = ''username'')' INTO column_exists_username; --- EXECUTE 'SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = ''' || current_table || ''' AND column_name = ''password'')' INTO column_exists_password; --- EXECUTE 'SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = ''' || current_table || ''' AND column_name = ''phone_number'')' INTO column_exists_phone_number; - --- IF column_exists_email THEN --- EXECUTE 'UPDATE ' || current_table || ' SET --- email = CASE WHEN email NOT LIKE ''%@speedandfunction.com'' AND email NOT LIKE ''%@opensupplyhub.org'' THEN md5(random()::text) || ''@'' || substring(email from position(''@'' in email) + 1) ELSE email END'; - --- IF column_exists_username THEN --- EXECUTE 'UPDATE ' || current_table || ' SET --- username = CASE WHEN email NOT LIKE ''%@speedandfunction.com'' AND email NOT LIKE ''%@opensupplyhub.org'' THEN substr(md5(random()::text), 1, 20) ELSE username END'; --- END IF; - --- IF column_exists_password THEN --- EXECUTE 'UPDATE ' || current_table || ' SET --- password = CASE WHEN email NOT LIKE ''%@speedandfunction.com'' AND email NOT LIKE ''%@opensupplyhub.org'' THEN md5(random()::text) ELSE password END'; --- END IF; - --- IF column_exists_phone_number THEN --- EXECUTE 'UPDATE ' || current_table || ' SET --- phone_number = CASE WHEN email NOT LIKE ''%@speedandfunction.com'' AND email NOT LIKE ''%@opensupplyhub.org'' THEN md5(random()::text) ELSE phone_number END'; --- END IF; --- END IF; --- END LOOP; --- END; diff --git a/src/anon-tools/do_dump.sh b/src/anon-tools/do_dump.sh index bc5bd0ab7..90720462b 100755 --- a/src/anon-tools/do_dump.sh +++ b/src/anon-tools/do_dump.sh @@ -69,7 +69,7 @@ pg_dump --clean --no-owner --no-privileges -Fc -d anondb -U anondb -f /dumps/os ls -la /dumps -echo "Finshed anonymization" +echo "Finished anonymization" AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_TEST \ AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_TEST \ diff --git a/src/anon-tools/initdb.sql b/src/anon-tools/initdb.sql deleted file mode 100644 index 4b2e2f148..000000000 --- a/src/anon-tools/initdb.sql +++ /dev/null @@ -1,20 +0,0 @@ --- CREATE EXTENSION plpgsql; -CREATE EXTENSION btree_gin; -CREATE EXTENSION pg_trgm; -CREATE EXTENSION pgcrypto; -CREATE EXTENSION unaccent; --- CREATE EXTENSION postgis; - --- CREATE EXTENSION postgis; --- -- enabling raster support --- CREATE EXTENSION postgis_raster; - --- -- enabling advanced 3d support --- CREATE EXTENSION postgis_sfcgal; --- -- enabling SQL/MM Net Topology --- CREATE EXTENSION postgis_topology; - --- -- using US census data for geocoding and standardization --- CREATE EXTENSION address_standardizer; --- CREATE EXTENSION fuzzystrmatch; --- CREATE EXTENSION postgis_tiger_geocoder; diff --git a/src/database/Dockerfile.local b/src/database/Dockerfile.local deleted file mode 100644 index 241989ddd..000000000 --- a/src/database/Dockerfile.local +++ /dev/null @@ -1,35 +0,0 @@ -FROM postgres:12.17 - -ENV POSTGRES_USER=opensupplyhub \ - POSTGRES_PASSWORD=opensupplyhub \ - POSTGRES_DB=opensupplyhub - - -# Build and install the PostGIS package from scratch to ensure it matches the -# LLVM version used by the Docker container based on postgres:12.17 image. -# See PostGIS installation details at https://postgis.net/docs/manual-3.4/postgis_installation.html - -# Install the requirements for building and using PostGIS. -RUN apt-get update && apt-get install -y \ - wget \ - tar \ - build-essential \ - libproj-dev \ - libgeos-dev \ - libjson-c-dev \ - libgdal-dev \ - libprotobuf-c-dev \ - protobuf-c-compiler \ - pkg-config \ - postgresql-server-dev-12 - -# Download the PostGIS source archive, build and install the PostGIS package. -RUN wget http://download.osgeo.org/postgis/source/postgis-3.4.0.tar.gz && \ - tar -xvzf postgis-3.4.0.tar.gz && \ - cd /postgis-3.4.0 && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf postgis-3.4.0 && \ - rm postgis-3.4.0.tar.gz diff --git a/src/django/api/migrations/0163_upgrade_postgres_extensions.py b/src/django/api/migrations/0163_upgrade_postgres_extensions.py new file mode 100644 index 000000000..073f57958 --- /dev/null +++ b/src/django/api/migrations/0163_upgrade_postgres_extensions.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.17 on 2025-01-14 11:17 + +from django.db.migrations import Migration, RunPython +from django.db import connection +from api.migrations._migration_helper import MigrationHelper + +helper = MigrationHelper(connection) + + +def perform_upgrading_pg_extensions(apps, schema_editor): + helper.run_sql_files([ + '0163_upgrade_postgres_extensions.sql' + ]) + + +class Migration(Migration): + ''' + This migration upgrades the PostgreSQL database extension versions. + + Currently, the database uses the following extensions: + 1. postgis + 2. unaccent + 3. pg_trgm + 4. plpgsql + 5. btree_gin + 6. pgcrypto + + Based on the available extension versions for PostgreSQL 13.15 in AWS RDS, + which will be used across all AWS environments after the database upgrade, + it was found that the `postgis` extension can be upgraded to version 3.4.2 + in Production and Staging. Additionally, the `pg_trgm` extension can be + upgraded to version 1.5 in Development, Test, Production, and Staging. + If the specified versions are already installed in the database, there + will be no issues. + ''' + + dependencies = [ + ('api', '0162_update_moderationevent_table_fields'), + ] + + operations = [ + RunPython(perform_upgrading_pg_extensions) + ] diff --git a/src/django/sqls/0163_upgrade_postgres_extensions.sql b/src/django/sqls/0163_upgrade_postgres_extensions.sql new file mode 100644 index 000000000..92c02e94b --- /dev/null +++ b/src/django/sqls/0163_upgrade_postgres_extensions.sql @@ -0,0 +1,2 @@ +ALTER EXTENSION postgis UPDATE TO '3.4.2'; +ALTER EXTENSION pg_trgm UPDATE TO '1.5'; \ No newline at end of file