[OSDEV-1514] Database. Upgrade the PostgreSQL version to 13

.github/workflows/deploy_to_aws.yml

@@ -224,8 +224,8 @@ jobs:
         uses: docker/build-push-action@v2
         if: ${{ steps.get_env_name.outputs.lowercase == 'production' }}
         with:
-          context: deployment/terraform/database_anonymizer_sheduled_task/docker
-          file: deployment/terraform/database_anonymizer_sheduled_task/docker/Dockerfile
+          context: deployment/terraform/database_anonymizer_scheduled_task/docker
+          file: deployment/terraform/database_anonymizer_scheduled_task/docker/Dockerfile
           push: true
           tags: ${{ vars.ECR_REGISTRY }}/${{ vars.IMAGE_NAME }}-database-anonymizer-${{ steps.get_env_name.outputs.lowercase }}:${{ env.GIT_COMMIT }}
 

deployment/environments/terraform-development.tfvars

@@ -14,15 +14,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "128"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.t3.micro"
 rds_database_identifier = "opensupplyhub-enc-stg"
 rds_database_name = "opensupplyhub"
 rds_multi_az = false
 rds_storage_encrypted = true
-snapshot_identifier   = ""
-rds_deletion_protection = true
+rds_allow_major_version_upgrade = true
+rds_apply_immediately = true
 
 app_ecs_desired_count = "1"
 app_ecs_deployment_min_percent = "100"

deployment/environments/terraform-preprod.tfvars

@@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.m6in.4xlarge"
 rds_database_identifier = "opensupplyhub-enc-pp"
 rds_database_name = "opensupplyhub"

deployment/environments/terraform-production.tfvars

@@ -13,13 +13,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.m6in.4xlarge"
 rds_database_identifier = "opensupplyhub-enc-prd"
 rds_database_name = "opensupplyhub"
 rds_multi_az = false
 rds_storage_encrypted = true
+rds_allow_major_version_upgrade = true
+rds_apply_immediately = true
 
 app_ecs_desired_count = "10"
 app_ecs_deployment_min_percent = "100"

deployment/environments/terraform-staging.tfvars

@@ -12,13 +12,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "128"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.t3.large"
 rds_database_identifier = "opensupplyhub-enc-stg"
 rds_database_name = "opensupplyhub"
 rds_multi_az = false
 rds_storage_encrypted = true
+rds_allow_major_version_upgrade = true
+rds_apply_immediately = true
 
 app_ecs_desired_count = "4"
 app_ecs_deployment_min_percent = "100"

deployment/environments/terraform-test.tfvars

@@ -14,13 +14,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.t3.xlarge"
 rds_database_identifier = "opensupplyhub-enc-tst"
 rds_database_name = "opensupplyhub"
 rds_multi_az = false
 rds_storage_encrypted = true
+rds_allow_major_version_upgrade = true
+rds_apply_immediately = true
 
 anonymized_database_instance_type = "db.t3.2xlarge"
 anonymized_database_identifier = "database-anonymizer"

deployment/terraform/anonymize_db_job.tf

@@ -1,7 +1,7 @@
 module "database_anonymizer" {
   count = var.database_anonymizer_enabled == true ? 1 : 0
 
-  source = "./database_anonymizer_sheduled_task"
+  source = "./database_anonymizer_scheduled_task"
 
   rds_database_identifier       = var.rds_database_identifier
   rds_database_name             = var.rds_database_name

deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgis/postgis:12-3.4-alpine
+FROM postgis/postgis:13-3.4-alpine
 
 WORKDIR /opt/
 

deployment/terraform/database.tf

@@ -78,28 +78,30 @@ resource "aws_db_parameter_group" "default" {
 module "database_enc" {
   source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.0.3"
 
-  vpc_id                     = module.vpc.id
-  allocated_storage          = var.rds_allocated_storage
-  engine_version             = var.rds_engine_version
-  instance_type              = var.rds_instance_type
-  storage_type               = var.rds_storage_type
-  database_identifier        = var.rds_database_identifier
-  database_name              = var.rds_database_name
-  database_username          = var.rds_database_username
-  database_password          = var.rds_database_password
-  backup_retention_period    = var.rds_backup_retention_period
-  backup_window              = var.rds_backup_window
-  maintenance_window         = var.rds_maintenance_window
-  auto_minor_version_upgrade = var.rds_auto_minor_version_upgrade
-  final_snapshot_identifier  = join("-", [var.rds_final_snapshot_identifier, formatdate("YYYYMMDDhhmmss", timestamp())])
-  skip_final_snapshot        = var.rds_skip_final_snapshot
-  copy_tags_to_snapshot      = var.rds_copy_tags_to_snapshot
-  multi_availability_zone    = var.rds_multi_az
-  storage_encrypted          = var.rds_storage_encrypted
-  subnet_group               = aws_db_subnet_group.default.name
-  parameter_group            = aws_db_parameter_group.default.name
-  deletion_protection        = var.rds_deletion_protection
-  snapshot_identifier        = var.snapshot_identifier
+  vpc_id                      = module.vpc.id
+  allocated_storage           = var.rds_allocated_storage
+  engine_version              = var.rds_engine_version
+  instance_type               = var.rds_instance_type
+  storage_type                = var.rds_storage_type
+  database_identifier         = var.rds_database_identifier
+  database_name               = var.rds_database_name
+  database_username           = var.rds_database_username
+  database_password           = var.rds_database_password
+  backup_retention_period     = var.rds_backup_retention_period
+  backup_window               = var.rds_backup_window
+  maintenance_window          = var.rds_maintenance_window
+  auto_minor_version_upgrade  = var.rds_auto_minor_version_upgrade
+  allow_major_version_upgrade = var.rds_allow_major_version_upgrade
+  apply_immediately           = var.rds_apply_immediately
+  final_snapshot_identifier   = join("-", [var.rds_final_snapshot_identifier, formatdate("YYYYMMDDhhmmss", timestamp())])
+  skip_final_snapshot         = var.rds_skip_final_snapshot
+  copy_tags_to_snapshot       = var.rds_copy_tags_to_snapshot
+  multi_availability_zone     = var.rds_multi_az
+  storage_encrypted           = var.rds_storage_encrypted
+  subnet_group                = aws_db_subnet_group.default.name
+  parameter_group             = aws_db_parameter_group.default.name
+  deletion_protection         = var.rds_deletion_protection
+  snapshot_identifier         = var.snapshot_identifier
 
   alarm_cpu_threshold                = var.rds_cpu_threshold_percent
   alarm_disk_queue_threshold         = var.rds_disk_queue_threshold

deployment/terraform/database_anonymizer_sheduled_task/docker/database_anonymizer.py → deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py

@@ -77,9 +77,6 @@
 )
 
 db = pg8000.native.Connection(**connection_information)
-# cur = db.cursor()
-# cur.execute(open("anonymize_script.sql", "r").read())
-# cur.commit()
 db.run(open("anonymize_script.sql", "r").read())
 print('Database anonymized successfully!')
 

deployment/terraform/variables.tf

@@ -70,11 +70,11 @@ variable "rds_allocated_storage" {
 }
 
 variable "rds_engine_version" {
-  default = "12.4"
+  default = "13"
 }
 
 variable "rds_parameter_group_family" {
-  default = "postgres12"
+  default = "postgres13"
 }
 
 variable "rds_instance_type" {
@@ -114,6 +114,18 @@ variable "rds_auto_minor_version_upgrade" {
   default = true
 }
 
+variable "rds_allow_major_version_upgrade" {
+  default     = false
+  type        = bool
+  description = "Indicates that major PostgreSQL engine version upgrades are allowed."
+}
+
+variable "rds_apply_immediately" {
+  default     = false
+  type        = bool
+  description = "Specifies whether any database modifications are applied immediately, or during the next maintenance window."
+}
+
 variable "rds_final_snapshot_identifier" {
   default = "osh-rds-snapshot"
 }

src/anon-tools/Dockerfile.dump

@@ -1,4 +1,4 @@
-FROM postgis/postgis:12-3.4-alpine
+FROM postgis/postgis:13-3.4-alpine
 
 WORKDIR /opt/
 
@@ -31,9 +31,8 @@ RUN chmod 644 ~/.ssh/known_hosts
 
 
 COPY ./do_dump.sh ./do_dump.sh
-COPY ./initdb.sql /docker-entrypoint-initdb.d
 
 VOLUME /keys
-# ENTRYPOINT [ "docker-entrypoint.sh" ]
+
 CMD ["sh", "do_dump.sh"]
 

src/anon-tools/Dockerfile.restore

@@ -1,4 +1,4 @@
-FROM postgis/postgis:12-3.4-alpine
+FROM postgis/postgis:13-3.4-alpine
 
 WORKDIR /opt/
 

src/anon-tools/do_dump.sh

@@ -69,7 +69,7 @@ pg_dump --clean --no-owner --no-privileges -Fc -d anondb -U anondb  -f /dumps/os
 
 ls -la /dumps
 
-echo "Finshed anonymization"
+echo "Finished anonymization"
 
 AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_TEST \
     AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_TEST \

src/django/api/migrations/0163_upgrade_postgres_extensions.py

@@ -0,0 +1,43 @@
+# Generated by Django 3.2.17 on 2025-01-14 11:17
+
+from django.db.migrations import Migration, RunPython
+from django.db import connection
+from api.migrations._migration_helper import MigrationHelper
+
+helper = MigrationHelper(connection)
+
+
+def perform_upgrading_pg_extensions(apps, schema_editor):
+    helper.run_sql_files([
+        '0163_upgrade_postgres_extensions.sql'
+    ])
+
+
+class Migration(Migration):
+    '''
+    This migration upgrades the PostgreSQL database extension versions.
+
+    Currently, the database uses the following extensions:
+    1. postgis
+    2. unaccent
+    3. pg_trgm
+    4. plpgsql
+    5. btree_gin
+    6. pgcrypto
+
+    Based on the available extension versions for PostgreSQL 13.15 in AWS RDS,
+    which will be used across all AWS environments after the database upgrade,
+    it was found that the `postgis` extension can be upgraded to version 3.4.2
+    in Production and Staging. Additionally, the `pg_trgm` extension can be
+    upgraded to version 1.5 in Development, Test, Production, and Staging.
+    If the specified versions are already installed in the database, there
+    will be no issues.
+    '''
+
+    dependencies = [
+        ('api', '0162_update_moderationevent_table_fields'),
+    ]
+
+    operations = [
+        RunPython(perform_upgrading_pg_extensions)
+    ]

src/django/sqls/0163_upgrade_postgres_extensions.sql

@@ -0,0 +1,2 @@
+ALTER EXTENSION postgis UPDATE TO '3.4.2';
+ALTER EXTENSION pg_trgm UPDATE TO '1.5';