diff --git a/deployment/environments/terraform-development.tfvars b/deployment/environments/terraform-development.tfvars index 3d574d3ef..682831566 100644 --- a/deployment/environments/terraform-development.tfvars +++ b/deployment/environments/terraform-development.tfvars @@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "128" -rds_engine_version = "13" -rds_parameter_group_family = "postgres13" +rds_engine_version = "16" +rds_parameter_group_family = "postgres16" rds_instance_type = "db.t3.micro" rds_database_identifier = "opensupplyhub-enc-dev" rds_database_name = "opensupplyhub" diff --git a/deployment/environments/terraform-preprod.tfvars b/deployment/environments/terraform-preprod.tfvars index 6b65c6bab..6002509a0 100644 --- a/deployment/environments/terraform-preprod.tfvars +++ b/deployment/environments/terraform-preprod.tfvars @@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "13" -rds_parameter_group_family = "postgres13" +rds_engine_version = "16" +rds_parameter_group_family = "postgres16" rds_instance_type = "db.m6in.4xlarge" rds_database_identifier = "opensupplyhub-enc-pp" rds_database_name = "opensupplyhub" diff --git a/deployment/environments/terraform-production.tfvars b/deployment/environments/terraform-production.tfvars index c77987158..7bf035ca0 100644 --- a/deployment/environments/terraform-production.tfvars +++ b/deployment/environments/terraform-production.tfvars @@ -13,8 +13,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "13" -rds_parameter_group_family = "postgres13" +rds_engine_version = "16" +rds_parameter_group_family = "postgres16" rds_instance_type = "db.m6in.4xlarge" rds_database_identifier = "opensupplyhub-enc-prd" rds_database_name = "opensupplyhub" diff --git a/deployment/environments/terraform-staging.tfvars b/deployment/environments/terraform-staging.tfvars index 616369e98..5cd690ef9 100644 --- a/deployment/environments/terraform-staging.tfvars +++ b/deployment/environments/terraform-staging.tfvars @@ -12,8 +12,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "128" -rds_engine_version = "13" -rds_parameter_group_family = "postgres13" +rds_engine_version = "16" +rds_parameter_group_family = "postgres16" rds_instance_type = "db.t3.large" rds_database_identifier = "opensupplyhub-enc-stg" rds_database_name = "opensupplyhub" diff --git a/deployment/environments/terraform-test.tfvars b/deployment/environments/terraform-test.tfvars index adccf0196..a0bab75a3 100644 --- a/deployment/environments/terraform-test.tfvars +++ b/deployment/environments/terraform-test.tfvars @@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "13" -rds_parameter_group_family = "postgres13" +rds_engine_version = "16" +rds_parameter_group_family = "postgres16" rds_instance_type = "db.t3.xlarge" rds_database_identifier = "opensupplyhub-enc-tst" rds_database_name = "opensupplyhub" diff --git a/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile b/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile index fc2f8e174..513728347 100644 --- a/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile +++ b/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM postgis/postgis:13-3.4-alpine +FROM postgis/postgis:16-3.4-alpine WORKDIR /opt/ @@ -9,4 +9,3 @@ RUN set -ex && \ COPY ./dump.sh ./dump.sh CMD ["bash", "dump.sh"] - diff --git a/deployment/terraform/database.tf b/deployment/terraform/database.tf index 3ef35d234..7c85fb55b 100644 --- a/deployment/terraform/database.tf +++ b/deployment/terraform/database.tf @@ -76,7 +76,7 @@ resource "aws_db_parameter_group" "default" { } module "database_enc" { - source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.1.0" + source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.2.0" vpc_id = module.vpc.id allocated_storage = var.rds_allocated_storage diff --git a/deployment/terraform/variables.tf b/deployment/terraform/variables.tf index eca9d1f54..6b8742c00 100644 --- a/deployment/terraform/variables.tf +++ b/deployment/terraform/variables.tf @@ -70,11 +70,11 @@ variable "rds_allocated_storage" { } variable "rds_engine_version" { - default = "13" + default = "16" } variable "rds_parameter_group_family" { - default = "postgres13" + default = "postgres16" } variable "rds_instance_type" { diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index c85efd303..102f3e2f1 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -10,10 +10,11 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * Release date: February 8, 2025 ### Database changes -* *Describe high-level database changes.* +* [OSDEV-1515](https://opensupplyhub.atlassian.net/browse/OSDEV-1515) - Upgraded the PostgreSQL version from 13 to 16 for the database used in local development, DB anonymization, DB restore setup, and environments in the AWS cloud. Additionally, the pg_trgm extension has been upgraded to version 1.6 based on the available extension version for PostgreSQL 16.3 in AWS RDS. For more information, see [Extensions supported for RDS for PostgreSQL 16](https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html#postgresql-extensions-16x). #### Migrations: -* *Describe migrations here.* +* 0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py - Updated the SQL script within the migration that upgrades the DB extension versions to handle previously failure cases when a higher version is available for upgrade or when the extension is not installed. This is primarily useful for local development or DB resets in the Development environment, where migrations are applied from scratch, one by one. This fix will not negatively affect other environments, as the migration has already been applied and will not be reapplied. Additionally, the changes are backward compatible. +* 0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.py - This migration refreshes the `pg_statistic` table after the upgrade to PostgreSQL 16 and upgrades the pg_trgm extension to version 1.6. The SQL script within the migration that upgrades the DB extension versions handles previously failure cases where a higher version is available for upgrade or where the extension is not installed. #### Schema changes * *Describe schema changes here.* @@ -37,6 +38,10 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * Ensure that the following commands are included in the `post_deployment` command: * `migrate` * `reindex_database` +* This release will upgrade PostgreSQL from version 13 to version 16. + * The upgrade will be performed automatically by Terrafrom and AWS, but some steps need to be completed **before** and **after** the upgrade. Please refer to [the Confluence article](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/640155649/PostgreSQL+database+upgrade+from+version+13+to+version+16) for detailed instructions. + * Steps to be completed before the upgrade are marked with the statement: "**This should be done before deploying the upgraded database.**". Post-upgrade tasks can be found under the [After the PostgreSQL major version upgrade](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/640155649/PostgreSQL+database+upgrade+from+version+13+to+version+16#After-the-PostgreSQL-major-version-upgrade) section. + * In case of an unsuccessful release along with the database upgrade, follow the instructions under the [Guide for rolling back the PostgreSQL major version upgrade](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/640155649/PostgreSQL+database+upgrade+from+version+13+to+version+16#Guide-for-rolling-back-the-PostgreSQL-major-version-upgrade) section. ## Release 1.28.1 diff --git a/src/anon-tools/Dockerfile.dump b/src/anon-tools/Dockerfile.dump index 8a0d946d4..208a17d35 100644 --- a/src/anon-tools/Dockerfile.dump +++ b/src/anon-tools/Dockerfile.dump @@ -1,4 +1,4 @@ -FROM postgis/postgis:13-3.4-alpine +FROM postgis/postgis:16-3.4-alpine WORKDIR /opt/ @@ -35,4 +35,3 @@ COPY ./do_dump.sh ./do_dump.sh VOLUME /keys CMD ["sh", "do_dump.sh"] - diff --git a/src/anon-tools/Dockerfile.restore b/src/anon-tools/Dockerfile.restore index 7236a5f5c..07c404089 100644 --- a/src/anon-tools/Dockerfile.restore +++ b/src/anon-tools/Dockerfile.restore @@ -1,4 +1,4 @@ -FROM postgis/postgis:13-3.4-alpine +FROM postgis/postgis:16-3.4-alpine WORKDIR /opt/ @@ -29,4 +29,3 @@ COPY ./do_restore.sh ./ VOLUME /keys CMD ["sh", "do_restore.sh"] - diff --git a/src/database/Dockerfile.local b/src/database/Dockerfile.local index dca21dceb..7060a7dd0 100644 --- a/src/database/Dockerfile.local +++ b/src/database/Dockerfile.local @@ -1,11 +1,12 @@ -# The latest supported version of PostgreSQL in AWS RDS is 13.15. -# However, the official PostgreSQL Docker registry provides only -# the latest minor version for PostgreSQL 13, and it is not -# possible to specify a specific minor version like 13.15. Therefore, -# it was decided to provide only the major version and let Docker -# determine which minor version to install. This should not cause -# incompatibilities, even if PostgreSQL 13.15 is used in AWS. -FROM postgres:13 +# The latest supported version of PostgreSQL in AWS RDS is 16.3 +# for PostgreSQL 16. However, the official PostgreSQL Docker +# registry provides only the latest minor version for PostgreSQL 16, +# and it is not possible to specify a specific minor version like +# 16.3. Therefore, it was decided to provide only the major version +# and let Docker determine which minor version to install. This +# should not cause incompatibilities, even if PostgreSQL 16.3 is +# currently used in AWS. +FROM postgres:16 ENV POSTGRES_USER=opensupplyhub \ POSTGRES_PASSWORD=opensupplyhub \ @@ -13,7 +14,7 @@ ENV POSTGRES_USER=opensupplyhub \ # Build and install the PostGIS package from scratch to ensure it matches the -# LLVM version used by the Docker container based on postgres:13 image. +# LLVM version used by the Docker container based on postgres:16 image. # See PostGIS installation details at https://postgis.net/docs/manual-3.4/postgis_installation.html # Install the requirements for building and using PostGIS. @@ -28,7 +29,7 @@ RUN apt-get update && apt-get install -y \ libprotobuf-c-dev \ protobuf-c-compiler \ pkg-config \ - postgresql-server-dev-13 + postgresql-server-dev-16 # Download the PostGIS source archive, build and install the PostGIS package. RUN wget https://download.osgeo.org/postgis/source/postgis-3.4.2.tar.gz && \ diff --git a/src/django/Dockerfile b/src/django/Dockerfile index 229d34a00..cf27d22dc 100644 --- a/src/django/Dockerfile +++ b/src/django/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 python:3.7-slim-bullseye +FROM --platform=linux/amd64 python:3.7-slim-bookworm RUN mkdir -p /usr/local/src/static/static WORKDIR /usr/local/src @@ -12,7 +12,7 @@ RUN set -ex \ && deps=" \ gdal-bin \ gettext \ - postgresql-client-13 \ + postgresql-client-15 \ " \ && apt-get update && apt-get install -y $buildDeps $deps --no-install-recommends \ && pip install --no-cache-dir -r requirements.txt \ diff --git a/src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py b/src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py index 5db520f23..f07a0d283 100644 --- a/src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py +++ b/src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py @@ -17,7 +17,7 @@ def refresh_pg_statistic_and_perform_upgrading_pg_extensions( class Migration(Migration): ''' This migration refreshes the pg_statistic table and upgrades the - PostgreSQL database extensions. + PostgreSQL database extensions in light of the upgrade to Postgres 13. Since this migration will be executed after the PostgreSQL major version upgrade, the ANALYZE operation should be run to refresh the pg_statistic diff --git a/src/django/api/migrations/0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.py b/src/django/api/migrations/0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.py new file mode 100644 index 000000000..63d8093b2 --- /dev/null +++ b/src/django/api/migrations/0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.py @@ -0,0 +1,51 @@ +# Generated by Django 3.2.17 on 2025-01-24 15:21 + +from django.db.migrations import Migration, RunPython +from django.db import connection +from api.migrations._migration_helper import MigrationHelper + +helper = MigrationHelper(connection) + + +def refresh_pg_statistic_and_perform_upgrading_pg_extensions( + apps, schema_editor): + helper.run_sql_files([ + ('0164_refresh_pg_statistic_and_upgrade_postgres_extensions' + '_after_db_upgrade_to_postgres_16.sql') + ]) + + +class Migration(Migration): + ''' + This migration refreshes the pg_statistic table and upgrades the + PostgreSQL database extensions in light of the upgrade to Postgres 16. + + Since this migration will be executed after the PostgreSQL major version + upgrade, the ANALYZE operation should be run to refresh the pg_statistic + table. Optimizer statistics aren't transferred during a major version + upgrade, so it is necessary to regenerate all statistics to avoid + performance issues. + + Currently, the database uses the following extensions: + 1. postgis + 2. unaccent + 3. pg_trgm + 4. plpgsql + 5. btree_gin + 6. pgcrypto + + Based on the available extension versions for PostgreSQL 16.3 in AWS RDS, + which will be used across all AWS environments after the database upgrade + to Postgres 16, it was found that the `pg_trgm` extensions can be upgraded + to version 1.6, respectively, in Production, Staging, Test, and + Development environments. If the specified versions are already installed + in the database, there will be no issues. + ''' + + dependencies = [ + ('api', '0163_refresh_pg_statistic_and_upgrade_postgres_extensions'), + ] + + operations = [ + RunPython(refresh_pg_statistic_and_perform_upgrading_pg_extensions) + ] diff --git a/src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql b/src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql index 7c4045c6f..f8a5b9896 100644 --- a/src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql +++ b/src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql @@ -1,4 +1,54 @@ ANALYZE VERBOSE; -ALTER EXTENSION postgis UPDATE TO '3.4.2'; -ALTER EXTENSION pg_trgm UPDATE TO '1.5'; +DO $$ +DECLARE +-- Variables for postgis. +current_version_postgis TEXT; +target_version_postgis TEXT := '3.4.2'; + +-- Variables for pg_trgm. +current_version_pg_trgm TEXT; +target_version_pg_trgm TEXT := '1.5'; + +BEGIN +-- Extension 1: Check and upgrade "postgis" to its target version. +SELECT + extversion +INTO + current_version_postgis +FROM + pg_extension +WHERE + extname = 'postgis'; + +IF current_version_postgis IS NULL THEN + RAISE EXCEPTION 'Extension "postgis" is not installed.'; +ELSIF current_version_postgis < target_version_postgis +THEN + EXECUTE FORMAT('ALTER EXTENSION postgis UPDATE TO %L', target_version_postgis); + RAISE NOTICE 'Extension "postgis" has been upgraded from % to %.', current_version_postgis, target_version_postgis; +ELSE + RAISE NOTICE 'Extension "postgis" is already at the target version or higher.'; +END IF; + +-- Extension 2: Check and upgrade "pg_trgm" to its target version. +SELECT + extversion +INTO + current_version_pg_trgm +FROM + pg_extension +WHERE + extname = 'pg_trgm'; + +IF current_version_pg_trgm IS NULL +THEN + RAISE EXCEPTION 'Extension "pg_trgm" is not installed.'; +ELSIF current_version_pg_trgm < target_version_pg_trgm +THEN + EXECUTE FORMAT('ALTER EXTENSION pg_trgm UPDATE TO %L', target_version_pg_trgm); + RAISE NOTICE 'Extension "pg_trgm" has been upgraded from % to %.', current_version_pg_trgm, target_version_pg_trgm; +ELSE + RAISE NOTICE 'Extension "pg_trgm" is already at the target version or higher.'; +END IF; +END $$; diff --git a/src/django/sqls/0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.sql b/src/django/sqls/0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.sql new file mode 100644 index 000000000..7a37dbf9c --- /dev/null +++ b/src/django/sqls/0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.sql @@ -0,0 +1,30 @@ +ANALYZE VERBOSE; + +DO $$ +DECLARE +-- Variables for pg_trgm. +current_version_pg_trgm TEXT; +target_version_pg_trgm TEXT := '1.6'; + +BEGIN +-- Extension 1: Check and upgrade "pg_trgm" to its target version. +SELECT + extversion +INTO + current_version_pg_trgm +FROM + pg_extension +WHERE + extname = 'pg_trgm'; + +IF current_version_pg_trgm IS NULL +THEN + RAISE EXCEPTION 'Extension "pg_trgm" is not installed.'; +ELSIF current_version_pg_trgm < target_version_pg_trgm +THEN + EXECUTE FORMAT('ALTER EXTENSION pg_trgm UPDATE TO %L', target_version_pg_trgm); + RAISE NOTICE 'Extension "pg_trgm" has been upgraded from % to %.', current_version_pg_trgm, target_version_pg_trgm; +ELSE + RAISE NOTICE 'Extension "pg_trgm" is already at the target version or higher.'; +END IF; +END $$;