From 9edaf28580348324f8b3b6970b1a31a4f5b7f8ca Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 6 Nov 2024 20:38:55 -0500 Subject: [PATCH 1/4] chore: adding support for x86_64-darwin (#1310) * chore: adding support for x86_64-darwin * fix: handle logic for various darwin platforms * fix: != &&, == || * fix: support system here too * feature: support added for x86_64-darwin * fix: increase timeout so that aws access does not expire * fix: duration was too high * chore: see if 2 hours will work * fix: try more than 2 less than 3 hrs * chore: try a bit lower * chore: bump down lower * chore: set to the apparent max * chore: limit nix build to 15 on x86_64-darwin --------- Co-authored-by: Sam Rose --- .github/workflows/nix-build.yml | 13 ++++++++----- ansible/vars.yml | 4 ++-- docker/nix/build_nix.sh | 18 ++++++++++++------ flake.nix | 1 + nix/ext/plv8.nix | 2 +- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 08c316bce..473579875 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -24,8 +24,10 @@ jobs: arch: arm64 - runner: macos-latest arch: arm64 + - runner: macos-13 + arch: amd64 runs-on: ${{ matrix.runner }} - + timeout-minutes: 180 steps: - name: Check out code @@ -40,6 +42,7 @@ jobs: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: "us-east-1" output-credentials: true + role-duration-seconds: 7200 - name: write secret key # use python so we don't interpolate the secret into the workflow logs, in case of bugs run: | @@ -47,23 +50,23 @@ jobs: env: NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: Log in to Docker Hub - if: matrix.runner != 'macos-latest' + if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build psql bundle with nix - if: matrix.runner != 'macos-latest' + if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13' run: docker build -t base_nix -f docker/nix/Dockerfile . - name: Run build psql bundle - if: matrix.runner != 'macos-latest' + if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13' run: | docker run -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \ -e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \ -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \ base_nix bash -c "./workspace/docker/nix/build_nix.sh" - name: Build psql bundle on macos - if: matrix.runner == 'macos-latest' + if: matrix.runner == 'macos-latest' || matrix.runner == 'macos-13' run: | curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \ --extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ diff --git a/ansible/vars.yml b/ansible/vars.yml index da1b61704..1017ef18c 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.006" - postgres16: "16.3.1.012" + postgres15: "15.8.1.007" + postgres16: "16.3.1.013" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 3cf8505c0..5a7896a46 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,9 +5,15 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi -nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link -nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link -nix build .#psql_15/bin -o psql_15 -nix build .#psql_16/bin -o psql_16 -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 +if [ $(nix-instantiate --eval -E builtins.currentSystem | tr -d '"') == "x86_64-darwin" ]; then + nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link + nix build .#psql_15/bin -o psql_15 + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 +else + nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link + nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link + nix build .#psql_15/bin -o psql_15 + nix build .#psql_16/bin -o psql_16 + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 +fi diff --git a/flake.nix b/flake.nix index 615df5da5..f1cff1abd 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,7 @@ system.x86_64-linux system.aarch64-linux system.aarch64-darwin + system.x86_64-darwin ]; in flake-utils.lib.eachSystem ourSystems (system: diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 785f5e601..a331a1127 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -120,7 +120,7 @@ stdenv.mkDerivation (finalAttrs: { description = "V8 Engine Javascript Procedural Language add-on for PostgreSQL"; homepage = "https://plv8.github.io/"; maintainers = with maintainers; [ samrose ]; - platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; + platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" "x86_64-darwin" ]; license = licenses.postgresql; }; }) From f5cbb070f29ad9f6ebc1ee728c0b42675537c1dd Mon Sep 17 00:00:00 2001 From: Artur Zakirov Date: Thu, 7 Nov 2024 17:53:17 +0100 Subject: [PATCH 2/4] pg_repack 1.5.2: Allow to run pg_repack by non-superuser (#1314) New version allows to run pg_repack by a non-superuser. PR on pg_repack repo: https://github.com/reorg/pg_repack/pull/431 --- ansible/vars.yml | 4 ++-- nix/ext/pg_repack.nix | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 1017ef18c..747cfd2ff 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.007" - postgres16: "16.3.1.013" + postgres15: "15.8.1.008" + postgres16: "16.3.1.014" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/nix/ext/pg_repack.nix b/nix/ext/pg_repack.nix index f3247378a..fac666149 100644 --- a/nix/ext/pg_repack.nix +++ b/nix/ext/pg_repack.nix @@ -11,15 +11,15 @@ stdenv.mkDerivation (finalAttrs: { pname = "pg_repack"; - version = "1.5.0"; + version = "1.5.2"; buildInputs = postgresql.buildInputs ++ [ postgresql ]; src = fetchFromGitHub { owner = "reorg"; repo = "pg_repack"; - rev = "ver_${finalAttrs.version}"; - hash = "sha256-do80phyMxwcRIkYyUt9z02z7byNQhK+pbSaCUmzG+4c="; + rev = "85b64c6d4f599b2988343c4e7121acab505c9006"; + hash = "sha256-lAuLI+vupusvn3uTzQ9OaLqkEfUVMCAwU9R70tTbb8Y="; }; installPhase = '' From 4a1d4726d52c8670e2d3c35a91626c58b957d81a Mon Sep 17 00:00:00 2001 From: Pavel Borisov <63344111+pashkinelfe@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:56:16 +0400 Subject: [PATCH 3/4] Upgrade pgvector to 0.8.0 (#1317) * Upgrade pgvector to 0.8.0 * Upgrade extensions_sql_interface.out for pgvector 0.8.0 * Bump AMI version in vars.yml --- ansible/vars.yml | 8 ++++---- nix/ext/pgvector.nix | 4 ++-- nix/tests/expected/extensions_sql_interface.out | 6 +++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 747cfd2ff..555887b62 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.008" - postgres16: "16.3.1.014" + postgres15: "15.8.1.010" + postgres16: "16.3.1.016" # Non Postgres Extensions pgbouncer_release: "1.19.0" @@ -149,8 +149,8 @@ hypopg_release_checksum: sha256:9afe6357fd389d8d33fad81703038ce520b09275ec00153c pg_repack_release: "1.5.0" pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52 -pgvector_release: "0.7.4" -pgvector_release_checksum: sha256:0341edf89b1924ae0d552f617e14fb7f8867c0194ed775bcc44fa40288642583 +pgvector_release: "0.8.0" +pgvector_release_checksum: sha256:867a2c328d4928a5a9d6f052cd3bc78c7d60228a9b914ad32aa3db88e9de27b0 pg_tle_release: "1.3.2" pg_tle_release_checksum: sha256:d04f72d88b21b954656609743560684ac42645b64a36c800d4d2f84d1f180de1 diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index aefeeea3a..a7d58ec61 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pgvector"; - version = "0.7.4"; + version = "0.8.0"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "pgvector"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-qwPaguQUdDHV8q6GDneLq5MuhVroPizpbqt7f08gKJI="; + hash = "sha256-JsZV+I4eRMypXTjGmjCtMBXDVpqTIPHQa28ogXncE/Q="; }; installPhase = '' diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index fff273a88..9097efd94 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -5032,6 +5032,10 @@ order by vector | public | array_to_halfvec | integer[], integer, boolean | halfvec vector | public | array_to_halfvec | numeric[], integer, boolean | halfvec vector | public | array_to_halfvec | real[], integer, boolean | halfvec + vector | public | array_to_sparsevec | double precision[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | integer[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | numeric[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | real[], integer, boolean | sparsevec vector | public | array_to_vector | double precision[], integer, boolean | vector vector | public | array_to_vector | integer[], integer, boolean | vector vector | public | array_to_vector | numeric[], integer, boolean | vector @@ -5194,7 +5198,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5030 rows) +(5034 rows) /* From ffa1052400519bce6447df5c19616e71f801448e Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Thu, 14 Nov 2024 11:45:52 +0200 Subject: [PATCH 4/4] chore: merge `release/15.6` changes into `develop` (#1320) * fix: account for `public` grantee * fix(ci): respect postgresVersion input (#1237) * feat: bump gotrue version to v2.162.0 (#1241) * fix: only grant pg_read_all_data if it exists (#1242) * fix: only grant pg_read_all_data if it exists * fix: prevent `public` from being casted into `regrole` * fix(15.6): account for pg_stat_monitor major version upgrade (#1247) * chore: release updates to run physical backup as a service to 15.6 image (#1248) * chore: updates to run physical backups as a service (#1235) * chore: init commence-backup service * chore: bump adminapi and adminmgr * chore: bump version * fix: provide correct filename * chore: bump postgres version * fix(15.6/pg_upgrade): retry commands within the cleanup step; wait until PG is ready to accept connections (#1250) * fix(15.6/upgrades): collision when patching wrappers lib locations for upgrades (#1252) * feat: bump auth 2.162.1 on 15.6 (#1256) * fix(15.6): disable pg_stat_monitor (#1260) * fix: disable pg_stat_monitor * chore: bump version * fix(15.6): disable pg_stat_monitor (#1262) * fix: disable pg_stat_monitor * chore: bump version * feat: bump gotrue to v2.162.2 (#1264) * chore: add timescaledb 2.9.1; wrappers upgrade fix; wrappers & plv8 naming fix (#1259) Co-authored-by: Bobbie Soedirgo Co-authored-by: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Co-authored-by: Kang Ming Co-authored-by: Stojan Dimitrovski Co-authored-by: Sam Rose fix(ci): respect postgresVersion input (#1237) fix: only grant pg_read_all_data if it exists (#1242) fix(15.6): disable pg_stat_monitor (#1260) * chore(15.6): bump pg version (#1273) * feat: bump auth to v2.163.0 on 15.6 (#1275) * fix: restart PG during pre-upgrade steps to shed hanging connections (#1271) * fix(upgrades): wrappers 4.2.0 -> wrappers 4.2.0 (#1278) * fix: handle supabase_admin authenticator membership snowflake * feat: add auth v2.163.1 to 15.6 (#1283) * feat: bump gotrue to v2.163.2 (#1287) * pg_net 0.11 on 15.6 release branch (#1290) * bump pg_net to 0.11.0 * bump image to 15.6.1.135 * fix: don't copy custom extension scripts during pg_upgrade (#1291) * fix: add recursive flag to custom extension script directory delete (#1292) * [GEN-11027] chore: reserve a fixed amount of blocks for the data volume; remove ansible pkg + ppa (#1295) * fix: grant predefined roles post-upgrade * fix: add more roles to reserved_roles & reserved_memberships (#1303) * fix: add more roles to reserved_roles & reserved_memberships * Update common-nix.vars.pkr.hcl * feat: update envoy lds config with auth jwks, oidc URLs, strip `sb-opk` header (#1296) * chore: udpate package repo for salt (#1307) * fix: use sudo for apt-get commands * Update pg net to 0.13.0 on pg 15.6 branch (#1315) * upgrade pg_net to 0.13.0 on 15.6 * bump postgres-version * chore: cleanup pgbouncer.get_auth ownership accordingly * chore: standardize * feat: filter timescaledb to only be included on v15 (#1321) * feat: filter timescaledb to only be included on v15 * chore: shuffle a bit to match what is needed for various versions * chore: small fixes on sb wrappers and plv8 to merge in all needed install function --------- Co-authored-by: Bobbie Soedirgo Co-authored-by: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Co-authored-by: Kang Ming Co-authored-by: angelico Co-authored-by: Stojan Dimitrovski Co-authored-by: Sam Rose Co-authored-by: Oliver Rice --- .../publish-nix-pgupgrade-scripts.yml | 3 + .../pg_upgrade_scripts/common.sh | 12 +++ .../pg_upgrade_scripts/complete.sh | 3 + .../pg_upgrade_scripts/initiate.sh | 81 ++++++------------- .../files/postgresql_config/supautils.conf.j2 | 4 +- ansible/vars.yml | 4 +- .../scripts/surrogate-bootstrap-nix.sh | 5 ++ flake.nix | 12 ++- nix/ext/plv8.nix | 42 +++++++--- nix/ext/timescaledb-2.9.1.nix | 51 ++++++++++++ nix/ext/wrappers/default.nix | 65 ++++++++++----- scripts/90-cleanup.sh | 5 +- scripts/nix-provision.sh | 6 +- 13 files changed, 200 insertions(+), 93 deletions(-) create mode 100644 nix/ext/timescaledb-2.9.1.nix diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 176f57a03..6fb7f1830 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -101,6 +101,9 @@ jobs: run: | VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + if [[ "${{ inputs.postgresVersion }}" != "" ]]; then + VERSION=${{ inputs.postgresVersion }} + fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index a11f8ff32..ea7217fcc 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -91,12 +91,24 @@ swap_postgres_and_supabase_admin() { alter database postgres connection limit 0; select pg_terminate_backend(pid) from pg_stat_activity where backend_type = 'client backend' and pid != pg_backend_pid(); EOSQL + + if [ -z "$IS_CI" ]; then + retry 5 systemctl restart postgresql + else + CI_start_postgres "" + fi + + retry 8 pg_isready -h localhost -U supabase_admin + run_sql <<'EOSQL' set statement_timeout = '600s'; begin; create role supabase_tmp superuser; set session authorization supabase_tmp; +-- to handle snowflakes that happened in the past +revoke supabase_admin from authenticator; + do $$ begin if exists (select from pg_extension where extname = 'timescaledb') then diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index d19b72422..c8542c685 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -78,6 +78,9 @@ EOF run_sql -c "$RECREATE_PG_CRON_QUERY" fi + + # #incident-2024-09-12-project-upgrades-are-temporarily-disabled + run_sql -c "grant pg_read_all_data, pg_signal_backend to postgres" } function complete_pg_upgrade { diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 235b4f4c1..88518766b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -39,6 +39,7 @@ MOUNT_POINT="/data_migration" LOG_FILE="/var/log/pg-upgrade-initiate.log" POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" +POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") @@ -47,7 +48,6 @@ SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" PGBINOLD="/usr/lib/postgresql/bin" -PGLIBOLD="/usr/lib/postgresql/lib" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" @@ -133,6 +133,22 @@ cleanup() { echo "Resetting postgres database connection limit" retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" + echo "Making sure postgres still has access to pg_shadow" + cat << EOF >> $POST_UPGRADE_POSTGRES_PERMS_SCRIPT +DO \$\$ +begin + if exists (select from pg_authid where rolname = 'pg_read_all_data') then + execute('grant pg_read_all_data to postgres'); + end if; +end +\$\$; +grant pg_signal_backend to postgres; +EOF + + if [ -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT ]; then + retry 5 run_sql -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT + fi + if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then echo "Unmounting data disk from ${MOUNT_POINT}" retry 3 umount $MOUNT_POINT @@ -148,6 +164,14 @@ cleanup() { } function handle_extensions { + if [ -z "$IS_CI" ]; then + retry 5 systemctl restart postgresql + else + CI_start_postgres + fi + + retry 8 pg_isready -h localhost -U supabase_admin + rm -f $POST_UPGRADE_EXTENSION_SCRIPT touch $POST_UPGRADE_EXTENSION_SCRIPT @@ -181,58 +205,6 @@ EOF done } -function patch_wrappers { - local IS_NIX_UPGRADE=$1 - - WRAPPERS_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'wrappers');") - if [ "$WRAPPERS_ENABLED" = "f" ]; then - echo "Wrappers extension not enabled. Skipping." - return - fi - - # This is a workaround for older versions of wrappers which don't have the expected - # naming scheme, containing the version in their library's file name - # e.g. wrappers-0.1.16.so, rather than wrappers.so - # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's - # library directory, so we're making sure the new version has the expected (old version's) - # file name. - # After the upgrade completes, the new version's library file is used. - # i.e. - # - old version: wrappers-0.1.16.so - # - new version: wrappers-0.1.18.so - # - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so - if [ "$IS_NIX_UPGRADE" = "true" ]; then - if [ -d "$PGLIBOLD" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - - find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do - if [ -f "$WRAPPERS_LIB_PATH" ]; then - WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true - fi - fi - done - fi - else - if [ -d "$PGLIBOLD" ]; then - WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit) - if [ -f "$WRAPPERS_LIB_PATH" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then - LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true - fi - fi - fi - fi - fi -} - function initiate_upgrade { mkdir -p "$MOUNT_POINT" SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/") @@ -409,8 +381,6 @@ function initiate_upgrade { export LD_LIBRARY_PATH="${PGLIBNEW}" fi - patch_wrappers "$IS_NIX_UPGRADE" - echo "9. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" @@ -473,6 +443,7 @@ EOF cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" # removing supautils config as to allow the latest one provided by the latest image to be used rm -f "$MOUNT_POINT/conf/supautils.conf" || true + rm -rf "$MOUNT_POINT/conf/extension-custom-scripts" || true # removing wal-g config as to allow it to be explicitly enabled on the new instance rm -f "$MOUNT_POINT/conf/wal-g.conf" diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index b2112cf98..086d6df9e 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -9,5 +9,5 @@ supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/ex supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres' supautils.privileged_role_allowed_configs = 'auto_explain.log_min_duration, auto_explain.log_nested_statements, log_lock_waits, log_min_messages, pg_net.batch_size, pg_net.ttl, pgaudit.log, pgaudit.log_catalog, pgaudit.log_client, pgaudit.log_level, pgaudit.log_relation, pgaudit.log_rows, pgaudit.log_statement, pgaudit.log_statement_once, pgaudit.role, pgrst.*, plan_filter.*, safeupdate.enabled, session_replication_role, track_io_timing' -supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, authenticator' -supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' +supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, authenticator' +supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' diff --git a/ansible/vars.yml b/ansible/vars.yml index 555887b62..d059b5f58 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -25,8 +25,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.162.0 -gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1 +gotrue_release: 2.163.2 +gotrue_release_checksum: sha1:31889bc8c498b924c2cb3b6c4084ef6e57ed97c0 aws_cli_release: "2.2.7" diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 45cbc8ac6..3b0bb93c8 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -110,6 +110,11 @@ function format_and_mount_rootfs { fi mkfs.ext4 /dev/xvdh + + # Explicitly reserving 100MiB worth of blocks for the data volume + RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096)) + tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT /dev/xvdh + mkdir -p /mnt/data mount -o defaults,discard /dev/xvdh /mnt/data } diff --git a/flake.nix b/flake.nix index f1cff1abd..b66ec3449 100644 --- a/flake.nix +++ b/flake.nix @@ -213,9 +213,15 @@ in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; makeOurPostgresPkgs = version: - let postgresql = getPostgresqlPackage version; - in map (path: pkgs.callPackage path { inherit postgresql; }) ourExtensions; - + let + postgresql = getPostgresqlPackage version; + extensions = if version == "15" + then ourExtensions ++ [ + ./nix/ext/timescaledb-2.9.1.nix + ] + else ourExtensions; + in + map (path: pkgs.callPackage path { inherit postgresql; }) extensions; # Create an attrset that contains all the extensions included in a server for the orioledb version of postgresql + extension. makeOurOrioleDbPostgresPkgsSet = version: patchedPostgres: (builtins.listToAttrs (map diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index a331a1127..340a07382 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -97,23 +97,43 @@ stdenv.mkDerivation (finalAttrs: { ''} ''; - postInstall = '' + postInstall = '' # Move the redirected to proper directory. # There appear to be no references to the install directories # so changing them does not cause issues. mv "$out/nix/store"/*/* "$out" rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" - ${lib.optionalString stdenv.isDarwin '' - install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} - install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} - install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} - install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} - ''} - - ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} - ''} + # Handle different PostgreSQL versions + if [ "${lib.versions.major postgresql.version}" = "15" ]; then + mv "$out/lib/plv8-${finalAttrs.version}.so" "$out/lib/plv8.so" + ln -s "$out/lib/plv8.so" "$out/lib/plv8-${finalAttrs.version}.so" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plv8.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plcoffee.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plls.control" + + ${lib.optionalString stdenv.isDarwin '' + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8.so + ''} + + ${lib.optionalString (!stdenv.isDarwin) '' + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so + ''} + else + ${lib.optionalString stdenv.isDarwin '' + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + ''} + + ${lib.optionalString (!stdenv.isDarwin) '' + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + ''} + fi ''; meta = with lib; { diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix new file mode 100644 index 000000000..ad955e86f --- /dev/null +++ b/nix/ext/timescaledb-2.9.1.nix @@ -0,0 +1,51 @@ +{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: + +stdenv.mkDerivation rec { + pname = "timescaledb-apache"; + version = "2.9.1"; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ postgresql openssl libkrb5 ]; + + src = fetchFromGitHub { + owner = "timescale"; + repo = "timescaledb"; + rev = version; + hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; + }; + + cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] + ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + + # Fix the install phase which tries to install into the pgsql extension dir, + # and cannot be manually overridden. This is rather fragile but works OK. + postPatch = '' + for x in CMakeLists.txt sql/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" + done + + for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" + done + ''; + + + # timescaledb-2.9.1.so already exists in the lib directory + # we have no need for the timescaledb.so or control file + postInstall = '' + rm $out/lib/timescaledb.so + rm $out/share/postgresql/extension/timescaledb.control + ''; + + meta = with lib; { + description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; + homepage = "https://www.timescale.com/"; + changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.asl20; + broken = versionOlder postgresql.version "13"; + }; +} diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index df44de35f..85e00aff3 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -78,24 +78,53 @@ buildPgrxExtension_0_12_6 rec { echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt ''; - postInstall = '' - echo "Creating SQL files for previous versions..." - current_version="${version}" - sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" - - if [ -f "$sql_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" - echo "Creating $new_file" - cp "$sql_file" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $sql_file not found" - fi - rm git_tags.txt - ''; + postInstall = '' + echo "Modifying main SQL file to use unversioned library name..." + current_version="${version}" + main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + if [ -f "$main_sql_file" ]; then + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" + echo "Modified $main_sql_file" + else + echo "Warning: $main_sql_file not found" + fi + echo "Creating and modifying SQL files for previous versions..." + + if [ -f "$main_sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$main_sql_file" "$new_file" + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" + echo "Modified $new_file" + fi + done < git_tags.txt + else + echo "Warning: $main_sql_file not found" + fi + mv $out/lib/wrappers-${version}${postgresql.dlSuffix} $out/lib/wrappers${postgresql.dlSuffix} + ln -s $out/lib/wrappers${postgresql.dlSuffix} $out/lib/wrappers-${version}${postgresql.dlSuffix} + + echo "Creating wrappers.so symlinks to support pg_upgrade..." + if [ -f "$out/lib/wrappers.so" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$out/lib/wrappers.so" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $out/lib/wrappers.so not found" + fi + + rm git_tags.txt + echo "Contents of updated wrappers.control:" + cat "$out/share/postgresql/extension/wrappers.control" + echo "List of generated SQL files:" + ls -l $out/share/postgresql/extension/wrappers--*.sql + ''; meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; diff --git a/scripts/90-cleanup.sh b/scripts/90-cleanup.sh index ebbf49080..f2e19686d 100644 --- a/scripts/90-cleanup.sh +++ b/scripts/90-cleanup.sh @@ -36,7 +36,10 @@ elif [ -n "$(command -v apt-get)" ]; then libicu-dev \ libcgal-dev \ libgcc-9-dev \ - libgcc-8-dev + libgcc-8-dev \ + ansible + + add-apt-repository --yes --remove ppa:ansible/ansible source /etc/os-release apt-get -y remove --purge linux-headers-5.11.0-1021-aws diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index c4776b3a8..a2cda7ffd 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -45,8 +45,12 @@ EOF $ARGS } +function cleanup_packages { + sudo apt-get -y remove --purge ansible + sudo add-apt-repository --yes --remove ppa:ansible/ansible +} install_packages install_nix execute_stage2_playbook - +cleanup_packages