Skip to content

Commit

Permalink
Merge branch 'develop' into sam/oriole17
Browse files Browse the repository at this point in the history
  • Loading branch information
samrose authored Nov 15, 2024
2 parents d29bd9c + ffa1052 commit 9a2d617
Show file tree
Hide file tree
Showing 17 changed files with 212 additions and 105 deletions.
12 changes: 7 additions & 5 deletions .github/workflows/nix-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,10 @@ jobs:
arch: arm64
- runner: macos-latest
arch: arm64
- runner: macos-13
arch: amd64
runs-on: ${{ matrix.runner }}

timeout-minutes: 180
steps:

- name: Check out code
Expand All @@ -48,23 +50,23 @@ jobs:
env:
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
- name: Log in to Docker Hub
if: matrix.runner != 'macos-latest'
if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13'
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build psql bundle with nix
if: matrix.runner != 'macos-latest'
if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13'
run: docker build -t base_nix -f docker/nix/Dockerfile .
- name: Run build psql bundle
if: matrix.runner != 'macos-latest'
if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13'
run: |
docker run -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
-e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
-e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
base_nix bash -c "./workspace/docker/nix/build_nix.sh"
- name: Build psql bundle on macos
if: matrix.runner == 'macos-latest'
if: matrix.runner == 'macos-latest' || matrix.runner == 'macos-13'
run: |
curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \
--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/publish-nix-pgupgrade-scripts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ jobs:
run: |
VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes
if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
VERSION=${{ inputs.postgresVersion }}
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
- name: Create a tarball containing pg_upgrade scripts
Expand Down
12 changes: 12 additions & 0 deletions ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -91,12 +91,24 @@ swap_postgres_and_supabase_admin() {
alter database postgres connection limit 0;
select pg_terminate_backend(pid) from pg_stat_activity where backend_type = 'client backend' and pid != pg_backend_pid();
EOSQL

if [ -z "$IS_CI" ]; then
retry 5 systemctl restart postgresql
else
CI_start_postgres ""
fi

retry 8 pg_isready -h localhost -U supabase_admin

run_sql <<'EOSQL'
set statement_timeout = '600s';
begin;
create role supabase_tmp superuser;
set session authorization supabase_tmp;
-- to handle snowflakes that happened in the past
revoke supabase_admin from authenticator;
do $$
begin
if exists (select from pg_extension where extname = 'timescaledb') then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ EOF

run_sql -c "$RECREATE_PG_CRON_QUERY"
fi

# #incident-2024-09-12-project-upgrades-are-temporarily-disabled
run_sql -c "grant pg_read_all_data, pg_signal_backend to postgres"
}

function complete_pg_upgrade {
Expand Down
81 changes: 26 additions & 55 deletions ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ MOUNT_POINT="/data_migration"
LOG_FILE="/var/log/pg-upgrade-initiate.log"

POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql"
POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql"
OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;")

SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;")
Expand All @@ -47,7 +48,6 @@ SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;")

POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf"
PGBINOLD="/usr/lib/postgresql/bin"
PGLIBOLD="/usr/lib/postgresql/lib"

PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION"
NIX_INSTALLER_PATH="/tmp/persistent/nix-installer"
Expand Down Expand Up @@ -133,6 +133,22 @@ cleanup() {
echo "Resetting postgres database connection limit"
retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;"

echo "Making sure postgres still has access to pg_shadow"
cat << EOF >> $POST_UPGRADE_POSTGRES_PERMS_SCRIPT
DO \$\$
begin
if exists (select from pg_authid where rolname = 'pg_read_all_data') then
execute('grant pg_read_all_data to postgres');
end if;
end
\$\$;
grant pg_signal_backend to postgres;
EOF

if [ -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT ]; then
retry 5 run_sql -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT
fi

if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then
echo "Unmounting data disk from ${MOUNT_POINT}"
retry 3 umount $MOUNT_POINT
Expand All @@ -148,6 +164,14 @@ cleanup() {
}

function handle_extensions {
if [ -z "$IS_CI" ]; then
retry 5 systemctl restart postgresql
else
CI_start_postgres
fi

retry 8 pg_isready -h localhost -U supabase_admin

rm -f $POST_UPGRADE_EXTENSION_SCRIPT
touch $POST_UPGRADE_EXTENSION_SCRIPT

Expand Down Expand Up @@ -181,58 +205,6 @@ EOF
done
}

function patch_wrappers {
local IS_NIX_UPGRADE=$1

WRAPPERS_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'wrappers');")
if [ "$WRAPPERS_ENABLED" = "f" ]; then
echo "Wrappers extension not enabled. Skipping."
return
fi

# This is a workaround for older versions of wrappers which don't have the expected
# naming scheme, containing the version in their library's file name
# e.g. wrappers-0.1.16.so, rather than wrappers.so
# pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's
# library directory, so we're making sure the new version has the expected (old version's)
# file name.
# After the upgrade completes, the new version's library file is used.
# i.e.
# - old version: wrappers-0.1.16.so
# - new version: wrappers-0.1.18.so
# - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so
if [ "$IS_NIX_UPGRADE" = "true" ]; then
if [ -d "$PGLIBOLD" ]; then
OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit)
OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")

find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do
if [ -f "$WRAPPERS_LIB_PATH" ]; then
WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH")
if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then
echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}"
cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true
fi
fi
done
fi
else
if [ -d "$PGLIBOLD" ]; then
WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit)
if [ -f "$WRAPPERS_LIB_PATH" ]; then
OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit)
if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then
LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")
if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then
echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}"
cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true
fi
fi
fi
fi
fi
}

function initiate_upgrade {
mkdir -p "$MOUNT_POINT"
SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/")
Expand Down Expand Up @@ -409,8 +381,6 @@ function initiate_upgrade {
export LD_LIBRARY_PATH="${PGLIBNEW}"
fi

patch_wrappers "$IS_NIX_UPGRADE"

echo "9. Creating new data directory, initializing database"
chown -R postgres:postgres "$MOUNT_POINT/"
rm -rf "${PGDATANEW:?}/"
Expand Down Expand Up @@ -473,6 +443,7 @@ EOF
cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/"
# removing supautils config as to allow the latest one provided by the latest image to be used
rm -f "$MOUNT_POINT/conf/supautils.conf" || true
rm -rf "$MOUNT_POINT/conf/extension-custom-scripts" || true

# removing wal-g config as to allow it to be explicitly enabled on the new instance
rm -f "$MOUNT_POINT/conf/wal-g.conf"
Expand Down
4 changes: 2 additions & 2 deletions ansible/files/postgresql_config/supautils.conf.j2
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/ex
supautils.privileged_extensions_superuser = 'supabase_admin'
supautils.privileged_role = 'postgres'
supautils.privileged_role_allowed_configs = 'auto_explain.log_min_duration, auto_explain.log_nested_statements, log_lock_waits, log_min_messages, pg_net.batch_size, pg_net.ttl, pgaudit.log, pgaudit.log_catalog, pgaudit.log_client, pgaudit.log_level, pgaudit.log_relation, pgaudit.log_rows, pgaudit.log_statement, pgaudit.log_statement_once, pgaudit.role, pgrst.*, plan_filter.*, safeupdate.enabled, session_replication_role, track_io_timing'
supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, authenticator'
supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*'
supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, authenticator'
supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*'
8 changes: 4 additions & 4 deletions ansible/vars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ postgrest_release: "12.2.3"
postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9
postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05

gotrue_release: 2.162.0
gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1
gotrue_release: 2.163.2
gotrue_release_checksum: sha1:31889bc8c498b924c2cb3b6c4084ef6e57ed97c0

aws_cli_release: "2.2.7"

Expand Down Expand Up @@ -149,8 +149,8 @@ hypopg_release_checksum: sha256:9afe6357fd389d8d33fad81703038ce520b09275ec00153c
pg_repack_release: "1.5.0"
pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52

pgvector_release: "0.7.4"
pgvector_release_checksum: sha256:0341edf89b1924ae0d552f617e14fb7f8867c0194ed775bcc44fa40288642583
pgvector_release: "0.8.0"
pgvector_release_checksum: sha256:867a2c328d4928a5a9d6f052cd3bc78c7d60228a9b914ad32aa3db88e9de27b0

pg_tle_release: "1.3.2"
pg_tle_release_checksum: sha256:d04f72d88b21b954656609743560684ac42645b64a36c800d4d2f84d1f180de1
Expand Down
5 changes: 5 additions & 0 deletions ebssurrogate/scripts/surrogate-bootstrap-nix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,11 @@ function format_and_mount_rootfs {
fi

mkfs.ext4 /dev/xvdh

# Explicitly reserving 100MiB worth of blocks for the data volume
RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096))
tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT /dev/xvdh

mkdir -p /mnt/data
mount -o defaults,discard /dev/xvdh /mnt/data
}
Expand Down
2 changes: 1 addition & 1 deletion flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
system.x86_64-linux
system.aarch64-linux
system.aarch64-darwin
system.x86_64-darwin
];
in
flake-utils.lib.eachSystem ourSystems (system:
Expand Down Expand Up @@ -199,7 +200,6 @@
else ourExtensions;
in map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse;


# Create an attrset that contains all the extensions included in a server.
makeOurPostgresPkgsSet = version:
(builtins.listToAttrs (map
Expand Down
6 changes: 3 additions & 3 deletions nix/ext/pg_repack.nix
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@

stdenv.mkDerivation (finalAttrs: {
pname = "pg_repack";
version = "1.5.0";
version = "1.5.2";

buildInputs = postgresql.buildInputs ++ [ postgresql ];

src = fetchFromGitHub {
owner = "reorg";
repo = "pg_repack";
rev = "ver_${finalAttrs.version}";
hash = "sha256-do80phyMxwcRIkYyUt9z02z7byNQhK+pbSaCUmzG+4c=";
rev = "85b64c6d4f599b2988343c4e7121acab505c9006";
hash = "sha256-lAuLI+vupusvn3uTzQ9OaLqkEfUVMCAwU9R70tTbb8Y=";
};

installPhase = ''
Expand Down
4 changes: 2 additions & 2 deletions nix/ext/pgvector.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@

stdenv.mkDerivation rec {
pname = "pgvector";
version = "0.7.4";
version = "0.8.0";

buildInputs = [ postgresql ];

src = fetchFromGitHub {
owner = "pgvector";
repo = pname;
rev = "refs/tags/v${version}";
hash = "sha256-qwPaguQUdDHV8q6GDneLq5MuhVroPizpbqt7f08gKJI=";
hash = "sha256-JsZV+I4eRMypXTjGmjCtMBXDVpqTIPHQa28ogXncE/Q=";
};

installPhase = ''
Expand Down
44 changes: 32 additions & 12 deletions nix/ext/plv8.nix
Original file line number Diff line number Diff line change
Expand Up @@ -97,30 +97,50 @@ stdenv.mkDerivation (finalAttrs: {
''}
'';

postInstall = ''
postInstall = ''
# Move the redirected to proper directory.
# There appear to be no references to the install directories
# so changing them does not cause issues.
mv "$out/nix/store"/*/* "$out"
rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix"
${lib.optionalString stdenv.isDarwin ''
install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
''}
${lib.optionalString (!stdenv.isDarwin) ''
${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
''}
# Handle different PostgreSQL versions
if [ "${lib.versions.major postgresql.version}" = "15" ]; then
mv "$out/lib/plv8-${finalAttrs.version}.so" "$out/lib/plv8.so"
ln -s "$out/lib/plv8.so" "$out/lib/plv8-${finalAttrs.version}.so"
sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plv8.control"
sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plcoffee.control"
sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plls.control"
${lib.optionalString stdenv.isDarwin ''
install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8.so
install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8.so
install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so
install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8.so
''}
${lib.optionalString (!stdenv.isDarwin) ''
${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so
''}
else
${lib.optionalString stdenv.isDarwin ''
install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
''}
${lib.optionalString (!stdenv.isDarwin) ''
${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix}
''}
fi
'';

meta = with lib; {
description = "V8 Engine Javascript Procedural Language add-on for PostgreSQL";
homepage = "https://plv8.github.io/";
maintainers = with maintainers; [ samrose ];
platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ];
platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" "x86_64-darwin" ];
license = licenses.postgresql;
};
})
Loading

0 comments on commit 9a2d617

Please sign in to comment.