From 0eb6ab2768d3d32d42becde4bfa5f54406f2abcb Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Fri, 20 Dec 2024 16:03:54 +0100 Subject: [PATCH 01/25] Do not create tenants unconditionally, check for existent ones instead --- .../compute_wrapper/shell/compute.sh | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/docker-compose/compute_wrapper/shell/compute.sh b/docker-compose/compute_wrapper/shell/compute.sh index 33455e458a0b..6e8b49079b07 100755 --- a/docker-compose/compute_wrapper/shell/compute.sh +++ b/docker-compose/compute_wrapper/shell/compute.sh @@ -20,27 +20,45 @@ while ! nc -z pageserver 6400; do done echo "Page server is ready." -echo "Create a tenant and timeline" -generate_id tenant_id +echo "Check if a tenant present" PARAMS=( - -X PUT + -X GET -H "Content-Type: application/json" - -d "{\"mode\": \"AttachedSingle\", \"generation\": 1, \"tenant_conf\": {}}" - "http://pageserver:9898/v1/tenant/${tenant_id}/location_config" + "http://pageserver:9898/v1/tenant" ) -result=$(curl "${PARAMS[@]}") -echo $result | jq . +tenant_id=$(curl "${PARAMS[@]}" | jq -r .[0].id) +if [ -z "${tenant_id}" ] || [ "${tenant_id}" = null ]; then + echo "Create a tenant" + generate_id tenant_id + PARAMS=( + -X PUT + -H "Content-Type: application/json" + -d "{\"mode\": \"AttachedSingle\", \"generation\": 1, \"tenant_conf\": {}}" + "http://pageserver:9898/v1/tenant/${tenant_id}/location_config" + ) + result=$(curl "${PARAMS[@]}") + echo $result | jq . +fi -generate_id timeline_id +echo "Check if a timeline present" PARAMS=( - -sbf - -X POST + -X GET -H "Content-Type: application/json" - -d "{\"new_timeline_id\": \"${timeline_id}\", \"pg_version\": ${PG_VERSION}}" - "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/" + "http://pageserver:9898/v1/tenant/${tenant_id}/timeline" ) -result=$(curl "${PARAMS[@]}") -echo $result | jq . +timeline_id=$(curl "${PARAMS[@]}" | jq -r .[0].timeline_id) +if [ -z "${timeline_id}" ] || [ "${timeline_id}" = null ]; then + generate_id timeline_id + PARAMS=( + -sbf + -X POST + -H "Content-Type: application/json" + -d "{\"new_timeline_id\": \"${timeline_id}\", \"pg_version\": ${PG_VERSION}}" + "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/" + ) + result=$(curl "${PARAMS[@]}") + echo $result | jq . +fi echo "Overwrite tenant id and timeline id in spec file" sed "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE_ORG} > ${SPEC_FILE} From 0ea62c79f1e7290464e837550e0058105d8d0e9d Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Thu, 26 Dec 2024 09:03:08 +0100 Subject: [PATCH 02/25] Add a concept for the test for extensions --- compute/compute-node.Dockerfile | 1 + docker-compose/collect_versions.sh | 55 +++++++++++++++++++++++++++ docker-compose/docker-compose.yml | 6 +-- docker-compose/docker_compose_test.sh | 3 -- 4 files changed, 59 insertions(+), 6 deletions(-) create mode 100755 docker-compose/collect_versions.sh diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 5e7b4e8287ce..f91caff609de 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1405,6 +1405,7 @@ RUN make PG_VERSION="${PG_VERSION}" -C compute FROM neon-pg-ext-build AS neon-pg-ext-test ARG PG_VERSION +RUN apt-get update && apt-get install -y curl jq RUN mkdir /ext-src #COPY --from=postgis-build /postgis.tar.gz /ext-src/ diff --git a/docker-compose/collect_versions.sh b/docker-compose/collect_versions.sh new file mode 100755 index 000000000000..db1aa273043c --- /dev/null +++ b/docker-compose/collect_versions.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -eux -o pipefail +export TAG=6659 +export PGUSER=cloud_admin +export PGPASSWORD=cloud_admin +export PGHOST=127.0.0.1 +export PGPORT=55433 +export PGDATABASE=postgres +docker volume prune -f +function wait_for_ready { + while ! docker compose logs compute_is_ready | grep -q "accepting connections"; do + sleep 1 + done +} +EXTENSIONS='[ +{"extname": "plv8", "extdir": "plv8-src"}, +{"extname": "vector", "extdir": "pgvector-src"} +]' +EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) +TAG=6660 docker compose up --build -d +wait_for_ready +for ext in $EXTNAMES; do + echo "CREATE EXTENSION IF NOT EXISTS ${ext};" +done | psql -X -v ON_ERROR_STOP=1 +query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /','}')" +new_vers=$(psql -Aqt -c "$query" ) +echo $new_vers +docker compose down +TAG=6659 docker compose --profile test-extensions up --build -d --force-recreate +wait_for_ready +for ext in $EXTNAMES; do + echo "CREATE EXTENSION IF NOT EXISTS ${ext};" +done | psql -X -v ON_ERROR_STOP=1 +query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion" +#query="select * from pg_extension" +echo $query +exts=$(psql -Aqt -c "$query") +if [ -z "${exts}" ]; then + echo "No extensions were upgraded" +else + for ext in ${exts}; do + echo Testing ${ext}... + EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') + docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && make installcheck" + TAG=6659 docker compose down compute + COMPUTE_TAG=6660 TAG=6659 docker compose up -d --build compute + wait_for_ready + docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && /usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type" + psql -d contrib_regression -c '\dx' + psql -d contrib_regression -c "alter extension ${ext} update" + psql -d contrib_regression -c '\dx' + docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && /usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type" + done +fi +docker compose --profile test-extensions down \ No newline at end of file diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 6e15fdbe0dde..458fd32e7d03 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -149,9 +149,9 @@ services: args: - REPOSITORY=${REPOSITORY:-neondatabase} - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-16} - - TAG=${TAG:-latest} - - http_proxy=$http_proxy - - https_proxy=$https_proxy + - TAG=${COMPUTE_TAG:-${TAG:-latest}} + - http_proxy=${http_proxy:-} + - https_proxy=${https_proxy:-} environment: - PG_VERSION=${PG_VERSION:-16} #- RUST_BACKTRACE=1 diff --git a/docker-compose/docker_compose_test.sh b/docker-compose/docker_compose_test.sh index 063664d0c67d..be8bfc121cc4 100755 --- a/docker-compose/docker_compose_test.sh +++ b/docker-compose/docker_compose_test.sh @@ -18,9 +18,6 @@ cd $(dirname $0) COMPUTE_CONTAINER_NAME=docker-compose-compute-1 TEST_CONTAINER_NAME=docker-compose-neon-test-extensions-1 PSQL_OPTION="-h localhost -U cloud_admin -p 55433 -d postgres" -: ${http_proxy:=} -: ${https_proxy:=} -export http_proxy https_proxy cleanup() { echo "show container information" From 70db6fe85d775297150dafbf6094b3ca30b9b963 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Mon, 6 Jan 2025 13:11:52 +0100 Subject: [PATCH 03/25] Add a concept for the test for extensions --- compute/compute-node.Dockerfile | 1 + docker-compose/collect_versions.sh | 55 ---------------- .../ext-src/pgvector-src/test-upgrade.sh | 4 ++ .../ext-src/plv8-src/test-upgrade.sh | 4 ++ .../postgresql-unit-src/test-upgrade.sh | 4 ++ docker-compose/test_extensions_upgrade.sh | 64 +++++++++++++++++++ 6 files changed, 77 insertions(+), 55 deletions(-) delete mode 100755 docker-compose/collect_versions.sh create mode 100755 docker-compose/ext-src/pgvector-src/test-upgrade.sh create mode 100755 docker-compose/ext-src/plv8-src/test-upgrade.sh create mode 100755 docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh create mode 100755 docker-compose/test_extensions_upgrade.sh diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index f91caff609de..a9ca30e92f22 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1462,6 +1462,7 @@ ENV PATH=/usr/local/pgsql/bin:$PATH ENV PGHOST=compute ENV PGPORT=55433 ENV PGUSER=cloud_admin +ENV PGPASSWORD=cloud_admin ENV PGDATABASE=postgres ######################################################################################### # diff --git a/docker-compose/collect_versions.sh b/docker-compose/collect_versions.sh deleted file mode 100755 index db1aa273043c..000000000000 --- a/docker-compose/collect_versions.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -eux -o pipefail -export TAG=6659 -export PGUSER=cloud_admin -export PGPASSWORD=cloud_admin -export PGHOST=127.0.0.1 -export PGPORT=55433 -export PGDATABASE=postgres -docker volume prune -f -function wait_for_ready { - while ! docker compose logs compute_is_ready | grep -q "accepting connections"; do - sleep 1 - done -} -EXTENSIONS='[ -{"extname": "plv8", "extdir": "plv8-src"}, -{"extname": "vector", "extdir": "pgvector-src"} -]' -EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) -TAG=6660 docker compose up --build -d -wait_for_ready -for ext in $EXTNAMES; do - echo "CREATE EXTENSION IF NOT EXISTS ${ext};" -done | psql -X -v ON_ERROR_STOP=1 -query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /','}')" -new_vers=$(psql -Aqt -c "$query" ) -echo $new_vers -docker compose down -TAG=6659 docker compose --profile test-extensions up --build -d --force-recreate -wait_for_ready -for ext in $EXTNAMES; do - echo "CREATE EXTENSION IF NOT EXISTS ${ext};" -done | psql -X -v ON_ERROR_STOP=1 -query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion" -#query="select * from pg_extension" -echo $query -exts=$(psql -Aqt -c "$query") -if [ -z "${exts}" ]; then - echo "No extensions were upgraded" -else - for ext in ${exts}; do - echo Testing ${ext}... - EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && make installcheck" - TAG=6659 docker compose down compute - COMPUTE_TAG=6660 TAG=6659 docker compose up -d --build compute - wait_for_ready - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && /usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type" - psql -d contrib_regression -c '\dx' - psql -d contrib_regression -c "alter extension ${ext} update" - psql -d contrib_regression -c '\dx' - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && /usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type" - done -fi -docker compose --profile test-extensions down \ No newline at end of file diff --git a/docker-compose/ext-src/pgvector-src/test-upgrade.sh b/docker-compose/ext-src/pgvector-src/test-upgrade.sh new file mode 100755 index 000000000000..51d2450c61b0 --- /dev/null +++ b/docker-compose/ext-src/pgvector-src/test-upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type \ No newline at end of file diff --git a/docker-compose/ext-src/plv8-src/test-upgrade.sh b/docker-compose/ext-src/plv8-src/test-upgrade.sh new file mode 100755 index 000000000000..38cc10a7f30e --- /dev/null +++ b/docker-compose/ext-src/plv8-src/test-upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --dbname=contrib_regression plv8 plv8-errors scalar_args inline json startup_pre startup varparam json_conv jsonb_conv window guc es6 arraybuffer composites currentresource startup_perms bytea find_function_perms memory_limits reset show array_spread regression dialect bigint procedure \ No newline at end of file diff --git a/docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh b/docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh new file mode 100755 index 000000000000..60c32655c195 --- /dev/null +++ b/docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --dbname=contrib_regression extension tables unit binary unicode prefix units time temperature functions language_functions round derived compare aggregate iec custom crosstab convert \ No newline at end of file diff --git a/docker-compose/test_extensions_upgrade.sh b/docker-compose/test_extensions_upgrade.sh new file mode 100755 index 000000000000..fae2884f6425 --- /dev/null +++ b/docker-compose/test_extensions_upgrade.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -eux -o pipefail +if [ -z ${OLDTAG+x} ] || [ -z ${NEWTAG+x} ] || [ -z "${OLDTAG}" ] || [ -z "${NEWTAG}" ]; then + echo OLDTAG and NEWTAG must be defined + exit 1 +fi +export PGUSER=cloud_admin +export PGPASSWORD=cloud_admin +export PGHOST=127.0.0.1 +export PGPORT=55433 +export PGDATABASE=postgres +docker volume prune -f +function wait_for_ready { + while ! docker compose logs compute_is_ready | grep -q "accepting connections"; do + sleep 1 + done +} +function create_extensions() { + for ext in ${1}; do + echo "CREATE EXTENSION IF NOT EXISTS ${ext};" + done | psql -X -v ON_ERROR_STOP=1 +} +EXTENSIONS='[ +{"extname": "plv8", "extdir": "plv8-src"}, +{"extname": "vector", "extdir": "pgvector-src"}, +{"extname": "unit", "extdir": "postgresql-unit-src"} +]' +EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) +TAG=${NEWTAG} docker compose up --build -d +wait_for_ready +create_extensions "${EXTNAMES}" +query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /','}')" +new_vers=$(psql -Aqt -c "$query" ) +echo $new_vers +docker compose down +TAG=${OLDTAG} docker compose --profile test-extensions up --build -d --force-recreate +wait_for_ready +# XXX this is about to be included into the image, for test only +docker compose cp ext-src neon-test-extensions:/ +for ext in $EXTNAMES; do + echo "CREATE EXTENSION IF NOT EXISTS ${ext};" +done | psql -X -v ON_ERROR_STOP=1 +query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion" +echo $query +exts=$(psql -Aqt -c "$query") +if [ -z "${exts}" ]; then + echo "No extensions were upgraded" +else + psql -c "CREATE DATABASE contrib_regression" + export PGDATABASE=contrib_regression + create_extensions "${exts}" + TAG=${OLDTAG} docker compose down compute compute_is_ready + COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} docker compose up -d --build compute compute_is_ready + wait_for_ready + for ext in ${exts}; do + echo Testing ${ext}... + EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') + psql -d contrib_regression -c "\dx ${ext}" + docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions sh -c /ext-src/${EXTDIR}/test-upgrade.sh + psql -d contrib_regression -c "alter extension ${ext} update" + psql -d contrib_regression -c "\dx ${ext}" + done +fi +docker compose --profile test-extensions down \ No newline at end of file From 9435e7fce71721bbef12de67eb1aebddd5120c97 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Wed, 8 Jan 2025 11:10:29 +0100 Subject: [PATCH 04/25] Add more extensions --- .../ext-src/hypopg-src/test-upgrade.patch | 27 ++++++++++++++++ .../ext-src/hypopg-src/test-upgrade.sh | 5 +++ .../ext-src/ip4r-src/test-upgrade.patch | 23 ++++++++++++++ .../ext-src/ip4r-src/test-upgrade.sh | 5 +++ .../ext-src/prefix-src/test-upgrade.sh | 4 +++ .../ext-src/rum-src/test-upgrade.patch | 19 ++++++++++++ .../ext-src/rum-src/test-upgrade.sh | 5 +++ docker-compose/test_extensions_upgrade.sh | 31 +++++++++++++------ 8 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 docker-compose/ext-src/hypopg-src/test-upgrade.patch create mode 100755 docker-compose/ext-src/hypopg-src/test-upgrade.sh create mode 100644 docker-compose/ext-src/ip4r-src/test-upgrade.patch create mode 100755 docker-compose/ext-src/ip4r-src/test-upgrade.sh create mode 100755 docker-compose/ext-src/prefix-src/test-upgrade.sh create mode 100644 docker-compose/ext-src/rum-src/test-upgrade.patch create mode 100755 docker-compose/ext-src/rum-src/test-upgrade.sh diff --git a/docker-compose/ext-src/hypopg-src/test-upgrade.patch b/docker-compose/ext-src/hypopg-src/test-upgrade.patch new file mode 100644 index 000000000000..71fe26b1643d --- /dev/null +++ b/docker-compose/ext-src/hypopg-src/test-upgrade.patch @@ -0,0 +1,27 @@ +diff --git a/expected/hypopg.out b/expected/hypopg.out +index 90121d0..859260b 100644 +--- a/expected/hypopg.out ++++ b/expected/hypopg.out +@@ -11,7 +11,8 @@ BEGIN + END; + $_$ + LANGUAGE plpgsql; +-CREATE EXTENSION hypopg; ++CREATE EXTENSION IF NOT EXISTS hypopg; ++NOTICE: extension "hypopg" already exists, skipping + CREATE TABLE hypo (id integer, val text, "Id2" bigint); + INSERT INTO hypo SELECT i, 'line ' || i + FROM generate_series(1,100000) f(i); +diff --git a/test/sql/hypopg.sql b/test/sql/hypopg.sql +index 99722b0..8d6bacb 100644 +--- a/test/sql/hypopg.sql ++++ b/test/sql/hypopg.sql +@@ -12,7 +12,7 @@ END; + $_$ + LANGUAGE plpgsql; + +-CREATE EXTENSION hypopg; ++CREATE EXTENSION IF NOT EXISTS hypopg; + + CREATE TABLE hypo (id integer, val text, "Id2" bigint); + diff --git a/docker-compose/ext-src/hypopg-src/test-upgrade.sh b/docker-compose/ext-src/hypopg-src/test-upgrade.sh new file mode 100755 index 000000000000..906f879d0474 --- /dev/null +++ b/docker-compose/ext-src/hypopg-src/test-upgrade.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +patch -p1 "${TENANT_ID_FILE}" + echo "${TIMELINE_ID}" > "${TIMELINE_ID_FILE}" + #XXX for ext in ${exts}; do + for ext in ${EXTNAMES}; do echo Testing ${ext}... EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') psql -d contrib_regression -c "\dx ${ext}" @@ -61,4 +71,5 @@ else psql -d contrib_regression -c "\dx ${ext}" done fi -docker compose --profile test-extensions down \ No newline at end of file +docker compose --profile test-extensions down +rm -f "${TENANT_ID_FILE}" "${TIMELINE_ID_FILE}" From 0cb337a129bb784db3806f753e870e11681ebf93 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Thu, 9 Jan 2025 11:38:58 +0100 Subject: [PATCH 05/25] Add two more --- .../compute_wrapper/shell/compute.sh | 71 ++++++++++--------- .../ext-src/hll-src/test-upgrade.sh | 4 ++ .../ext-src/pg_cron-src/test-upgrade.patch | 45 ++++++++++++ .../ext-src/pg_cron-src/test-upgrade.sh | 5 ++ .../pg_hint_plan-src/test-upgrade.patch | 21 ++++++ .../ext-src/pg_hint_plan-src/test-upgrade.sh | 4 ++ docker-compose/test_extensions_upgrade.sh | 46 +++++++++--- 7 files changed, 153 insertions(+), 43 deletions(-) create mode 100755 docker-compose/ext-src/hll-src/test-upgrade.sh create mode 100644 docker-compose/ext-src/pg_cron-src/test-upgrade.patch create mode 100755 docker-compose/ext-src/pg_cron-src/test-upgrade.sh create mode 100644 docker-compose/ext-src/pg_hint_plan-src/test-upgrade.patch create mode 100644 docker-compose/ext-src/pg_hint_plan-src/test-upgrade.sh diff --git a/docker-compose/compute_wrapper/shell/compute.sh b/docker-compose/compute_wrapper/shell/compute.sh index 6e8b49079b07..ba3c63846b8f 100755 --- a/docker-compose/compute_wrapper/shell/compute.sh +++ b/docker-compose/compute_wrapper/shell/compute.sh @@ -13,6 +13,8 @@ PG_VERSION=${PG_VERSION:-14} SPEC_FILE_ORG=/var/db/postgres/specs/spec.json SPEC_FILE=/tmp/spec.json +TENANT_ID_FILE=/var/db/postgres/specs/tenant_id +TIMELINE_ID_FILE=/var/db/postgres/specs/timeline_id echo "Waiting pageserver become ready." while ! nc -z pageserver 6400; do @@ -20,44 +22,49 @@ while ! nc -z pageserver 6400; do done echo "Page server is ready." -echo "Check if a tenant present" -PARAMS=( - -X GET - -H "Content-Type: application/json" - "http://pageserver:9898/v1/tenant" -) -tenant_id=$(curl "${PARAMS[@]}" | jq -r .[0].id) -if [ -z "${tenant_id}" ] || [ "${tenant_id}" = null ]; then - echo "Create a tenant" - generate_id tenant_id + if [ -f "${TENANT_ID_FILE}" ] && [ -f "${TIMELINE_ID_FILE}" ]; then + tenant_id=$(cat "${TENANT_ID_FILE}") + timeline_id=$(cat "${TIMELINE_ID_FILE}") +else + echo "Check if a tenant present" PARAMS=( - -X PUT + -X GET -H "Content-Type: application/json" - -d "{\"mode\": \"AttachedSingle\", \"generation\": 1, \"tenant_conf\": {}}" - "http://pageserver:9898/v1/tenant/${tenant_id}/location_config" + "http://pageserver:9898/v1/tenant" ) - result=$(curl "${PARAMS[@]}") - echo $result | jq . -fi + tenant_id=$(curl "${PARAMS[@]}" | jq -r .[0].id) + if [ -z "${tenant_id}" ] || [ "${tenant_id}" = null ]; then + echo "Create a tenant" + generate_id tenant_id + PARAMS=( + -X PUT + -H "Content-Type: application/json" + -d "{\"mode\": \"AttachedSingle\", \"generation\": 1, \"tenant_conf\": {}}" + "http://pageserver:9898/v1/tenant/${tenant_id}/location_config" + ) + result=$(curl "${PARAMS[@]}") + echo $result | jq . + fi -echo "Check if a timeline present" -PARAMS=( - -X GET - -H "Content-Type: application/json" - "http://pageserver:9898/v1/tenant/${tenant_id}/timeline" -) -timeline_id=$(curl "${PARAMS[@]}" | jq -r .[0].timeline_id) -if [ -z "${timeline_id}" ] || [ "${timeline_id}" = null ]; then - generate_id timeline_id + echo "Check if a timeline present" PARAMS=( - -sbf - -X POST - -H "Content-Type: application/json" - -d "{\"new_timeline_id\": \"${timeline_id}\", \"pg_version\": ${PG_VERSION}}" - "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/" + -X GET + -H "Content-Type: application/json" + "http://pageserver:9898/v1/tenant/${tenant_id}/timeline" ) - result=$(curl "${PARAMS[@]}") - echo $result | jq . + timeline_id=$(curl "${PARAMS[@]}" | jq -r .[0].timeline_id) + if [ -z "${timeline_id}" ] || [ "${timeline_id}" = null ]; then + generate_id timeline_id + PARAMS=( + -sbf + -X POST + -H "Content-Type: application/json" + -d "{\"new_timeline_id\": \"${timeline_id}\", \"pg_version\": ${PG_VERSION}}" + "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/" + ) + result=$(curl "${PARAMS[@]}") + echo $result | jq . + fi fi echo "Overwrite tenant id and timeline id in spec file" diff --git a/docker-compose/ext-src/hll-src/test-upgrade.sh b/docker-compose/ext-src/hll-src/test-upgrade.sh new file mode 100755 index 000000000000..fbc206b9092f --- /dev/null +++ b/docker-compose/ext-src/hll-src/test-upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin' --dbname=contrib_regression add_agg agg_oob auto_sparse card_op cast_shape copy_binary cumulative_add_cardinality_correction cumulative_add_comprehensive_promotion cumulative_add_sparse_edge cumulative_add_sparse_random cumulative_add_sparse_step cumulative_union_comprehensive cumulative_union_explicit_explicit cumulative_union_explicit_promotion cumulative_union_probabilistic_probabilistic cumulative_union_sparse_full_representation cumulative_union_sparse_promotion cumulative_union_sparse_sparse disable_hashagg equal explicit_thresh hash hash_any meta_func murmur_bigint murmur_bytea nosparse notequal scalar_oob storedproc transaction typmod typmod_insert union_op \ No newline at end of file diff --git a/docker-compose/ext-src/pg_cron-src/test-upgrade.patch b/docker-compose/ext-src/pg_cron-src/test-upgrade.patch new file mode 100644 index 000000000000..9edaa9a930c1 --- /dev/null +++ b/docker-compose/ext-src/pg_cron-src/test-upgrade.patch @@ -0,0 +1,45 @@ +diff --git a/expected/pg_cron-test.out b/expected/pg_cron-test.out +index d79d542..4239a6e 100644 +--- a/expected/pg_cron-test.out ++++ b/expected/pg_cron-test.out +@@ -1,12 +1,3 @@ +-CREATE EXTENSION pg_cron VERSION '1.0'; +-SELECT extversion FROM pg_extension WHERE extname='pg_cron'; +- extversion +------------- +- 1.0 +-(1 row) +- +--- Test binary compatibility with v1.4 function signature. +-ALTER EXTENSION pg_cron UPDATE TO '1.4'; + SELECT cron.unschedule(job_name := 'no_such_job'); + ERROR: could not find valid entry for job 'no_such_job' + SELECT cron.schedule('testjob', '* * * * *', 'SELECT 1'); +@@ -301,7 +292,6 @@ SELECT cron.schedule('bad-last-dom-job1', '0 11 $foo * *', 'VACUUM FULL'); + ERROR: invalid schedule: 0 11 $foo * * + HINT: Use cron format (e.g. 5 4 * * *), or interval format '[1-59] seconds' + -- cleaning +-DROP EXTENSION pg_cron; + drop user pgcron_cront; + drop database pgcron_dbno; + drop database pgcron_dbyes; +diff --git a/sql/pg_cron-test.sql b/sql/pg_cron-test.sql +index 45f94d9..8753c36 100644 +--- a/sql/pg_cron-test.sql ++++ b/sql/pg_cron-test.sql +@@ -1,7 +1,3 @@ +-CREATE EXTENSION pg_cron VERSION '1.0'; +-SELECT extversion FROM pg_extension WHERE extname='pg_cron'; +--- Test binary compatibility with v1.4 function signature. +-ALTER EXTENSION pg_cron UPDATE TO '1.4'; + SELECT cron.unschedule(job_name := 'no_such_job'); + SELECT cron.schedule('testjob', '* * * * *', 'SELECT 1'); + SELECT cron.unschedule('testjob'); +@@ -157,7 +153,6 @@ SELECT jobid, jobname, schedule, command FROM cron.job ORDER BY jobid; + SELECT cron.schedule('bad-last-dom-job1', '0 11 $foo * *', 'VACUUM FULL'); + + -- cleaning +-DROP EXTENSION pg_cron; + drop user pgcron_cront; + drop database pgcron_dbno; + drop database pgcron_dbyes; \ No newline at end of file diff --git a/docker-compose/ext-src/pg_cron-src/test-upgrade.sh b/docker-compose/ext-src/pg_cron-src/test-upgrade.sh new file mode 100755 index 000000000000..23481a4aaee8 --- /dev/null +++ b/docker-compose/ext-src/pg_cron-src/test-upgrade.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +patch -p1 "${TENANT_ID_FILE}" - echo "${TIMELINE_ID}" > "${TIMELINE_ID_FILE}" + tenant_id=$(psql -Aqt -c "SHOW neon.tenant_id") + timeline_id=$(psql -Aqt -c "SHOW neon.timeline_id") #XXX for ext in ${exts}; do for ext in ${EXTNAMES}; do echo Testing ${ext}... EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') + generate_id new_timeline_id + PARAMS=( + -sbf + -X POST + -H "Content-Type: application/json" + -d "{\"new_timeline_id\": \"${new_timeline_id}\", \"pg_version\": ${PG_VERSION}, \"ancestor_timeline_id\": \"${timeline_id}\"}" + "http://127.0.0.1:9898/v1/tenant/${tenant_id}/timeline/" + ) + result=$(curl "${PARAMS[@]}") + echo $result | jq . + echo "${tenant_id}" > "${TENANT_ID_FILE}" + echo "${new_timeline_id}" > "${TIMELINE_ID_FILE}" + TAG=${OLDTAG} docker compose down compute compute_is_ready + COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} docker compose up -d --build compute compute_is_ready + wait_for_ready + psql -c "SHOW neon.timeline_id" + if [ ${ext} == "pg_hintplan" ]; then + TMPDIR=$(mktemp -d) + # The following block does the same for the pg_hintplan test + docker compose cp neon-test-extensions:/ext-src/pg_hint_plan-src/data $TMPDIR/data + docker compose cp $TMPDIR/data compute:/ext-src/pg_hint_plan-src/ + rm -rf $TMPDIR + fi psql -d contrib_regression -c "\dx ${ext}" docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions sh -c /ext-src/${EXTDIR}/test-upgrade.sh psql -d contrib_regression -c "alter extension ${ext} update" From c98e6a9b998c694fdfa6dce18553349029264823 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Thu, 9 Jan 2025 14:23:31 +0100 Subject: [PATCH 06/25] Fix pg_cron --- .../ext-src/pg_cron-src/test-upgrade.patch | 74 +++++++++++++------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/docker-compose/ext-src/pg_cron-src/test-upgrade.patch b/docker-compose/ext-src/pg_cron-src/test-upgrade.patch index 9edaa9a930c1..7f187be6e284 100644 --- a/docker-compose/ext-src/pg_cron-src/test-upgrade.patch +++ b/docker-compose/ext-src/pg_cron-src/test-upgrade.patch @@ -1,45 +1,75 @@ diff --git a/expected/pg_cron-test.out b/expected/pg_cron-test.out -index d79d542..4239a6e 100644 +index d79d542..1663886 100644 --- a/expected/pg_cron-test.out +++ b/expected/pg_cron-test.out -@@ -1,12 +1,3 @@ +@@ -1,30 +1,3 @@ -CREATE EXTENSION pg_cron VERSION '1.0'; -SELECT extversion FROM pg_extension WHERE extname='pg_cron'; -- extversion +- extversion ------------- - 1.0 -(1 row) - --- Test binary compatibility with v1.4 function signature. -ALTER EXTENSION pg_cron UPDATE TO '1.4'; - SELECT cron.unschedule(job_name := 'no_such_job'); - ERROR: could not find valid entry for job 'no_such_job' - SELECT cron.schedule('testjob', '* * * * *', 'SELECT 1'); -@@ -301,7 +292,6 @@ SELECT cron.schedule('bad-last-dom-job1', '0 11 $foo * *', 'VACUUM FULL'); +-SELECT cron.unschedule(job_name := 'no_such_job'); +-ERROR: could not find valid entry for job 'no_such_job' +-SELECT cron.schedule('testjob', '* * * * *', 'SELECT 1'); +- schedule +----------- +- 1 +-(1 row) +- +-SELECT cron.unschedule('testjob'); +- unschedule +------------- +- t +-(1 row) +- +--- Test cache invalidation +-DROP EXTENSION pg_cron; +-CREATE EXTENSION pg_cron VERSION '1.4'; +-ALTER EXTENSION pg_cron UPDATE; + -- Vacuum every day at 10:00am (GMT) + SELECT cron.schedule('0 10 * * *', 'VACUUM'); + schedule +@@ -300,8 +273,3 @@ SELECT jobid, jobname, schedule, command FROM cron.job ORDER BY jobid; + SELECT cron.schedule('bad-last-dom-job1', '0 11 $foo * *', 'VACUUM FULL'); ERROR: invalid schedule: 0 11 $foo * * HINT: Use cron format (e.g. 5 4 * * *), or interval format '[1-59] seconds' - -- cleaning +--- cleaning -DROP EXTENSION pg_cron; - drop user pgcron_cront; - drop database pgcron_dbno; - drop database pgcron_dbyes; +-drop user pgcron_cront; +-drop database pgcron_dbno; +-drop database pgcron_dbyes; diff --git a/sql/pg_cron-test.sql b/sql/pg_cron-test.sql -index 45f94d9..8753c36 100644 +index 45f94d9..241cf73 100644 --- a/sql/pg_cron-test.sql +++ b/sql/pg_cron-test.sql -@@ -1,7 +1,3 @@ +@@ -1,17 +1,3 @@ -CREATE EXTENSION pg_cron VERSION '1.0'; -SELECT extversion FROM pg_extension WHERE extname='pg_cron'; --- Test binary compatibility with v1.4 function signature. -ALTER EXTENSION pg_cron UPDATE TO '1.4'; - SELECT cron.unschedule(job_name := 'no_such_job'); - SELECT cron.schedule('testjob', '* * * * *', 'SELECT 1'); - SELECT cron.unschedule('testjob'); -@@ -157,7 +153,6 @@ SELECT jobid, jobname, schedule, command FROM cron.job ORDER BY jobid; +-SELECT cron.unschedule(job_name := 'no_such_job'); +-SELECT cron.schedule('testjob', '* * * * *', 'SELECT 1'); +-SELECT cron.unschedule('testjob'); +- +--- Test cache invalidation +-DROP EXTENSION pg_cron; +-CREATE EXTENSION pg_cron VERSION '1.4'; +- +-ALTER EXTENSION pg_cron UPDATE; +- + -- Vacuum every day at 10:00am (GMT) + SELECT cron.schedule('0 10 * * *', 'VACUUM'); + +@@ -156,8 +142,3 @@ SELECT jobid, jobname, schedule, command FROM cron.job ORDER BY jobid; + -- invalid last of day job SELECT cron.schedule('bad-last-dom-job1', '0 11 $foo * *', 'VACUUM FULL'); - - -- cleaning + +--- cleaning -DROP EXTENSION pg_cron; - drop user pgcron_cront; - drop database pgcron_dbno; - drop database pgcron_dbyes; \ No newline at end of file +-drop user pgcron_cront; +-drop database pgcron_dbno; +-drop database pgcron_dbyes; From b19319baff0278ac3df1025d4b2a7f9ccb7c26a3 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Thu, 9 Jan 2025 15:06:52 +0100 Subject: [PATCH 07/25] use environment variables instead of files --- docker-compose/compute_wrapper/shell/compute.sh | 8 +++----- docker-compose/docker-compose.yml | 2 ++ docker-compose/test_extensions_upgrade.sh | 12 +++++++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docker-compose/compute_wrapper/shell/compute.sh b/docker-compose/compute_wrapper/shell/compute.sh index ba3c63846b8f..bfa01d028bde 100755 --- a/docker-compose/compute_wrapper/shell/compute.sh +++ b/docker-compose/compute_wrapper/shell/compute.sh @@ -13,8 +13,6 @@ PG_VERSION=${PG_VERSION:-14} SPEC_FILE_ORG=/var/db/postgres/specs/spec.json SPEC_FILE=/tmp/spec.json -TENANT_ID_FILE=/var/db/postgres/specs/tenant_id -TIMELINE_ID_FILE=/var/db/postgres/specs/timeline_id echo "Waiting pageserver become ready." while ! nc -z pageserver 6400; do @@ -22,9 +20,9 @@ while ! nc -z pageserver 6400; do done echo "Page server is ready." - if [ -f "${TENANT_ID_FILE}" ] && [ -f "${TIMELINE_ID_FILE}" ]; then - tenant_id=$(cat "${TENANT_ID_FILE}") - timeline_id=$(cat "${TIMELINE_ID_FILE}") + if [ -n "${TENANT_ID:-}" ] && [ -n "${TIMELINE_ID:-}" ]; then + tenant_id=${TENANT_ID} + timeline_id=${TIMELINE_ID} else echo "Check if a tenant present" PARAMS=( diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 458fd32e7d03..007d4edd2c12 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -154,6 +154,8 @@ services: - https_proxy=${https_proxy:-} environment: - PG_VERSION=${PG_VERSION:-16} + - TENANT_ID=${TENANT_ID:-} + - TIMELINE_ID=${TIMELINE_ID:-} #- RUST_BACKTRACE=1 # Mount the test files directly, for faster editing cycle. volumes: diff --git a/docker-compose/test_extensions_upgrade.sh b/docker-compose/test_extensions_upgrade.sh index 81e2a3a8ac37..d1d9d9836482 100755 --- a/docker-compose/test_extensions_upgrade.sh +++ b/docker-compose/test_extensions_upgrade.sh @@ -76,12 +76,14 @@ else ) result=$(curl "${PARAMS[@]}") echo $result | jq . - echo "${tenant_id}" > "${TENANT_ID_FILE}" - echo "${new_timeline_id}" > "${TIMELINE_ID_FILE}" - TAG=${OLDTAG} docker compose down compute compute_is_ready - COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} docker compose up -d --build compute compute_is_ready + TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} TAG=${OLDTAG} docker compose down compute compute_is_ready + COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} docker compose up -d --build compute compute_is_ready wait_for_ready - psql -c "SHOW neon.timeline_id" + TID=$(psql -Aqt -c "SHOW neon.timeline_id") + if [ ${TID} != ${new_timeline_id} ]; then + echo Timeline mismatch + exit 1 + fi if [ ${ext} == "pg_hintplan" ]; then TMPDIR=$(mktemp -d) # The following block does the same for the pg_hintplan test From 79479e5f49b24101bae4aaafe7b7028315739c72 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Thu, 9 Jan 2025 15:37:55 +0100 Subject: [PATCH 08/25] Add pg_uuidv7 --- docker-compose/ext-src/pg_uuidv7-src/test-upgrade.sh | 4 ++++ docker-compose/test_extensions_upgrade.sh | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100755 docker-compose/ext-src/pg_uuidv7-src/test-upgrade.sh diff --git a/docker-compose/ext-src/pg_uuidv7-src/test-upgrade.sh b/docker-compose/ext-src/pg_uuidv7-src/test-upgrade.sh new file mode 100755 index 000000000000..efbcbceb0f63 --- /dev/null +++ b/docker-compose/ext-src/pg_uuidv7-src/test-upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --dbname=contrib_regression 002_uuid_generate_v7 003_uuid_v7_to_timestamptz 004_uuid_timestamptz_to_v7 005_uuid_v7_to_timestamp 006_uuid_timestamp_to_v7 \ No newline at end of file diff --git a/docker-compose/test_extensions_upgrade.sh b/docker-compose/test_extensions_upgrade.sh index d1d9d9836482..a950a5f253e3 100755 --- a/docker-compose/test_extensions_upgrade.sh +++ b/docker-compose/test_extensions_upgrade.sh @@ -38,7 +38,8 @@ EXTENSIONS='[ {"extname": "ip4r", "extdir": "ip4r-src"}, {"extname": "prefix", "extdir": "prefix-src"}, {"extname": "hll", "extdir": "hll-src"}, -{"extname": "pg_cron", "extdir": "pg_cron-src"} +{"extname": "pg_cron", "extdir": "pg_cron-src"}, +{"extname": "pg_uuidv7", "extdir": "pg_uuidv7-src"} ]' EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) TAG=${NEWTAG} docker compose up --build -d @@ -98,4 +99,3 @@ else done fi docker compose --profile test-extensions down -rm -f "${TENANT_ID_FILE}" "${TIMELINE_ID_FILE}" From c6ab6ab3d4fe795a84faead5f5b569f51ac7b1b9 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Fri, 10 Jan 2025 10:56:11 +0100 Subject: [PATCH 09/25] Add more extensions --- .../ext-src/pg_hint_plan-src/test-upgrade.sh | 0 .../ext-src/pg_ivm-src/test-upgrade.patch | 18 +++++++++++++ .../ext-src/pg_ivm-src/test-upgrade.sh | 5 ++++ .../pg_roaringbitmap-src/test-upgrade.patch | 25 +++++++++++++++++++ .../pg_roaringbitmap-src/test-upgrade.sh | 5 ++++ .../ext-src/pg_semver-src/test-upgrade.patch | 24 ++++++++++++++++++ .../ext-src/pg_semver-src/test-upgrade.sh | 5 ++++ docker-compose/test_extensions_upgrade.sh | 5 +++- 8 files changed, 86 insertions(+), 1 deletion(-) mode change 100644 => 100755 docker-compose/ext-src/pg_hint_plan-src/test-upgrade.sh create mode 100644 docker-compose/ext-src/pg_ivm-src/test-upgrade.patch create mode 100755 docker-compose/ext-src/pg_ivm-src/test-upgrade.sh create mode 100644 docker-compose/ext-src/pg_roaringbitmap-src/test-upgrade.patch create mode 100755 docker-compose/ext-src/pg_roaringbitmap-src/test-upgrade.sh create mode 100644 docker-compose/ext-src/pg_semver-src/test-upgrade.patch create mode 100755 docker-compose/ext-src/pg_semver-src/test-upgrade.sh diff --git a/docker-compose/ext-src/pg_hint_plan-src/test-upgrade.sh b/docker-compose/ext-src/pg_hint_plan-src/test-upgrade.sh old mode 100644 new mode 100755 diff --git a/docker-compose/ext-src/pg_ivm-src/test-upgrade.patch b/docker-compose/ext-src/pg_ivm-src/test-upgrade.patch new file mode 100644 index 000000000000..77de3accfdc6 --- /dev/null +++ b/docker-compose/ext-src/pg_ivm-src/test-upgrade.patch @@ -0,0 +1,18 @@ +diff --git a/expected/pg_ivm.out b/expected/pg_ivm.out +index e8798ee..cca58d0 100644 +--- a/expected/pg_ivm.out ++++ b/expected/pg_ivm.out +@@ -1,4 +1,3 @@ +-CREATE EXTENSION pg_ivm; + GRANT ALL ON SCHEMA public TO public; + -- create a table to use as a basis for views and materialized views in various combinations + CREATE TABLE mv_base_a (i int, j int); +diff --git a/sql/pg_ivm.sql b/sql/pg_ivm.sql +index d3c1a01..9382d7f 100644 +--- a/sql/pg_ivm.sql ++++ b/sql/pg_ivm.sql +@@ -1,4 +1,3 @@ +-CREATE EXTENSION pg_ivm; + GRANT ALL ON SCHEMA public TO public; + + -- create a table to use as a basis for views and materialized views in various combinations diff --git a/docker-compose/ext-src/pg_ivm-src/test-upgrade.sh b/docker-compose/ext-src/pg_ivm-src/test-upgrade.sh new file mode 100755 index 000000000000..fee9689f82de --- /dev/null +++ b/docker-compose/ext-src/pg_ivm-src/test-upgrade.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +patch -p1 Date: Fri, 10 Jan 2025 11:05:23 +0100 Subject: [PATCH 10/25] Move pg_anon for PGv16 to compute --- docker-compose/compute_wrapper/shell/compute.sh | 8 +++++++- docker-compose/docker_compose_test.sh | 5 ----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker-compose/compute_wrapper/shell/compute.sh b/docker-compose/compute_wrapper/shell/compute.sh index bfa01d028bde..76dd9a1f5d9a 100755 --- a/docker-compose/compute_wrapper/shell/compute.sh +++ b/docker-compose/compute_wrapper/shell/compute.sh @@ -20,6 +20,12 @@ while ! nc -z pageserver 6400; do done echo "Page server is ready." +if [ ${PG_VERSION} == 16 ]; then + jq '.cluster.settings += [{"name": "session_preload_libraries","value": "anon","vartype": "string"}]' ${SPEC_FILE_ORG} > ${SPEC_FILE} +else + cp ${SPEC_FILE_ORG} ${SPEC_FILE} +fi + if [ -n "${TENANT_ID:-}" ] && [ -n "${TIMELINE_ID:-}" ]; then tenant_id=${TENANT_ID} timeline_id=${TIMELINE_ID} @@ -66,7 +72,7 @@ else fi echo "Overwrite tenant id and timeline id in spec file" -sed "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE_ORG} > ${SPEC_FILE} +sed -i "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE} sed -i "s/TIMELINE_ID/${timeline_id}/" ${SPEC_FILE} cat ${SPEC_FILE} diff --git a/docker-compose/docker_compose_test.sh b/docker-compose/docker_compose_test.sh index be8bfc121cc4..6ca36d76a1e3 100755 --- a/docker-compose/docker_compose_test.sh +++ b/docker-compose/docker_compose_test.sh @@ -33,11 +33,6 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do cleanup PG_TEST_VERSION=$((pg_version < 16 ? 16 : pg_version)) # The support of pg_anon not yet added to PG17, so we have to add the corresponding option for other PG versions - if [ "${pg_version}" -ne 17 ]; then - SPEC_PATH="compute_wrapper/var/db/postgres/specs" - mv $SPEC_PATH/spec.json $SPEC_PATH/spec.bak - jq '.cluster.settings += [{"name": "session_preload_libraries","value": "anon","vartype": "string"}]' "${SPEC_PATH}/spec.bak" > "${SPEC_PATH}/spec.json" - fi PG_VERSION=$pg_version PG_TEST_VERSION=$PG_TEST_VERSION docker compose --profile test-extensions -f $COMPOSE_FILE up --build -d echo "wait until the compute is ready. timeout after 60s. " From 2c346628312c5246f49d8faba557613053253481 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Fri, 10 Jan 2025 11:21:38 +0100 Subject: [PATCH 11/25] Some cleanup --- docker-compose/docker_compose_test.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose/docker_compose_test.sh b/docker-compose/docker_compose_test.sh index 6ca36d76a1e3..2ac93132273e 100755 --- a/docker-compose/docker_compose_test.sh +++ b/docker-compose/docker_compose_test.sh @@ -98,8 +98,4 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do fi fi cleanup - # Restore the original spec.json - if [ "$pg_version" -ne 17 ]; then - mv "$SPEC_PATH/spec.bak" "$SPEC_PATH/spec.json" - fi done From 0bbe722ef3999dbf54f096fd224618901a207922 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Fri, 10 Jan 2025 14:59:43 +0100 Subject: [PATCH 12/25] Add the test to workflow --- .github/workflows/build_and_test.yml | 12 ++++++++++++ compute/compute-node.Dockerfile | 1 + docker-compose/test_extensions_upgrade.sh | 23 ++++++++--------------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 55c4bf08b90d..67ab5e860731 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -924,6 +924,18 @@ jobs: TEST_VERSION_ONLY: ${{ matrix.pg_version }} run: ./docker-compose/docker_compose_test.sh + - name: Get the prevous staging build tag + id: oldtag + run: echo oldtag=$(git rev-list --count main) >>$GITHUB_OUTPUT + + - name: Test extension upgrade + timeout-minutes: 20 + if: ${{ needs.tag.outputs.build-tag == github.run_id }} + env: + NEWTAG: ${{ needs.tag.outputs.build-tag }} + OLDTAG: ${{ steps.oldtag.outputs.oldtag }} + run: ./docker-compose/test_extensions_upgrade.sh + - name: Print logs and clean up if: always() run: | diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index a9ca30e92f22..99c5b9d249d1 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1458,6 +1458,7 @@ RUN case "${PG_VERSION}" in "v17") \ echo "postgresql_anonymizer does not yet support PG17" && exit 0;; \ esac && patch -p1 Date: Fri, 10 Jan 2025 15:24:07 +0100 Subject: [PATCH 13/25] Change copy --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 99c5b9d249d1..e9db0d79f1e3 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1458,7 +1458,7 @@ RUN case "${PG_VERSION}" in "v17") \ echo "postgresql_anonymizer does not yet support PG17" && exit 0;; \ esac && patch -p1 Date: Fri, 10 Jan 2025 21:47:29 +0100 Subject: [PATCH 14/25] Change copy --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index b060ea8f3bde..1a0932b330c7 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1444,7 +1444,7 @@ RUN case "${PG_VERSION}" in "v17") \ echo "postgresql_anonymizer does not yet support PG17" && exit 0;; \ esac && patch -p1 Date: Mon, 13 Jan 2025 08:42:23 +0100 Subject: [PATCH 15/25] Workaround the problem of cache --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 1a0932b330c7..bffad05933e8 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1444,7 +1444,7 @@ RUN case "${PG_VERSION}" in "v17") \ echo "postgresql_anonymizer does not yet support PG17" && exit 0;; \ esac && patch -p1 Date: Mon, 13 Jan 2025 10:04:10 +0100 Subject: [PATCH 16/25] Fix the computing the main tag --- .github/workflows/build_and_test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 67eb34322dec..8266bb447bb3 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -886,6 +886,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193 - uses: docker/login-action@v3 @@ -926,7 +928,7 @@ jobs: - name: Get the prevous staging build tag id: oldtag - run: echo oldtag=$(git rev-list --count main) >>$GITHUB_OUTPUT + run: echo oldtag=$(git rev-list --count remotes/origin/main) >>$GITHUB_OUTPUT - name: Test extension upgrade timeout-minutes: 20 From a711d5015398dd0a5c673a7c8564291b9cce2ae3 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Mon, 13 Jan 2025 10:32:47 +0100 Subject: [PATCH 17/25] Add chdir --- docker-compose/test_extensions_upgrade.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose/test_extensions_upgrade.sh b/docker-compose/test_extensions_upgrade.sh index 82c7883c6dd6..b297e2a480fb 100755 --- a/docker-compose/test_extensions_upgrade.sh +++ b/docker-compose/test_extensions_upgrade.sh @@ -1,5 +1,6 @@ #!/bin/bash set -eux -o pipefail +cd "$(dirname "${0}")" # Takes a variable name as argument. The result is stored in that variable. generate_id() { local -n resvar=$1 From 5dbacbd0f4e1dde0ba5123cb163845bd52dd788a Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Mon, 13 Jan 2025 12:08:50 +0100 Subject: [PATCH 18/25] Do not rely on local psql --- docker-compose/test_extensions_upgrade.sh | 33 ++++++++++------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/docker-compose/test_extensions_upgrade.sh b/docker-compose/test_extensions_upgrade.sh index b297e2a480fb..441b4dfa1c4b 100755 --- a/docker-compose/test_extensions_upgrade.sh +++ b/docker-compose/test_extensions_upgrade.sh @@ -11,11 +11,6 @@ if [ -z ${OLDTAG+x} ] || [ -z ${NEWTAG+x} ] || [ -z "${OLDTAG}" ] || [ -z "${NEW exit 1 fi export PG_VERSION=${PG_VERSION:-16} -export PGUSER=cloud_admin -export PGPASSWORD=cloud_admin -export PGHOST=127.0.0.1 -export PGPORT=55433 -export PGDATABASE=postgres function wait_for_ready { TIME=0 while ! docker compose logs compute_is_ready | grep -q "accepting connections" && [ ${TIME} -le 300 ] ; do @@ -29,8 +24,8 @@ function wait_for_ready { } function create_extensions() { for ext in ${1}; do - echo "CREATE EXTENSION IF NOT EXISTS ${ext};" - done | psql -X -v ON_ERROR_STOP=1 + docker compose exec neon-test-extensions psql -X -v ON_ERROR_STOP=1 -d contrib_regression -c "CREATE EXTENSION IF NOT EXISTS ${ext}" + done } EXTENSIONS='[ {"extname": "plv8", "extdir": "plv8-src"}, @@ -48,26 +43,26 @@ EXTENSIONS='[ {"extname": "pg_ivm", "extdir": "pg_ivm-src"} ]' EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) -TAG=${NEWTAG} docker compose up --build -d +TAG=${NEWTAG} docker compose --profile test-extensions up --build -d wait_for_ready +docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression" create_extensions "${EXTNAMES}" query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /\',\'}')" -new_vers=$(psql -Aqt -c "$query" ) -docker compose down +new_vers=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query") +docker compose --profile test-extensions down TAG=${OLDTAG} docker compose --profile test-extensions up --build -d --force-recreate wait_for_ready # XXX this is about to be included into the image, for test only docker compose cp ext-src neon-test-extensions:/ -psql -c "CREATE DATABASE contrib_regression" -export PGDATABASE=contrib_regression +docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression" create_extensions "${EXTNAMES}" query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion" -exts=$(psql -Aqt -c "$query") +exts=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query") if [ -z "${exts}" ]; then echo "No extensions were upgraded" else - tenant_id=$(psql -Aqt -c "SHOW neon.tenant_id") - timeline_id=$(psql -Aqt -c "SHOW neon.timeline_id") + tenant_id=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.tenant_id") + timeline_id=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") for ext in ${exts}; do echo Testing ${ext}... EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') @@ -84,15 +79,15 @@ else TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} TAG=${OLDTAG} docker compose down compute compute_is_ready COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} docker compose up -d --build compute compute_is_ready wait_for_ready - TID=$(psql -Aqt -c "SHOW neon.timeline_id") + TID=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") if [ ${TID} != ${new_timeline_id} ]; then echo Timeline mismatch exit 1 fi - psql -d contrib_regression -c "\dx ${ext}" + docker compose exec neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions sh -c /ext-src/${EXTDIR}/test-upgrade.sh - psql -d contrib_regression -c "alter extension ${ext} update" - psql -d contrib_regression -c "\dx ${ext}" + docker compose exec neon-test-extensions psql -d contrib_regression -c "alter extension ${ext} update" + docker compose exec neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" done fi docker compose --profile test-extensions down From 6bb14b9184e1fe6312d28b9de752d32f5af7e615 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Mon, 13 Jan 2025 12:49:24 +0100 Subject: [PATCH 19/25] Use the previous release as old --- .github/workflows/build_and_test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 8266bb447bb3..5eca2d252f2b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -928,7 +928,10 @@ jobs: - name: Get the prevous staging build tag id: oldtag - run: echo oldtag=$(git rev-list --count remotes/origin/main) >>$GITHUB_OUTPUT + run: | + oldtag=$(git rev-list --count remotes/origin/main) + ((oldtag -= 1)) + echo oldtag=$oldtag >>$GITHUB_OUTPUT - name: Test extension upgrade timeout-minutes: 20 From 0f42dbaccdf926aa5c76a8dcedc8c17b520f2616 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Mon, 13 Jan 2025 13:32:01 +0100 Subject: [PATCH 20/25] Add an environment variable as a workaround for the "chicken or the egg" problem --- .github/workflows/build_and_test.yml | 9 +-------- docker-compose/test_extensions_upgrade.sh | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 5eca2d252f2b..e3a0b0744397 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -926,19 +926,12 @@ jobs: TEST_VERSION_ONLY: ${{ matrix.pg_version }} run: ./docker-compose/docker_compose_test.sh - - name: Get the prevous staging build tag - id: oldtag - run: | - oldtag=$(git rev-list --count remotes/origin/main) - ((oldtag -= 1)) - echo oldtag=$oldtag >>$GITHUB_OUTPUT - - name: Test extension upgrade timeout-minutes: 20 if: ${{ needs.tag.outputs.build-tag == github.run_id }} env: NEWTAG: ${{ needs.tag.outputs.build-tag }} - OLDTAG: ${{ steps.oldtag.outputs.oldtag }} + OLDTAG: latest run: ./docker-compose/test_extensions_upgrade.sh - name: Print logs and clean up diff --git a/docker-compose/test_extensions_upgrade.sh b/docker-compose/test_extensions_upgrade.sh index 441b4dfa1c4b..6608f0c7e3c9 100755 --- a/docker-compose/test_extensions_upgrade.sh +++ b/docker-compose/test_extensions_upgrade.sh @@ -45,24 +45,24 @@ EXTENSIONS='[ EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) TAG=${NEWTAG} docker compose --profile test-extensions up --build -d wait_for_ready -docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression" +docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -c "CREATE DATABASE contrib_regression" create_extensions "${EXTNAMES}" query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /\',\'}')" -new_vers=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query") +new_vers=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -d contrib_regression -c "$query") docker compose --profile test-extensions down TAG=${OLDTAG} docker compose --profile test-extensions up --build -d --force-recreate wait_for_ready # XXX this is about to be included into the image, for test only docker compose cp ext-src neon-test-extensions:/ -docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression" +docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -c "CREATE DATABASE contrib_regression" create_extensions "${EXTNAMES}" query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion" -exts=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query") +exts=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -d contrib_regression -c "$query") if [ -z "${exts}" ]; then echo "No extensions were upgraded" else - tenant_id=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.tenant_id") - timeline_id=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") + tenant_id=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -c "SHOW neon.tenant_id") + timeline_id=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") for ext in ${exts}; do echo Testing ${ext}... EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') @@ -79,15 +79,15 @@ else TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} TAG=${OLDTAG} docker compose down compute compute_is_ready COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} docker compose up -d --build compute compute_is_ready wait_for_ready - TID=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") + TID=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") if [ ${TID} != ${new_timeline_id} ]; then echo Timeline mismatch exit 1 fi - docker compose exec neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" + docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions sh -c /ext-src/${EXTDIR}/test-upgrade.sh - docker compose exec neon-test-extensions psql -d contrib_regression -c "alter extension ${ext} update" - docker compose exec neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" + docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -d contrib_regression -c "alter extension ${ext} update" + docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" done fi docker compose --profile test-extensions down From ce56d495ab55de3a233d759a1d4a68bf86f5d529 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Mon, 13 Jan 2025 13:43:29 +0100 Subject: [PATCH 21/25] Change the place for the password --- docker-compose/docker-compose.yml | 2 ++ docker-compose/test_extensions_upgrade.sh | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 007d4edd2c12..489d60f38c93 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -187,6 +187,8 @@ services: neon-test-extensions: profiles: ["test-extensions"] image: ${REPOSITORY:-neondatabase}/neon-test-extensions-v${PG_TEST_VERSION:-16}:${TAG:-latest} + environment: + - PGPASSWORD=cloud_admin entrypoint: - "/bin/bash" - "-c" diff --git a/docker-compose/test_extensions_upgrade.sh b/docker-compose/test_extensions_upgrade.sh index 6608f0c7e3c9..b1acaec6a4b2 100755 --- a/docker-compose/test_extensions_upgrade.sh +++ b/docker-compose/test_extensions_upgrade.sh @@ -45,24 +45,24 @@ EXTENSIONS='[ EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) TAG=${NEWTAG} docker compose --profile test-extensions up --build -d wait_for_ready -docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -c "CREATE DATABASE contrib_regression" +docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression" create_extensions "${EXTNAMES}" query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /\',\'}')" -new_vers=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -d contrib_regression -c "$query") +new_vers=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query") docker compose --profile test-extensions down TAG=${OLDTAG} docker compose --profile test-extensions up --build -d --force-recreate wait_for_ready # XXX this is about to be included into the image, for test only docker compose cp ext-src neon-test-extensions:/ -docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -c "CREATE DATABASE contrib_regression" +docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression" create_extensions "${EXTNAMES}" query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion" -exts=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -d contrib_regression -c "$query") +exts=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query") if [ -z "${exts}" ]; then echo "No extensions were upgraded" else - tenant_id=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -c "SHOW neon.tenant_id") - timeline_id=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") + tenant_id=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.tenant_id") + timeline_id=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") for ext in ${exts}; do echo Testing ${ext}... EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') @@ -79,15 +79,15 @@ else TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} TAG=${OLDTAG} docker compose down compute compute_is_ready COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} docker compose up -d --build compute compute_is_ready wait_for_ready - TID=$(docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") + TID=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") if [ ${TID} != ${new_timeline_id} ]; then echo Timeline mismatch exit 1 fi - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions sh -c /ext-src/${EXTDIR}/test-upgrade.sh - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -d contrib_regression -c "alter extension ${ext} update" - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" + docker compose exec neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" + docker compose exec neon-test-extensions sh -c /ext-src/${EXTDIR}/test-upgrade.sh + docker compose exec neon-test-extensions psql -d contrib_regression -c "alter extension ${ext} update" + docker compose exec neon-test-extensions psql -d contrib_regression -c "\dx ${ext}" done fi docker compose --profile test-extensions down From d37c3e584b7eadb678b03f667c635c6e19b5f250 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Mon, 13 Jan 2025 14:49:14 +0100 Subject: [PATCH 22/25] Change the batch files --- .github/workflows/build_and_test.yml | 2 -- docker-compose/ext-src/hll-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/hypopg-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/ip4r-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/pg_cron-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/pg_hint_plan-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/pg_ivm-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/pg_roaringbitmap-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/pg_semver-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/pg_uuidv7-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/pgvector-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/plv8-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/prefix-src/test-upgrade.sh | 3 ++- docker-compose/ext-src/rum-src/test-upgrade.sh | 3 ++- 15 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index e3a0b0744397..d4a416abba0f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -886,8 +886,6 @@ jobs: steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193 - uses: docker/login-action@v3 diff --git a/docker-compose/ext-src/hll-src/test-upgrade.sh b/docker-compose/ext-src/hll-src/test-upgrade.sh index fbc206b9092f..f9e9aedcb2a4 100755 --- a/docker-compose/ext-src/hll-src/test-upgrade.sh +++ b/docker-compose/ext-src/hll-src/test-upgrade.sh @@ -1,4 +1,5 @@ #!/bin/sh set -ex cd "$(dirname ${0})" -/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin' --dbname=contrib_regression add_agg agg_oob auto_sparse card_op cast_shape copy_binary cumulative_add_cardinality_correction cumulative_add_comprehensive_promotion cumulative_add_sparse_edge cumulative_add_sparse_random cumulative_add_sparse_step cumulative_union_comprehensive cumulative_union_explicit_explicit cumulative_union_explicit_promotion cumulative_union_probabilistic_probabilistic cumulative_union_sparse_full_representation cumulative_union_sparse_promotion cumulative_union_sparse_sparse disable_hashagg equal explicit_thresh hash hash_any meta_func murmur_bigint murmur_bytea nosparse notequal scalar_oob storedproc transaction typmod typmod_insert union_op \ No newline at end of file +PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress +${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin' --dbname=contrib_regression add_agg agg_oob auto_sparse card_op cast_shape copy_binary cumulative_add_cardinality_correction cumulative_add_comprehensive_promotion cumulative_add_sparse_edge cumulative_add_sparse_random cumulative_add_sparse_step cumulative_union_comprehensive cumulative_union_explicit_explicit cumulative_union_explicit_promotion cumulative_union_probabilistic_probabilistic cumulative_union_sparse_full_representation cumulative_union_sparse_promotion cumulative_union_sparse_sparse disable_hashagg equal explicit_thresh hash hash_any meta_func murmur_bigint murmur_bytea nosparse notequal scalar_oob storedproc transaction typmod typmod_insert union_op \ No newline at end of file diff --git a/docker-compose/ext-src/hypopg-src/test-upgrade.sh b/docker-compose/ext-src/hypopg-src/test-upgrade.sh index 906f879d0474..066ac3329e5b 100755 --- a/docker-compose/ext-src/hypopg-src/test-upgrade.sh +++ b/docker-compose/ext-src/hypopg-src/test-upgrade.sh @@ -2,4 +2,5 @@ set -ex cd "$(dirname ${0})" patch -p1 Date: Tue, 14 Jan 2025 14:29:58 +0100 Subject: [PATCH 23/25] Remove the pg_anon-related stuff. --- docker-compose/compute_wrapper/shell/compute.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker-compose/compute_wrapper/shell/compute.sh b/docker-compose/compute_wrapper/shell/compute.sh index 76dd9a1f5d9a..b4f8d3d66abc 100755 --- a/docker-compose/compute_wrapper/shell/compute.sh +++ b/docker-compose/compute_wrapper/shell/compute.sh @@ -20,11 +20,7 @@ while ! nc -z pageserver 6400; do done echo "Page server is ready." -if [ ${PG_VERSION} == 16 ]; then - jq '.cluster.settings += [{"name": "session_preload_libraries","value": "anon","vartype": "string"}]' ${SPEC_FILE_ORG} > ${SPEC_FILE} -else - cp ${SPEC_FILE_ORG} ${SPEC_FILE} -fi +cp ${SPEC_FILE_ORG} ${SPEC_FILE} if [ -n "${TENANT_ID:-}" ] && [ -n "${TIMELINE_ID:-}" ]; then tenant_id=${TENANT_ID} From e76d66a13ce8fdf824a93df837d56f66db796906 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Tue, 14 Jan 2025 14:34:46 +0100 Subject: [PATCH 24/25] Cleanup of Dockerfile --- compute/compute-node.Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index bffad05933e8..e9fc899ebb93 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1444,12 +1444,10 @@ RUN case "${PG_VERSION}" in "v17") \ echo "postgresql_anonymizer does not yet support PG17" && exit 0;; \ esac && patch -p1 Date: Tue, 14 Jan 2025 14:37:54 +0100 Subject: [PATCH 25/25] Cleanup of Dockerfile --- compute/compute-node.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index e9fc899ebb93..06aaf9e7f423 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1391,7 +1391,6 @@ RUN make PG_VERSION="${PG_VERSION}" -C compute FROM neon-pg-ext-build AS neon-pg-ext-test ARG PG_VERSION -RUN apt-get update && apt-get install -y curl jq RUN mkdir /ext-src #COPY --from=postgis-build /postgis.tar.gz /ext-src/